tests, report unknown cfg, rm catalog vector flds (#551)

* clean up reporting of the un-used config params - instead of printing,
collect them and print in one place if needed (allows testing too)
* remove `vector_layer` in catalog - too verbose, not needed - can be
received via tilejson for individual source
* clean up tests so that they all use the same config yaml

src/config.rs

@@ -6,7 +6,6 @@ use std::path::Path;
 use std::pin::Pin;
 
 use futures::future::try_join_all;
-use log::warn;
 use serde::{Deserialize, Serialize};
 use serde_yaml::Value;
 use subst::VariableMap;
@@ -40,12 +39,13 @@ pub struct Config {
 
 impl Config {
     /// Apply defaults to the config, and validate if there is a connection string
-    pub fn finalize(&mut self) -> Result<()> {
+    pub fn finalize(&mut self) -> Result<Unrecognized> {
-        report_unrecognized_config("", &self.unrecognized);
+        let mut res = Unrecognized::new();
+        copy_unrecognized_config(&mut res, "", &self.unrecognized);
 
         let mut any = if let Some(pg) = &mut self.postgres {
             for pg in pg.iter_mut() {
-                pg.finalize()?;
+                res.extend(pg.finalize()?);
             }
             !pg.is_empty()
         } else {
@@ -53,21 +53,21 @@ impl Config {
         };
 
         any |= if let Some(cfg) = &mut self.pmtiles {
-            cfg.finalize("pmtiles.")?;
+            res.extend(cfg.finalize("pmtiles.")?);
             !cfg.is_empty()
         } else {
             false
         };
 
         any |= if let Some(cfg) = &mut self.mbtiles {
-            cfg.finalize("mbtiles.")?;
+            res.extend(cfg.finalize("mbtiles.")?);
             !cfg.is_empty()
         } else {
             false
         };
 
         if any {
-            Ok(())
+            Ok(res)
         } else {
             Err(NoSources)
         }
@@ -103,10 +103,18 @@ impl Config {
     }
 }
 
-pub fn report_unrecognized_config(prefix: &str, unrecognized: &HashMap<String, Value>) {
+pub type Unrecognized = HashMap<String, Value>;
-    for key in unrecognized.keys() {
+
-        warn!("Unrecognized config key: {prefix}{key}");
+pub fn copy_unrecognized_config(
-    }
+    result: &mut Unrecognized,
+    prefix: &str,
+    unrecognized: &Unrecognized,
+) {
+    result.extend(
+        unrecognized
+            .iter()
+            .map(|(k, v)| (format!("{prefix}{k}"), v.clone())),
+    );
 }
 
 /// Read config from a file
@@ -142,7 +150,8 @@ pub mod tests {
 
     pub fn assert_config(yaml: &str, expected: &Config) {
         let mut config = parse_cfg(yaml);
-        config.finalize().unwrap();
+        let res = config.finalize().unwrap();
+        assert!(res.is_empty(), "unrecognized config: {res:?}");
         assert_eq!(&config, expected);
     }
 }

src/file_config.rs

@@ -8,7 +8,7 @@ use log::{info, warn};
 use serde::{Deserialize, Serialize};
 use serde_yaml::Value;
 
-use crate::config::report_unrecognized_config;
+use crate::config::{copy_unrecognized_config, Unrecognized};
 use crate::file_config::FileError::{InvalidFilePath, InvalidSourceFilePath};
 use crate::utils::sorted_opt_map;
 use crate::OneOrMany::{Many, One};
@@ -84,11 +84,12 @@ pub struct FileConfigSource {
 }
 
 impl FileConfigEnum {
-    pub fn finalize(&self, prefix: &str) -> Result<&Self, Error> {
+    pub fn finalize(&self, prefix: &str) -> Result<Unrecognized, Error> {
+        let mut res = Unrecognized::new();
         if let Self::Config(cfg) = self {
-            report_unrecognized_config(prefix, &cfg.unrecognized);
+            copy_unrecognized_config(&mut res, prefix, &cfg.unrecognized);
         }
-        Ok(self)
+        Ok(res)
     }
 
     #[must_use]
@@ -239,7 +240,8 @@ mod tests {
                   path: /tmp/file.ext
         "})
         .unwrap();
-        cfg.finalize("").unwrap();
+        let res = cfg.finalize("").unwrap();
+        assert!(res.is_empty(), "unrecognized config: {res:?}");
         let FileConfigEnum::Config(cfg) = cfg else {
             panic!();
         };

src/pg/config.rs

@@ -2,7 +2,7 @@ use futures::future::try_join;
 use serde::{Deserialize, Serialize};
 use tilejson::TileJSON;
 
-use crate::config::report_unrecognized_config;
+use crate::config::{copy_unrecognized_config, Unrecognized};
 use crate::pg::config_function::FuncInfoSources;
 use crate::pg::config_table::TableInfoSources;
 use crate::pg::configurator::PgBuilder;
@@ -55,22 +55,23 @@ pub struct PgCfgPublishType {
 
 impl PgConfig {
     /// Apply defaults to the config, and validate if there is a connection string
-    pub fn finalize(&mut self) -> Result<&Self> {
+    pub fn finalize(&mut self) -> Result<Unrecognized> {
+        let mut res = Unrecognized::new();
         if let Some(ref ts) = self.tables {
             for (k, v) in ts {
-                report_unrecognized_config(&format!("tables.{k}."), &v.unrecognized);
+                copy_unrecognized_config(&mut res, &format!("tables.{k}."), &v.unrecognized);
             }
         }
         if let Some(ref fs) = self.functions {
             for (k, v) in fs {
-                report_unrecognized_config(&format!("functions.{k}."), &v.unrecognized);
+                copy_unrecognized_config(&mut res, &format!("functions.{k}."), &v.unrecognized);
             }
         }
         if self.tables.is_none() && self.functions.is_none() && self.auto_publish.is_none() {
             self.auto_publish = Some(BoolOrObject::Bool(true));
         }
 
-        Ok(self)
+        Ok(res)
     }
 
     pub async fn resolve(&mut self, id_resolver: IdResolver) -> crate::Result<Sources> {

src/srv/server.rs

@@ -18,7 +18,7 @@ use itertools::Itertools;
 use log::{debug, error};
 use martin_tile_utils::DataFormat;
 use serde::{Deserialize, Serialize};
-use tilejson::{TileJSON, VectorLayer};
+use tilejson::TileJSON;
 
 use crate::source::{Source, Sources, UrlQuery, Xyz};
 use crate::srv::config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT};
@@ -106,8 +106,6 @@ pub struct IndexEntry {
     pub description: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub attribution: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub vector_layer: Option<Vec<VectorLayer>>,
 }
 
 impl PartialOrd<Self> for IndexEntry {
@@ -159,7 +157,6 @@ async fn get_catalog(state: Data<AppState>) -> impl Responder {
                 name: tilejson.name,
                 description: tilejson.description,
                 attribution: tilejson.attribution,
-                vector_layer: tilejson.vector_layers,
             }
         })
         .sorted()

tests/expected/auto/catalog_auto.json

@@ -126,17 +126,6 @@
     "content_type": "application/x-protobuf",
     "description": "Major cities from Natural Earth data",
     "id": "world_cities",
-    "name": "Major cities from Natural Earth data",
+    "name": "Major cities from Natural Earth data"
-    "vector_layer": [
-      {
-        "description": "",
-        "fields": {
-          "name": "String"
-        },
-        "id": "cities",
-        "maxzoom": 6,
-        "minzoom": 0
-      }
-    ]
   }
 ]

tests/pg_function_source_test.rs

@@ -1,7 +1,6 @@
 use ctor::ctor;
 use indoc::indoc;
 use itertools::Itertools;
-use martin::pg::get_function_sources;
 use martin::Xyz;
 
 pub mod utils;
@@ -12,26 +11,6 @@ fn init() {
     let _ = env_logger::builder().is_test(true).try_init();
 }
 
-#[actix_rt::test]
-async fn get_function_sources_ok() {
-    let pool = mock_pool().await;
-    let sources = get_function_sources(&pool).await.unwrap();
-
-    assert!(!sources.is_empty());
-
-    let funcs = sources.get("public").unwrap();
-    let source = funcs.get("function_zxy_query").unwrap();
-    assert_eq!(source.1.schema, "public");
-    assert_eq!(source.1.function, "function_zxy_query");
-    assert_eq!(source.1.minzoom, None);
-    assert_eq!(source.1.maxzoom, None);
-    assert_eq!(source.1.bounds, None);
-
-    let source = funcs.get("function_zxy_query_jsonb").unwrap();
-    assert_eq!(source.1.schema, "public");
-    assert_eq!(source.1.function, "function_zxy_query_jsonb");
-}
-
 #[actix_rt::test]
 async fn function_source_tilejson() {
     let mock = mock_sources(mock_pgcfg("connection_string: $DATABASE_URL")).await;

tests/pg_server_test.rs

@@ -16,8 +16,9 @@ fn init() {
 }
 
 macro_rules! create_app {
-    ($sources:literal) => {{
+    ($sources:expr) => {{
-        let sources = mock_sources(mock_pgcfg($sources)).await.0;
+        let cfg = mock_cfg(indoc::indoc!($sources));
+        let sources = mock_sources(cfg).await.0;
         let state = crate::utils::mock_app_data(sources).await;
         ::actix_web::test::init_service(
             ::actix_web::App::new()
@@ -34,7 +35,10 @@ fn test_get(path: &str) -> Request {
 
 #[actix_rt::test]
 async fn pg_get_catalog_ok() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+   connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/catalog");
     let response = call_service(&app, req).await;
@@ -55,8 +59,9 @@ async fn pg_get_catalog_ok() {
 #[actix_rt::test]
 async fn pg_get_table_source_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     bad_srid:
       schema: public
       table: table_source
@@ -84,6 +89,24 @@ tables:
     let req = test_get("/bad_srid");
     let response = call_service(&app, req).await;
     assert_eq!(response.status(), StatusCode::NOT_FOUND);
+}
+
+#[actix_rt::test]
+async fn pg_get_table_source_ok_rewrite() {
+    let app = create_app! { "
+postgres:
+  connection_string: $DATABASE_URL
+  tables:
+    table_source:
+      schema: public
+      table: table_source
+      srid: 4326
+      geometry_column: geom
+      bounds: [-180.0, -90.0, 180.0, 90.0]
+      geometry_type: GEOMETRY
+      properties:
+        gid: int4
+" };
 
     let req = TestRequest::get()
         .uri("/table_source?token=martin")
@@ -101,8 +124,9 @@ tables:
 #[actix_rt::test]
 async fn pg_get_table_source_tile_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     points2:
       schema: public
       table: points2
@@ -192,8 +216,9 @@ tables:
 #[actix_rt::test]
 async fn pg_get_table_source_multiple_geom_tile_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     points2:
       schema: public
       table: points2
@@ -283,8 +308,9 @@ tables:
 #[actix_rt::test]
 async fn pg_get_table_source_tile_minmax_zoom_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     points3857:
       schema: public
       table: points3857
@@ -389,7 +415,10 @@ tables:
 
 #[actix_rt::test]
 async fn pg_get_function_tiles() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+   connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/function_zoom_xy/6/38/20");
     assert!(call_service(&app, req).await.status().is_success());
@@ -419,8 +448,9 @@ async fn pg_get_function_tiles() {
 #[actix_rt::test]
 async fn pg_get_composite_source_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     table_source_multiple_geom.geom2:
       schema: public
       table: table_source_multiple_geom
@@ -509,8 +539,9 @@ tables:
 #[actix_rt::test]
 async fn pg_get_composite_source_tile_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     points_empty_srid:
       schema: public
       table: points_empty_srid
@@ -600,8 +631,9 @@ tables:
 #[actix_rt::test]
 async fn pg_get_composite_source_tile_minmax_zoom_ok() {
     let app = create_app! { "
-connection_string: $DATABASE_URL
+postgres:
-tables:
+  connection_string: $DATABASE_URL
+  tables:
     points1:
       schema: public
       table: points1
@@ -664,7 +696,10 @@ tables:
 
 #[actix_rt::test]
 async fn pg_null_functions() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+   connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/function_null/0/0/0");
     let response = call_service(&app, req).await;
@@ -681,7 +716,10 @@ async fn pg_null_functions() {
 
 #[actix_rt::test]
 async fn pg_get_function_source_ok() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+   connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/non_existent");
     let response = call_service(&app, req).await;
@@ -718,6 +756,14 @@ async fn pg_get_function_source_ok() {
     let req = test_get("/function_zxy_row_key");
     let response = call_service(&app, req).await;
     assert!(response.status().is_success());
+}
+
+#[actix_rt::test]
+async fn pg_get_function_source_ok_rewrite() {
+    let app = create_app! { "
+postgres:
+  connection_string: $DATABASE_URL
+"};
 
     let req = TestRequest::get()
         .uri("/function_zxy_query?token=martin")
@@ -745,7 +791,10 @@ async fn pg_get_function_source_ok() {
 
 #[actix_rt::test]
 async fn pg_get_function_source_tile_ok() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+  connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/function_zxy_query/0/0/0");
     let response = call_service(&app, req).await;
@@ -755,8 +804,9 @@ async fn pg_get_function_source_tile_ok() {
 #[actix_rt::test]
 async fn pg_get_function_source_tile_minmax_zoom_ok() {
     let app = create_app! {"
-connection_string: $DATABASE_URL
+postgres:
-functions:
+  connection_string: $DATABASE_URL
+  functions:
     function_source1:
       schema: public
       function: function_zxy_query
@@ -811,7 +861,10 @@ functions:
 
 #[actix_rt::test]
 async fn pg_get_function_source_query_params_ok() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+  connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/function_zxy_query_test/0/0/0");
     let response = call_service(&app, req).await;
@@ -825,7 +878,10 @@ async fn pg_get_function_source_query_params_ok() {
 
 #[actix_rt::test]
 async fn pg_get_health_returns_ok() {
-    let app = create_app! { "connection_string: $DATABASE_URL" };
+    let app = create_app! { "
+postgres:
+  connection_string: $DATABASE_URL
+"};
 
     let req = test_get("/health");
     let response = call_service(&app, req).await;
@@ -903,7 +959,7 @@ tables:
 
     // --------------------------------------------
 
-    let state = crate::utils::mock_app_data(mock.0).await;
+    let state = mock_app_data(mock.0).await;
     let app = ::actix_web::test::init_service(
         ::actix_web::App::new()
             .app_data(state)

tests/utils/mod.rs

@@ -25,6 +25,7 @@ pub fn mock_cfg(yaml: &str) -> Config {
     };
     let env = FauxEnv(vec![("DATABASE_URL", db_url.into())].into_iter().collect());
     let mut cfg: Config = subst::yaml::from_str(yaml, &env).unwrap();
-    cfg.finalize().unwrap();
+    let res = cfg.finalize().unwrap();
+    assert!(res.is_empty(), "unrecognized config: {res:?}");
     cfg
 }

tests/utils/pg_utils.rs

@@ -1,9 +1,9 @@
+use indoc::formatdoc;
 pub use martin::args::Env;
-use martin::pg::{PgConfig, Pool, TableInfo};
+use martin::pg::TableInfo;
-use martin::OneOrMany::One;
+use martin::{Config, IdResolver, Source, Sources};
-use martin::{Config, IdResolver, OneOrMany, Source, Sources};
 
-use crate::FauxEnv;
+use crate::mock_cfg;
 
 //
 // This file is used by many tests and benchmarks.
@@ -15,25 +15,10 @@ pub type MockSource = (Sources, Config);
 #[allow(dead_code)]
 #[must_use]
 pub fn mock_pgcfg(yaml: &str) -> Config {
-    let Ok(db_url) = std::env::var("DATABASE_URL") else {
+    mock_cfg(&formatdoc! {"
-        panic!("DATABASE_URL env var is not set. Unable to do integration tests");
+        postgres:
-    };
+          {}
-    let env = FauxEnv(vec![("DATABASE_URL", db_url.into())].into_iter().collect());
+    ", yaml.replace('\n', "\n  ")})
-    let cfg: PgConfig = subst::yaml::from_str(yaml, &env).unwrap();
-    let mut config = Config {
-        postgres: Some(One(cfg)),
-        ..Default::default()
-    };
-    config.finalize().unwrap();
-    config
-}
-
-#[allow(dead_code)]
-pub async fn mock_pool() -> Pool {
-    let cfg = mock_pgcfg("connection_string: $DATABASE_URL");
-    let OneOrMany::One(cfg) = cfg.postgres.unwrap() else { panic!() };
-    let res = Pool::new(&cfg).await;
-    res.expect("Failed to create pool")
 }
 
 #[allow(dead_code)]