Fix mbtiles validation, CI, and logging (#903)

Cargo.lock

@@ -1799,6 +1799,7 @@ dependencies = [
  "actix-rt",
  "anyhow",
  "clap",
+ "env_logger",
  "futures",
  "log",
  "martin-tile-utils",

justfile

@@ -110,6 +110,7 @@ test-int: clean-test install-sqlx
 
 # Run integration tests and save its output as the new expected output
 bless: start clean-test
+    rm -rf tests/temp
     cargo test --features bless-tests
     tests/test.sh
     rm -rf tests/expected
@@ -216,7 +217,7 @@ git-pre-push: stop start
 # Update sqlite database schema.
 prepare-sqlite: install-sqlx
     mkdir -p martin-mbtiles/.sqlx
-    cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles -- --lib --tests
+    cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/mbtiles/world_cities.mbtiles -- --lib --tests
 
 # Install SQLX cli if not already installed.
 [private]

martin-mbtiles/Cargo.toml

@@ -12,7 +12,7 @@ rust-version.workspace = true
 [features]
 # TODO: Disable "cli" feature in default builds
 default = ["cli", "native-tls"]
-cli = ["dep:anyhow", "dep:clap", "dep:tokio"]
+cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:tokio"]
 # One of the following two must be used
 native-tls = ["sqlx/runtime-tokio-native-tls"]
 rustls = ["sqlx/runtime-tokio-rustls"]
@@ -30,6 +30,7 @@ tilejson.workspace = true
 # Bin dependencies
 anyhow = { workspace = true, optional = true }
 clap = { workspace = true, optional = true }
+env_logger = { workspace = true, optional = true }
 serde_yaml.workspace = true
 sqlite-hashes.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }

martin-mbtiles/src/bin/main.rs

@@ -1,11 +1,10 @@
 use std::path::{Path, PathBuf};
 
 use clap::{Parser, Subcommand};
+use log::{error, LevelFilter};
 use martin_mbtiles::{
     apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions,
 };
-use sqlx::sqlite::SqliteConnectOptions;
-use sqlx::{Connection, SqliteConnection};
 
 #[derive(Parser, PartialEq, Eq, Debug)]
 #[command(
@@ -73,9 +72,23 @@ enum Commands {
 }
 
 #[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    let args = Args::parse();
+async fn main() {
+    env_logger::builder()
+        .filter_level(LevelFilter::Info)
+        .format_indent(None)
+        .format_module_path(false)
+        .format_target(false)
+        .format_timestamp(None)
+        .init();
 
+    if let Err(err) = main_int().await {
+        error!("{err}");
+        std::process::exit(1);
+    }
+}
+
+async fn main_int() -> anyhow::Result<()> {
+    let args = Args::parse();
     match args.command {
         Commands::MetaAll { file } => {
             meta_print_all(file.as_path()).await?;
@@ -109,8 +122,7 @@ async fn main() -> anyhow::Result<()> {
 
 async fn meta_print_all(file: &Path) -> anyhow::Result<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(true).await?;
     let metadata = mbt.get_metadata(&mut conn).await?;
     println!("{}", serde_yaml::to_string(&metadata)?);
     Ok(())
@@ -118,8 +130,7 @@ async fn meta_print_all(file: &Path) -> anyhow::Result<()> {
 
 async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(true).await?;
     if let Some(s) = mbt.get_metadata_value(&mut conn, key).await? {
         println!("{s}");
     }
@@ -128,8 +139,7 @@ async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
 
 async fn meta_set_value(file: &Path, key: &str, value: Option<String>) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(false).await?;
     mbt.set_metadata_value(&mut conn, key, value).await
 }
 
@@ -139,8 +149,7 @@ async fn validate_mbtiles(
     update_agg_tiles_hash: bool,
 ) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(!update_agg_tiles_hash).await?;
     mbt.check_integrity(&mut conn, check_type).await?;
     mbt.check_each_tile_hash(&mut conn).await?;
     if update_agg_tiles_hash {

martin-mbtiles/src/mbtiles.rs

@@ -435,7 +435,9 @@ impl Mbtiles {
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
+        let filepath = self.filepath();
         if integrity_check == IntegrityCheckType::Off {
+            info!("Skipping integrity check for {filepath}");
             return Ok(());
         }
 
@@ -452,13 +454,14 @@ impl Mbtiles {
 
         if result.len() > 1
             || result.get(0).ok_or(FailedIntegrityCheck(
-                self.filepath().to_string(),
+                filepath.to_string(),
                 vec!["SQLite could not perform integrity check".to_string()],
             ))? != "ok"
         {
             return Err(FailedIntegrityCheck(self.filepath().to_string(), result));
         }
 
+        info!("{integrity_check:?} integrity check passed for {filepath}");
         Ok(())
     }
 
@@ -466,16 +469,17 @@ impl Mbtiles {
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
+        let filepath = self.filepath();
         let Some(stored) = self.get_agg_tiles_hash(&mut *conn).await? else {
-            return Err(AggHashValueNotFound(self.filepath().to_string()));
+            return Err(AggHashValueNotFound(filepath.to_string()));
         };
-
         let computed = calc_agg_tiles_hash(&mut *conn).await?;
         if stored != computed {
-            let file = self.filepath().to_string();
+            let file = filepath.to_string();
             return Err(AggHashMismatch(computed, stored, file));
         }
 
+        info!("The agg_tiles_hashes={computed} has been verified for {filepath}");
         Ok(())
     }
 
@@ -486,23 +490,15 @@ impl Mbtiles {
     {
         let old_hash = self.get_agg_tiles_hash(&mut *conn).await?;
         let hash = calc_agg_tiles_hash(&mut *conn).await?;
+        let path = self.filepath();
         if old_hash.as_ref() == Some(&hash) {
-            info!(
-                "agg_tiles_hash is already set to the correct value `{hash}` in {}",
-                self.filepath()
-            );
+            info!("agg_tiles_hash is already set to the correct value `{hash}` in {path}");
             Ok(())
         } else {
             if let Some(old_hash) = old_hash {
-                info!(
-                    "Updating agg_tiles_hash from {old_hash} to {hash} in {}",
-                    self.filepath()
-                );
+                info!("Updating agg_tiles_hash from {old_hash} to {hash} in {path}");
             } else {
-                info!(
-                    "Initializing agg_tiles_hash to {hash} in {}",
-                    self.filepath()
-                );
+                info!("Creating new metadata value agg_tiles_hash = {hash} in {path}");
             }
             self.set_metadata_value(&mut *conn, "agg_tiles_hash", Some(hash))
                 .await
@@ -550,7 +546,10 @@ impl Mbtiles {
                     v.get(0),
                     v.get(1),
                 ))
-            })
+            })?;
+
+        info!("All tile hashes are valid for {}", self.filepath());
+        Ok(())
     }
 }
 
@@ -610,7 +609,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn mbtiles_meta() {
-        let filepath = "../tests/fixtures/files/geography-class-jpg.mbtiles";
+        let filepath = "../tests/fixtures/mbtiles/geography-class-jpg.mbtiles";
         let mbt = Mbtiles::new(filepath).unwrap();
         assert_eq!(mbt.filepath(), filepath);
         assert_eq!(mbt.filename(), "geography-class-jpg");
@@ -618,7 +617,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn metadata_jpeg() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await;
         let metadata = mbt.get_metadata(&mut conn).await.unwrap();
         let tj = metadata.tilejson;
 
@@ -635,7 +634,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn metadata_mvt() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
         let metadata = mbt.get_metadata(&mut conn).await.unwrap();
         let tj = metadata.tilejson;
 
@@ -666,7 +665,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn metadata_get_key() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
 
         let res = mbt.get_metadata_value(&mut conn, "bounds").await.unwrap();
         assert_eq!(res.unwrap(), "-123.123590,-37.818085,174.763027,59.352706");
@@ -726,15 +725,15 @@ mod tests {
 
     #[actix_rt::test]
     async fn detect_type() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
         assert_eq!(res, MbtType::Flat);
 
-        let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
         assert_eq!(res, MbtType::FlatWithHash);
 
-        let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
         assert_eq!(res, MbtType::Normalized);
 
@@ -745,7 +744,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn validate_valid_file() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await;
 
         mbt.check_integrity(&mut conn, IntegrityCheckType::Quick)
             .await
@@ -755,7 +754,7 @@ mod tests {
     #[actix_rt::test]
     async fn validate_invalid_file() {
         let (mut conn, mbt) =
-            open("../tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles").await;
+            open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await;
         let result = mbt.check_agg_tiles_hashes(&mut conn).await;
         assert!(matches!(result, Err(MbtError::AggHashMismatch(..))));
     }

martin-mbtiles/src/tile_copier.rs

@@ -603,14 +603,14 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_flat_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst = PathBuf::from("file:copy_flat_tables_mem_db?mode=memory&cache=shared");
         verify_copy_all(src, dst, None, Flat).await
     }
 
     #[actix_rt::test]
     async fn copy_flat_from_flat_with_hash_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles");
         let dst = PathBuf::from(
             "file:copy_flat_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared",
         );
@@ -619,7 +619,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_flat_from_normalized_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles");
         let dst =
             PathBuf::from("file:copy_flat_from_normalized_tables_mem_db?mode=memory&cache=shared");
         verify_copy_all(src, dst, Some(Flat), Flat).await
@@ -627,14 +627,14 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_flat_with_hash_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles");
         let dst = PathBuf::from("file:copy_flat_with_hash_tables_mem_db?mode=memory&cache=shared");
         verify_copy_all(src, dst, None, FlatWithHash).await
     }
 
     #[actix_rt::test]
     async fn copy_flat_with_hash_from_flat_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst = PathBuf::from(
             "file:copy_flat_with_hash_from_flat_tables_mem_db?mode=memory&cache=shared",
         );
@@ -643,7 +643,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_flat_with_hash_from_normalized_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles");
         let dst = PathBuf::from(
             "file:copy_flat_with_hash_from_normalized_tables_mem_db?mode=memory&cache=shared",
         );
@@ -652,14 +652,14 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_normalized_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles");
         let dst = PathBuf::from("file:copy_normalized_tables_mem_db?mode=memory&cache=shared");
         verify_copy_all(src, dst, None, Normalized).await
     }
 
     #[actix_rt::test]
     async fn copy_normalized_from_flat_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst =
             PathBuf::from("file:copy_normalized_from_flat_tables_mem_db?mode=memory&cache=shared");
         verify_copy_all(src, dst, Some(Normalized), Normalized).await
@@ -667,7 +667,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_normalized_from_flat_with_hash_tables() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles");
         let dst = PathBuf::from(
             "file:copy_normalized_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared",
         );
@@ -676,7 +676,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_with_min_max_zoom() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared");
         let opt = TileCopierOptions::new(src, dst)
             .min_zoom(Some(2))
@@ -686,7 +686,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_with_zoom_levels() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared");
         let opt = TileCopierOptions::new(src, dst)
             .min_zoom(Some(2))
@@ -697,11 +697,11 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_with_diff_with_file() -> MbtResult<()> {
-        let src = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles");
         let dst = PathBuf::from("file:copy_with_diff_with_file_mem_db?mode=memory&cache=shared");
 
         let diff_file =
-            PathBuf::from("../tests/fixtures/files/geography-class-jpg-modified.mbtiles");
+            PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles");
 
         let copy_opts =
             TileCopierOptions::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone());
@@ -744,10 +744,10 @@ mod tests {
 
     #[actix_rt::test]
     async fn ignore_dst_type_when_copy_to_existing() -> MbtResult<()> {
-        let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles");
+        let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles");
 
         // Copy the dst file to an in-memory DB
-        let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst = PathBuf::from(
             "file:ignore_dst_type_when_copy_to_existing_mem_db?mode=memory&cache=shared",
         );
@@ -761,8 +761,8 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_to_existing_abort_mode() {
-        let src = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles");
-        let dst = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles");
+        let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
 
         let copy_opts =
             TileCopierOptions::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort);
@@ -775,10 +775,10 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_to_existing_override_mode() -> MbtResult<()> {
-        let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles");
+        let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles");
 
         // Copy the dst file to an in-memory DB
-        let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst =
             PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared");
 
@@ -804,10 +804,10 @@ mod tests {
 
     #[actix_rt::test]
     async fn copy_to_existing_ignore_mode() -> MbtResult<()> {
-        let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles");
+        let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles");
 
         // Copy the dst file to an in-memory DB
-        let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let dst =
             PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared");
 
@@ -858,7 +858,7 @@ mod tests {
     #[actix_rt::test]
     async fn apply_flat_diff_file() -> MbtResult<()> {
         // Copy the src file to an in-memory DB
-        let src_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
         let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared");
 
         let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone())
@@ -866,11 +866,11 @@ mod tests {
             .await?;
 
         // Apply diff to the src data in in-memory DB
-        let diff_file = PathBuf::from("../tests/fixtures/files/world_cities_diff.mbtiles");
+        let diff_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles");
         apply_mbtiles_diff(src, diff_file).await?;
 
         // Verify the data is the same as the file the diff was generated from
-        let path = "../tests/fixtures/files/world_cities_modified.mbtiles";
+        let path = "../tests/fixtures/mbtiles/world_cities_modified.mbtiles";
         attach_other_db(&mut src_conn, path).await?;
 
         assert!(
@@ -886,7 +886,7 @@ mod tests {
     #[actix_rt::test]
     async fn apply_normalized_diff_file() -> MbtResult<()> {
         // Copy the src file to an in-memory DB
-        let src_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles");
+        let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles");
         let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared");
 
         let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone())
@@ -894,11 +894,11 @@ mod tests {
             .await?;
 
         // Apply diff to the src data in in-memory DB
-        let diff_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg-diff.mbtiles");
+        let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles");
         apply_mbtiles_diff(src, diff_file).await?;
 
         // Verify the data is the same as the file the diff was generated from
-        let path = "../tests/fixtures/files/geography-class-jpg-modified.mbtiles";
+        let path = "../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles";
         attach_other_db(&mut src_conn, path).await?;
 
         assert!(

tests/config.yaml

@@ -162,7 +162,7 @@ postgres:
 
 pmtiles:
   sources:
-    pmt: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
+    pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
 
 sprites:
   paths: tests/fixtures/sprites/src1

tests/expected/generated_config.yaml

@@ -133,23 +133,27 @@ postgres:
       schema: public
       function: function_zxy_row_key
 pmtiles:
-  paths: tests/fixtures/files
+  paths:
+  - tests/fixtures/mbtiles
+  - tests/fixtures/pmtiles
   sources:
-    png: tests/fixtures/files/png.pmtiles
-    stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
-    webp2: tests/fixtures/files/webp2.pmtiles
+    png: tests/fixtures/pmtiles/png.pmtiles
+    stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
+    webp2: tests/fixtures/pmtiles/webp2.pmtiles
 mbtiles:
-  paths: tests/fixtures/files
+  paths:
+  - tests/fixtures/mbtiles
+  - tests/fixtures/pmtiles
   sources:
-    geography-class-jpg: tests/fixtures/files/geography-class-jpg.mbtiles
-    geography-class-jpg-diff: tests/fixtures/files/geography-class-jpg-diff.mbtiles
-    geography-class-jpg-modified: tests/fixtures/files/geography-class-jpg-modified.mbtiles
-    geography-class-png: tests/fixtures/files/geography-class-png.mbtiles
-    geography-class-png-no-bounds: tests/fixtures/files/geography-class-png-no-bounds.mbtiles
-    json: tests/fixtures/files/json.mbtiles
-    uncompressed_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles
-    webp: tests/fixtures/files/webp.mbtiles
-    world_cities: tests/fixtures/files/world_cities.mbtiles
-    world_cities_diff: tests/fixtures/files/world_cities_diff.mbtiles
-    world_cities_modified: tests/fixtures/files/world_cities_modified.mbtiles
-    zoomed_world_cities: tests/fixtures/files/zoomed_world_cities.mbtiles
+    geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles
+    geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles
+    geography-class-jpg-modified: tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles
+    geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles
+    geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles
+    json: tests/fixtures/mbtiles/json.mbtiles
+    uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles
+    webp: tests/fixtures/mbtiles/webp.mbtiles
+    world_cities: tests/fixtures/mbtiles/world_cities.mbtiles
+    world_cities_diff: tests/fixtures/mbtiles/world_cities_diff.mbtiles
+    world_cities_modified: tests/fixtures/mbtiles/world_cities_modified.mbtiles
+    zoomed_world_cities: tests/fixtures/mbtiles/zoomed_world_cities.mbtiles

tests/expected/given_config.yaml

@@ -159,7 +159,7 @@ postgres:
       - 90.0
 pmtiles:
   sources:
-    pmt: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
+    pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
 sprites:
   paths: tests/fixtures/sprites/src1
   sources:

tests/expected/mbtiles/copy_diff.txt

@@ -0,0 +1 @@
+[INFO ] Creating new metadata value agg_tiles_hash = C7E2E5A9BA04693994DB1F57D1DF5646 in tests/temp/world_cities_diff.mbtiles

tests/expected/mbtiles/copy_diff2.txt

@@ -0,0 +1 @@
+[INFO ] Creating new metadata value agg_tiles_hash = D41D8CD98F00B204E9800998ECF8427E in tests/temp/world_cities_diff_modified.mbtiles

tests/expected/mbtiles/validate-bad.txt

@@ -0,0 +1,3 @@
+[INFO ] Quick integrity check passed for ./tests/fixtures/files/bad_hash.mbtiles
+[INFO ] All tile hashes are valid for ./tests/fixtures/files/bad_hash.mbtiles
+[ERROR] Computed aggregate tiles hash D4E1030D57751A0B45A28A71267E46B8 does not match tile data in metadata CAFEC0DEDEADBEEFDEADBEEFDEADBEEF for MBTile file ./tests/fixtures/files/bad_hash.mbtiles

tests/expected/mbtiles/validate-fix.txt

@@ -0,0 +1,3 @@
+[INFO ] Quick integrity check passed for tests/temp/fix_bad_hash.mbtiles
+[INFO ] All tile hashes are valid for tests/temp/fix_bad_hash.mbtiles
+[INFO ] Updating agg_tiles_hash from CAFEC0DEDEADBEEFDEADBEEFDEADBEEF to D4E1030D57751A0B45A28A71267E46B8 in tests/temp/fix_bad_hash.mbtiles

tests/expected/mbtiles/validate-fix2.txt

@@ -0,0 +1,3 @@
+[INFO ] Quick integrity check passed for tests/temp/fix_bad_hash.mbtiles
+[INFO ] All tile hashes are valid for tests/temp/fix_bad_hash.mbtiles
+[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for tests/temp/fix_bad_hash.mbtiles

tests/expected/mbtiles/validate-ok.txt

@@ -0,0 +1,3 @@
+[INFO ] Quick integrity check passed for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles
+[INFO ] All tile hashes are valid for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles
+[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles

tests/mb_server_test.rs

@@ -34,10 +34,10 @@ fn test_get(path: &str) -> TestRequest {
 const CONFIG: &str = indoc! {"
         mbtiles:
             sources:
-                m_json: tests/fixtures/files/json.mbtiles
-                m_mvt: tests/fixtures/files/world_cities.mbtiles
-                m_raw_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles
-                m_webp: tests/fixtures/files/webp.mbtiles
+                m_json: tests/fixtures/mbtiles/json.mbtiles
+                m_mvt: tests/fixtures/mbtiles/world_cities.mbtiles
+                m_raw_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles
+                m_webp: tests/fixtures/mbtiles/webp.mbtiles
     "};
 
 #[actix_rt::test]

tests/pg_table_source_test.rs

@@ -113,5 +113,5 @@ async fn table_source_schemas() {
           functions: false
     "});
     let sources = mock_sources(cfg).await.0;
-    assert_eq!(sources.keys().collect::<Vec<_>>(), vec!["MixPoints"],);
+    assert_eq!(sources.keys().collect::<Vec<_>>(), vec!["MixPoints"]);
 }

tests/pmt_server_test.rs

@@ -34,12 +34,12 @@ fn test_get(path: &str) -> TestRequest {
 const CONFIG: &str = indoc! {"
         pmtiles:
             sources:
-                p_png: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
+                p_png: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
     "};
 
 #[actix_rt::test]
 async fn pmt_get_catalog() {
-    let path = "pmtiles: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles";
+    let path = "pmtiles: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles";
     let app = create_app! { path };
 
     let req = test_get("/catalog").to_request();

tests/test.sh

@@ -150,7 +150,7 @@ echo "Test auto configured Martin"
 TEST_OUT_DIR="$(dirname "$0")/output/auto"
 mkdir -p "$TEST_OUT_DIR"
 
-ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/files)
+ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles)
 set -x
 $MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_1.txt &
 PROCESS_ID=`jobs -p`
@@ -289,21 +289,34 @@ if [[ "$MBTILES_BIN" != "-" ]]; then
 
   $MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt"
   $MBTILES_BIN meta-all --help 2>&1 | tee "$TEST_OUT_DIR/meta-all_help.txt"
-  $MBTILES_BIN meta-all ./tests/fixtures/files/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt"
+  $MBTILES_BIN meta-all ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt"
   $MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt"
-  $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt"
-  $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt"
+  $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt"
+  $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt"
+  $MBTILES_BIN validate ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-ok.txt"
+
+  set +e
+  $MBTILES_BIN validate ./tests/fixtures/files/bad_hash.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-bad.txt"
+  if [[ $? -eq 0 ]]; then
+    echo "ERROR: validate with bad_hash should have failed"
+    exit 1
+  fi
+  set -e
+
+  cp ./tests/fixtures/files/bad_hash.mbtiles "$TEST_TEMP_DIR/fix_bad_hash.mbtiles"
+  $MBTILES_BIN validate --update-agg-tiles-hash "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" 2>&1 | tee "$TEST_OUT_DIR/validate-fix.txt"
+  $MBTILES_BIN validate "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" 2>&1 | tee "$TEST_OUT_DIR/validate-fix2.txt"
 
   # Create diff file
   $MBTILES_BIN copy \
-    ./tests/fixtures/files/world_cities.mbtiles \
+    ./tests/fixtures/mbtiles/world_cities.mbtiles \
     "$TEST_TEMP_DIR/world_cities_diff.mbtiles" \
-    --diff-with-file ./tests/fixtures/files/world_cities_modified.mbtiles \
+    --diff-with-file ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \
     2>&1 | tee "$TEST_OUT_DIR/copy_diff.txt"
 
   if command -v sqlite3 > /dev/null; then
     # Apply this diff to the original version of the file
-    cp ./tests/fixtures/files/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles"
+    cp ./tests/fixtures/mbtiles/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles"
 
     sqlite3 "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \
       -bail \
@@ -315,7 +328,7 @@ if [[ "$MBTILES_BIN" != "-" ]]; then
     # Ensure that applying the diff resulted in the modified version of the file
     $MBTILES_BIN copy \
       --diff-with-file "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \
-      ./tests/fixtures/files/world_cities_modified.mbtiles \
+      ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \
       "$TEST_TEMP_DIR/world_cities_diff_modified.mbtiles" \
       2>&1 | tee "$TEST_OUT_DIR/copy_diff2.txt"