Improve mbtiles update, update min/max zoom on martin-cp (#1096)

* `mbtiles update` now allows different types of zoom updates - reset to
content, grow only, or skip (dry run)
* `martin-cp` will now update (grow-only) metadata zooms

Addresses a few concerns in the #1081

Cargo.toml

@@ -89,6 +89,7 @@ insta.opt-level = 3
 similar.opt-level = 3
 
 #[patch.crates-io]
+#enum-display = { path = "../enum-display" }
 #pmtiles = { path = "../pmtiles-rs" }
 #sqlite-hashes = { path = "../sqlite-hashes" }
 #tilejson = { path = "../tilejson" }

martin/src/bin/martin-cp.rs

@@ -19,6 +19,7 @@ use martin::{
 };
 use martin_tile_utils::{bbox_to_xyz, TileInfo};
 use mbtiles::sqlx::SqliteConnection;
+use mbtiles::UpdateZoomType::GrowOnly;
 use mbtiles::{
     init_mbtiles_schema, is_empty_database, CopyDuplicateMode, MbtError, MbtType, MbtTypeCli,
     Mbtiles,
@@ -353,6 +354,8 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()>
 
     info!("{progress}");
 
+    mbt.update_metadata(&mut conn, GrowOnly).await?;
+
     for (key, value) in args.set_meta {
         info!("Setting metadata key={key} value={value}");
         mbt.set_metadata_value(&mut conn, &key, value).await?;

mbtiles/.sqlx/query-96f3201d2151fbef63593c0e87648a2991e05060e71aa96141ece0867afa2d6c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "SQLite",
-  "query": "\n    SELECT min(zoom_level) AS min_zoom,\n           max(zoom_level) AS max_zoom\n    FROM tiles",
+  "query": "\nSELECT min(zoom_level) AS min_zoom,\n       max(zoom_level) AS max_zoom\nFROM tiles;",
   "describe": {
     "columns": [
       {
@@ -22,5 +22,5 @@
       true
     ]
   },
-  "hash": "47bdc12fe7b34fb2e4e1fc3b937bba64268170ab6e5381abfe07df24d8133229"
+  "hash": "96f3201d2151fbef63593c0e87648a2991e05060e71aa96141ece0867afa2d6c"
 }

mbtiles/src/bin/mbtiles.rs

@@ -4,7 +4,7 @@ use clap::{Parser, Subcommand};
 use log::error;
 use mbtiles::{
     apply_patch, AggHashType, CopyDuplicateMode, CopyType, IntegrityCheckType, MbtResult,
-    MbtTypeCli, Mbtiles, MbtilesCopier,
+    MbtTypeCli, Mbtiles, MbtilesCopier, UpdateZoomType,
 };
 use tilejson::Bounds;
 
@@ -68,6 +68,9 @@ enum Commands {
     UpdateMetadata {
         /// MBTiles file to validate
         file: PathBuf,
+        /// Update the min and max zoom levels in the metadata table to match the tiles table.
+        #[arg(long, value_enum, default_value_t=UpdateZoomType::default())]
+        update_zoom: UpdateZoomType,
     },
     /// Validate tile data if hash of tile data exists in file
     #[command(name = "validate", alias = "check", alias = "verify")]
@@ -179,9 +182,10 @@ async fn main_int() -> anyhow::Result<()> {
         } => {
             apply_patch(src_file, diff_file).await?;
         }
-        Commands::UpdateMetadata { file } => {
+        Commands::UpdateMetadata { file, update_zoom } => {
             let mbt = Mbtiles::new(file.as_path())?;
-            mbt.update_metadata().await?;
+            let mut conn = mbt.open().await?;
+            mbt.update_metadata(&mut conn, update_zoom).await?;
         }
         Commands::Validate {
             file,

mbtiles/src/errors.rs

@@ -1,6 +1,6 @@
 use std::path::PathBuf;
 
-use martin_tile_utils::TileInfo;
+use martin_tile_utils::{TileInfo, MAX_ZOOM};
 use sqlite_hashes::rusqlite;
 
 use crate::MbtType;
@@ -74,6 +74,9 @@ pub enum MbtError {
 
     #[error("Unless  --on-duplicate (override|ignore|abort)  is set, writing tiles to an existing non-empty MBTiles file is disabled. Either set --on-duplicate flag, or delete {}", .0.display())]
     DestinationFileExists(PathBuf),
+
+    #[error("Invalid zoom value {0}={1}, expecting an integer between 0..{MAX_ZOOM}")]
+    InvalidZoomValue(&'static str, String),
 }
 
 pub type MbtResult<T> = Result<T, MbtError>;

mbtiles/src/lib.rs

@@ -27,6 +27,7 @@ pub use queries::*;
 mod summary;
 
 mod update;
+pub use update::UpdateZoomType;
 
 mod validation;
 pub use validation::{

mbtiles/src/metadata.rs

@@ -11,6 +11,7 @@ use sqlx::{query, SqliteExecutor};
 use tilejson::{tilejson, Bounds, Center, TileJSON};
 
 use crate::errors::MbtResult;
+use crate::MbtError::InvalidZoomValue;
 use crate::Mbtiles;
 
 #[serde_with::skip_serializing_none]
@@ -63,6 +64,20 @@ impl Mbtiles {
         Ok(None)
     }
 
+    pub async fn get_metadata_zoom_value<T>(
+        &self,
+        conn: &mut T,
+        zoom_name: &'static str,
+    ) -> MbtResult<Option<u8>>
+    where
+        for<'e> &'e mut T: SqliteExecutor<'e>,
+    {
+        self.get_metadata_value(conn, zoom_name)
+            .await?
+            .map(|v| v.parse().map_err(|_| InvalidZoomValue(zoom_name, v)))
+            .transpose()
+    }
+
     pub async fn set_metadata_value<T, S>(&self, conn: &mut T, key: &str, value: S) -> MbtResult<()>
     where
         S: ToString,

mbtiles/src/queries.rs

@@ -1,7 +1,9 @@
 use log::debug;
+use martin_tile_utils::MAX_ZOOM;
 use sqlx::{query, Executor as _, SqliteExecutor};
 
 use crate::errors::MbtResult;
+use crate::MbtError::InvalidZoomValue;
 use crate::MbtType;
 
 /// Returns true if the database is empty (no tables/indexes/...)
@@ -308,3 +310,38 @@ where
         .await?;
     Ok(())
 }
+
+fn validate_zoom(zoom: Option<i32>, zoom_name: &'static str) -> MbtResult<Option<u8>> {
+    if let Some(zoom) = zoom {
+        let z = u8::try_from(zoom).ok().filter(|v| *v <= MAX_ZOOM);
+        if z.is_none() {
+            Err(InvalidZoomValue(zoom_name, zoom.to_string()))
+        } else {
+            Ok(z)
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+pub async fn compute_min_max_zoom<T>(conn: &mut T) -> MbtResult<Option<(u8, u8)>>
+where
+    for<'e> &'e mut T: SqliteExecutor<'e>,
+{
+    let info = query!(
+        "
+SELECT min(zoom_level) AS min_zoom,
+       max(zoom_level) AS max_zoom
+FROM tiles;"
+    )
+    .fetch_one(conn)
+    .await?;
+
+    let min_zoom = validate_zoom(info.min_zoom, "zoom_level")?;
+    let max_zoom = validate_zoom(info.max_zoom, "zoom_level")?;
+
+    match (min_zoom, max_zoom) {
+        (Some(min_zoom), Some(max_zoom)) => Ok(Some((min_zoom, max_zoom))),
+        _ => Ok(None),
+    }
+}

mbtiles/src/update.rs

@@ -1,31 +1,100 @@
-use log::info;
-use sqlx::query;
+// See https://github.com/SeedyROM/enum-display/issues/1
+#![allow(unused_qualifications)]
 
+use enum_display::EnumDisplay;
+use log::{info, warn};
+use sqlx::SqliteExecutor;
+
+use self::UpdateZoomType::{GrowOnly, Reset, Skip};
 use crate::errors::MbtResult;
-use crate::Mbtiles;
+use crate::MbtError::InvalidZoomValue;
+use crate::{compute_min_max_zoom, Mbtiles};
+
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)]
+#[enum_display(case = "Kebab")]
+#[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
+pub enum UpdateZoomType {
+    /// Reset the minzoom and maxzoom metadata values to match the content of the tiles table
+    #[default]
+    Reset,
+    /// Only update minzoom and maxzoom if the zooms in the tiles table are outside the range set in the metadata
+    GrowOnly,
+    /// Perform a dry run and print result, without updating the minzoom and maxzoom metadata values
+    Skip,
+}
 
 impl Mbtiles {
-    pub async fn update_metadata(&self) -> MbtResult<()> {
-        let mut conn = self.open().await?;
-
-        let info = query!(
-            "
-    SELECT min(zoom_level) AS min_zoom,
-           max(zoom_level) AS max_zoom
-    FROM tiles"
-        )
-        .fetch_one(&mut conn)
-        .await?;
-
-        if let Some(min_zoom) = info.min_zoom {
-            info!("Updating minzoom to {min_zoom}");
-            self.set_metadata_value(&mut conn, "minzoom", &min_zoom)
-                .await?;
+    async fn set_zoom_value<T>(
+        &self,
+        conn: &mut T,
+        is_max_zoom: bool,
+        calc_zoom: u8,
+        update_zoom: UpdateZoomType,
+    ) -> MbtResult<()>
+    where
+        for<'e> &'e mut T: SqliteExecutor<'e>,
+    {
+        let zoom_name = if is_max_zoom { "maxzoom" } else { "minzoom" };
+        match self.get_metadata_zoom_value(conn, zoom_name).await {
+            Ok(Some(meta_zoom)) => {
+                let is_outside_range = if is_max_zoom {
+                    meta_zoom < calc_zoom
+                } else {
+                    meta_zoom > calc_zoom
+                };
+                if meta_zoom == calc_zoom {
+                    info!("Metadata value {zoom_name} is already set to correct value {meta_zoom}");
+                } else if update_zoom == Skip {
+                    info!("Metadata value {zoom_name} is set to {meta_zoom}, but should be set to {calc_zoom}. Skipping update");
+                } else if is_outside_range || update_zoom == Reset {
+                    info!("Updating metadata {zoom_name} from {meta_zoom} to {calc_zoom}");
+                    self.set_metadata_value(conn, zoom_name, calc_zoom).await?;
+                } else if is_max_zoom {
+                    info!("Metadata value {zoom_name}={meta_zoom} is greater than the computed {zoom_name} {calc_zoom} in tiles table, not updating");
+                } else {
+                    info!("Metadata value {zoom_name}={meta_zoom} is less than the computed {zoom_name} {calc_zoom} in tiles table, not updating");
+                }
+            }
+            Ok(None) => {
+                info!("Setting metadata value {zoom_name} to {calc_zoom}");
+                self.set_metadata_value(conn, zoom_name, calc_zoom).await?;
+            }
+            Err(InvalidZoomValue(_, val)) => {
+                warn!("Overriding invalid metadata value {zoom_name}='{val}' to {calc_zoom}");
+                self.set_metadata_value(conn, zoom_name, calc_zoom).await?;
+            }
+            Err(e) => Err(e)?,
         }
-        if let Some(max_zoom) = info.max_zoom {
-            info!("Updating maxzoom to {max_zoom}");
-            self.set_metadata_value(&mut conn, "maxzoom", &max_zoom)
-                .await?;
+        Ok(())
+    }
+
+    /// Update the metadata table with the min and max zoom levels
+    /// from the tiles table.
+    /// If `grow_only` is true, only update the metadata if the
+    /// new min or max zoom is outside the current range.
+    pub async fn update_metadata<T>(
+        &self,
+        conn: &mut T,
+        update_zoom: UpdateZoomType,
+    ) -> MbtResult<()>
+    where
+        for<'e> &'e mut T: SqliteExecutor<'e>,
+    {
+        match (update_zoom, compute_min_max_zoom(&mut *conn).await?) {
+            (_, Some((min_zoom, max_zoom))) => {
+                self.set_zoom_value(&mut *conn, false, min_zoom, update_zoom)
+                    .await?;
+                self.set_zoom_value(&mut *conn, true, max_zoom, update_zoom)
+                    .await?;
+            }
+            (GrowOnly | Skip, None) => {
+                info!("No tiles found in the tiles table, skipping metadata min/max zoom update");
+            }
+            (Reset, None) => {
+                info!("No tiles found in the tiles table, deleting minzoom and maxzoom if exist");
+                self.delete_metadata_value(&mut *conn, "minzoom").await?;
+                self.delete_metadata_value(&mut *conn, "maxzoom").await?;
+            }
         }
 
         Ok(())

mbtiles/tests/copy.rs

@@ -12,7 +12,7 @@ use mbtiles::IntegrityCheckType::Off;
 use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized};
 use mbtiles::{
     apply_patch, init_mbtiles_schema, invert_y_value, CopyType, MbtResult, MbtTypeCli, Mbtiles,
-    MbtilesCopier,
+    MbtilesCopier, UpdateZoomType,
 };
 use pretty_assertions::assert_eq as pretty_assert_eq;
 use rstest::{fixture, rstest};
@@ -249,7 +249,7 @@ fn databases() -> Databases {
 #[actix_rt::test]
 async fn update() -> MbtResult<()> {
     let (mbt, mut cn) = new_file_no_hash!(databases, Flat, METADATA_V1, TILES_V1, "update");
-    mbt.update_metadata().await?;
+    mbt.update_metadata(&mut cn, UpdateZoomType::Reset).await?;
     let dmp = dump(&mut cn).await?;
     assert_snapshot!(&dmp, "update");