From f52cd3c611c3435fad5cb6472ad6c48a39f5c517 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 28 Sep 2023 18:19:05 -0400 Subject: [PATCH] Minor mbtiles doc cleanups --- docs/src/tools.md | 4 +++- martin-mbtiles/src/bin/main.rs | 23 +++++++++++------------ martin-mbtiles/src/lib.rs | 2 +- martin-mbtiles/src/mbtiles.rs | 1 - 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/src/tools.md b/docs/src/tools.md index a2e35b5e..bbb63082 100644 --- a/docs/src/tools.md +++ b/docs/src/tools.md @@ -74,7 +74,9 @@ The original [MBTiles specification](https://github.com/mapbox/mbtiles-spec#read A typical Normalized schema generated by tools like [tilelive-copy](https://github.com/mapbox/TileLive#bintilelive-copy) use MD5 hash in the `tile_id` column. The Martin's `mbtiles` tool can use this hash to verify the content of each tile. We also define a new `flat-with-hash` schema that stores the hash and tile data in the same table. This schema is more efficient than the `normalized` schema when data has no duplicate tiles (see below). Per tile validation is not available for `flat` schema. -Per-tile validation will catch individual invalid tiles, but it will not detect overall datastore corruption (e.g. missing tiles or tiles that shouldn't exist, or tiles with incorrect z/x/y values). For that, Martin `mbtiles` tool defines a new metadata value called `agg_tiles_hash`. The value is computed by hashing `cast(zoom_level AS text), cast(tile_column AS text), cast(tile_row AS text), cast(tile_data as blob)` combined for all rows in the `tiles` table/view, ordered by z,x,y. In case there are no rows or all are NULL, the hash value of an empty string is used. +Per-tile validation will catch individual invalid tiles, but it will not detect overall datastore corruption (e.g. missing tiles or tiles that shouldn't exist, or tiles with incorrect z/x/y values). +For that, Martin `mbtiles` tool defines a new metadata value called `agg_tiles_hash`. The value is computed by hashing `cast(zoom_level AS text), cast(tile_column AS text), cast(tile_row AS text), tile_data` combined for all rows in the `tiles` table/view, ordered by z,x,y. +In case there are no rows or all are NULL, the hash value of an empty string is used. Note that SQLite allows any value type to be stored as in any column, so if `tile_data` accidentally contains non-blob/text/null value, validation will fail. The `mbtiles` tool will compute `agg_tiles_hash` value when copying or validating mbtiles files. diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 39579703..5a7b0f54 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -1,8 +1,9 @@ use std::path::{Path, PathBuf}; -use anyhow::Result; use clap::{Parser, Subcommand}; -use martin_mbtiles::{apply_mbtiles_diff, IntegrityCheckType, Mbtiles, TileCopierOptions}; +use martin_mbtiles::{ + apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions, +}; use sqlx::sqlite::SqliteConnectOptions; use sqlx::{Connection, SqliteConnection}; @@ -72,7 +73,7 @@ enum Commands { } #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> anyhow::Result<()> { let args = Args::parse(); match args.command { @@ -106,7 +107,7 @@ async fn main() -> Result<()> { Ok(()) } -async fn meta_print_all(file: &Path) -> Result<()> { +async fn meta_print_all(file: &Path) -> anyhow::Result<()> { let mbt = Mbtiles::new(file)?; let opt = SqliteConnectOptions::new().filename(file).read_only(true); let mut conn = SqliteConnection::connect_with(&opt).await?; @@ -115,7 +116,7 @@ async fn meta_print_all(file: &Path) -> Result<()> { Ok(()) } -async fn meta_get_value(file: &Path, key: &str) -> Result<()> { +async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; let opt = SqliteConnectOptions::new().filename(file).read_only(true); let mut conn = SqliteConnection::connect_with(&opt).await?; @@ -125,30 +126,28 @@ async fn meta_get_value(file: &Path, key: &str) -> Result<()> { Ok(()) } -async fn meta_set_value(file: &Path, key: &str, value: Option) -> Result<()> { +async fn meta_set_value(file: &Path, key: &str, value: Option) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; let opt = SqliteConnectOptions::new().filename(file); let mut conn = SqliteConnection::connect_with(&opt).await?; - mbt.set_metadata_value(&mut conn, key, value).await?; - Ok(()) + mbt.set_metadata_value(&mut conn, key, value).await } async fn validate_mbtiles( file: &Path, check_type: IntegrityCheckType, update_agg_tiles_hash: bool, -) -> Result<()> { +) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; let opt = SqliteConnectOptions::new().filename(file).read_only(true); let mut conn = SqliteConnection::connect_with(&opt).await?; mbt.check_integrity(&mut conn, check_type).await?; mbt.check_each_tile_hash(&mut conn).await?; if update_agg_tiles_hash { - mbt.update_agg_tiles_hash(&mut conn).await?; + mbt.update_agg_tiles_hash(&mut conn).await } else { - mbt.check_agg_tiles_hashes(&mut conn).await?; + mbt.check_agg_tiles_hashes(&mut conn).await } - Ok(()) } #[cfg(test)] diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index 4d90e423..16a9747d 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -6,7 +6,7 @@ mod mbtiles_pool; mod mbtiles_queries; mod tile_copier; -pub use errors::MbtError; +pub use errors::{MbtError, MbtResult}; pub use mbtiles::{IntegrityCheckType, Mbtiles, Metadata}; pub use mbtiles_pool::MbtilesPool; pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions}; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index add902fe..6a1c3724 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -470,7 +470,6 @@ impl Mbtiles { return Err(AggHashValueNotFound(self.filepath().to_string())); }; - // let conn = self.open_with_hashes(true)?; let computed = calc_agg_tiles_hash(&mut *conn).await?; if stored != computed { let file = self.filepath().to_string();