mirror of
https://github.com/maplibre/martin.git
synced 2024-12-19 12:51:37 +03:00
Minor mbtiles doc cleanups
This commit is contained in:
parent
1a386b7071
commit
f52cd3c611
@ -74,7 +74,9 @@ The original [MBTiles specification](https://github.com/mapbox/mbtiles-spec#read
|
|||||||
|
|
||||||
A typical Normalized schema generated by tools like [tilelive-copy](https://github.com/mapbox/TileLive#bintilelive-copy) use MD5 hash in the `tile_id` column. The Martin's `mbtiles` tool can use this hash to verify the content of each tile. We also define a new `flat-with-hash` schema that stores the hash and tile data in the same table. This schema is more efficient than the `normalized` schema when data has no duplicate tiles (see below). Per tile validation is not available for `flat` schema.
|
A typical Normalized schema generated by tools like [tilelive-copy](https://github.com/mapbox/TileLive#bintilelive-copy) use MD5 hash in the `tile_id` column. The Martin's `mbtiles` tool can use this hash to verify the content of each tile. We also define a new `flat-with-hash` schema that stores the hash and tile data in the same table. This schema is more efficient than the `normalized` schema when data has no duplicate tiles (see below). Per tile validation is not available for `flat` schema.
|
||||||
|
|
||||||
Per-tile validation will catch individual invalid tiles, but it will not detect overall datastore corruption (e.g. missing tiles or tiles that shouldn't exist, or tiles with incorrect z/x/y values). For that, Martin `mbtiles` tool defines a new metadata value called `agg_tiles_hash`. The value is computed by hashing `cast(zoom_level AS text), cast(tile_column AS text), cast(tile_row AS text), cast(tile_data as blob)` combined for all rows in the `tiles` table/view, ordered by z,x,y. In case there are no rows or all are NULL, the hash value of an empty string is used.
|
Per-tile validation will catch individual invalid tiles, but it will not detect overall datastore corruption (e.g. missing tiles or tiles that shouldn't exist, or tiles with incorrect z/x/y values).
|
||||||
|
For that, Martin `mbtiles` tool defines a new metadata value called `agg_tiles_hash`. The value is computed by hashing `cast(zoom_level AS text), cast(tile_column AS text), cast(tile_row AS text), tile_data` combined for all rows in the `tiles` table/view, ordered by z,x,y.
|
||||||
|
In case there are no rows or all are NULL, the hash value of an empty string is used. Note that SQLite allows any value type to be stored as in any column, so if `tile_data` accidentally contains non-blob/text/null value, validation will fail.
|
||||||
|
|
||||||
The `mbtiles` tool will compute `agg_tiles_hash` value when copying or validating mbtiles files.
|
The `mbtiles` tool will compute `agg_tiles_hash` value when copying or validating mbtiles files.
|
||||||
|
|
||||||
|
@ -1,8 +1,9 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::Result;
|
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use martin_mbtiles::{apply_mbtiles_diff, IntegrityCheckType, Mbtiles, TileCopierOptions};
|
use martin_mbtiles::{
|
||||||
|
apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions,
|
||||||
|
};
|
||||||
use sqlx::sqlite::SqliteConnectOptions;
|
use sqlx::sqlite::SqliteConnectOptions;
|
||||||
use sqlx::{Connection, SqliteConnection};
|
use sqlx::{Connection, SqliteConnection};
|
||||||
|
|
||||||
@ -72,7 +73,7 @@ enum Commands {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
|
||||||
match args.command {
|
match args.command {
|
||||||
@ -106,7 +107,7 @@ async fn main() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn meta_print_all(file: &Path) -> Result<()> {
|
async fn meta_print_all(file: &Path) -> anyhow::Result<()> {
|
||||||
let mbt = Mbtiles::new(file)?;
|
let mbt = Mbtiles::new(file)?;
|
||||||
let opt = SqliteConnectOptions::new().filename(file).read_only(true);
|
let opt = SqliteConnectOptions::new().filename(file).read_only(true);
|
||||||
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
||||||
@ -115,7 +116,7 @@ async fn meta_print_all(file: &Path) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn meta_get_value(file: &Path, key: &str) -> Result<()> {
|
async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
|
||||||
let mbt = Mbtiles::new(file)?;
|
let mbt = Mbtiles::new(file)?;
|
||||||
let opt = SqliteConnectOptions::new().filename(file).read_only(true);
|
let opt = SqliteConnectOptions::new().filename(file).read_only(true);
|
||||||
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
||||||
@ -125,30 +126,28 @@ async fn meta_get_value(file: &Path, key: &str) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn meta_set_value(file: &Path, key: &str, value: Option<String>) -> Result<()> {
|
async fn meta_set_value(file: &Path, key: &str, value: Option<String>) -> MbtResult<()> {
|
||||||
let mbt = Mbtiles::new(file)?;
|
let mbt = Mbtiles::new(file)?;
|
||||||
let opt = SqliteConnectOptions::new().filename(file);
|
let opt = SqliteConnectOptions::new().filename(file);
|
||||||
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
||||||
mbt.set_metadata_value(&mut conn, key, value).await?;
|
mbt.set_metadata_value(&mut conn, key, value).await
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn validate_mbtiles(
|
async fn validate_mbtiles(
|
||||||
file: &Path,
|
file: &Path,
|
||||||
check_type: IntegrityCheckType,
|
check_type: IntegrityCheckType,
|
||||||
update_agg_tiles_hash: bool,
|
update_agg_tiles_hash: bool,
|
||||||
) -> Result<()> {
|
) -> MbtResult<()> {
|
||||||
let mbt = Mbtiles::new(file)?;
|
let mbt = Mbtiles::new(file)?;
|
||||||
let opt = SqliteConnectOptions::new().filename(file).read_only(true);
|
let opt = SqliteConnectOptions::new().filename(file).read_only(true);
|
||||||
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
let mut conn = SqliteConnection::connect_with(&opt).await?;
|
||||||
mbt.check_integrity(&mut conn, check_type).await?;
|
mbt.check_integrity(&mut conn, check_type).await?;
|
||||||
mbt.check_each_tile_hash(&mut conn).await?;
|
mbt.check_each_tile_hash(&mut conn).await?;
|
||||||
if update_agg_tiles_hash {
|
if update_agg_tiles_hash {
|
||||||
mbt.update_agg_tiles_hash(&mut conn).await?;
|
mbt.update_agg_tiles_hash(&mut conn).await
|
||||||
} else {
|
} else {
|
||||||
mbt.check_agg_tiles_hashes(&mut conn).await?;
|
mbt.check_agg_tiles_hashes(&mut conn).await
|
||||||
}
|
}
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -6,7 +6,7 @@ mod mbtiles_pool;
|
|||||||
mod mbtiles_queries;
|
mod mbtiles_queries;
|
||||||
mod tile_copier;
|
mod tile_copier;
|
||||||
|
|
||||||
pub use errors::MbtError;
|
pub use errors::{MbtError, MbtResult};
|
||||||
pub use mbtiles::{IntegrityCheckType, Mbtiles, Metadata};
|
pub use mbtiles::{IntegrityCheckType, Mbtiles, Metadata};
|
||||||
pub use mbtiles_pool::MbtilesPool;
|
pub use mbtiles_pool::MbtilesPool;
|
||||||
pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions};
|
pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions};
|
||||||
|
@ -470,7 +470,6 @@ impl Mbtiles {
|
|||||||
return Err(AggHashValueNotFound(self.filepath().to_string()));
|
return Err(AggHashValueNotFound(self.filepath().to_string()));
|
||||||
};
|
};
|
||||||
|
|
||||||
// let conn = self.open_with_hashes(true)?;
|
|
||||||
let computed = calc_agg_tiles_hash(&mut *conn).await?;
|
let computed = calc_agg_tiles_hash(&mut *conn).await?;
|
||||||
if stored != computed {
|
if stored != computed {
|
||||||
let file = self.filepath().to_string();
|
let file = self.filepath().to_string();
|
||||||
|
Loading…
Reference in New Issue
Block a user