From 6f32f0e7b4ffe526c46f688ef1575599aada3fb1 Mon Sep 17 00:00:00 2001 From: rstanciu <55859815+upsicleclown@users.noreply.github.com> Date: Tue, 13 Jun 2023 17:04:11 -0700 Subject: [PATCH] Add feature to detect Mbtiles format (#713) Feature for `Mbtiles` struct to detect which format a `.mbtiles` file is in according to the [MBTiles specfication](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#database). The function `detect_type` identifies whether the `.mbtiles` file contains a `tiles` _table_ OR if it contains `map` and `images` table (which provide the data for a `tiles` _view_). See also #667 --- martin-mbtiles/sqlx-data.json | 36 ++++++++++++ martin-mbtiles/src/errors.rs | 3 + martin-mbtiles/src/lib.rs | 1 + martin-mbtiles/src/mbtiles.rs | 35 ++++++++++++ martin-mbtiles/src/mbtiles_queries.rs | 79 +++++++++++++++++++++++++++ 5 files changed, 154 insertions(+) create mode 100644 martin-mbtiles/src/mbtiles_queries.rs diff --git a/martin-mbtiles/sqlx-data.json b/martin-mbtiles/sqlx-data.json index 65d31b5b..752e8a2f 100644 --- a/martin-mbtiles/sqlx-data.json +++ b/martin-mbtiles/sqlx-data.json @@ -1,5 +1,23 @@ { "db": "SQLite", + "09e15d4479a96829f8dcd93e6f40f7e5f487f6c33614aa82ae3716e3bb932dfa": { + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "nullable": [ + false + ], + "parameters": { + "Right": 0 + } + }, + "query": "SELECT (\n -- Has a \"map\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- \"map\" table's columns and their types are as expected:\n -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE \"notnull\" = 0\n AND ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_id\" AND type = \"TEXT\"))\n --\n ) AND (\n -- Has a \"images\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- \"images\" table's columns and their types are as expected:\n -- 2 non-null columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE \"notnull\" = 0\n AND ((name = \"tile_id\" AND type = \"TEXT\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) AS is_valid;\n" + }, "386a375cf65c3e5aef51deffc99d23bd852ba445c1058aed380fe83bed618c29": { "describe": { "columns": [ @@ -102,6 +120,24 @@ }, "query": "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level >= 0 LIMIT 1" }, + "78d1356063c080d9bcea05a5ad95ffb771de5adb62873d794be09062506451d3": { + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "nullable": [ + false + ], + "parameters": { + "Right": 0 + } + }, + "query": "SELECT (\n -- Has a \"tiles\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles\" table's columns and their types are as expected:\n -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE \"notnull\" = 0\n AND ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) as is_valid;\n" + }, "d6ac76a234c97d0dc1fc4331d8b2cd90903d5401f8f0956245e5163bedd23a4d": { "describe": { "columns": [ diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs index eef1e10a..9962371b 100644 --- a/martin-mbtiles/src/errors.rs +++ b/martin-mbtiles/src/errors.rs @@ -13,6 +13,9 @@ pub enum MbtError { #[error("Inconsistent tile formats detected: {0} vs {1}")] InconsistentMetadata(TileInfo, TileInfo), + #[error("Invalid data storage format for MBTile file {0}")] + InvalidDataStorageFormat(String), + #[error("No tiles found")] NoTilesFound, } diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index a69f020b..7d033356 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -3,6 +3,7 @@ mod errors; mod mbtiles; mod mbtiles_pool; +mod mbtiles_queries; pub use errors::MbtError; pub use mbtiles::{Mbtiles, Metadata}; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index 506a1964..b4db25cd 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -15,6 +15,7 @@ use sqlx::{query, SqliteExecutor}; use tilejson::{tilejson, Bounds, Center, TileJSON}; use crate::errors::{MbtError, MbtResult}; +use crate::mbtiles_queries::{is_deduplicated_type, is_tile_tables_type}; #[derive(Clone, Debug, PartialEq)] pub struct Metadata { @@ -25,6 +26,12 @@ pub struct Metadata { pub json: Option, } +#[derive(Debug, PartialEq)] +pub enum Type { + TileTables, + DeDuplicated, +} + #[derive(Clone, Debug)] pub struct Mbtiles { filepath: String, @@ -268,6 +275,19 @@ impl Mbtiles { } Ok(None) } + + pub async fn detect_type(&self, conn: &mut T) -> MbtResult + where + for<'e> &'e mut T: SqliteExecutor<'e>, + { + if is_deduplicated_type(&mut *conn).await? { + Ok(Type::DeDuplicated) + } else if is_tile_tables_type(&mut *conn).await? { + Ok(Type::TileTables) + } else { + Err(MbtError::InvalidDataStorageFormat(self.filepath.clone())) + } + } } #[cfg(test)] @@ -359,4 +379,19 @@ mod tests { let res = mbt.get_metadata_value(&mut conn, "").await; assert_eq!(res.unwrap(), None); } + + #[actix_rt::test] + async fn detect_type() { + let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let res = mbt.detect_type(&mut conn).await.unwrap(); + assert_eq!(res, Type::TileTables); + + let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; + let res = mbt.detect_type(&mut conn).await.unwrap(); + assert_eq!(res, Type::DeDuplicated); + + let (mut conn, mbt) = open(":memory:").await; + let res = mbt.detect_type(&mut conn).await; + assert!(matches!(res, Err(MbtError::InvalidDataStorageFormat(_)))); + } } diff --git a/martin-mbtiles/src/mbtiles_queries.rs b/martin-mbtiles/src/mbtiles_queries.rs new file mode 100644 index 00000000..d76b5634 --- /dev/null +++ b/martin-mbtiles/src/mbtiles_queries.rs @@ -0,0 +1,79 @@ +use crate::errors::MbtResult; +use sqlx::{query, SqliteExecutor}; + +pub async fn is_deduplicated_type(conn: &mut T) -> MbtResult +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + let sql = query!( + r#"SELECT ( + -- Has a "map" table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'map' + AND type = 'table' + -- + ) AND ( + -- "map" table's columns and their types are as expected: + -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_id). + -- The order is not important + SELECT COUNT(*) = 4 + FROM pragma_table_info('map') + WHERE "notnull" = 0 + AND ((name = "zoom_level" AND type = "INTEGER") + OR (name = "tile_column" AND type = "INTEGER") + OR (name = "tile_row" AND type = "INTEGER") + OR (name = "tile_id" AND type = "TEXT")) + -- + ) AND ( + -- Has a "images" table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'images' + AND type = 'table' + -- + ) AND ( + -- "images" table's columns and their types are as expected: + -- 2 non-null columns (tile_id, tile_data). + -- The order is not important + SELECT COUNT(*) = 2 + FROM pragma_table_info('images') + WHERE "notnull" = 0 + AND ((name = "tile_id" AND type = "TEXT") + OR (name = "tile_data" AND type = "BLOB")) + -- + ) AS is_valid; +"# + ); + Ok(sql.fetch_one(&mut *conn).await?.is_valid == 1) +} + +pub async fn is_tile_tables_type(conn: &mut T) -> MbtResult +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + let sql = query!( + r#"SELECT ( + -- Has a "tiles" table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'tiles' + AND type = 'table' + -- + ) AND ( + -- "tiles" table's columns and their types are as expected: + -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_data). + -- The order is not important + SELECT COUNT(*) = 4 + FROM pragma_table_info('tiles') + WHERE "notnull" = 0 + AND ((name = "zoom_level" AND type = "INTEGER") + OR (name = "tile_column" AND type = "INTEGER") + OR (name = "tile_row" AND type = "INTEGER") + OR (name = "tile_data" AND type = "BLOB")) + -- + ) as is_valid; +"# + ); + Ok(sql.fetch_one(&mut *conn).await?.is_valid == 1) +}