diff --git a/Cargo.lock b/Cargo.lock index 2218bb5a..676ee609 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,6 +1919,7 @@ dependencies = [ "env_logger", "futures", "insta", + "itertools 0.12.0", "log", "martin-tile-utils", "pretty_assertions", diff --git a/justfile b/justfile index 755b091c..2a13a4dc 100644 --- a/justfile +++ b/justfile @@ -264,6 +264,10 @@ fmt-md: fmt2: cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate +# Run cargo check +check: + cargo check --workspace --all-targets --bins --tests --lib --benches + # Run cargo clippy clippy: cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index efbbfa9c..2b187e7e 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -11,7 +11,7 @@ use tile_grid::{tms, Tms, Xyz}; pub const EARTH_CIRCUMFERENCE: f64 = 40_075_016.685_578_5; pub const EARTH_RADIUS: f64 = EARTH_CIRCUMFERENCE / 2.0 / PI; -pub const MAX_ZOOM: u8 = 30; +pub const MAX_ZOOM: u8 = 24; use std::sync::OnceLock; fn web_merc() -> &'static Tms { @@ -209,6 +209,8 @@ pub fn tile_index(lon: f64, lat: f64, zoom: u8) -> (u32, u32) { #[must_use] pub fn xyz_to_bbox(zoom: u8, min_x: u32, min_y: u32, max_x: u32, max_y: u32) -> [f64; 4] { assert!(zoom <= MAX_ZOOM, "zoom {zoom} must be <= {MAX_ZOOM}"); + assert!(min_x <= max_x, "min_x {min_x} must be <= max_x {max_x}"); + assert!(min_y <= max_y, "min_y {min_y} must be <= max_y {max_y}"); let left_top_bounds = web_merc().xy_bounds(&Xyz::new(u64::from(min_x), u64::from(min_y), zoom)); let right_bottom_bounds = web_merc().xy_bounds(&Xyz::new(u64::from(max_x), u64::from(max_y), zoom)); diff --git a/martin/src/fonts/mod.rs b/martin/src/fonts/mod.rs index c60c4a06..c776f1ca 100644 --- a/martin/src/fonts/mod.rs +++ b/martin/src/fonts/mod.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use std::sync::OnceLock; use bit_set::BitSet; -use itertools::Itertools; +use itertools::Itertools as _; use log::{debug, info, warn}; use pbf_font_tools::freetype::{Face, Library}; use pbf_font_tools::protobuf::Message; @@ -335,7 +335,6 @@ fn parse_font( } else { format!("{s:02X}-{e:02X}") }) - .collect::>() .join(", "), ); diff --git a/martin/src/pg/configurator.rs b/martin/src/pg/configurator.rs index 60b7afe9..9e0bde7a 100644 --- a/martin/src/pg/configurator.rs +++ b/martin/src/pg/configurator.rs @@ -2,7 +2,7 @@ use std::cmp::Ordering; use std::collections::HashSet; use futures::future::join_all; -use itertools::Itertools; +use itertools::Itertools as _; use log::{debug, error, info, warn}; use crate::args::BoundsCalcType; diff --git a/martin/src/pg/function_source.rs b/martin/src/pg/function_source.rs index 6b3fc852..0fa921e1 100644 --- a/martin/src/pg/function_source.rs +++ b/martin/src/pg/function_source.rs @@ -1,4 +1,4 @@ -use std::fmt::Write; +use std::fmt::Write as _; use std::iter::zip; use log::{debug, warn}; diff --git a/martin/src/pg/utils.rs b/martin/src/pg/utils.rs index 80036708..8d79c214 100755 --- a/martin/src/pg/utils.rs +++ b/martin/src/pg/utils.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, HashMap}; use deadpool_postgres::tokio_postgres::types::Json; +use itertools::Itertools as _; use log::{error, info, warn}; use postgis::{ewkb, LineString, Point, Polygon}; use tilejson::{Bounds, TileJSON}; @@ -104,7 +105,7 @@ fn find_info_kv<'a, T>( match find_kv_ignore_case(map, key) { Ok(None) => { warn!("Unable to configure source {id} because {info} '{key}' was not found. Possible values are: {}", - map.keys().map(String::as_str).collect::>().join(", ")); + map.keys().map(String::as_str).join(", ")); None } Ok(Some(result)) => { diff --git a/martin/src/utils/error.rs b/martin/src/utils/error.rs index 701d332d..3167e8a6 100644 --- a/martin/src/utils/error.rs +++ b/martin/src/utils/error.rs @@ -1,5 +1,5 @@ use std::error::Error; -use std::fmt::Write; +use std::fmt::Write as _; use std::io; use std::path::PathBuf; diff --git a/martin/src/utils/id_resolver.rs b/martin/src/utils/id_resolver.rs index 7c91d14b..3942717e 100644 --- a/martin/src/utils/id_resolver.rs +++ b/martin/src/utils/id_resolver.rs @@ -1,6 +1,6 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; -use std::fmt::Write; +use std::fmt::Write as _; use std::sync::{Arc, Mutex}; use log::warn; diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index cb5c4f76..f87f0fd3 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -19,10 +19,11 @@ cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio" [dependencies] enum-display.workspace = true futures.workspace = true +itertools.workspace = true log.workspace = true martin-tile-utils.workspace = true -serde_json.workspace = true serde.workspace = true +serde_json.workspace = true serde_with.workspace = true size_format.workspace = true sqlite-hashes.workspace = true diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 75e6f5eb..76c11048 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use log::error; use mbtiles::{apply_patch, AggHashType, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier}; -#[derive(Parser, PartialEq, Eq, Debug)] +#[derive(Parser, PartialEq, Debug)] #[command( version, name = "mbtiles", @@ -19,7 +19,7 @@ pub struct Args { command: Commands, } -#[derive(Subcommand, PartialEq, Eq, Debug)] +#[derive(Subcommand, PartialEq, Debug)] enum Commands { /// Show MBTiles file summary statistics #[command(name = "summary", alias = "info")] diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index 7c4ba4cf..a5a576f9 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -1,13 +1,16 @@ -use std::collections::HashSet; +use std::fmt::Write as _; use std::path::PathBuf; #[cfg(feature = "cli")] use clap::{Args, ValueEnum}; use enum_display::EnumDisplay; -use log::{debug, info}; +use itertools::Itertools as _; +use log::{debug, info, trace}; +use martin_tile_utils::{bbox_to_xyz, MAX_ZOOM}; use serde::{Deserialize, Serialize}; -use sqlite_hashes::rusqlite::{params_from_iter, Connection}; +use sqlite_hashes::rusqlite::Connection; use sqlx::{query, Executor as _, Row, SqliteConnection}; +use tilejson::Bounds; use crate::errors::MbtResult; use crate::queries::{ @@ -15,7 +18,7 @@ use crate::queries::{ }; use crate::MbtType::{Flat, FlatWithHash, Normalized}; use crate::{ - reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, + invert_y_value, reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF, }; @@ -39,7 +42,7 @@ impl CopyDuplicateMode { } } -#[derive(Clone, Default, PartialEq, Eq, Debug)] +#[derive(Clone, Default, PartialEq, Debug)] #[cfg_attr(feature = "cli", derive(Args))] pub struct MbtilesCopier { /// MBTiles file to read from @@ -73,6 +76,9 @@ pub struct MbtilesCopier { /// List of zoom levels to copy #[cfg_attr(feature = "cli", arg(long, value_delimiter = ','))] pub zoom_levels: Vec, + /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. + #[cfg_attr(feature = "cli", arg(long))] + pub bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[cfg_attr(feature = "cli", arg(long, conflicts_with("apply_patch")))] @@ -105,6 +111,7 @@ impl MbtilesCopier { min_zoom: None, max_zoom: None, zoom_levels: Vec::default(), + bbox: vec![], diff_with_file: None, apply_patch: None, skip_agg_tiles_hash: false, @@ -216,7 +223,7 @@ impl MbtileCopierInt { Self::get_select_from(src_type, dst_type).to_string() }; - let (where_clause, query_args) = self.get_where_clause(); + let where_clause = self.get_where_clause(); let select_from = format!("{select_from} {where_clause}"); let on_dupl = on_duplicate.to_sql(); let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type); @@ -232,14 +239,7 @@ impl MbtileCopierInt { // SAFETY: this is safe as long as handle_lock is valid. We will drop the lock. let rusqlite_conn = unsafe { Connection::from_handle(handle) }?; - Self::copy_tiles( - &rusqlite_conn, - dst_type, - &query_args, - on_dupl, - &select_from, - &sql_cond, - )?; + Self::copy_tiles(&rusqlite_conn, dst_type, on_dupl, &select_from, &sql_cond)?; self.copy_metadata(&rusqlite_conn, &dif, on_dupl)?; } @@ -316,7 +316,6 @@ impl MbtileCopierInt { fn copy_tiles( rusqlite_conn: &Connection, dst_type: MbtType, - query_args: &Vec, on_dupl: &str, select_from: &str, sql_cond: &str, @@ -346,8 +345,8 @@ impl MbtileCopierInt { SELECT tile_hash as tile_id, tile_data FROM ({select_from})" ); - debug!("Copying to {dst_type} with {sql} {query_args:?}"); - rusqlite_conn.execute(&sql, params_from_iter(query_args))?; + debug!("Copying to {dst_type} with {sql}"); + rusqlite_conn.execute(&sql, [])?; format!( " @@ -359,8 +358,8 @@ impl MbtileCopierInt { } }; - debug!("Copying to {dst_type} with {sql} {query_args:?}"); - rusqlite_conn.execute(&sql, params_from_iter(query_args))?; + debug!("Copying to {dst_type} with {sql}"); + rusqlite_conn.execute(&sql, [])?; Ok(()) } @@ -585,32 +584,49 @@ impl MbtileCopierInt { } } - fn get_where_clause(&self) -> (String, Vec) { - let mut query_args = vec![]; - - let sql = if !&self.options.zoom_levels.is_empty() { - let zooms: HashSet = self.options.zoom_levels.iter().copied().collect(); - for z in &zooms { - query_args.push(*z); - } - format!(" AND zoom_level IN ({})", vec!["?"; zooms.len()].join(",")) + /// Format SQL WHERE clause and return it along with the query arguments. + /// Note that there is no risk of SQL injection here, as the arguments are integers. + fn get_where_clause(&self) -> String { + let mut sql = if !&self.options.zoom_levels.is_empty() { + let zooms = self.options.zoom_levels.iter().join(","); + format!(" AND zoom_level IN ({zooms})") } else if let Some(min_zoom) = self.options.min_zoom { if let Some(max_zoom) = self.options.max_zoom { - query_args.push(min_zoom); - query_args.push(max_zoom); - " AND zoom_level BETWEEN ? AND ?".to_string() + format!(" AND zoom_level BETWEEN {min_zoom} AND {max_zoom}") } else { - query_args.push(min_zoom); - " AND zoom_level >= ?".to_string() + format!(" AND zoom_level >= {min_zoom}") } } else if let Some(max_zoom) = self.options.max_zoom { - query_args.push(max_zoom); - " AND zoom_level <= ?".to_string() + format!(" AND zoom_level <= {max_zoom}") } else { String::new() }; - (sql, query_args) + if !self.options.bbox.is_empty() { + sql.push_str(" AND (\n"); + for (idx, bbox) in self.options.bbox.iter().enumerate() { + // Use maximum zoom value for easy filtering, + // converting it on the fly to the actual zoom level + let (min_x, min_y, max_x, max_y) = + bbox_to_xyz(bbox.left, bbox.bottom, bbox.right, bbox.top, MAX_ZOOM); + trace!("Bounding box {bbox} converted to {min_x},{min_y},{max_x},{max_y} at zoom {MAX_ZOOM}"); + let (min_y, max_y) = ( + invert_y_value(MAX_ZOOM, max_y), + invert_y_value(MAX_ZOOM, min_y), + ); + + if idx > 0 { + sql.push_str(" OR\n"); + } + writeln!( + sql, + "((tile_column * (1 << ({MAX_ZOOM} - zoom_level))) BETWEEN {min_x} AND {max_x} AND (tile_row * (1 << ({MAX_ZOOM} - zoom_level))) BETWEEN {min_y} AND {max_y})", + ).unwrap(); + } + sql.push(')'); + } + + sql } } diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 407c1571..b5d59a16 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -4,11 +4,15 @@ use std::str::from_utf8; use ctor::ctor; use insta::{allow_duplicates, assert_display_snapshot}; +use itertools::Itertools as _; use log::info; +use martin_tile_utils::xyz_to_bbox; use mbtiles::AggHashType::Verify; use mbtiles::IntegrityCheckType::Off; use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized}; -use mbtiles::{apply_patch, init_mbtiles_schema, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier}; +use mbtiles::{ + apply_patch, init_mbtiles_schema, invert_y_value, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier, +}; use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; use serde::Serialize; @@ -255,6 +259,20 @@ async fn convert( let z6only = dump(&mut opt.run().await?).await?; assert_snapshot!(z6only, "v1__z6__{frm}-{to}"); + let mut opt = copier(&frm_mbt, &mem); + opt.dst_type_cli = Some(dst_type); + + // Filter (0, 0, 2, 2) in mbtiles coordinates, which is (0, 2^5-1-2, 2, 2^5-1-0) = (0, 29, 2, 31) in XYZ coordinates, and slightly decrease it + let mut bbox = xyz_to_bbox(5, 0, invert_y_value(5, 2), 2, invert_y_value(5, 0)); + bbox[0] += 180.0 * 0.1 / f64::from(1 << 5); + bbox[1] += 90.0 * 0.1 / f64::from(1 << 5); + bbox[2] -= 180.0 * 0.1 / f64::from(1 << 5); + bbox[3] -= 90.0 * 0.1 / f64::from(1 << 5); + opt.bbox.push(bbox.into()); + + let dmp = dump(&mut opt.run().await?).await?; + assert_snapshot!(dmp, "v1__bbox__{frm}-{to}"); + let mut opt = copier(&frm_mbt, &mem); opt.dst_type_cli = Some(dst_type); opt.min_zoom = Some(6); @@ -370,11 +388,16 @@ async fn patch_on_copy( #[actix_rt::test] #[ignore] async fn test_one() { + let db = Databases::default(); + + // Test convert + convert(Flat, Flat, &db).await.unwrap(); + + // Test diff patch copy let src_type = FlatWithHash; let dif_type = FlatWithHash; // let dst_type = Some(FlatWithHash); let dst_type = None; - let db = databases(); diff_and_patch(src_type, dif_type, dst_type, &db) .await @@ -465,7 +488,6 @@ async fn dump(conn: &mut SqliteConnection) -> MbtResult> { }) .unwrap_or("NULL".to_string()) }) - .collect::>() .join(", "); format!("( {val} )") }) diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap new file mode 100644 index 00000000..510f5754 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap @@ -0,0 +1,42 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 2, 2, blob(remove) )', + '( 6, 1, 4, blob(edit-v1) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap new file mode 100644 index 00000000..d7da6423 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap new file mode 100644 index 00000000..ae37bc9b --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap @@ -0,0 +1,85 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap new file mode 100644 index 00000000..510f5754 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap @@ -0,0 +1,42 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 2, 2, blob(remove) )', + '( 6, 1, 4, blob(edit-v1) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap new file mode 100644 index 00000000..d7da6423 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap new file mode 100644 index 00000000..ae37bc9b --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap @@ -0,0 +1,85 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap new file mode 100644 index 00000000..510f5754 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap @@ -0,0 +1,42 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 2, 2, blob(remove) )', + '( 6, 1, 4, blob(edit-v1) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap new file mode 100644 index 00000000..d7da6423 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap new file mode 100644 index 00000000..ae37bc9b --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap @@ -0,0 +1,85 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/validate.rs b/mbtiles/tests/validate.rs index 7192d453..d682f111 100644 --- a/mbtiles/tests/validate.rs +++ b/mbtiles/tests/validate.rs @@ -1,4 +1,7 @@ -use martin_tile_utils::MAX_ZOOM; +#![allow(clippy::unreadable_literal)] + +use insta::assert_snapshot; +use martin_tile_utils::{bbox_to_xyz, MAX_ZOOM}; use mbtiles::MbtError::InvalidTileIndex; use mbtiles::{create_metadata_table, Mbtiles}; use rstest::rstest; @@ -92,13 +95,16 @@ async fn tile_coordinate(#[case] prefix: &str, #[case] suffix: &str) { ok!("1, {prefix} 1 {suffix}"); ok!("2, {prefix} 3 {suffix}"); ok!("3, {prefix} 7 {suffix}"); - ok!("30, {prefix} 0 {suffix}"); - ok!("30, {prefix} 1073741823 {suffix}"); + ok!("24, {prefix} 0 {suffix}"); + ok!("24, {prefix} 16777215 {suffix}"); + // ok!("30, {prefix} 0 {suffix}"); + // ok!("30, {prefix} 1073741823 {suffix}"); err!("0, {prefix} 1 {suffix}"); err!("1, {prefix} 2 {suffix}"); err!("2, {prefix} 4 {suffix}"); err!("3, {prefix} 8 {suffix}"); + err!("24, {prefix} 16777216 {suffix}"); err!("30, {prefix} 1073741824 {suffix}"); err!("{MAX_ZOOM}, {prefix} 1073741824 {suffix}"); err!("{}, {prefix} 0 {suffix}", MAX_ZOOM + 1); // unsupported zoom @@ -117,3 +123,45 @@ async fn tile_data() { err!("0, 0, 0, CAST('abc' AS TEXT)"); err!("0, 0, 0, CAST(123 AS TEXT)"); } + +#[test] +fn test_box() { + fn tst(left: f64, bottom: f64, right: f64, top: f64, zoom: u8) -> String { + let (x0, y0, x1, y1) = bbox_to_xyz(left, bottom, right, top, zoom); + format!("({x0}, {y0}, {x1}, {y1})") + } + + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 0), @"(0, 0, 0, 0)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 1), @"(0, 1, 0, 1)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 2), @"(0, 3, 0, 3)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 3), @"(0, 7, 0, 7)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 4), @"(0, 14, 1, 15)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 5), @"(0, 29, 2, 31)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 6), @"(0, 58, 5, 63)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 7), @"(0, 116, 11, 126)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 8), @"(0, 233, 23, 253)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 9), @"(0, 466, 47, 507)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 10), @"(1, 933, 94, 1014)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 11), @"(3, 1866, 188, 2029)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 12), @"(6, 3732, 377, 4059)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 13), @"(12, 7465, 755, 8119)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 14), @"(25, 14931, 1510, 16239)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 15), @"(51, 29863, 3020, 32479)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 16), @"(102, 59727, 6041, 64958)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 17), @"(204, 119455, 12083, 129917)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 18), @"(409, 238911, 24166, 259834)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 19), @"(819, 477823, 48332, 519669)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 20), @"(1638, 955647, 96665, 1039339)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 21), @"(3276, 1911295, 193331, 2078678)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 22), @"(6553, 3822590, 386662, 4157356)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 23), @"(13107, 7645181, 773324, 8314713)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 24), @"(26214, 15290363, 1546649, 16629427)"); + + // All these are incorrect + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 25), @"(33554431, 33554431, 33554431, 33554431)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 26), @"(67108863, 67108863, 67108863, 67108863)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 27), @"(134217727, 134217727, 134217727, 134217727)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 28), @"(268435455, 268435455, 268435455, 268435455)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 29), @"(536870911, 536870911, 536870911, 536870911)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 30), @"(1073741823, 1073741823, 1073741823, 1073741823)"); +}