diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index ebfb5e63..4c746016 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -3,8 +3,8 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use log::error; use mbtiles::{ - apply_patch, AggHashType, CopyDuplicateMode, IntegrityCheckType, MbtResult, MbtTypeCli, - Mbtiles, MbtilesCopier, + apply_patch, AggHashType, CopyDuplicateMode, CopyType, IntegrityCheckType, MbtResult, + MbtTypeCli, Mbtiles, MbtilesCopier, }; use tilejson::Bounds; @@ -83,38 +83,42 @@ enum Commands { #[derive(Clone, Default, PartialEq, Debug, clap::Args)] pub struct CopyArgs { /// MBTiles file to read from - pub src_file: PathBuf, + src_file: PathBuf, /// MBTiles file to write to - pub dst_file: PathBuf, + dst_file: PathBuf, + /// Limit what gets copied. + /// When copying tiles only, the agg_tiles_hash will still be updated unless --skip-agg-tiles-hash is set. + #[arg(long, value_name = "TYPE", default_value_t=CopyType::default())] + copy: CopyType, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source #[arg(long, alias = "dst-type", alias = "dst_type", value_name = "SCHEMA")] - pub mbtiles_type: Option, + mbtiles_type: Option, /// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists #[arg(long, value_enum)] - pub on_duplicate: Option, + on_duplicate: Option, /// Minimum zoom level to copy #[arg(long, conflicts_with("zoom_levels"))] - pub min_zoom: Option, + min_zoom: Option, /// Maximum zoom level to copy #[arg(long, conflicts_with("zoom_levels"))] - pub max_zoom: Option, + max_zoom: Option, /// List of zoom levels to copy #[arg(long, value_delimiter = ',')] - pub zoom_levels: Vec, + zoom_levels: Vec, /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. #[arg(long)] - pub bbox: Vec, + bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[arg(long, conflicts_with("apply_patch"))] - pub diff_with_file: Option, + diff_with_file: Option, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[arg(long, conflicts_with("diff_with_file"))] - pub apply_patch: Option, + apply_patch: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. #[arg(long)] - pub skip_agg_tiles_hash: bool, + skip_agg_tiles_hash: bool, } #[tokio::main] @@ -149,6 +153,7 @@ async fn main_int() -> anyhow::Result<()> { let opts = MbtilesCopier { src_file: opts.src_file, dst_file: opts.dst_file, + copy: opts.copy, dst_type_cli: opts.mbtiles_type, dst_type: None, on_duplicate: opts.on_duplicate, @@ -396,6 +401,22 @@ mod tests { ); } + #[test] + fn test_copy_limit() { + assert_eq!( + Args::parse_from(["mbtiles", "copy", "src_file", "dst_file", "--copy", "metadata"]), + Args { + verbose: false, + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + copy: CopyType::Metadata, + ..Default::default() + }) + } + ); + } + #[test] fn test_meta_get_no_arguments() { assert_eq!( diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index e59763ab..7270ef39 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -16,11 +16,11 @@ use crate::queries::{ }; use crate::MbtType::{Flat, FlatWithHash, Normalized}; use crate::{ - invert_y_value, reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, - AGG_TILES_HASH_IN_DIFF, + invert_y_value, reset_db_settings, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles, + AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF, }; -#[derive(PartialEq, Eq, Debug, Clone, Copy, EnumDisplay, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum CopyDuplicateMode { @@ -46,6 +46,8 @@ pub struct MbtilesCopier { pub src_file: PathBuf, /// MBTiles file to write to pub dst_file: PathBuf, + /// Limit what gets copied + pub copy: CopyType, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source pub dst_type_cli: Option, /// Destination type with options @@ -78,24 +80,6 @@ struct MbtileCopierInt { } impl MbtilesCopier { - #[must_use] - pub fn new(src_filepath: PathBuf, dst_filepath: PathBuf) -> Self { - Self { - src_file: src_filepath, - dst_file: dst_filepath, - dst_type_cli: None, - dst_type: None, - on_duplicate: None, - min_zoom: None, - max_zoom: None, - zoom_levels: Vec::default(), - bbox: vec![], - diff_with_file: None, - apply_patch: None, - skip_agg_tiles_hash: false, - } - } - pub async fn run(self) -> MbtResult { MbtileCopierInt::new(self)?.run().await } @@ -166,6 +150,11 @@ impl MbtileCopierInt { src_mbt.attach_to(&mut conn, "sourceDb").await?; + let what = match self.options.copy { + CopyType::All => "", + CopyType::Tiles => "tiles data ", + CopyType::Metadata => "metadata ", + }; let dst_type: MbtType; if let Some((dif_mbt, dif_type, _)) = &dif { if !is_empty_db { @@ -175,39 +164,24 @@ impl MbtileCopierInt { dif_mbt.attach_to(&mut conn, "diffDb").await?; let dif_path = dif_mbt.filepath(); if self.options.diff_with_file.is_some() { - info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) into a new file {dst_mbt} ({dst_type})"); + info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) {what}into a new file {dst_mbt} ({dst_type})"); } else { - info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) into a new file {dst_mbt} ({dst_type})"); + info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) {what}into a new file {dst_mbt} ({dst_type})"); } } else if is_empty_db { dst_type = self.options.dst_type().unwrap_or(src_type); - info!("Copying {src_mbt} ({src_type}) to a new file {dst_mbt} ({dst_type})"); + info!("Copying {src_mbt} ({src_type}) {what}to a new file {dst_mbt} ({dst_type})"); } else { dst_type = self.validate_dst_type(dst_mbt.detect_type(&mut conn).await?)?; - info!("Copying {src_mbt} ({src_type}) to an existing file {dst_mbt} ({dst_type})"); + info!( + "Copying {src_mbt} ({src_type}) {what}to an existing file {dst_mbt} ({dst_type})" + ); } if is_empty_db { self.init_new_schema(&mut conn, src_type, dst_type).await?; } - let select_from = if let Some((_, dif_type, _)) = &dif { - if self.options.diff_with_file.is_some() { - Self::get_select_from_with_diff(*dif_type, dst_type) - } else { - Self::get_select_from_apply_patch(src_type, *dif_type, dst_type) - } - } else { - Self::get_select_from(src_type, dst_type).to_string() - }; - - let where_clause = self.get_where_clause(); - let select_from = format!("{select_from} {where_clause}"); - let on_dupl = on_duplicate.to_sql(); - let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type); - - debug!("Copying tiles with 'INSERT {on_dupl}' {src_type} -> {dst_type} ({sql_cond})"); - { // SAFETY: This must be scoped to make sure the handle is dropped before we continue using conn // Make sure not to execute any other queries while the handle is locked @@ -217,12 +191,20 @@ impl MbtileCopierInt { // SAFETY: this is safe as long as handle_lock is valid. We will drop the lock. let rusqlite_conn = unsafe { Connection::from_handle(handle) }?; - Self::copy_tiles(&rusqlite_conn, dst_type, on_dupl, &select_from, &sql_cond)?; + if self.options.copy.copy_tiles() { + self.copy_tiles(&rusqlite_conn, &dif, src_type, dst_type, on_duplicate)?; + } else { + debug!("Skipping copying tiles"); + } - self.copy_metadata(&rusqlite_conn, &dif, on_dupl)?; + if self.options.copy.copy_metadata() { + self.copy_metadata(&rusqlite_conn, &dif, on_duplicate)?; + } else { + debug!("Skipping copying metadata"); + } } - if !self.options.skip_agg_tiles_hash { + if self.options.copy.copy_tiles() && !self.options.skip_agg_tiles_hash { dst_mbt.update_agg_tiles_hash(&mut conn).await?; } @@ -237,8 +219,9 @@ impl MbtileCopierInt { &self, rusqlite_conn: &Connection, dif: &Option<(Mbtiles, MbtType, MbtType)>, - on_dupl: &str, + on_duplicate: CopyDuplicateMode, ) -> Result<(), MbtError> { + let on_dupl = on_duplicate.to_sql(); let sql; if dif.is_some() { // Insert all rows from diffDb.metadata if they do not exist or are different in sourceDb.metadata. @@ -292,19 +275,35 @@ impl MbtileCopierInt { } fn copy_tiles( + &self, rusqlite_conn: &Connection, + dif: &Option<(Mbtiles, MbtType, MbtType)>, + src_type: MbtType, dst_type: MbtType, - on_dupl: &str, - select_from: &str, - sql_cond: &str, + on_duplicate: CopyDuplicateMode, ) -> Result<(), MbtError> { + let on_dupl = on_duplicate.to_sql(); + + let select_from = if let Some((_, dif_type, _)) = &dif { + if self.options.diff_with_file.is_some() { + Self::get_select_from_with_diff(*dif_type, dst_type) + } else { + Self::get_select_from_apply_patch(src_type, *dif_type, dst_type) + } + } else { + Self::get_select_from(src_type, dst_type).to_string() + }; + + let where_clause = self.get_where_clause(); + let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type); + let sql = match dst_type { Flat => { format!( " INSERT {on_dupl} INTO tiles (zoom_level, tile_column, tile_row, tile_data) - {select_from} {sql_cond}" + {select_from} {where_clause} {sql_cond}" ) } FlatWithHash => { @@ -312,7 +311,7 @@ impl MbtileCopierInt { " INSERT {on_dupl} INTO tiles_with_hash (zoom_level, tile_column, tile_row, tile_data, tile_hash) - {select_from} {sql_cond}" + {select_from} {where_clause} {sql_cond}" ) } Normalized { .. } => { @@ -321,7 +320,7 @@ impl MbtileCopierInt { INSERT OR IGNORE INTO images (tile_id, tile_data) SELECT tile_hash as tile_id, tile_data - FROM ({select_from})" + FROM ({select_from} {where_clause})" ); debug!("Copying to {dst_type} with {sql}"); rusqlite_conn.execute(&sql, [])?; @@ -331,7 +330,7 @@ impl MbtileCopierInt { INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id) SELECT zoom_level, tile_column, tile_row, tile_hash as tile_id - FROM ({select_from} {sql_cond})" + FROM ({select_from} {where_clause} {sql_cond})" ) } }; @@ -634,8 +633,12 @@ mod tests { dst_type_cli: Option, expected_dst_type: MbtType, ) -> MbtResult<()> { - let mut opt = MbtilesCopier::new(src_filepath.clone(), dst_filepath.clone()); - opt.dst_type_cli = dst_type_cli; + let opt = MbtilesCopier { + src_file: src_filepath.clone(), + dst_file: dst_filepath.clone(), + dst_type_cli, + ..Default::default() + }; let mut dst_conn = opt.run().await?; Mbtiles::new(src_filepath)? @@ -750,22 +753,26 @@ mod tests { #[actix_rt::test] async fn copy_with_min_max_zoom() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); - let mut opt = MbtilesCopier::new(src, dst); - opt.min_zoom = Some(2); - opt.max_zoom = Some(4); + let opt = MbtilesCopier { + src_file: PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"), + dst_file: PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"), + min_zoom: Some(2), + max_zoom: Some(4), + ..Default::default() + }; verify_copy_with_zoom_filter(opt, 3).await } #[actix_rt::test] async fn copy_with_zoom_levels() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); - let mut opt = MbtilesCopier::new(src, dst); - opt.min_zoom = Some(2); - opt.max_zoom = Some(4); - opt.zoom_levels.extend(&[1, 6]); + let opt = MbtilesCopier { + src_file: PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"), + dst_file: PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"), + min_zoom: Some(2), + max_zoom: Some(4), + zoom_levels: vec![1, 6], + ..Default::default() + }; verify_copy_with_zoom_filter(opt, 2).await } @@ -777,8 +784,12 @@ mod tests { let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"); - let mut opt = MbtilesCopier::new(src.clone(), dst.clone()); - opt.diff_with_file = Some(diff_file.clone()); + let opt = MbtilesCopier { + src_file: src.clone(), + dst_file: dst.clone(), + diff_with_file: Some(diff_file.clone()), + ..Default::default() + }; let mut dst_conn = opt.run().await?; assert!(dst_conn @@ -820,8 +831,12 @@ mod tests { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let mut opt = MbtilesCopier::new(src.clone(), dst.clone()); - opt.on_duplicate = Some(CopyDuplicateMode::Abort); + let opt = MbtilesCopier { + src_file: src.clone(), + dst_file: dst.clone(), + on_duplicate: Some(CopyDuplicateMode::Abort), + ..Default::default() + }; assert!(matches!( opt.run().await.unwrap_err(), @@ -838,12 +853,20 @@ mod tests { let dst = PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared"); - let _dst_conn = MbtilesCopier::new(dst_file.clone(), dst.clone()) - .run() - .await?; + let _dst_conn = MbtilesCopier { + src_file: dst_file.clone(), + dst_file: dst.clone(), + ..Default::default() + } + .run() + .await?; - let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone()); - opt.on_duplicate = Some(CopyDuplicateMode::Override); + let opt = MbtilesCopier { + src_file: src_file.clone(), + dst_file: dst.clone(), + on_duplicate: Some(CopyDuplicateMode::Override), + ..Default::default() + }; let mut dst_conn = opt.run().await?; // Verify the tiles in the destination file is a superset of the tiles in the source file @@ -867,12 +890,20 @@ mod tests { let dst = PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared"); - let _dst_conn = MbtilesCopier::new(dst_file.clone(), dst.clone()) - .run() - .await?; + let _dst_conn = MbtilesCopier { + src_file: dst_file.clone(), + dst_file: dst.clone(), + ..Default::default() + } + .run() + .await?; - let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone()); - opt.on_duplicate = Some(CopyDuplicateMode::Ignore); + let opt = MbtilesCopier { + src_file: src_file.clone(), + dst_file: dst.clone(), + on_duplicate: Some(CopyDuplicateMode::Ignore), + ..Default::default() + }; let mut dst_conn = opt.run().await?; // Verify the tiles in the destination file are the same as those in the source file except for those with duplicate (zoom_level, tile_column, tile_row) diff --git a/mbtiles/src/lib.rs b/mbtiles/src/lib.rs index 7cdc1a8d..30be6032 100644 --- a/mbtiles/src/lib.rs +++ b/mbtiles/src/lib.rs @@ -10,7 +10,7 @@ mod errors; pub use errors::{MbtError, MbtResult}; mod mbtiles; -pub use mbtiles::{MbtTypeCli, Mbtiles}; +pub use mbtiles::{CopyType, MbtTypeCli, Mbtiles}; mod metadata; pub use metadata::Metadata; diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index 64dcb16e..fe62c7e6 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -21,6 +21,27 @@ pub enum MbtTypeCli { Normalized, } +#[derive(Default, Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] +#[enum_display(case = "Kebab")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum CopyType { + #[default] + All, + Metadata, + Tiles, +} + +impl CopyType { + #[must_use] + pub fn copy_tiles(&self) -> bool { + matches!(self, Self::All | Self::Tiles) + } + #[must_use] + pub fn copy_metadata(&self) -> bool { + matches!(self, Self::All | Self::Metadata) + } +} + #[derive(Clone, Debug)] pub struct Mbtiles { filepath: String, diff --git a/mbtiles/src/patcher.rs b/mbtiles/src/patcher.rs index c1b68b30..06333320 100644 --- a/mbtiles/src/patcher.rs +++ b/mbtiles/src/patcher.rs @@ -148,9 +148,13 @@ mod tests { let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared"); - let mut src_conn = MbtilesCopier::new(src_file.clone(), src.clone()) - .run() - .await?; + let mut src_conn = MbtilesCopier { + src_file: src_file.clone(), + dst_file: src.clone(), + ..Default::default() + } + .run() + .await?; // Apply patch to the src data in in-memory DB let patch_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles"); @@ -175,9 +179,13 @@ mod tests { let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared"); - let mut src_conn = MbtilesCopier::new(src_file.clone(), src.clone()) - .run() - .await?; + let mut src_conn = MbtilesCopier { + src_file: src_file.clone(), + dst_file: src.clone(), + ..Default::default() + } + .run() + .await?; // Apply patch to the src data in in-memory DB let patch_file = diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index 57e9ebbc..34ce483a 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -48,7 +48,7 @@ impl MbtType { } } -#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum IntegrityCheckType { @@ -58,7 +58,7 @@ pub enum IntegrityCheckType { Off, } -#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum AggHashType { diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index e99125a4..3166fa16 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -11,7 +11,8 @@ use mbtiles::AggHashType::Verify; use mbtiles::IntegrityCheckType::Off; use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized}; use mbtiles::{ - apply_patch, init_mbtiles_schema, invert_y_value, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier, + apply_patch, init_mbtiles_schema, invert_y_value, CopyType, MbtResult, MbtTypeCli, Mbtiles, + MbtilesCopier, }; use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; @@ -87,10 +88,6 @@ fn path(mbt: &Mbtiles) -> PathBuf { PathBuf::from(mbt.filepath()) } -fn copier(src: &Mbtiles, dst: &Mbtiles) -> MbtilesCopier { - MbtilesCopier::new(path(src), path(dst)) -} - fn shorten(v: MbtTypeCli) -> &'static str { match v { Flat => "flat", @@ -135,9 +132,13 @@ macro_rules! new_file { cn_tmp.execute($sql_meta).await.unwrap(); let (dst_mbt, cn_dst) = open!($function, $($arg)*); - let mut opt = copier(&tmp_mbt, &dst_mbt); - opt.dst_type_cli = Some($dst_type_cli); - opt.skip_agg_tiles_hash = $skip_agg; + let opt = MbtilesCopier { + src_file: path(&tmp_mbt), + dst_file: path(&dst_mbt), + dst_type_cli: Some($dst_type_cli), + skip_agg_tiles_hash: $skip_agg, + ..Default::default() + }; opt.run().await.unwrap(); (dst_mbt, cn_dst) @@ -199,7 +200,12 @@ fn databases() -> Databases { let (v1_mbt, mut v1_cn) = open!(databases, "{typ}__v1"); let raw_mbt = result.mbtiles("v1_no_hash", mbt_typ); - copier(raw_mbt, &v1_mbt).run().await.unwrap(); + let opt = MbtilesCopier { + src_file: path(raw_mbt), + dst_file: path(&v1_mbt), + ..Default::default() + }; + opt.run().await.unwrap(); let dmp = dump(&mut v1_cn).await.unwrap(); assert_snapshot!(&dmp, "{typ}__v1"); let hash = v1_mbt.validate(Off, Verify).await.unwrap(); @@ -220,9 +226,13 @@ fn databases() -> Databases { let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif"); let v1_mbt = result.mbtiles("v1", mbt_typ); - let mut opt = copier(v1_mbt, &dif_mbt); let v2_mbt = result.mbtiles("v2", mbt_typ); - opt.diff_with_file = Some(path(v2_mbt)); + let opt = MbtilesCopier { + src_file: path(v1_mbt), + dst_file: path(&dif_mbt), + diff_with_file: Some(path(v2_mbt)), + ..Default::default() + }; opt.run().await.unwrap(); let dmp = dump(&mut dif_cn).await.unwrap(); assert_snapshot!(&dmp, "{typ}__dif"); @@ -248,44 +258,87 @@ async fn convert( let mem = Mbtiles::new(":memory:")?; let (frm_mbt, _frm_cn) = new_file!(convert, frm_type, METADATA_V1, TILES_V1, "{frm}-{to}"); - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + ..Default::default() + }; let dmp = dump(&mut opt.run().await?).await?; pretty_assert_eq!(databases.dump("v1", dst_type), &dmp); - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - opt.zoom_levels.push(6); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + copy: CopyType::Metadata, + dst_type_cli: Some(dst_type), + ..Default::default() + }; + let dmp = dump(&mut opt.run().await?).await?; + allow_duplicates! { + assert_snapshot!(dmp, "v1__meta__{to}"); + }; + + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + copy: CopyType::Tiles, + dst_type_cli: Some(dst_type), + ..Default::default() + }; + let dmp = dump(&mut opt.run().await?).await?; + allow_duplicates! { + assert_snapshot!(dmp, "v1__tiles__{to}"); + } + + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + zoom_levels: vec![6], + ..Default::default() + }; let z6only = dump(&mut opt.run().await?).await?; allow_duplicates! { assert_snapshot!(z6only, "v1__z6__{to}"); } - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - // Filter (0, 0, 2, 2) in mbtiles coordinates, which is (0, 2^5-1-2, 2, 2^5-1-0) = (0, 29, 2, 31) in XYZ coordinates, and slightly decrease it let mut bbox = xyz_to_bbox(5, 0, invert_y_value(5, 2), 2, invert_y_value(5, 0)); - bbox[0] += 180.0 * 0.1 / f64::from(1 << 5); - bbox[1] += 90.0 * 0.1 / f64::from(1 << 5); - bbox[2] -= 180.0 * 0.1 / f64::from(1 << 5); - bbox[3] -= 90.0 * 0.1 / f64::from(1 << 5); - opt.bbox.push(bbox.into()); - + let adjust = 90.0 * 0.1 / f64::from(1 << 5); + bbox[0] += adjust; + bbox[1] += adjust; + bbox[2] -= adjust; + bbox[3] -= adjust; + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + bbox: vec![bbox.into()], + ..Default::default() + }; let dmp = dump(&mut opt.run().await?).await?; allow_duplicates! { assert_snapshot!(dmp, "v1__bbox__{to}"); } - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - opt.min_zoom = Some(6); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + min_zoom: Some(6), + ..Default::default() + }; pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?); - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - opt.min_zoom = Some(6); - opt.max_zoom = Some(6); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + min_zoom: Some(6), + max_zoom: Some(6), + ..Default::default() + }; pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?); Ok(()) @@ -309,8 +362,12 @@ async fn diff_and_patch( let (dif_mbt, mut dif_cn) = open!(diff_and_patchdiff_and_patch, "{prefix}__dif"); info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)"); - let mut opt = copier(v1_mbt, &dif_mbt); - opt.diff_with_file = Some(path(v2_mbt)); + let mut opt = MbtilesCopier { + src_file: path(v1_mbt), + dst_file: path(&dif_mbt), + diff_with_file: Some(path(v2_mbt)), + ..Default::default() + }; if let Some(dif_type) = dif_type { opt.dst_type_cli = Some(dif_type); } @@ -374,8 +431,12 @@ async fn patch_on_copy( let (v2_mbt, mut v2_cn) = open!(patch_on_copy, "{prefix}__v2"); info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)"); - let mut opt = copier(v1_mbt, &v2_mbt); - opt.apply_patch = Some(path(dif_mbt)); + let mut opt = MbtilesCopier { + src_file: path(v1_mbt), + dst_file: path(&v2_mbt), + apply_patch: Some(path(dif_mbt)), + ..Default::default() + }; if let Some(v2_type) = v2_type { opt.dst_type_cli = Some(v2_type); } @@ -506,9 +567,14 @@ async fn dump(conn: &mut SqliteConnection) -> MbtResult> { } #[allow(dead_code)] -async fn save_to_file(source_mbt: &Mbtiles, path: &str) -> MbtResult<()> { - let mut opt = copier(source_mbt, &Mbtiles::new(path)?); - opt.skip_agg_tiles_hash = true; +async fn save_to_file(source_mbt: &Mbtiles, path_mbt: &str) -> MbtResult<()> { + let dst = &Mbtiles::new(path_mbt)?; + let opt = MbtilesCopier { + src_file: path(source_mbt), + dst_file: path(dst), + skip_agg_tiles_hash: true, + ..Default::default() + }; opt.run().await?; Ok(()) } diff --git a/mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap new file mode 100644 index 00000000..910232f8 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap @@ -0,0 +1,37 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap new file mode 100644 index 00000000..548e2328 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap @@ -0,0 +1,45 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap new file mode 100644 index 00000000..3aa32c6c --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap @@ -0,0 +1,76 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap new file mode 100644 index 00000000..865b95c6 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap @@ -0,0 +1,45 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )'] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(root) )', + '( 5, 0, 0, blob(same) )', + '( 5, 0, 1, blob() )', + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 1, 3, blob(non-empty) )', + '( 5, 2, 2, blob(remove) )', + '( 5, 2, 3, blob() )', + '( 6, 0, 3, blob(same) )', + '( 6, 0, 5, blob(1-keep-1-rm) )', + '( 6, 1, 4, blob(edit-v1) )', + '( 6, 2, 6, blob(1-keep-1-rm) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap new file mode 100644 index 00000000..3c5f391e --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap @@ -0,0 +1,53 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )'] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(root), "63A9F0EA7BB98050796B649E85481845" )', + '( 5, 0, 0, blob(same), "51037A4A37730F52C8732586D3AAA316" )', + '( 5, 0, 1, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 3, blob(non-empty), "720C02778717818CC0A869955BA2AFB6" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 5, 2, 3, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 6, 0, 3, blob(same), "51037A4A37730F52C8732586D3AAA316" )', + '( 6, 0, 5, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 6, 2, 6, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap new file mode 100644 index 00000000..02a44e39 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap @@ -0,0 +1,92 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "51037A4A37730F52C8732586D3AAA316", blob(same) )', + '( "535A5575B48444EDEB926815AB26EC9B", blob(1-keep-1-rm) )', + '( "63A9F0EA7BB98050796B649E85481845", blob(root) )', + '( "720C02778717818CC0A869955BA2AFB6", blob(non-empty) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, "63A9F0EA7BB98050796B649E85481845" )', + '( 5, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 5, 0, 1, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 3, "720C02778717818CC0A869955BA2AFB6" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 5, 2, 3, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 6, 0, 3, "51037A4A37730F52C8732586D3AAA316" )', + '( 6, 0, 5, "535A5575B48444EDEB926815AB26EC9B" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 6, 2, 6, "535A5575B48444EDEB926815AB26EC9B" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )'] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id'''