maplibre/martin
1
1
Fork 0
mirror of https://github.com/maplibre/martin.git synced 2024-12-21 05:41:55 +03:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

New mbtiles copy --copy (all|tiles|metadata) flag to limit what gets copied (#1073)

Browse Source 
Limit what gets copied from one mbtiles to another.

Closes #1069
This commit is contained in:
Yuri Astrakhan 2023-12-16 14:59:52 -05:00 committed by GitHub
parent 15a521dcdc
commit de6b681d74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 636 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
mbtiles/src/bin/mbtiles.rs
View File

@ -3,8 +3,8 @@ use std::path::{Path, PathBuf};
use clap::{Parser, Subcommand};
use log::error;
use mbtiles::{
apply_patch, AggHashType, CopyDuplicateMode, IntegrityCheckType, MbtResult, MbtTypeCli,
Mbtiles, MbtilesCopier,
apply_patch, AggHashType, CopyDuplicateMode, CopyType, IntegrityCheckType, MbtResult,
MbtTypeCli, Mbtiles, MbtilesCopier,
};
use tilejson::Bounds;
@ -83,38 +83,42 @@ enum Commands {
#[derive(Clone, Default, PartialEq, Debug, clap::Args)]
pub struct CopyArgs {
/// MBTiles file to read from
pub src_file: PathBuf,
src_file: PathBuf,
/// MBTiles file to write to
pub dst_file: PathBuf,
dst_file: PathBuf,
/// Limit what gets copied.
/// When copying tiles only, the agg_tiles_hash will still be updated unless --skip-agg-tiles-hash is set.
#[arg(long, value_name = "TYPE", default_value_t=CopyType::default())]
copy: CopyType,
/// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source
#[arg(long, alias = "dst-type", alias = "dst_type", value_name = "SCHEMA")]
pub mbtiles_type: Option<MbtTypeCli>,
mbtiles_type: Option<MbtTypeCli>,
/// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists
#[arg(long, value_enum)]
pub on_duplicate: Option<CopyDuplicateMode>,
on_duplicate: Option<CopyDuplicateMode>,
/// Minimum zoom level to copy
#[arg(long, conflicts_with("zoom_levels"))]
pub min_zoom: Option<u8>,
min_zoom: Option<u8>,
/// Maximum zoom level to copy
#[arg(long, conflicts_with("zoom_levels"))]
pub max_zoom: Option<u8>,
max_zoom: Option<u8>,
/// List of zoom levels to copy
#[arg(long, value_delimiter = ',')]
pub zoom_levels: Vec<u8>,
zoom_levels: Vec<u8>,
/// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times.
#[arg(long)]
pub bbox: Vec<Bounds>,
bbox: Vec<Bounds>,
/// Compare source file with this file, and only copy non-identical tiles to destination.
/// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file.
#[arg(long, conflicts_with("apply_patch"))]
pub diff_with_file: Option<PathBuf>,
diff_with_file: Option<PathBuf>,
/// Compare source file with this file, and only copy non-identical tiles to destination.
/// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file.
#[arg(long, conflicts_with("diff_with_file"))]
pub apply_patch: Option<PathBuf>,
apply_patch: Option<PathBuf>,
/// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value.
#[arg(long)]
pub skip_agg_tiles_hash: bool,
skip_agg_tiles_hash: bool,
}
#[tokio::main]
@ -149,6 +153,7 @@ async fn main_int() -> anyhow::Result<()> {
let opts = MbtilesCopier {
src_file: opts.src_file,
dst_file: opts.dst_file,
copy: opts.copy,
dst_type_cli: opts.mbtiles_type,
dst_type: None,
on_duplicate: opts.on_duplicate,
@ -396,6 +401,22 @@ mod tests {
);
}
#[test]
fn test_copy_limit() {
assert_eq!(
Args::parse_from(["mbtiles", "copy", "src_file", "dst_file", "--copy", "metadata"]),
Args {
verbose: false,
command: Copy(CopyArgs {
src_file: PathBuf::from("src_file"),
dst_file: PathBuf::from("dst_file"),
copy: CopyType::Metadata,
..Default::default()
})
}
);
}
#[test]
fn test_meta_get_no_arguments() {
assert_eq!(

191
mbtiles/src/copier.rs
View File

@ -16,11 +16,11 @@ use crate::queries::{
};
use crate::MbtType::{Flat, FlatWithHash, Normalized};
use crate::{
invert_y_value, reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH,
AGG_TILES_HASH_IN_DIFF,
invert_y_value, reset_db_settings, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles,
AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF,
};
#[derive(PartialEq, Eq, Debug, Clone, Copy, EnumDisplay, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)]
#[enum_display(case = "Kebab")]
#[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
pub enum CopyDuplicateMode {
@ -46,6 +46,8 @@ pub struct MbtilesCopier {
pub src_file: PathBuf,
/// MBTiles file to write to
pub dst_file: PathBuf,
/// Limit what gets copied
pub copy: CopyType,
/// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source
pub dst_type_cli: Option<MbtTypeCli>,
/// Destination type with options
@ -78,24 +80,6 @@ struct MbtileCopierInt {
}
impl MbtilesCopier {
#[must_use]
pub fn new(src_filepath: PathBuf, dst_filepath: PathBuf) -> Self {
Self {
src_file: src_filepath,
dst_file: dst_filepath,
dst_type_cli: None,
dst_type: None,
on_duplicate: None,
min_zoom: None,
max_zoom: None,
zoom_levels: Vec::default(),
bbox: vec![],
diff_with_file: None,
apply_patch: None,
skip_agg_tiles_hash: false,
}
}
pub async fn run(self) -> MbtResult<SqliteConnection> {
MbtileCopierInt::new(self)?.run().await
}
@ -166,6 +150,11 @@ impl MbtileCopierInt {
src_mbt.attach_to(&mut conn, "sourceDb").await?;
let what = match self.options.copy {
CopyType::All => "",
CopyType::Tiles => "tiles data ",
CopyType::Metadata => "metadata ",
};
let dst_type: MbtType;
if let Some((dif_mbt, dif_type, _)) = &dif {
if !is_empty_db {
@ -175,39 +164,24 @@ impl MbtileCopierInt {
dif_mbt.attach_to(&mut conn, "diffDb").await?;
let dif_path = dif_mbt.filepath();
if self.options.diff_with_file.is_some() {
info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) into a new file {dst_mbt} ({dst_type})");
info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) {what}into a new file {dst_mbt} ({dst_type})");
} else {
info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) into a new file {dst_mbt} ({dst_type})");
info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) {what}into a new file {dst_mbt} ({dst_type})");
}
} else if is_empty_db {
dst_type = self.options.dst_type().unwrap_or(src_type);
info!("Copying {src_mbt} ({src_type}) to a new file {dst_mbt} ({dst_type})");
info!("Copying {src_mbt} ({src_type}) {what}to a new file {dst_mbt} ({dst_type})");
} else {
dst_type = self.validate_dst_type(dst_mbt.detect_type(&mut conn).await?)?;
info!("Copying {src_mbt} ({src_type}) to an existing file {dst_mbt} ({dst_type})");
info!(
"Copying {src_mbt} ({src_type}) {what}to an existing file {dst_mbt} ({dst_type})"
);
}
if is_empty_db {
self.init_new_schema(&mut conn, src_type, dst_type).await?;
}
let select_from = if let Some((_, dif_type, _)) = &dif {
if self.options.diff_with_file.is_some() {
Self::get_select_from_with_diff(*dif_type, dst_type)
} else {
Self::get_select_from_apply_patch(src_type, *dif_type, dst_type)
}
} else {
Self::get_select_from(src_type, dst_type).to_string()
};
let where_clause = self.get_where_clause();
let select_from = format!("{select_from} {where_clause}");
let on_dupl = on_duplicate.to_sql();
let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type);
debug!("Copying tiles with 'INSERT {on_dupl}' {src_type} -> {dst_type} ({sql_cond})");
{
// SAFETY: This must be scoped to make sure the handle is dropped before we continue using conn
// Make sure not to execute any other queries while the handle is locked
@ -217,12 +191,20 @@ impl MbtileCopierInt {
// SAFETY: this is safe as long as handle_lock is valid. We will drop the lock.
let rusqlite_conn = unsafe { Connection::from_handle(handle) }?;
Self::copy_tiles(&rusqlite_conn, dst_type, on_dupl, &select_from, &sql_cond)?;
if self.options.copy.copy_tiles() {
self.copy_tiles(&rusqlite_conn, &dif, src_type, dst_type, on_duplicate)?;
} else {
debug!("Skipping copying tiles");
}
self.copy_metadata(&rusqlite_conn, &dif, on_dupl)?;
if self.options.copy.copy_metadata() {
self.copy_metadata(&rusqlite_conn, &dif, on_duplicate)?;
} else {
debug!("Skipping copying metadata");
}
}
if !self.options.skip_agg_tiles_hash {
if self.options.copy.copy_tiles() && !self.options.skip_agg_tiles_hash {
dst_mbt.update_agg_tiles_hash(&mut conn).await?;
}
@ -237,8 +219,9 @@ impl MbtileCopierInt {
&self,
rusqlite_conn: &Connection,
dif: &Option<(Mbtiles, MbtType, MbtType)>,
on_dupl: &str,
on_duplicate: CopyDuplicateMode,
) -> Result<(), MbtError> {
let on_dupl = on_duplicate.to_sql();
let sql;
if dif.is_some() {
// Insert all rows from diffDb.metadata if they do not exist or are different in sourceDb.metadata.
@ -292,19 +275,35 @@ impl MbtileCopierInt {
}
fn copy_tiles(
&self,
rusqlite_conn: &Connection,
dif: &Option<(Mbtiles, MbtType, MbtType)>,
src_type: MbtType,
dst_type: MbtType,
on_dupl: &str,
select_from: &str,
sql_cond: &str,
on_duplicate: CopyDuplicateMode,
) -> Result<(), MbtError> {
let on_dupl = on_duplicate.to_sql();
let select_from = if let Some((_, dif_type, _)) = &dif {
if self.options.diff_with_file.is_some() {
Self::get_select_from_with_diff(*dif_type, dst_type)
} else {
Self::get_select_from_apply_patch(src_type, *dif_type, dst_type)
}
} else {
Self::get_select_from(src_type, dst_type).to_string()
};
let where_clause = self.get_where_clause();
let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type);
let sql = match dst_type {
Flat => {
format!(
"
INSERT {on_dupl} INTO tiles
(zoom_level, tile_column, tile_row, tile_data)
{select_from} {sql_cond}"
{select_from} {where_clause} {sql_cond}"
)
}
FlatWithHash => {
@ -312,7 +311,7 @@ impl MbtileCopierInt {
"
INSERT {on_dupl} INTO tiles_with_hash
(zoom_level, tile_column, tile_row, tile_data, tile_hash)
{select_from} {sql_cond}"
{select_from} {where_clause} {sql_cond}"
)
}
Normalized { .. } => {
@ -321,7 +320,7 @@ impl MbtileCopierInt {
INSERT OR IGNORE INTO images
(tile_id, tile_data)
SELECT tile_hash as tile_id, tile_data
FROM ({select_from})"
FROM ({select_from} {where_clause})"
);
debug!("Copying to {dst_type} with {sql}");
rusqlite_conn.execute(&sql, [])?;
@ -331,7 +330,7 @@ impl MbtileCopierInt {
INSERT {on_dupl} INTO map
(zoom_level, tile_column, tile_row, tile_id)
SELECT zoom_level, tile_column, tile_row, tile_hash as tile_id
FROM ({select_from} {sql_cond})"
FROM ({select_from} {where_clause} {sql_cond})"
)
}
};
@ -634,8 +633,12 @@ mod tests {
dst_type_cli: Option<MbtTypeCli>,
expected_dst_type: MbtType,
) -> MbtResult<()> {
let mut opt = MbtilesCopier::new(src_filepath.clone(), dst_filepath.clone());
opt.dst_type_cli = dst_type_cli;
let opt = MbtilesCopier {
src_file: src_filepath.clone(),
dst_file: dst_filepath.clone(),
dst_type_cli,
..Default::default()
};
let mut dst_conn = opt.run().await?;
Mbtiles::new(src_filepath)?
@ -750,22 +753,26 @@ mod tests {
#[actix_rt::test]
async fn copy_with_min_max_zoom() -> MbtResult<()> {
let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared");
let mut opt = MbtilesCopier::new(src, dst);
opt.min_zoom = Some(2);
opt.max_zoom = Some(4);
let opt = MbtilesCopier {
src_file: PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"),
dst_file: PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"),
min_zoom: Some(2),
max_zoom: Some(4),
..Default::default()
};
verify_copy_with_zoom_filter(opt, 3).await
}
#[actix_rt::test]
async fn copy_with_zoom_levels() -> MbtResult<()> {
let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared");
let mut opt = MbtilesCopier::new(src, dst);
opt.min_zoom = Some(2);
opt.max_zoom = Some(4);
opt.zoom_levels.extend(&[1, 6]);
let opt = MbtilesCopier {
src_file: PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"),
dst_file: PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"),
min_zoom: Some(2),
max_zoom: Some(4),
zoom_levels: vec![1, 6],
..Default::default()
};
verify_copy_with_zoom_filter(opt, 2).await
}
@ -777,8 +784,12 @@ mod tests {
let diff_file =
PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles");
let mut opt = MbtilesCopier::new(src.clone(), dst.clone());
opt.diff_with_file = Some(diff_file.clone());
let opt = MbtilesCopier {
src_file: src.clone(),
dst_file: dst.clone(),
diff_with_file: Some(diff_file.clone()),
..Default::default()
};
let mut dst_conn = opt.run().await?;
assert!(dst_conn
@ -820,8 +831,12 @@ mod tests {
let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles");
let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let mut opt = MbtilesCopier::new(src.clone(), dst.clone());
opt.on_duplicate = Some(CopyDuplicateMode::Abort);
let opt = MbtilesCopier {
src_file: src.clone(),
dst_file: dst.clone(),
on_duplicate: Some(CopyDuplicateMode::Abort),
..Default::default()
};
assert!(matches!(
opt.run().await.unwrap_err(),
@ -838,12 +853,20 @@ mod tests {
let dst =
PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared");
let _dst_conn = MbtilesCopier::new(dst_file.clone(), dst.clone())
.run()
.await?;
let _dst_conn = MbtilesCopier {
src_file: dst_file.clone(),
dst_file: dst.clone(),
..Default::default()
}
.run()
.await?;
let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone());
opt.on_duplicate = Some(CopyDuplicateMode::Override);
let opt = MbtilesCopier {
src_file: src_file.clone(),
dst_file: dst.clone(),
on_duplicate: Some(CopyDuplicateMode::Override),
..Default::default()
};
let mut dst_conn = opt.run().await?;
// Verify the tiles in the destination file is a superset of the tiles in the source file
@ -867,12 +890,20 @@ mod tests {
let dst =
PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared");
let _dst_conn = MbtilesCopier::new(dst_file.clone(), dst.clone())
.run()
.await?;
let _dst_conn = MbtilesCopier {
src_file: dst_file.clone(),
dst_file: dst.clone(),
..Default::default()
}
.run()
.await?;
let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone());
opt.on_duplicate = Some(CopyDuplicateMode::Ignore);
let opt = MbtilesCopier {
src_file: src_file.clone(),
dst_file: dst.clone(),
on_duplicate: Some(CopyDuplicateMode::Ignore),
..Default::default()
};
let mut dst_conn = opt.run().await?;
// Verify the tiles in the destination file are the same as those in the source file except for those with duplicate (zoom_level, tile_column, tile_row)

2
mbtiles/src/lib.rs
View File

@ -10,7 +10,7 @@ mod errors;
pub use errors::{MbtError, MbtResult};
mod mbtiles;
pub use mbtiles::{MbtTypeCli, Mbtiles};
pub use mbtiles::{CopyType, MbtTypeCli, Mbtiles};
mod metadata;
pub use metadata::Metadata;

21
mbtiles/src/mbtiles.rs
View File

@ -21,6 +21,27 @@ pub enum MbtTypeCli {
Normalized,
}
#[derive(Default, Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)]
#[enum_display(case = "Kebab")]
#[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
pub enum CopyType {
#[default]
All,
Metadata,
Tiles,
}
impl CopyType {
#[must_use]
pub fn copy_tiles(&self) -> bool {
matches!(self, Self::All | Self::Tiles)
}
#[must_use]
pub fn copy_metadata(&self) -> bool {
matches!(self, Self::All | Self::Metadata)
}
}
#[derive(Clone, Debug)]
pub struct Mbtiles {
filepath: String,

20
mbtiles/src/patcher.rs
View File

@ -148,9 +148,13 @@ mod tests {
let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared");
let mut src_conn = MbtilesCopier::new(src_file.clone(), src.clone())
.run()
.await?;
let mut src_conn = MbtilesCopier {
src_file: src_file.clone(),
dst_file: src.clone(),
..Default::default()
}
.run()
.await?;
// Apply patch to the src data in in-memory DB
let patch_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles");
@ -175,9 +179,13 @@ mod tests {
let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles");
let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared");
let mut src_conn = MbtilesCopier::new(src_file.clone(), src.clone())
.run()
.await?;
let mut src_conn = MbtilesCopier {
src_file: src_file.clone(),
dst_file: src.clone(),
..Default::default()
}
.run()
.await?;
// Apply patch to the src data in in-memory DB
let patch_file =

4
mbtiles/src/validation.rs
View File

@ -48,7 +48,7 @@ impl MbtType {
}
}
#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)]
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)]
#[enum_display(case = "Kebab")]
#[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
pub enum IntegrityCheckType {
@ -58,7 +58,7 @@ pub enum IntegrityCheckType {
Off,
}
#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)]
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)]
#[enum_display(case = "Kebab")]
#[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
pub enum AggHashType {

144
mbtiles/tests/copy.rs
View File

@ -11,7 +11,8 @@ use mbtiles::AggHashType::Verify;
use mbtiles::IntegrityCheckType::Off;
use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized};
use mbtiles::{
apply_patch, init_mbtiles_schema, invert_y_value, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier,
apply_patch, init_mbtiles_schema, invert_y_value, CopyType, MbtResult, MbtTypeCli, Mbtiles,
MbtilesCopier,
};
use pretty_assertions::assert_eq as pretty_assert_eq;
use rstest::{fixture, rstest};
@ -87,10 +88,6 @@ fn path(mbt: &Mbtiles) -> PathBuf {
PathBuf::from(mbt.filepath())
}
fn copier(src: &Mbtiles, dst: &Mbtiles) -> MbtilesCopier {
MbtilesCopier::new(path(src), path(dst))
}
fn shorten(v: MbtTypeCli) -> &'static str {
match v {
Flat => "flat",
@ -135,9 +132,13 @@ macro_rules! new_file {
cn_tmp.execute($sql_meta).await.unwrap();
let (dst_mbt, cn_dst) = open!($function, $($arg)*);
let mut opt = copier(&tmp_mbt, &dst_mbt);
opt.dst_type_cli = Some($dst_type_cli);
opt.skip_agg_tiles_hash = $skip_agg;
let opt = MbtilesCopier {
src_file: path(&tmp_mbt),
dst_file: path(&dst_mbt),
dst_type_cli: Some($dst_type_cli),
skip_agg_tiles_hash: $skip_agg,
..Default::default()
};
opt.run().await.unwrap();
(dst_mbt, cn_dst)
@ -199,7 +200,12 @@ fn databases() -> Databases {
let (v1_mbt, mut v1_cn) = open!(databases, "{typ}__v1");
let raw_mbt = result.mbtiles("v1_no_hash", mbt_typ);
copier(raw_mbt, &v1_mbt).run().await.unwrap();
let opt = MbtilesCopier {
src_file: path(raw_mbt),
dst_file: path(&v1_mbt),
..Default::default()
};
opt.run().await.unwrap();
let dmp = dump(&mut v1_cn).await.unwrap();
assert_snapshot!(&dmp, "{typ}__v1");
let hash = v1_mbt.validate(Off, Verify).await.unwrap();
@ -220,9 +226,13 @@ fn databases() -> Databases {
let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif");
let v1_mbt = result.mbtiles("v1", mbt_typ);
let mut opt = copier(v1_mbt, &dif_mbt);
let v2_mbt = result.mbtiles("v2", mbt_typ);
opt.diff_with_file = Some(path(v2_mbt));
let opt = MbtilesCopier {
src_file: path(v1_mbt),
dst_file: path(&dif_mbt),
diff_with_file: Some(path(v2_mbt)),
..Default::default()
};
opt.run().await.unwrap();
let dmp = dump(&mut dif_cn).await.unwrap();
assert_snapshot!(&dmp, "{typ}__dif");
@ -248,44 +258,87 @@ async fn convert(
let mem = Mbtiles::new(":memory:")?;
let (frm_mbt, _frm_cn) = new_file!(convert, frm_type, METADATA_V1, TILES_V1, "{frm}-{to}");
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
dst_type_cli: Some(dst_type),
..Default::default()
};
let dmp = dump(&mut opt.run().await?).await?;
pretty_assert_eq!(databases.dump("v1", dst_type), &dmp);
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
opt.zoom_levels.push(6);
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
copy: CopyType::Metadata,
dst_type_cli: Some(dst_type),
..Default::default()
};
let dmp = dump(&mut opt.run().await?).await?;
allow_duplicates! {
assert_snapshot!(dmp, "v1__meta__{to}");
};
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
copy: CopyType::Tiles,
dst_type_cli: Some(dst_type),
..Default::default()
};
let dmp = dump(&mut opt.run().await?).await?;
allow_duplicates! {
assert_snapshot!(dmp, "v1__tiles__{to}");
}
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
dst_type_cli: Some(dst_type),
zoom_levels: vec![6],
..Default::default()
};
let z6only = dump(&mut opt.run().await?).await?;
allow_duplicates! {
assert_snapshot!(z6only, "v1__z6__{to}");
}
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
// Filter (0, 0, 2, 2) in mbtiles coordinates, which is (0, 2^5-1-2, 2, 2^5-1-0) = (0, 29, 2, 31) in XYZ coordinates, and slightly decrease it
let mut bbox = xyz_to_bbox(5, 0, invert_y_value(5, 2), 2, invert_y_value(5, 0));
bbox[0] += 180.0 * 0.1 / f64::from(1 << 5);
bbox[1] += 90.0 * 0.1 / f64::from(1 << 5);
bbox[2] -= 180.0 * 0.1 / f64::from(1 << 5);
bbox[3] -= 90.0 * 0.1 / f64::from(1 << 5);
opt.bbox.push(bbox.into());
let adjust = 90.0 * 0.1 / f64::from(1 << 5);
bbox[0] += adjust;
bbox[1] += adjust;
bbox[2] -= adjust;
bbox[3] -= adjust;
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
dst_type_cli: Some(dst_type),
bbox: vec![bbox.into()],
..Default::default()
};
let dmp = dump(&mut opt.run().await?).await?;
allow_duplicates! {
assert_snapshot!(dmp, "v1__bbox__{to}");
}
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
opt.min_zoom = Some(6);
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
dst_type_cli: Some(dst_type),
min_zoom: Some(6),
..Default::default()
};
pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?);
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
opt.min_zoom = Some(6);
opt.max_zoom = Some(6);
let opt = MbtilesCopier {
src_file: path(&frm_mbt),
dst_file: path(&mem),
dst_type_cli: Some(dst_type),
min_zoom: Some(6),
max_zoom: Some(6),
..Default::default()
};
pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?);
Ok(())
@ -309,8 +362,12 @@ async fn diff_and_patch(
let (dif_mbt, mut dif_cn) = open!(diff_and_patchdiff_and_patch, "{prefix}__dif");
info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)");
let mut opt = copier(v1_mbt, &dif_mbt);
opt.diff_with_file = Some(path(v2_mbt));
let mut opt = MbtilesCopier {
src_file: path(v1_mbt),
dst_file: path(&dif_mbt),
diff_with_file: Some(path(v2_mbt)),
..Default::default()
};
if let Some(dif_type) = dif_type {
opt.dst_type_cli = Some(dif_type);
}
@ -374,8 +431,12 @@ async fn patch_on_copy(
let (v2_mbt, mut v2_cn) = open!(patch_on_copy, "{prefix}__v2");
info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)");
let mut opt = copier(v1_mbt, &v2_mbt);
opt.apply_patch = Some(path(dif_mbt));
let mut opt = MbtilesCopier {
src_file: path(v1_mbt),
dst_file: path(&v2_mbt),
apply_patch: Some(path(dif_mbt)),
..Default::default()
};
if let Some(v2_type) = v2_type {
opt.dst_type_cli = Some(v2_type);
}
@ -506,9 +567,14 @@ async fn dump(conn: &mut SqliteConnection) -> MbtResult<Vec<SqliteEntry>> {
}
#[allow(dead_code)]
async fn save_to_file(source_mbt: &Mbtiles, path: &str) -> MbtResult<()> {
let mut opt = copier(source_mbt, &Mbtiles::new(path)?);
opt.skip_agg_tiles_hash = true;
async fn save_to_file(source_mbt: &Mbtiles, path_mbt: &str) -> MbtResult<()> {
let dst = &Mbtiles::new(path_mbt)?;
let opt = MbtilesCopier {
src_file: path(source_mbt),
dst_file: path(dst),
skip_agg_tiles_hash: true,
..Default::default()
};
opt.run().await?;
Ok(())
}

37
mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap Normal file
View File

@ -0,0 +1,37 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles'
sql = '''
CREATE TABLE tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = []
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles'

45
mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap Normal file
View File

@ -0,0 +1,45 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE TABLE tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = []
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles_with_hash'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash'''

76
mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap Normal file
View File

@ -0,0 +1,76 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'images'
sql = '''
CREATE TABLE images (
tile_id text NOT NULL PRIMARY KEY,
tile_data blob)'''
values = []
[[]]
type = 'table'
tbl_name = 'map'
sql = '''
CREATE TABLE map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = []
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'index'
tbl_name = 'images'
[[]]
type = 'index'
tbl_name = 'map'
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id'''
[[]]
type = 'view'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE VIEW tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id'''

45
mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap Normal file
View File

@ -0,0 +1,45 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )']
[[]]
type = 'table'
tbl_name = 'tiles'
sql = '''
CREATE TABLE tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 3, 6, 7, blob(root) )',
'( 5, 0, 0, blob(same) )',
'( 5, 0, 1, blob() )',
'( 5, 1, 1, blob(edit-v1) )',
'( 5, 1, 2, blob() )',
'( 5, 1, 3, blob(non-empty) )',
'( 5, 2, 2, blob(remove) )',
'( 5, 2, 3, blob() )',
'( 6, 0, 3, blob(same) )',
'( 6, 0, 5, blob(1-keep-1-rm) )',
'( 6, 1, 4, blob(edit-v1) )',
'( 6, 2, 6, blob(1-keep-1-rm) )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles'

53
mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap Normal file
View File

@ -0,0 +1,53 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )']
[[]]
type = 'table'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE TABLE tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 3, 6, 7, blob(root), "63A9F0EA7BB98050796B649E85481845" )',
'( 5, 0, 0, blob(same), "51037A4A37730F52C8732586D3AAA316" )',
'( 5, 0, 1, blob(), "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 1, 3, blob(non-empty), "720C02778717818CC0A869955BA2AFB6" )',
'( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 5, 2, 3, blob(), "D41D8CD98F00B204E9800998ECF8427E" )',
'( 6, 0, 3, blob(same), "51037A4A37730F52C8732586D3AAA316" )',
'( 6, 0, 5, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )',
'( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 6, 2, 6, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles_with_hash'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash'''

92
mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap Normal file
View File

@ -0,0 +1,92 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'images'
sql = '''
CREATE TABLE images (
tile_id text NOT NULL PRIMARY KEY,
tile_data blob)'''
values = [
'( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )',
'( "51037A4A37730F52C8732586D3AAA316", blob(same) )',
'( "535A5575B48444EDEB926815AB26EC9B", blob(1-keep-1-rm) )',
'( "63A9F0EA7BB98050796B649E85481845", blob(root) )',
'( "720C02778717818CC0A869955BA2AFB6", blob(non-empty) )',
'( "D41D8CD98F00B204E9800998ECF8427E", blob() )',
'( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )',
]
[[]]
type = 'table'
tbl_name = 'map'
sql = '''
CREATE TABLE map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 3, 6, 7, "63A9F0EA7BB98050796B649E85481845" )',
'( 5, 0, 0, "51037A4A37730F52C8732586D3AAA316" )',
'( 5, 0, 1, "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 1, 3, "720C02778717818CC0A869955BA2AFB6" )',
'( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 5, 2, 3, "D41D8CD98F00B204E9800998ECF8427E" )',
'( 6, 0, 3, "51037A4A37730F52C8732586D3AAA316" )',
'( 6, 0, 5, "535A5575B48444EDEB926815AB26EC9B" )',
'( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 6, 2, 6, "535A5575B48444EDEB926815AB26EC9B" )',
]
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )']
[[]]
type = 'index'
tbl_name = 'images'
[[]]
type = 'index'
tbl_name = 'map'
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id'''
[[]]
type = 'view'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE VIEW tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id'''