From 1342b38e758fecf0be12cd7232ac91a5fbff4e57 Mon Sep 17 00:00:00 2001 From: rstanciu <55859815+upsicleclown@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:38:03 -0700 Subject: [PATCH] Add apply-diff command (#747) Add command `apply-diff` to apply diff file generated from `copy` command --- docs/src/tools.md | 12 ++- martin-mbtiles/src/bin/main.rs | 32 +++++++- martin-mbtiles/src/errors.rs | 4 + martin-mbtiles/src/lib.rs | 2 +- martin-mbtiles/src/tile_copier.rs | 72 ++++++++++++++++++ tests/expected/auto/catalog_auto.json | 6 ++ tests/expected/generated_config.yaml | 1 + tests/expected/mbtiles/help.txt | 7 +- .../fixtures/files/world_cities_diff.mbtiles | Bin 0 -> 3584 bytes 9 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/files/world_cities_diff.mbtiles diff --git a/docs/src/tools.md b/docs/src/tools.md index 7eaca850..6597f37d 100644 --- a/docs/src/tools.md +++ b/docs/src/tools.md @@ -26,10 +26,13 @@ Copy command can also be used to compare two mbtiles files and generate a diff. mbtiles copy src_file.mbtiles diff_file.mbtiles --force-simple --diff-with-file modified_file.mbtiles ``` -The `diff_file.mbtiles` can be applied to the `src_file.mbtiles` elsewhere to avoid copying/transmitting the entire modified dataset. - -One way to apply the diff is to use the `sqlite3` command line tool directly. Here, we assume that the `src_file.mbtiles` is in the simple tables format, and that the `diff_file.mbtiles` is the output of the `mbtiles copy` command above. This SQL will delete all tiles from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles from `diff_file.mbtiles` into `src_file.mbtiles`. The name of the diff file is passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements. +### apply-diff +Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset. +```shell +mbtiles apply_diff src_file.mbtiles diff_file.mbtiles +``` +Another way to apply the diff is to use the `sqlite3` command line tool directly. This SQL will delete all tiles from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles from `diff_file.mbtiles` into `src_file.mbtiles`. The name of the diff file is passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements. ```shell sqlite3 src_file.mbtiles \ -bail \ @@ -38,3 +41,6 @@ sqlite3 src_file.mbtiles \ "DELETE FROM tiles WHERE (zoom_level, tile_column, tile_row) IN (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL);" \ "INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;" ``` + +**_NOTE:_** Both of these methods for applying a diff _only_ work for mbtiles files in the simple tables format; they do _not_ work for mbtiles files in deduplicated format. + diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 0560cd2a..e393bc15 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use clap::{Parser, Subcommand}; -use martin_mbtiles::{copy_mbtiles_file, Mbtiles, TileCopierOptions}; +use martin_mbtiles::{apply_mbtiles_diff, copy_mbtiles_file, Mbtiles, TileCopierOptions}; use sqlx::sqlite::SqliteConnectOptions; use sqlx::{Connection, SqliteConnection}; @@ -45,6 +45,14 @@ enum Commands { /// Copy tiles from one mbtiles file to another. #[command(name = "copy")] Copy(TileCopierOptions), + /// Apply diff file generated from 'copy' command + #[command(name = "apply-diff")] + ApplyDiff { + /// MBTiles file to apply diff to + src_file: PathBuf, + /// Diff file + diff_file: PathBuf, + }, } #[tokio::main] @@ -58,6 +66,12 @@ async fn main() -> Result<()> { Commands::Copy(opts) => { copy_mbtiles_file(opts).await?; } + Commands::ApplyDiff { + src_file, + diff_file, + } => { + apply_mbtiles_diff(src_file, diff_file).await?; + } } Ok(()) @@ -82,7 +96,7 @@ mod tests { use martin_mbtiles::TileCopierOptions; use crate::Args; - use crate::Commands::{Copy, MetaGetValue}; + use crate::Commands::{ApplyDiff, Copy, MetaGetValue}; #[test] fn test_copy_no_arguments() { @@ -254,4 +268,18 @@ mod tests { } ); } + + #[test] + fn test_apply_diff_with_arguments() { + assert_eq!( + Args::parse_from(["mbtiles", "apply-diff", "src_file", "diff_file"]), + Args { + verbose: false, + command: ApplyDiff { + src_file: PathBuf::from("src_file"), + diff_file: PathBuf::from("diff_file"), + } + } + ); + } } diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs index 4ddf9531..f058656d 100644 --- a/martin-mbtiles/src/errors.rs +++ b/martin-mbtiles/src/errors.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use crate::mbtiles::MbtType; use martin_tile_utils::TileInfo; #[derive(thiserror::Error, Debug)] @@ -16,6 +17,9 @@ pub enum MbtError { #[error("Invalid data format for MBTile file {0}")] InvalidDataFormat(String), + #[error("Incorrect data format for MBTile file {0}; expected {1:?} and got {2:?}")] + IncorrectDataFormat(String, MbtType, MbtType), + #[error(r#"Filename "{0}" passed to SQLite must be valid UTF-8"#)] InvalidFilenameType(PathBuf), diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index bee37927..198ffd59 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -9,4 +9,4 @@ mod tile_copier; pub use errors::MbtError; pub use mbtiles::{Mbtiles, Metadata}; pub use mbtiles_pool::MbtilesPool; -pub use tile_copier::{copy_mbtiles_file, TileCopierOptions}; +pub use tile_copier::{apply_mbtiles_diff, copy_mbtiles_file, TileCopierOptions}; diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs index 1257f785..67539518 100644 --- a/martin-mbtiles/src/tile_copier.rs +++ b/martin-mbtiles/src/tile_copier.rs @@ -10,6 +10,7 @@ use sqlx::{query, query_with, Arguments, Connection, Row, SqliteConnection}; use crate::errors::MbtResult; use crate::mbtiles::MbtType; +use crate::mbtiles::MbtType::TileTables; use crate::{MbtError, Mbtiles}; #[derive(Clone, Default, PartialEq, Eq, Debug)] @@ -290,6 +291,47 @@ async fn open_and_detect_type(mbtiles: &Mbtiles) -> MbtResult { mbtiles.detect_type(&mut conn).await } +pub async fn apply_mbtiles_diff( + src_file: PathBuf, + diff_file: PathBuf, +) -> MbtResult { + let src_mbtiles = Mbtiles::new(src_file)?; + let diff_mbtiles = Mbtiles::new(diff_file)?; + + let opt = SqliteConnectOptions::new().filename(src_mbtiles.filepath()); + let mut conn = SqliteConnection::connect_with(&opt).await?; + let src_type = src_mbtiles.detect_type(&mut conn).await?; + + if src_type != MbtType::TileTables { + return Err(MbtError::IncorrectDataFormat( + src_mbtiles.filepath().to_string(), + TileTables, + src_type, + )); + } + + open_and_detect_type(&diff_mbtiles).await?; + + query("ATTACH DATABASE ? AS diffDb") + .bind(diff_mbtiles.filepath()) + .execute(&mut conn) + .await?; + + query( + " + DELETE FROM tiles + WHERE (zoom_level, tile_column, tile_row) IN + (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL); + + INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) + SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;", + ) + .execute(&mut conn) + .await?; + + Ok(conn) +} + pub async fn copy_mbtiles_file(opts: TileCopierOptions) -> MbtResult { let tile_copier = TileCopier::new(opts)?; @@ -464,4 +506,34 @@ mod tests { .await .is_none()); } + + #[actix_rt::test] + async fn apply_diff_file() { + // Copy the src file to an in-memory DB + let src_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("file::memory:?cache=shared"); + + let _src_conn = copy_mbtiles_file(TileCopierOptions::new(src_file.clone(), src.clone())) + .await + .unwrap(); + + // Apply diff to the src data in in-memory DB + let diff_file = PathBuf::from("../tests/fixtures/files/world_cities_diff.mbtiles"); + let mut src_conn = apply_mbtiles_diff(src, diff_file).await.unwrap(); + + // Verify the data is the same as the file the diff was generated from + let modified_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let _ = query("ATTACH DATABASE ? AS otherDb") + .bind(modified_file.clone().to_str().unwrap()) + .execute(&mut src_conn) + .await; + + assert!( + query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;") + .fetch_optional(&mut src_conn) + .await + .unwrap() + .is_none() + ); + } } diff --git a/tests/expected/auto/catalog_auto.json b/tests/expected/auto/catalog_auto.json index 8afd3baf..fe1ebdd2 100644 --- a/tests/expected/auto/catalog_auto.json +++ b/tests/expected/auto/catalog_auto.json @@ -165,6 +165,12 @@ "name": "Major cities from Natural Earth data", "description": "Major cities from Natural Earth data" }, + { + "id": "world_cities_diff", + "content_type": "application/x-protobuf", + "name": "Major cities from Natural Earth data", + "description": "Major cities from Natural Earth data" + }, { "id": "world_cities_modified", "content_type": "application/x-protobuf", diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index f05a44c2..cb6f99d3 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -158,4 +158,5 @@ mbtiles: uncompressed_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles webp: tests/fixtures/files/webp.mbtiles world_cities: tests/fixtures/files/world_cities.mbtiles + world_cities_diff: tests/fixtures/files/world_cities_diff.mbtiles world_cities_modified: tests/fixtures/files/world_cities_modified.mbtiles diff --git a/tests/expected/mbtiles/help.txt b/tests/expected/mbtiles/help.txt index 1094cdd7..832fc449 100644 --- a/tests/expected/mbtiles/help.txt +++ b/tests/expected/mbtiles/help.txt @@ -3,9 +3,10 @@ A utility to work with .mbtiles file content Usage: mbtiles Commands: - meta-get Gets a single value from the MBTiles metadata table - copy Copy tiles from one mbtiles file to another - help Print this message or the help of the given subcommand(s) + meta-get Gets a single value from the MBTiles metadata table + copy Copy tiles from one mbtiles file to another + apply-diff Apply diff file generated from 'copy' command + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help diff --git a/tests/fixtures/files/world_cities_diff.mbtiles b/tests/fixtures/files/world_cities_diff.mbtiles new file mode 100644 index 0000000000000000000000000000000000000000..275f3316553c4b7e350951419655f2b24e7cc710 GIT binary patch literal 3584 zcmeHJ&2Jnv6d&*GW|J;Zp(=`^s=}RK8ZlXIvQ2iw0g-G1>t0@$Wr9zxVu} z{p|L&HDNg2Q95CU)}4Ywi1Qkyj^mUEf9Py0!M8ZT*pU^zOnth?;*487fw?*3{N#`y$>-z~@&P#@H_1ivB5{HFe{CODD#g(| z=WeG;9(sE`FiLO7tj~4oy+Yrihtv}xATJQG!lCCz-0^ey0od*AyfEyJo;UqnuJ*Ww zNmt75c+gRq4AaS}>3S9ZGjlb6vfijRr<%3qj6c=5QfCC){~8y>Ri1sQ>*#)X0_35Hs`MRvz3y2{%pbyaHGyvibx2FM6Kci0k%`27p_K}y5oje+8Xg{ovk&K=eC&|l$vyHj`GI^-z9CdlRR$jTSQUj_47+<$L*ih?!i*UcfE5NDW9%`QD=o`7p<* zAivo>-wOtUvrXTNxPoVl?xVm}1#c%9X9i4lGQ&@3e8{uc{NW=RoM>GcCt4gh9af&9oAuBZ9YliCJ$MH*mSvscmar+GtH>rts!uGC&7f z;8GN37KwmGIBk*jn8uz`c4zlO_{5r_9O zjIOg#!-EWg6fp{}^BC%rmKeKrejoxxTf+2_5-h<*tSfw5X>NHc4b(m|vm{^Q28OPK zavOY~F7mj0jNeuo({Pn(fu^quMMF+knB;k|uZunlyEp=S4!Smz^mPqk7GNW%jmQJs z=E_E>4PlA8HneR>E!)j=-Zm=ebybXerri&vb+vu>3q`N8EJj_MOzd)WY?Gzk9y}Hi z@-A3Ipn82Pq%LyGDqmVYAE>>GrVBP&x-~Bdu7Zrc^Q3~nv MZXMz8=t-sRPc`^X+W-In literal 0 HcmV?d00001