Add apply-diff command (#747)

Add command `apply-diff` to apply diff file generated from `copy`
command

docs/src/tools.md

@@ -26,10 +26,13 @@ Copy command can also be used to compare two mbtiles files and generate a diff.
 mbtiles copy src_file.mbtiles diff_file.mbtiles --force-simple --diff-with-file modified_file.mbtiles
 ```
 
-The `diff_file.mbtiles` can be applied to the `src_file.mbtiles` elsewhere to avoid copying/transmitting the entire modified dataset.
+### apply-diff
-
+Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.
-One way to apply the diff is to use the `sqlite3` command line tool directly. Here, we assume that the `src_file.mbtiles` is in the simple tables format, and that the `diff_file.mbtiles` is the output of the `mbtiles copy` command above. This SQL will delete all tiles from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles from `diff_file.mbtiles` into `src_file.mbtiles`. The name of the diff file is passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements.
+```shell
+mbtiles apply_diff src_file.mbtiles diff_file.mbtiles
+```
 
+Another way to apply the diff is to use the `sqlite3` command line tool directly. This SQL will delete all tiles from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles from `diff_file.mbtiles` into `src_file.mbtiles`. The name of the diff file is passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements.
 ```shell
 sqlite3 src_file.mbtiles \
   -bail \
@@ -38,3 +41,6 @@ sqlite3 src_file.mbtiles \
   "DELETE FROM tiles WHERE (zoom_level, tile_column, tile_row) IN (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL);" \
   "INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;"
 ```
+
+**_NOTE:_** Both of these methods for applying a diff _only_ work for mbtiles files in the simple tables format; they do _not_ work for mbtiles files in deduplicated format.
+

martin-mbtiles/src/bin/main.rs

@@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
-use martin_mbtiles::{copy_mbtiles_file, Mbtiles, TileCopierOptions};
+use martin_mbtiles::{apply_mbtiles_diff, copy_mbtiles_file, Mbtiles, TileCopierOptions};
 use sqlx::sqlite::SqliteConnectOptions;
 use sqlx::{Connection, SqliteConnection};
 
@@ -45,6 +45,14 @@ enum Commands {
     /// Copy tiles from one mbtiles file to another.
     #[command(name = "copy")]
     Copy(TileCopierOptions),
+    /// Apply diff file generated from 'copy' command
+    #[command(name = "apply-diff")]
+    ApplyDiff {
+        /// MBTiles file to apply diff to
+        src_file: PathBuf,
+        /// Diff file
+        diff_file: PathBuf,
+    },
 }
 
 #[tokio::main]
@@ -58,6 +66,12 @@ async fn main() -> Result<()> {
         Commands::Copy(opts) => {
             copy_mbtiles_file(opts).await?;
         }
+        Commands::ApplyDiff {
+            src_file,
+            diff_file,
+        } => {
+            apply_mbtiles_diff(src_file, diff_file).await?;
+        }
     }
 
     Ok(())
@@ -82,7 +96,7 @@ mod tests {
     use martin_mbtiles::TileCopierOptions;
 
     use crate::Args;
-    use crate::Commands::{Copy, MetaGetValue};
+    use crate::Commands::{ApplyDiff, Copy, MetaGetValue};
 
     #[test]
     fn test_copy_no_arguments() {
@@ -254,4 +268,18 @@ mod tests {
             }
         );
     }
+
+    #[test]
+    fn test_apply_diff_with_arguments() {
+        assert_eq!(
+            Args::parse_from(["mbtiles", "apply-diff", "src_file", "diff_file"]),
+            Args {
+                verbose: false,
+                command: ApplyDiff {
+                    src_file: PathBuf::from("src_file"),
+                    diff_file: PathBuf::from("diff_file"),
+                }
+            }
+        );
+    }
 }

martin-mbtiles/src/errors.rs

@@ -1,5 +1,6 @@
 use std::path::PathBuf;
 
+use crate::mbtiles::MbtType;
 use martin_tile_utils::TileInfo;
 
 #[derive(thiserror::Error, Debug)]
@@ -16,6 +17,9 @@ pub enum MbtError {
     #[error("Invalid data format for MBTile file {0}")]
     InvalidDataFormat(String),
 
+    #[error("Incorrect data format for MBTile file {0}; expected {1:?} and got {2:?}")]
+    IncorrectDataFormat(String, MbtType, MbtType),
+
     #[error(r#"Filename "{0}" passed to SQLite must be valid UTF-8"#)]
     InvalidFilenameType(PathBuf),
 

martin-mbtiles/src/lib.rs

@@ -9,4 +9,4 @@ mod tile_copier;
 pub use errors::MbtError;
 pub use mbtiles::{Mbtiles, Metadata};
 pub use mbtiles_pool::MbtilesPool;
-pub use tile_copier::{copy_mbtiles_file, TileCopierOptions};
+pub use tile_copier::{apply_mbtiles_diff, copy_mbtiles_file, TileCopierOptions};

martin-mbtiles/src/tile_copier.rs

@@ -10,6 +10,7 @@ use sqlx::{query, query_with, Arguments, Connection, Row, SqliteConnection};
 
 use crate::errors::MbtResult;
 use crate::mbtiles::MbtType;
+use crate::mbtiles::MbtType::TileTables;
 use crate::{MbtError, Mbtiles};
 
 #[derive(Clone, Default, PartialEq, Eq, Debug)]
@@ -290,6 +291,47 @@ async fn open_and_detect_type(mbtiles: &Mbtiles) -> MbtResult<MbtType> {
     mbtiles.detect_type(&mut conn).await
 }
 
+pub async fn apply_mbtiles_diff(
+    src_file: PathBuf,
+    diff_file: PathBuf,
+) -> MbtResult<SqliteConnection> {
+    let src_mbtiles = Mbtiles::new(src_file)?;
+    let diff_mbtiles = Mbtiles::new(diff_file)?;
+
+    let opt = SqliteConnectOptions::new().filename(src_mbtiles.filepath());
+    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let src_type = src_mbtiles.detect_type(&mut conn).await?;
+
+    if src_type != MbtType::TileTables {
+        return Err(MbtError::IncorrectDataFormat(
+            src_mbtiles.filepath().to_string(),
+            TileTables,
+            src_type,
+        ));
+    }
+
+    open_and_detect_type(&diff_mbtiles).await?;
+
+    query("ATTACH DATABASE ? AS diffDb")
+        .bind(diff_mbtiles.filepath())
+        .execute(&mut conn)
+        .await?;
+
+    query(
+        "
+    DELETE FROM tiles 
+    WHERE (zoom_level, tile_column, tile_row) IN 
+        (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL);
+
+    INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) 
+    SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;",
+    )
+    .execute(&mut conn)
+    .await?;
+
+    Ok(conn)
+}
+
 pub async fn copy_mbtiles_file(opts: TileCopierOptions) -> MbtResult<SqliteConnection> {
     let tile_copier = TileCopier::new(opts)?;
 
@@ -464,4 +506,34 @@ mod tests {
         .await
         .is_none());
     }
+
+    #[actix_rt::test]
+    async fn apply_diff_file() {
+        // Copy the src file to an in-memory DB
+        let src_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let src = PathBuf::from("file::memory:?cache=shared");
+
+        let _src_conn = copy_mbtiles_file(TileCopierOptions::new(src_file.clone(), src.clone()))
+            .await
+            .unwrap();
+
+        // Apply diff to the src data in in-memory DB
+        let diff_file = PathBuf::from("../tests/fixtures/files/world_cities_diff.mbtiles");
+        let mut src_conn = apply_mbtiles_diff(src, diff_file).await.unwrap();
+
+        // Verify the data is the same as the file the diff was generated from
+        let modified_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles");
+        let _ = query("ATTACH DATABASE ? AS otherDb")
+            .bind(modified_file.clone().to_str().unwrap())
+            .execute(&mut src_conn)
+            .await;
+
+        assert!(
+            query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")
+                .fetch_optional(&mut src_conn)
+                .await
+                .unwrap()
+                .is_none()
+        );
+    }
 }

tests/expected/auto/catalog_auto.json

@@ -165,6 +165,12 @@
     "name": "Major cities from Natural Earth data",
     "description": "Major cities from Natural Earth data"
   },
+  {
+    "id": "world_cities_diff",
+    "content_type": "application/x-protobuf",
+    "name": "Major cities from Natural Earth data",
+    "description": "Major cities from Natural Earth data"
+  },
   {
     "id": "world_cities_modified",
     "content_type": "application/x-protobuf",

tests/expected/generated_config.yaml

@@ -158,4 +158,5 @@ mbtiles:
     uncompressed_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles
     webp: tests/fixtures/files/webp.mbtiles
     world_cities: tests/fixtures/files/world_cities.mbtiles
+    world_cities_diff: tests/fixtures/files/world_cities_diff.mbtiles
     world_cities_modified: tests/fixtures/files/world_cities_modified.mbtiles

tests/expected/mbtiles/help.txt

@@ -5,6 +5,7 @@ Usage: mbtiles <COMMAND>
 Commands:
   meta-get    Gets a single value from the MBTiles metadata table
   copy        Copy tiles from one mbtiles file to another
+  apply-diff  Apply diff file generated from 'copy' command
   help        Print this message or the help of the given subcommand(s)
 
 Options: