Add basic copying functionality (#712)

Copy an existing `.mbtiles` file to a new file.

```shell
mbtiles copy  <src_file.mbtiles> <dst_file.mbtiles>
```

Optionally filters by zooms.  Supports de-duplicated and simple mbtiles

---------

Co-authored-by: Yuri Astrakhan <yuriastrakhan@gmail.com>

docs/src/tools.md

@@ -13,3 +13,10 @@ Retrieve raw metadata value by its name. The value is printed to stdout without
 ```shell
 mbtiles meta-get <file.mbtiles> <key>
 ```
+
+### copy
+Copy existing `.mbtiles` file to a new, non-existent file.
+
+```shell
+mbtiles copy  <src_file.mbtiles> <dst_file.mbtiles>
+```

martin-mbtiles/src/bin/main.rs

@@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
-use martin_mbtiles::Mbtiles;
+use martin_mbtiles::{Mbtiles, TileCopier, TileCopierOptions};
 use sqlx::sqlite::SqliteConnectOptions;
 use sqlx::{Connection, SqliteConnection};
 
@@ -13,6 +13,9 @@ use sqlx::{Connection, SqliteConnection};
     about = "A utility to work with .mbtiles file content"
 )]
 pub struct Args {
+    /// Display detailed information
+    #[arg(short, long, hide = true)]
+    verbose: bool,
     #[command(subcommand)]
     command: Commands,
 }
@@ -39,13 +42,23 @@ enum Commands {
     //     /// MBTiles file to modify
     //     file: PathBuf,
     // },
-    // /// Copy tiles from one mbtiles file to another.
-    // Copy {
-    //     /// MBTiles file to read from
-    //     src_file: PathBuf,
-    //     /// MBTiles file to write to
-    //     dst_file: PathBuf,
-    // },
+    /// Copy tiles from one mbtiles file to another.
+    #[command(name = "copy")]
+    Copy {
+        /// MBTiles file to read from
+        src_file: PathBuf,
+        /// MBTiles file to write to
+        dst_file: PathBuf,
+        /// Minimum zoom level to copy
+        #[arg(long)]
+        min_zoom: Option<u8>,
+        /// Maximum zoom level to copy
+        #[arg(long)]
+        max_zoom: Option<u8>,
+        /// List of zoom levels to copy; if provided, min-zoom and max-zoom will be ignored
+        #[arg(long, value_delimiter(','))]
+        zoom_levels: Vec<u8>,
+    },
 }
 
 #[tokio::main]
@@ -56,6 +69,23 @@ async fn main() -> Result<()> {
         Commands::MetaGetValue { file, key } => {
             meta_get_value(file.as_path(), &key).await?;
         }
+        Commands::Copy {
+            src_file,
+            dst_file,
+            min_zoom,
+            max_zoom,
+            zoom_levels,
+        } => {
+            let copy_opts = TileCopierOptions::new()
+                .verbose(args.verbose)
+                .min_zoom(min_zoom)
+                .max_zoom(max_zoom)
+                .zooms(zoom_levels);
+
+            let tile_copier = TileCopier::new(src_file, dst_file, copy_opts)?;
+
+            tile_copier.run().await?;
+        }
     }
 
     Ok(())

martin-mbtiles/src/errors.rs

@@ -13,11 +13,17 @@ pub enum MbtError {
     #[error("Inconsistent tile formats detected: {0} vs {1}")]
     InconsistentMetadata(TileInfo, TileInfo),
 
-    #[error("Invalid data storage format for MBTile file {0}")]
-    InvalidDataStorageFormat(String),
+    #[error("Invalid data format for MBTile file {0}")]
+    InvalidDataFormat(String),
+
+    #[error(r#"Filename "{0}" passed to SQLite must be valid UTF-8"#)]
+    InvalidFilenameType(PathBuf),
 
     #[error("No tiles found")]
     NoTilesFound,
+
+    #[error("The destination file {0} is non-empty")]
+    NonEmptyTargetFile(PathBuf),
 }
 
 pub type MbtResult<T> = Result<T, MbtError>;

martin-mbtiles/src/lib.rs

@@ -4,7 +4,9 @@ mod errors;
 mod mbtiles;
 mod mbtiles_pool;
 mod mbtiles_queries;
+mod tile_copier;
 
 pub use errors::MbtError;
 pub use mbtiles::{Mbtiles, Metadata};
 pub use mbtiles_pool::MbtilesPool;
+pub use tile_copier::{TileCopier, TileCopierOptions};

martin-mbtiles/src/mbtiles.rs

@@ -27,7 +27,7 @@ pub struct Metadata {
 }
 
 #[derive(Debug, PartialEq)]
-pub enum Type {
+pub enum MbtType {
     TileTables,
     DeDuplicated,
 }
@@ -276,16 +276,16 @@ impl Mbtiles {
         Ok(None)
     }
 
-    pub async fn detect_type<T>(&self, conn: &mut T) -> MbtResult<Type>
+    pub async fn detect_type<T>(&self, conn: &mut T) -> MbtResult<MbtType>
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
         if is_deduplicated_type(&mut *conn).await? {
-            Ok(Type::DeDuplicated)
+            Ok(MbtType::DeDuplicated)
         } else if is_tile_tables_type(&mut *conn).await? {
-            Ok(Type::TileTables)
+            Ok(MbtType::TileTables)
         } else {
-            Err(MbtError::InvalidDataStorageFormat(self.filepath.clone()))
+            Err(MbtError::InvalidDataFormat(self.filepath.clone()))
         }
     }
 }
@@ -384,14 +384,14 @@ mod tests {
     async fn detect_type() {
         let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
-        assert_eq!(res, Type::TileTables);
+        assert_eq!(res, MbtType::TileTables);
 
         let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
-        assert_eq!(res, Type::DeDuplicated);
+        assert_eq!(res, MbtType::DeDuplicated);
 
         let (mut conn, mbt) = open(":memory:").await;
         let res = mbt.detect_type(&mut conn).await;
-        assert!(matches!(res, Err(MbtError::InvalidDataStorageFormat(_))));
+        assert!(matches!(res, Err(MbtError::InvalidDataFormat(_))));
     }
 }

martin-mbtiles/src/tile_copier.rs

@@ -0,0 +1,311 @@
+extern crate core;
+
+use crate::errors::MbtResult;
+use crate::mbtiles::MbtType;
+use crate::{MbtError, Mbtiles};
+use sqlx::sqlite::SqliteConnectOptions;
+use sqlx::{query, Connection, Row, SqliteConnection};
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+#[derive(Clone, Default, Debug)]
+pub struct TileCopierOptions {
+    zooms: HashSet<u8>,
+    min_zoom: Option<u8>,
+    max_zoom: Option<u8>,
+    //self.bbox = bbox
+    verbose: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct TileCopier {
+    src_mbtiles: Mbtiles,
+    dst_filepath: PathBuf,
+    options: TileCopierOptions,
+}
+
+impl TileCopierOptions {
+    pub fn new() -> Self {
+        Self {
+            zooms: HashSet::new(),
+            min_zoom: None,
+            max_zoom: None,
+            verbose: false,
+        }
+    }
+
+    pub fn zooms(mut self, zooms: Vec<u8>) -> Self {
+        for zoom in zooms {
+            self.zooms.insert(zoom);
+        }
+        self
+    }
+
+    pub fn min_zoom(mut self, min_zoom: Option<u8>) -> Self {
+        self.min_zoom = min_zoom;
+        self
+    }
+
+    pub fn max_zoom(mut self, max_zoom: Option<u8>) -> Self {
+        self.max_zoom = max_zoom;
+        self
+    }
+
+    pub fn verbose(mut self, verbose: bool) -> Self {
+        self.verbose = verbose;
+        self
+    }
+}
+
+impl TileCopier {
+    pub fn new(
+        src_filepath: PathBuf,
+        dst_filepath: PathBuf,
+        options: TileCopierOptions,
+    ) -> MbtResult<Self> {
+        Ok(TileCopier {
+            src_mbtiles: Mbtiles::new(src_filepath)?,
+            dst_filepath,
+            options,
+        })
+    }
+
+    pub async fn run(self) -> MbtResult<()> {
+        let opt = SqliteConnectOptions::new()
+            .read_only(true)
+            .filename(PathBuf::from(&self.src_mbtiles.filepath()));
+        let mut conn = SqliteConnection::connect_with(&opt).await?;
+        let storage_type = self.src_mbtiles.detect_type(&mut conn).await?;
+
+        let opt = SqliteConnectOptions::new()
+            .create_if_missing(true)
+            .filename(&self.dst_filepath);
+        let mut conn = SqliteConnection::connect_with(&opt).await?;
+
+        if query("SELECT 1 FROM sqlite_schema LIMIT 1")
+            .fetch_optional(&mut conn)
+            .await?
+            .is_some()
+        {
+            return Err(MbtError::NonEmptyTargetFile(self.dst_filepath.clone()));
+        }
+
+        query("PRAGMA page_size = 512").execute(&mut conn).await?;
+        query("VACUUM").execute(&mut conn).await?;
+
+        query("ATTACH DATABASE ? AS sourceDb")
+            .bind(self.src_mbtiles.filepath())
+            .execute(&mut conn)
+            .await?;
+
+        let schema = query("SELECT sql FROM sourceDb.sqlite_schema WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images')")
+            .fetch_all(&mut conn)
+            .await?;
+
+        for row in &schema {
+            let row: String = row.get(0);
+            query(row.as_str()).execute(&mut conn).await?;
+        }
+
+        query("INSERT INTO metadata SELECT * FROM sourceDb.metadata")
+            .execute(&mut conn)
+            .await?;
+
+        match storage_type {
+            MbtType::TileTables => self.copy_tile_tables(&mut conn).await,
+            MbtType::DeDuplicated => self.copy_deduplicated(&mut conn).await,
+        }
+    }
+
+    async fn copy_tile_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
+        self.run_query_with_options(conn, "INSERT INTO tiles SELECT * FROM sourceDb.tiles")
+            .await?;
+
+        Ok(())
+    }
+
+    async fn copy_deduplicated(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
+        query("INSERT INTO map SELECT * FROM sourceDb.map")
+            .execute(&mut *conn)
+            .await?;
+
+        self.run_query_with_options(
+            conn,
+            "INSERT INTO images
+                SELECT images.tile_data, images.tile_id
+                FROM sourceDb.images
+                  JOIN sourceDb.map
+                  ON images.tile_id = map.tile_id",
+        )
+        .await?;
+
+        Ok(())
+    }
+
+    async fn run_query_with_options(
+        &self,
+        conn: &mut SqliteConnection,
+        sql: &str,
+    ) -> MbtResult<()> {
+        let mut params: Vec<String> = vec![];
+
+        let sql = if !&self.options.zooms.is_empty() {
+            params.extend(self.options.zooms.iter().map(|z| z.to_string()));
+            format!(
+                "{sql} WHERE zoom_level IN ({})",
+                vec!["?"; self.options.zooms.len()].join(",")
+            )
+        } else if let Some(min_zoom) = &self.options.min_zoom {
+            if let Some(max_zoom) = &self.options.max_zoom {
+                params.push(min_zoom.to_string());
+                params.push(max_zoom.to_string());
+                format!("{sql} WHERE zoom_level BETWEEN ? AND ?")
+            } else {
+                params.push(min_zoom.to_string());
+                format!("{sql} WHERE zoom_level >= ?")
+            }
+        } else if let Some(max_zoom) = &self.options.max_zoom {
+            params.push(max_zoom.to_string());
+            format!("{sql} WHERE zoom_level <= ? ")
+        } else {
+            sql.to_string()
+        };
+
+        let mut query = query(sql.as_str());
+
+        for param in params {
+            query = query.bind(param);
+        }
+
+        query.execute(conn).await?;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs::remove_file;
+
+    use sqlx::{Connection, SqliteConnection};
+
+    use super::*;
+
+    async fn verify_copy_all(src_filepath: PathBuf, dst_filepath: PathBuf) {
+        let copy_opts = TileCopierOptions::new();
+        let tile_copier =
+            TileCopier::new(src_filepath.clone(), dst_filepath.clone(), copy_opts).unwrap();
+
+        tile_copier.run().await.unwrap();
+
+        let mut src_conn = SqliteConnection::connect_with(
+            &SqliteConnectOptions::new().filename(src_filepath.clone()),
+        )
+        .await
+        .unwrap();
+        let mut dst_conn = SqliteConnection::connect_with(
+            &SqliteConnectOptions::new().filename(dst_filepath.clone()),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(
+            query("SELECT COUNT(*) FROM tiles;")
+                .fetch_one(&mut src_conn)
+                .await
+                .unwrap()
+                .get::<i32, _>(0),
+            query("SELECT COUNT(*) FROM tiles;")
+                .fetch_one(&mut dst_conn)
+                .await
+                .unwrap()
+                .get::<i32, _>(0)
+        );
+
+        remove_file(dst_filepath).unwrap();
+    }
+
+    async fn verify_copy_with_zoom_filter(
+        src_filepath: PathBuf,
+        dst_filepath: PathBuf,
+        opts: TileCopierOptions,
+        expected_zoom_levels: u8,
+    ) {
+        let tile_copier =
+            TileCopier::new(src_filepath.clone(), dst_filepath.clone(), opts).unwrap();
+
+        tile_copier.run().await.unwrap();
+
+        let mut dst_conn = SqliteConnection::connect_with(
+            &SqliteConnectOptions::new().filename(dst_filepath.clone()),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(
+            query("SELECT COUNT(DISTINCT zoom_level) FROM tiles;")
+                .fetch_one(&mut dst_conn)
+                .await
+                .unwrap()
+                .get::<u8, _>(0),
+            expected_zoom_levels
+        );
+
+        remove_file(dst_filepath).unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn copy_tile_tables() {
+        let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let temp_filepath = PathBuf::from("../tests/tmp_tile_tables.mbtiles");
+
+        verify_copy_all(src_filepath, temp_filepath).await;
+    }
+
+    #[actix_rt::test]
+    async fn non_empty_target_file() {
+        let copy_opts = TileCopierOptions::new();
+        let tile_copier = TileCopier::new(
+            PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"),
+            PathBuf::from("../tests/fixtures/files/json.mbtiles"),
+            copy_opts,
+        )
+        .unwrap();
+
+        assert!(matches!(
+            tile_copier.run().await,
+            Err(MbtError::NonEmptyTargetFile(_))
+        ));
+    }
+
+    #[actix_rt::test]
+    async fn copy_deduplicated() {
+        let src_filepath = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles");
+        let temp_filepath = PathBuf::from("../tests/tmp_deduplicated.mbtiles");
+
+        verify_copy_all(src_filepath, temp_filepath).await;
+    }
+
+    #[actix_rt::test]
+    async fn copy_with_min_max_zoom() {
+        let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let temp_filepath = PathBuf::from("../tests/tmp_min_max_zoom.mbtiles");
+
+        let copy_opts = TileCopierOptions::new().min_zoom(Some(2)).max_zoom(Some(4));
+
+        verify_copy_with_zoom_filter(src_filepath, temp_filepath, copy_opts, 3).await;
+    }
+
+    #[actix_rt::test]
+    async fn copy_with_zoom_levels() {
+        let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
+        let temp_filepath = PathBuf::from("../tests/tmp_zoom_levels.mbtiles");
+
+        let copy_opts = TileCopierOptions::new()
+            .min_zoom(Some(2))
+            .max_zoom(Some(4))
+            .zooms(vec![1, 6]);
+
+        verify_copy_with_zoom_filter(src_filepath, temp_filepath, copy_opts, 2).await;
+    }
+}

tests/expected/mbtiles/help.txt

@@ -4,6 +4,7 @@ Usage: mbtiles <COMMAND>
 
 Commands:
   meta-get  Gets a single value from the MBTiles metadata table
+  copy      Copy tiles from one mbtiles file to another
   help      Print this message or the help of the given subcommand(s)
 
 Options: