diff --git a/docs/src/tools.md b/docs/src/tools.md index a6871c6e..8f9ad2b6 100644 --- a/docs/src/tools.md +++ b/docs/src/tools.md @@ -13,3 +13,10 @@ Retrieve raw metadata value by its name. The value is printed to stdout without ```shell mbtiles meta-get ``` + +### copy +Copy existing `.mbtiles` file to a new, non-existent file. + +```shell +mbtiles copy +``` diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 70c35858..7f82eb56 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use clap::{Parser, Subcommand}; -use martin_mbtiles::Mbtiles; +use martin_mbtiles::{Mbtiles, TileCopier, TileCopierOptions}; use sqlx::sqlite::SqliteConnectOptions; use sqlx::{Connection, SqliteConnection}; @@ -13,6 +13,9 @@ use sqlx::{Connection, SqliteConnection}; about = "A utility to work with .mbtiles file content" )] pub struct Args { + /// Display detailed information + #[arg(short, long, hide = true)] + verbose: bool, #[command(subcommand)] command: Commands, } @@ -39,13 +42,23 @@ enum Commands { // /// MBTiles file to modify // file: PathBuf, // }, - // /// Copy tiles from one mbtiles file to another. - // Copy { - // /// MBTiles file to read from - // src_file: PathBuf, - // /// MBTiles file to write to - // dst_file: PathBuf, - // }, + /// Copy tiles from one mbtiles file to another. + #[command(name = "copy")] + Copy { + /// MBTiles file to read from + src_file: PathBuf, + /// MBTiles file to write to + dst_file: PathBuf, + /// Minimum zoom level to copy + #[arg(long)] + min_zoom: Option, + /// Maximum zoom level to copy + #[arg(long)] + max_zoom: Option, + /// List of zoom levels to copy; if provided, min-zoom and max-zoom will be ignored + #[arg(long, value_delimiter(','))] + zoom_levels: Vec, + }, } #[tokio::main] @@ -56,6 +69,23 @@ async fn main() -> Result<()> { Commands::MetaGetValue { file, key } => { meta_get_value(file.as_path(), &key).await?; } + Commands::Copy { + src_file, + dst_file, + min_zoom, + max_zoom, + zoom_levels, + } => { + let copy_opts = TileCopierOptions::new() + .verbose(args.verbose) + .min_zoom(min_zoom) + .max_zoom(max_zoom) + .zooms(zoom_levels); + + let tile_copier = TileCopier::new(src_file, dst_file, copy_opts)?; + + tile_copier.run().await?; + } } Ok(()) diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs index 9962371b..4ddf9531 100644 --- a/martin-mbtiles/src/errors.rs +++ b/martin-mbtiles/src/errors.rs @@ -13,11 +13,17 @@ pub enum MbtError { #[error("Inconsistent tile formats detected: {0} vs {1}")] InconsistentMetadata(TileInfo, TileInfo), - #[error("Invalid data storage format for MBTile file {0}")] - InvalidDataStorageFormat(String), + #[error("Invalid data format for MBTile file {0}")] + InvalidDataFormat(String), + + #[error(r#"Filename "{0}" passed to SQLite must be valid UTF-8"#)] + InvalidFilenameType(PathBuf), #[error("No tiles found")] NoTilesFound, + + #[error("The destination file {0} is non-empty")] + NonEmptyTargetFile(PathBuf), } pub type MbtResult = Result; diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index 7d033356..2117d8bf 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -4,7 +4,9 @@ mod errors; mod mbtiles; mod mbtiles_pool; mod mbtiles_queries; +mod tile_copier; pub use errors::MbtError; pub use mbtiles::{Mbtiles, Metadata}; pub use mbtiles_pool::MbtilesPool; +pub use tile_copier::{TileCopier, TileCopierOptions}; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index b4db25cd..73079ec8 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -27,7 +27,7 @@ pub struct Metadata { } #[derive(Debug, PartialEq)] -pub enum Type { +pub enum MbtType { TileTables, DeDuplicated, } @@ -276,16 +276,16 @@ impl Mbtiles { Ok(None) } - pub async fn detect_type(&self, conn: &mut T) -> MbtResult + pub async fn detect_type(&self, conn: &mut T) -> MbtResult where for<'e> &'e mut T: SqliteExecutor<'e>, { if is_deduplicated_type(&mut *conn).await? { - Ok(Type::DeDuplicated) + Ok(MbtType::DeDuplicated) } else if is_tile_tables_type(&mut *conn).await? { - Ok(Type::TileTables) + Ok(MbtType::TileTables) } else { - Err(MbtError::InvalidDataStorageFormat(self.filepath.clone())) + Err(MbtError::InvalidDataFormat(self.filepath.clone())) } } } @@ -384,14 +384,14 @@ mod tests { async fn detect_type() { let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); - assert_eq!(res, Type::TileTables); + assert_eq!(res, MbtType::TileTables); let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); - assert_eq!(res, Type::DeDuplicated); + assert_eq!(res, MbtType::DeDuplicated); let (mut conn, mbt) = open(":memory:").await; let res = mbt.detect_type(&mut conn).await; - assert!(matches!(res, Err(MbtError::InvalidDataStorageFormat(_)))); + assert!(matches!(res, Err(MbtError::InvalidDataFormat(_)))); } } diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs new file mode 100644 index 00000000..ce7bd79e --- /dev/null +++ b/martin-mbtiles/src/tile_copier.rs @@ -0,0 +1,311 @@ +extern crate core; + +use crate::errors::MbtResult; +use crate::mbtiles::MbtType; +use crate::{MbtError, Mbtiles}; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{query, Connection, Row, SqliteConnection}; +use std::collections::HashSet; +use std::path::PathBuf; + +#[derive(Clone, Default, Debug)] +pub struct TileCopierOptions { + zooms: HashSet, + min_zoom: Option, + max_zoom: Option, + //self.bbox = bbox + verbose: bool, +} + +#[derive(Clone, Debug)] +pub struct TileCopier { + src_mbtiles: Mbtiles, + dst_filepath: PathBuf, + options: TileCopierOptions, +} + +impl TileCopierOptions { + pub fn new() -> Self { + Self { + zooms: HashSet::new(), + min_zoom: None, + max_zoom: None, + verbose: false, + } + } + + pub fn zooms(mut self, zooms: Vec) -> Self { + for zoom in zooms { + self.zooms.insert(zoom); + } + self + } + + pub fn min_zoom(mut self, min_zoom: Option) -> Self { + self.min_zoom = min_zoom; + self + } + + pub fn max_zoom(mut self, max_zoom: Option) -> Self { + self.max_zoom = max_zoom; + self + } + + pub fn verbose(mut self, verbose: bool) -> Self { + self.verbose = verbose; + self + } +} + +impl TileCopier { + pub fn new( + src_filepath: PathBuf, + dst_filepath: PathBuf, + options: TileCopierOptions, + ) -> MbtResult { + Ok(TileCopier { + src_mbtiles: Mbtiles::new(src_filepath)?, + dst_filepath, + options, + }) + } + + pub async fn run(self) -> MbtResult<()> { + let opt = SqliteConnectOptions::new() + .read_only(true) + .filename(PathBuf::from(&self.src_mbtiles.filepath())); + let mut conn = SqliteConnection::connect_with(&opt).await?; + let storage_type = self.src_mbtiles.detect_type(&mut conn).await?; + + let opt = SqliteConnectOptions::new() + .create_if_missing(true) + .filename(&self.dst_filepath); + let mut conn = SqliteConnection::connect_with(&opt).await?; + + if query("SELECT 1 FROM sqlite_schema LIMIT 1") + .fetch_optional(&mut conn) + .await? + .is_some() + { + return Err(MbtError::NonEmptyTargetFile(self.dst_filepath.clone())); + } + + query("PRAGMA page_size = 512").execute(&mut conn).await?; + query("VACUUM").execute(&mut conn).await?; + + query("ATTACH DATABASE ? AS sourceDb") + .bind(self.src_mbtiles.filepath()) + .execute(&mut conn) + .await?; + + let schema = query("SELECT sql FROM sourceDb.sqlite_schema WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images')") + .fetch_all(&mut conn) + .await?; + + for row in &schema { + let row: String = row.get(0); + query(row.as_str()).execute(&mut conn).await?; + } + + query("INSERT INTO metadata SELECT * FROM sourceDb.metadata") + .execute(&mut conn) + .await?; + + match storage_type { + MbtType::TileTables => self.copy_tile_tables(&mut conn).await, + MbtType::DeDuplicated => self.copy_deduplicated(&mut conn).await, + } + } + + async fn copy_tile_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { + self.run_query_with_options(conn, "INSERT INTO tiles SELECT * FROM sourceDb.tiles") + .await?; + + Ok(()) + } + + async fn copy_deduplicated(&self, conn: &mut SqliteConnection) -> MbtResult<()> { + query("INSERT INTO map SELECT * FROM sourceDb.map") + .execute(&mut *conn) + .await?; + + self.run_query_with_options( + conn, + "INSERT INTO images + SELECT images.tile_data, images.tile_id + FROM sourceDb.images + JOIN sourceDb.map + ON images.tile_id = map.tile_id", + ) + .await?; + + Ok(()) + } + + async fn run_query_with_options( + &self, + conn: &mut SqliteConnection, + sql: &str, + ) -> MbtResult<()> { + let mut params: Vec = vec![]; + + let sql = if !&self.options.zooms.is_empty() { + params.extend(self.options.zooms.iter().map(|z| z.to_string())); + format!( + "{sql} WHERE zoom_level IN ({})", + vec!["?"; self.options.zooms.len()].join(",") + ) + } else if let Some(min_zoom) = &self.options.min_zoom { + if let Some(max_zoom) = &self.options.max_zoom { + params.push(min_zoom.to_string()); + params.push(max_zoom.to_string()); + format!("{sql} WHERE zoom_level BETWEEN ? AND ?") + } else { + params.push(min_zoom.to_string()); + format!("{sql} WHERE zoom_level >= ?") + } + } else if let Some(max_zoom) = &self.options.max_zoom { + params.push(max_zoom.to_string()); + format!("{sql} WHERE zoom_level <= ? ") + } else { + sql.to_string() + }; + + let mut query = query(sql.as_str()); + + for param in params { + query = query.bind(param); + } + + query.execute(conn).await?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::fs::remove_file; + + use sqlx::{Connection, SqliteConnection}; + + use super::*; + + async fn verify_copy_all(src_filepath: PathBuf, dst_filepath: PathBuf) { + let copy_opts = TileCopierOptions::new(); + let tile_copier = + TileCopier::new(src_filepath.clone(), dst_filepath.clone(), copy_opts).unwrap(); + + tile_copier.run().await.unwrap(); + + let mut src_conn = SqliteConnection::connect_with( + &SqliteConnectOptions::new().filename(src_filepath.clone()), + ) + .await + .unwrap(); + let mut dst_conn = SqliteConnection::connect_with( + &SqliteConnectOptions::new().filename(dst_filepath.clone()), + ) + .await + .unwrap(); + + assert_eq!( + query("SELECT COUNT(*) FROM tiles;") + .fetch_one(&mut src_conn) + .await + .unwrap() + .get::(0), + query("SELECT COUNT(*) FROM tiles;") + .fetch_one(&mut dst_conn) + .await + .unwrap() + .get::(0) + ); + + remove_file(dst_filepath).unwrap(); + } + + async fn verify_copy_with_zoom_filter( + src_filepath: PathBuf, + dst_filepath: PathBuf, + opts: TileCopierOptions, + expected_zoom_levels: u8, + ) { + let tile_copier = + TileCopier::new(src_filepath.clone(), dst_filepath.clone(), opts).unwrap(); + + tile_copier.run().await.unwrap(); + + let mut dst_conn = SqliteConnection::connect_with( + &SqliteConnectOptions::new().filename(dst_filepath.clone()), + ) + .await + .unwrap(); + + assert_eq!( + query("SELECT COUNT(DISTINCT zoom_level) FROM tiles;") + .fetch_one(&mut dst_conn) + .await + .unwrap() + .get::(0), + expected_zoom_levels + ); + + remove_file(dst_filepath).unwrap(); + } + + #[actix_rt::test] + async fn copy_tile_tables() { + let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let temp_filepath = PathBuf::from("../tests/tmp_tile_tables.mbtiles"); + + verify_copy_all(src_filepath, temp_filepath).await; + } + + #[actix_rt::test] + async fn non_empty_target_file() { + let copy_opts = TileCopierOptions::new(); + let tile_copier = TileCopier::new( + PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"), + PathBuf::from("../tests/fixtures/files/json.mbtiles"), + copy_opts, + ) + .unwrap(); + + assert!(matches!( + tile_copier.run().await, + Err(MbtError::NonEmptyTargetFile(_)) + )); + } + + #[actix_rt::test] + async fn copy_deduplicated() { + let src_filepath = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let temp_filepath = PathBuf::from("../tests/tmp_deduplicated.mbtiles"); + + verify_copy_all(src_filepath, temp_filepath).await; + } + + #[actix_rt::test] + async fn copy_with_min_max_zoom() { + let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let temp_filepath = PathBuf::from("../tests/tmp_min_max_zoom.mbtiles"); + + let copy_opts = TileCopierOptions::new().min_zoom(Some(2)).max_zoom(Some(4)); + + verify_copy_with_zoom_filter(src_filepath, temp_filepath, copy_opts, 3).await; + } + + #[actix_rt::test] + async fn copy_with_zoom_levels() { + let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let temp_filepath = PathBuf::from("../tests/tmp_zoom_levels.mbtiles"); + + let copy_opts = TileCopierOptions::new() + .min_zoom(Some(2)) + .max_zoom(Some(4)) + .zooms(vec![1, 6]); + + verify_copy_with_zoom_filter(src_filepath, temp_filepath, copy_opts, 2).await; + } +} diff --git a/tests/expected/mbtiles/help.txt b/tests/expected/mbtiles/help.txt index 8a81adae..1094cdd7 100644 --- a/tests/expected/mbtiles/help.txt +++ b/tests/expected/mbtiles/help.txt @@ -4,6 +4,7 @@ Usage: mbtiles Commands: meta-get Gets a single value from the MBTiles metadata table + copy Copy tiles from one mbtiles file to another help Print this message or the help of the given subcommand(s) Options: