maplibre/martin
1
1
Fork 0
mirror of https://github.com/maplibre/martin.git synced 2024-12-19 21:01:45 +03:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Add basic copying functionality (#712)

Browse Source 
Copy an existing `.mbtiles` file to a new file.

```shell
mbtiles copy  <src_file.mbtiles> <dst_file.mbtiles>
```

Optionally filters by zooms.  Supports de-duplicated and simple mbtiles

---------

Co-authored-by: Yuri Astrakhan <yuriastrakhan@gmail.com>
This commit is contained in:
rstanciu 2023-06-16 17:00:46 -07:00 committed by GitHub
parent a187e6e1da
commit bc140e0216
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 375 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/src/tools.md
View File

@ -13,3 +13,10 @@ Retrieve raw metadata value by its name. The value is printed to stdout without
```shell
mbtiles meta-get <file.mbtiles> <key>
```
### copy
Copy existing `.mbtiles` file to a new, non-existent file.
```shell
mbtiles copy <src_file.mbtiles> <dst_file.mbtiles>
```

46
martin-mbtiles/src/bin/main.rs
View File

@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
use anyhow::Result;
use clap::{Parser, Subcommand};
use martin_mbtiles::Mbtiles;
use martin_mbtiles::{Mbtiles, TileCopier, TileCopierOptions};
use sqlx::sqlite::SqliteConnectOptions;
use sqlx::{Connection, SqliteConnection};
@ -13,6 +13,9 @@ use sqlx::{Connection, SqliteConnection};
about = "A utility to work with .mbtiles file content"
)]
pub struct Args {
/// Display detailed information
#[arg(short, long, hide = true)]
verbose: bool,
#[command(subcommand)]
command: Commands,
}
@ -39,13 +42,23 @@ enum Commands {
// /// MBTiles file to modify
// file: PathBuf,
// },
// /// Copy tiles from one mbtiles file to another.
// Copy {
// /// MBTiles file to read from
// src_file: PathBuf,
// /// MBTiles file to write to
// dst_file: PathBuf,
// },
/// Copy tiles from one mbtiles file to another.
#[command(name = "copy")]
Copy {
/// MBTiles file to read from
src_file: PathBuf,
/// MBTiles file to write to
dst_file: PathBuf,
/// Minimum zoom level to copy
#[arg(long)]
min_zoom: Option<u8>,
/// Maximum zoom level to copy
#[arg(long)]
max_zoom: Option<u8>,
/// List of zoom levels to copy; if provided, min-zoom and max-zoom will be ignored
#[arg(long, value_delimiter(','))]
zoom_levels: Vec<u8>,
},
}
#[tokio::main]
@ -56,6 +69,23 @@ async fn main() -> Result<()> {
Commands::MetaGetValue { file, key } => {
meta_get_value(file.as_path(), &key).await?;
}
Commands::Copy {
src_file,
dst_file,
min_zoom,
max_zoom,
zoom_levels,
} => {
let copy_opts = TileCopierOptions::new()
.verbose(args.verbose)
.min_zoom(min_zoom)
.max_zoom(max_zoom)
.zooms(zoom_levels);
let tile_copier = TileCopier::new(src_file, dst_file, copy_opts)?;
tile_copier.run().await?;
}
}
Ok(())

10
martin-mbtiles/src/errors.rs
View File

@ -13,11 +13,17 @@ pub enum MbtError {
#[error("Inconsistent tile formats detected: {0} vs {1}")]
InconsistentMetadata(TileInfo, TileInfo),
#[error("Invalid data storage format for MBTile file {0}")]
InvalidDataStorageFormat(String),
#[error("Invalid data format for MBTile file {0}")]
InvalidDataFormat(String),
#[error(r#"Filename "{0}" passed to SQLite must be valid UTF-8"#)]
InvalidFilenameType(PathBuf),
#[error("No tiles found")]
NoTilesFound,
#[error("The destination file {0} is non-empty")]
NonEmptyTargetFile(PathBuf),
}
pub type MbtResult<T> = Result<T, MbtError>;

2
martin-mbtiles/src/lib.rs
View File

@ -4,7 +4,9 @@ mod errors;
mod mbtiles;
mod mbtiles_pool;
mod mbtiles_queries;
mod tile_copier;
pub use errors::MbtError;
pub use mbtiles::{Mbtiles, Metadata};
pub use mbtiles_pool::MbtilesPool;
pub use tile_copier::{TileCopier, TileCopierOptions};

16
martin-mbtiles/src/mbtiles.rs
View File

@ -27,7 +27,7 @@ pub struct Metadata {
}
#[derive(Debug, PartialEq)]
pub enum Type {
pub enum MbtType {
TileTables,
DeDuplicated,
}
@ -276,16 +276,16 @@ impl Mbtiles {
Ok(None)
}
pub async fn detect_type<T>(&self, conn: &mut T) -> MbtResult<Type>
pub async fn detect_type<T>(&self, conn: &mut T) -> MbtResult<MbtType>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
if is_deduplicated_type(&mut *conn).await? {
Ok(Type::DeDuplicated)
Ok(MbtType::DeDuplicated)
} else if is_tile_tables_type(&mut *conn).await? {
Ok(Type::TileTables)
Ok(MbtType::TileTables)
} else {
Err(MbtError::InvalidDataStorageFormat(self.filepath.clone()))
Err(MbtError::InvalidDataFormat(self.filepath.clone()))
}
}
}
@ -384,14 +384,14 @@ mod tests {
async fn detect_type() {
let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
let res = mbt.detect_type(&mut conn).await.unwrap();
assert_eq!(res, Type::TileTables);
assert_eq!(res, MbtType::TileTables);
let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await;
let res = mbt.detect_type(&mut conn).await.unwrap();
assert_eq!(res, Type::DeDuplicated);
assert_eq!(res, MbtType::DeDuplicated);
let (mut conn, mbt) = open(":memory:").await;
let res = mbt.detect_type(&mut conn).await;
assert!(matches!(res, Err(MbtError::InvalidDataStorageFormat(_))));
assert!(matches!(res, Err(MbtError::InvalidDataFormat(_))));
}
}

311
martin-mbtiles/src/tile_copier.rs Normal file
View File

@ -0,0 +1,311 @@
extern crate core;
use crate::errors::MbtResult;
use crate::mbtiles::MbtType;
use crate::{MbtError, Mbtiles};
use sqlx::sqlite::SqliteConnectOptions;
use sqlx::{query, Connection, Row, SqliteConnection};
use std::collections::HashSet;
use std::path::PathBuf;
#[derive(Clone, Default, Debug)]
pub struct TileCopierOptions {
zooms: HashSet<u8>,
min_zoom: Option<u8>,
max_zoom: Option<u8>,
//self.bbox = bbox
verbose: bool,
}
#[derive(Clone, Debug)]
pub struct TileCopier {
src_mbtiles: Mbtiles,
dst_filepath: PathBuf,
options: TileCopierOptions,
}
impl TileCopierOptions {
pub fn new() -> Self {
Self {
zooms: HashSet::new(),
min_zoom: None,
max_zoom: None,
verbose: false,
}
}
pub fn zooms(mut self, zooms: Vec<u8>) -> Self {
for zoom in zooms {
self.zooms.insert(zoom);
}
self
}
pub fn min_zoom(mut self, min_zoom: Option<u8>) -> Self {
self.min_zoom = min_zoom;
self
}
pub fn max_zoom(mut self, max_zoom: Option<u8>) -> Self {
self.max_zoom = max_zoom;
self
}
pub fn verbose(mut self, verbose: bool) -> Self {
self.verbose = verbose;
self
}
}
impl TileCopier {
pub fn new(
src_filepath: PathBuf,
dst_filepath: PathBuf,
options: TileCopierOptions,
) -> MbtResult<Self> {
Ok(TileCopier {
src_mbtiles: Mbtiles::new(src_filepath)?,
dst_filepath,
options,
})
}
pub async fn run(self) -> MbtResult<()> {
let opt = SqliteConnectOptions::new()
.read_only(true)
.filename(PathBuf::from(&self.src_mbtiles.filepath()));
let mut conn = SqliteConnection::connect_with(&opt).await?;
let storage_type = self.src_mbtiles.detect_type(&mut conn).await?;
let opt = SqliteConnectOptions::new()
.create_if_missing(true)
.filename(&self.dst_filepath);
let mut conn = SqliteConnection::connect_with(&opt).await?;
if query("SELECT 1 FROM sqlite_schema LIMIT 1")
.fetch_optional(&mut conn)
.await?
.is_some()
{
return Err(MbtError::NonEmptyTargetFile(self.dst_filepath.clone()));
}
query("PRAGMA page_size = 512").execute(&mut conn).await?;
query("VACUUM").execute(&mut conn).await?;
query("ATTACH DATABASE ? AS sourceDb")
.bind(self.src_mbtiles.filepath())
.execute(&mut conn)
.await?;
let schema = query("SELECT sql FROM sourceDb.sqlite_schema WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images')")
.fetch_all(&mut conn)
.await?;
for row in &schema {
let row: String = row.get(0);
query(row.as_str()).execute(&mut conn).await?;
}
query("INSERT INTO metadata SELECT * FROM sourceDb.metadata")
.execute(&mut conn)
.await?;
match storage_type {
MbtType::TileTables => self.copy_tile_tables(&mut conn).await,
MbtType::DeDuplicated => self.copy_deduplicated(&mut conn).await,
}
}
async fn copy_tile_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
self.run_query_with_options(conn, "INSERT INTO tiles SELECT * FROM sourceDb.tiles")
.await?;
Ok(())
}
async fn copy_deduplicated(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
query("INSERT INTO map SELECT * FROM sourceDb.map")
.execute(&mut *conn)
.await?;
self.run_query_with_options(
conn,
"INSERT INTO images
SELECT images.tile_data, images.tile_id
FROM sourceDb.images
JOIN sourceDb.map
ON images.tile_id = map.tile_id",
)
.await?;
Ok(())
}
async fn run_query_with_options(
&self,
conn: &mut SqliteConnection,
sql: &str,
) -> MbtResult<()> {
let mut params: Vec<String> = vec![];
let sql = if !&self.options.zooms.is_empty() {
params.extend(self.options.zooms.iter().map(|z| z.to_string()));
format!(
"{sql} WHERE zoom_level IN ({})",
vec!["?"; self.options.zooms.len()].join(",")
)
} else if let Some(min_zoom) = &self.options.min_zoom {
if let Some(max_zoom) = &self.options.max_zoom {
params.push(min_zoom.to_string());
params.push(max_zoom.to_string());
format!("{sql} WHERE zoom_level BETWEEN ? AND ?")
} else {
params.push(min_zoom.to_string());
format!("{sql} WHERE zoom_level >= ?")
}
} else if let Some(max_zoom) = &self.options.max_zoom {
params.push(max_zoom.to_string());
format!("{sql} WHERE zoom_level <= ? ")
} else {
sql.to_string()
};
let mut query = query(sql.as_str());
for param in params {
query = query.bind(param);
}
query.execute(conn).await?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::fs::remove_file;
use sqlx::{Connection, SqliteConnection};
use super::*;
async fn verify_copy_all(src_filepath: PathBuf, dst_filepath: PathBuf) {
let copy_opts = TileCopierOptions::new();
let tile_copier =
TileCopier::new(src_filepath.clone(), dst_filepath.clone(), copy_opts).unwrap();
tile_copier.run().await.unwrap();
let mut src_conn = SqliteConnection::connect_with(
&SqliteConnectOptions::new().filename(src_filepath.clone()),
)
.await
.unwrap();
let mut dst_conn = SqliteConnection::connect_with(
&SqliteConnectOptions::new().filename(dst_filepath.clone()),
)
.await
.unwrap();
assert_eq!(
query("SELECT COUNT(*) FROM tiles;")
.fetch_one(&mut src_conn)
.await
.unwrap()
.get::<i32, _>(0),
query("SELECT COUNT(*) FROM tiles;")
.fetch_one(&mut dst_conn)
.await
.unwrap()
.get::<i32, _>(0)
);
remove_file(dst_filepath).unwrap();
}
async fn verify_copy_with_zoom_filter(
src_filepath: PathBuf,
dst_filepath: PathBuf,
opts: TileCopierOptions,
expected_zoom_levels: u8,
) {
let tile_copier =
TileCopier::new(src_filepath.clone(), dst_filepath.clone(), opts).unwrap();
tile_copier.run().await.unwrap();
let mut dst_conn = SqliteConnection::connect_with(
&SqliteConnectOptions::new().filename(dst_filepath.clone()),
)
.await
.unwrap();
assert_eq!(
query("SELECT COUNT(DISTINCT zoom_level) FROM tiles;")
.fetch_one(&mut dst_conn)
.await
.unwrap()
.get::<u8, _>(0),
expected_zoom_levels
);
remove_file(dst_filepath).unwrap();
}
#[actix_rt::test]
async fn copy_tile_tables() {
let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
let temp_filepath = PathBuf::from("../tests/tmp_tile_tables.mbtiles");
verify_copy_all(src_filepath, temp_filepath).await;
}
#[actix_rt::test]
async fn non_empty_target_file() {
let copy_opts = TileCopierOptions::new();
let tile_copier = TileCopier::new(
PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"),
PathBuf::from("../tests/fixtures/files/json.mbtiles"),
copy_opts,
)
.unwrap();
assert!(matches!(
tile_copier.run().await,
Err(MbtError::NonEmptyTargetFile(_))
));
}
#[actix_rt::test]
async fn copy_deduplicated() {
let src_filepath = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles");
let temp_filepath = PathBuf::from("../tests/tmp_deduplicated.mbtiles");
verify_copy_all(src_filepath, temp_filepath).await;
}
#[actix_rt::test]
async fn copy_with_min_max_zoom() {
let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
let temp_filepath = PathBuf::from("../tests/tmp_min_max_zoom.mbtiles");
let copy_opts = TileCopierOptions::new().min_zoom(Some(2)).max_zoom(Some(4));
verify_copy_with_zoom_filter(src_filepath, temp_filepath, copy_opts, 3).await;
}
#[actix_rt::test]
async fn copy_with_zoom_levels() {
let src_filepath = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles");
let temp_filepath = PathBuf::from("../tests/tmp_zoom_levels.mbtiles");
let copy_opts = TileCopierOptions::new()
.min_zoom(Some(2))
.max_zoom(Some(4))
.zooms(vec![1, 6]);
verify_copy_with_zoom_filter(src_filepath, temp_filepath, copy_opts, 2).await;
}
}

1
tests/expected/mbtiles/help.txt
View File

@ -4,6 +4,7 @@ Usage: mbtiles <COMMAND>
Commands:
meta-get Gets a single value from the MBTiles metadata table
copy Copy tiles from one mbtiles file to another
help Print this message or the help of the given subcommand(s)
Options: