maplibre/martin
1
1
Fork 0
mirror of https://github.com/maplibre/martin.git synced 2024-12-18 20:31:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Cleanup mbtiles, rename TileCopierOptions, testing (#916)

Browse Source 
* Rename `TileCopierOptions` -> `TileCopier`
* remove a few un-needed sqlite open to detect mbtiles type
* move `open_and_detect_type` to `MBTiles`
* add `attach_to` to `MBTiles`
* move various table creation fn to mbtiles_queries file
* a few sql format
This commit is contained in:
Yuri Astrakhan 2023-10-01 23:28:43 -04:00 committed by GitHub
parent 14ea5cb2f6
commit 6b7bcabe49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 433 additions and 471 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -1781,7 +1781,7 @@ dependencies = [
[[package]]
name = "martin-mbtiles"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"actix-rt",
"anyhow",

2
Cargo.toml
View File

@ -31,7 +31,7 @@ indoc = "2"
itertools = "0.11"
json-patch = "1.1"
log = "0.4"
martin-mbtiles = { path = "./martin-mbtiles", version = "0.5.0", default-features = false }
martin-mbtiles = { path = "./martin-mbtiles", version = "0.6.0", default-features = false }
martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" }
num_cpus = "1"
pmtiles = { version = "0.3", features = ["mmap-async-tokio", "tilejson"] }

20
martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json
View File

@ -1,20 +0,0 @@
{
"db_name": "SQLite",
"query": "SELECT (\n -- Has a \"map\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- \"map\" table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_id\" AND type = \"TEXT\"))\n --\n ) AND (\n -- Has a \"images\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- \"images\" table's columns and their types are as expected:\n -- 2 columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE ((name = \"tile_id\" AND type = \"TEXT\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) AS is_valid;\n",
"describe": {
"columns": [
{
"name": "is_valid",
"ordinal": 0,
"type_info": "Int"
}
],
"parameters": {
"Right": 0
},
"nullable": [
null
]
},
"hash": "14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628"
}

20
martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json
View File

@ -1,20 +0,0 @@
{
"db_name": "SQLite",
"query": "SELECT (\n -- Has a \"tiles\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles\" table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) as is_valid;\n",
"describe": {
"columns": [
{
"name": "is_valid",
"ordinal": 0,
"type_info": "Int"
}
],
"parameters": {
"Right": 0
},
"nullable": [
null
]
},
"hash": "177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd"
}

20
martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json
View File

@ -1,20 +0,0 @@
{
"db_name": "SQLite",
"query": "SELECT (\n -- Has a \"tiles_with_hash\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles_with_hash'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles_with_hash\" table's columns and their types are as expected:\n -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash).\n -- The order is not important\n SELECT COUNT(*) = 5\n FROM pragma_table_info('tiles_with_hash')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\")\n OR (name = \"tile_hash\" AND type = \"TEXT\"))\n --\n ) as is_valid;\n",
"describe": {
"columns": [
{
"name": "is_valid",
"ordinal": 0,
"type_info": "Int"
}
],
"parameters": {
"Right": 0
},
"nullable": [
null
]
},
"hash": "3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401"
}

12
martin-mbtiles/.sqlx/query-3b2930e8d61f31ea1bf32efe340b7766f876ddb9a357a512ab3a37914bea003c.json
View File

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "ATTACH DATABASE ? AS sourceDb",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "3b2930e8d61f31ea1bf32efe340b7766f876ddb9a357a512ab3a37914bea003c"
}

12
martin-mbtiles/.sqlx/query-45de99a3628a53940ef80b0e2603c46f61ff92ffbc6ec3bba4860abd60d224cb.json
View File

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "ATTACH DATABASE ? AS srcDb",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "45de99a3628a53940ef80b0e2603c46f61ff92ffbc6ec3bba4860abd60d224cb"
}

20
martin-mbtiles/.sqlx/query-4905d37cd3818e2fe9f65fdd20437901cbe4b6421bac3cf671e86d4b5d8dc0f3.json Normal file
View File

@ -0,0 +1,20 @@
{
"db_name": "SQLite",
"query": "SELECT (\n -- Has a 'tiles_with_hash' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles_with_hash'\n AND type = 'table'\n --\n ) AND (\n -- 'tiles_with_hash' table's columns and their types are as expected:\n -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash).\n -- The order is not important\n SELECT COUNT(*) = 5\n FROM pragma_table_info('tiles_with_hash')\n WHERE ((name = 'zoom_level' AND type = 'INTEGER')\n OR (name = 'tile_column' AND type = 'INTEGER')\n OR (name = 'tile_row' AND type = 'INTEGER')\n OR (name = 'tile_data' AND type = 'BLOB')\n OR (name = 'tile_hash' AND type = 'TEXT'))\n --\n ) as is_valid;",
"describe": {
"columns": [
{
"name": "is_valid",
"ordinal": 0,
"type_info": "Int"
}
],
"parameters": {
"Right": 0
},
"nullable": [
null
]
},
"hash": "4905d37cd3818e2fe9f65fdd20437901cbe4b6421bac3cf671e86d4b5d8dc0f3"
}

20
martin-mbtiles/.sqlx/query-7341bfc10beb4719811556a57ae8098085994c8fba93e0293359afd43079c50c.json Normal file
View File

@ -0,0 +1,20 @@
{
"db_name": "SQLite",
"query": "SELECT (\n -- Has a 'tiles' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- 'tiles' table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE ((name = 'zoom_level' AND type = 'INTEGER')\n OR (name = 'tile_column' AND type = 'INTEGER')\n OR (name = 'tile_row' AND type = 'INTEGER')\n OR (name = 'tile_data' AND type = 'BLOB'))\n --\n ) as is_valid;",
"describe": {
"columns": [
{
"name": "is_valid",
"ordinal": 0,
"type_info": "Int"
}
],
"parameters": {
"Right": 0
},
"nullable": [
null
]
},
"hash": "7341bfc10beb4719811556a57ae8098085994c8fba93e0293359afd43079c50c"
}

20
martin-mbtiles/.sqlx/query-809e89c3b223e28c6716d405e13ba30fbf018805fe9ca2acd2b2e225183d1f13.json Normal file
View File

@ -0,0 +1,20 @@
{
"db_name": "SQLite",
"query": "SELECT (\n -- Has a 'map' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- 'map' table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE ((name = 'zoom_level' AND type = 'INTEGER')\n OR (name = 'tile_column' AND type = 'INTEGER')\n OR (name = 'tile_row' AND type = 'INTEGER')\n OR (name = 'tile_id' AND type = 'TEXT'))\n --\n ) AND (\n -- Has a 'images' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- 'images' table's columns and their types are as expected:\n -- 2 columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE ((name = 'tile_id' AND type = 'TEXT')\n OR (name = 'tile_data' AND type = 'BLOB'))\n --\n ) AS is_valid;",
"describe": {
"columns": [
{
"name": "is_valid",
"ordinal": 0,
"type_info": "Int"
}
],
"parameters": {
"Right": 0
},
"nullable": [
null
]
},
"hash": "809e89c3b223e28c6716d405e13ba30fbf018805fe9ca2acd2b2e225183d1f13"
}

12
martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json
View File

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "ATTACH DATABASE ? AS newDb",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b"
}

12
martin-mbtiles/.sqlx/query-b3aaef71d6a26404c3bebcc6ee8ad480aaa224721cd9ddb4ac5859f71a57727e.json
View File

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "ATTACH DATABASE ? AS otherDb",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "b3aaef71d6a26404c3bebcc6ee8ad480aaa224721cd9ddb4ac5859f71a57727e"
}

12
martin-mbtiles/.sqlx/query-d1d61dfa7c34dafb4588f78e23b2ee47cfc72b56f6ed275a0b0688047405498f.json
View File

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "ATTACH DATABASE ? AS originalDb",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "d1d61dfa7c34dafb4588f78e23b2ee47cfc72b56f6ed275a0b0688047405498f"
}

12
martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json
View File

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "ATTACH DATABASE ? AS diffDb",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c"
}

2
martin-mbtiles/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "martin-mbtiles"
version = "0.5.0"
version = "0.6.0"
authors = ["Yuri Astrakhan <YuriAstrakhan@gmail.com>", "MapLibre contributors"]
description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics."
keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"]

22
martin-mbtiles/src/bin/main.rs
View File

@ -2,9 +2,7 @@ use std::path::{Path, PathBuf};
use clap::{Parser, Subcommand};
use log::{error, LevelFilter};
use martin_mbtiles::{
apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions,
};
use martin_mbtiles::{apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopier};
#[derive(Parser, PartialEq, Eq, Debug)]
#[command(
@ -48,7 +46,7 @@ enum Commands {
},
/// Copy tiles from one mbtiles file to another.
#[command(name = "copy")]
Copy(TileCopierOptions),
Copy(TileCopier),
/// Apply diff file generated from 'copy' command
#[command(name = "apply-diff")]
ApplyDiff {
@ -165,7 +163,7 @@ mod tests {
use clap::error::ErrorKind;
use clap::Parser;
use martin_mbtiles::{CopyDuplicateMode, TileCopierOptions};
use martin_mbtiles::{CopyDuplicateMode, TileCopier};
use crate::Commands::{ApplyDiff, Copy, MetaGetValue, MetaSetValue, Validate};
use crate::{Args, IntegrityCheckType};
@ -186,7 +184,7 @@ mod tests {
Args::parse_from(["mbtiles", "copy", "src_file", "dst_file"]),
Args {
verbose: false,
command: Copy(TileCopierOptions::new(
command: Copy(TileCopier::new(
PathBuf::from("src_file"),
PathBuf::from("dst_file")
))
@ -210,7 +208,7 @@ mod tests {
Args {
verbose: false,
command: Copy(
TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
.min_zoom(Some(1))
.max_zoom(Some(100))
)
@ -270,7 +268,7 @@ mod tests {
Args {
verbose: false,
command: Copy(
TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
.zoom_levels(vec![1, 3, 7])
)
}
@ -291,7 +289,7 @@ mod tests {
Args {
verbose: false,
command: Copy(
TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
.diff_with_file(PathBuf::from("no_file"))
)
}
@ -312,7 +310,7 @@ mod tests {
Args {
verbose: false,
command: Copy(
TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
.on_duplicate(CopyDuplicateMode::Override)
)
}
@ -333,7 +331,7 @@ mod tests {
Args {
verbose: false,
command: Copy(
TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
.on_duplicate(CopyDuplicateMode::Ignore)
)
}
@ -354,7 +352,7 @@ mod tests {
Args {
verbose: false,
command: Copy(
TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
.on_duplicate(CopyDuplicateMode::Abort)
)
}

2
martin-mbtiles/src/lib.rs
View File

@ -10,6 +10,6 @@ mod mbtiles_pool;
pub use mbtiles_pool::MbtilesPool;
mod tile_copier;
pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions};
pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopier};
mod mbtiles_queries;

141
martin-mbtiles/src/mbtiles.rs
View File

@ -123,6 +123,19 @@ impl Mbtiles {
}
}
/// Attach this MBTiles file to the given SQLite connection as a given name
pub async fn attach_to<T>(&self, conn: &mut T, name: &str) -> MbtResult<()>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
query(&format!("ATTACH DATABASE ? AS {name}"))
.bind(self.filepath())
.execute(conn)
.await?;
Ok(())
}
/// Get a single metadata value from the metadata table
pub async fn get_metadata_value<T>(&self, conn: &mut T, key: &str) -> MbtResult<Option<String>>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
@ -359,6 +372,11 @@ impl Mbtiles {
Ok(None)
}
pub async fn open_and_detect_type(&self) -> MbtResult<MbtType> {
let mut conn = self.open_with_hashes(true).await?;
self.detect_type(&mut conn).await
}
pub async fn detect_type<T>(&self, conn: &mut T) -> MbtResult<MbtType>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
@ -583,7 +601,8 @@ where
}
pub async fn attach_hash_fn(conn: &mut SqliteConnection) -> MbtResult<()> {
let handle = conn.lock_handle().await?.as_raw_handle().as_ptr();
let mut handle_lock = conn.lock_handle().await?;
let handle = handle_lock.as_raw_handle().as_ptr();
// Safety: we know that the handle is a SQLite connection is locked and is not used anywhere else.
// The registered functions will be dropped when SQLX drops DB connection.
let rc = unsafe { sqlite_hashes::rusqlite::Connection::from_handle(handle) }?;
@ -600,25 +619,26 @@ mod tests {
use super::*;
async fn open(filepath: &str) -> (SqliteConnection, Mbtiles) {
let mbt = Mbtiles::new(filepath).unwrap();
let mut conn = SqliteConnection::connect(mbt.filepath()).await.unwrap();
attach_hash_fn(&mut conn).await.unwrap();
(conn, mbt)
async fn open(filepath: &str) -> MbtResult<(SqliteConnection, Mbtiles)> {
let mbt = Mbtiles::new(filepath)?;
let mut conn = SqliteConnection::connect(mbt.filepath()).await?;
attach_hash_fn(&mut conn).await?;
Ok((conn, mbt))
}
#[actix_rt::test]
async fn mbtiles_meta() {
async fn mbtiles_meta() -> MbtResult<()> {
let filepath = "../tests/fixtures/mbtiles/geography-class-jpg.mbtiles";
let mbt = Mbtiles::new(filepath).unwrap();
let mbt = Mbtiles::new(filepath)?;
assert_eq!(mbt.filepath(), filepath);
assert_eq!(mbt.filename(), "geography-class-jpg");
Ok(())
}
#[actix_rt::test]
async fn metadata_jpeg() {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await;
let metadata = mbt.get_metadata(&mut conn).await.unwrap();
async fn metadata_jpeg() -> MbtResult<()> {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await?;
let metadata = mbt.get_metadata(&mut conn).await?;
let tj = metadata.tilejson;
assert_eq!(tj.description.unwrap(), "One of the example maps that comes with TileMill - a bright & colorful world map that blends retro and high-tech with its folded paper texture and interactive flag tooltips. ");
@ -630,12 +650,13 @@ mod tests {
assert_eq!(tj.version.unwrap(), "1.0.0");
assert_eq!(metadata.id, "geography-class-jpg");
assert_eq!(metadata.tile_info, Format::Jpeg.into());
Ok(())
}
#[actix_rt::test]
async fn metadata_mvt() {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
let metadata = mbt.get_metadata(&mut conn).await.unwrap();
async fn metadata_mvt() -> MbtResult<()> {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?;
let metadata = mbt.get_metadata(&mut conn).await?;
let tj = metadata.tilejson;
assert_eq!(tj.maxzoom.unwrap(), 6);
@ -661,41 +682,38 @@ mod tests {
TileInfo::new(Format::Mvt, Encoding::Gzip)
);
assert_eq!(metadata.layer_type, Some("overlay".to_string()));
Ok(())
}
#[actix_rt::test]
async fn metadata_get_key() {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
async fn metadata_get_key() -> MbtResult<()> {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?;
let res = mbt.get_metadata_value(&mut conn, "bounds").await.unwrap();
assert_eq!(res.unwrap(), "-123.123590,-37.818085,174.763027,59.352706");
let res = mbt.get_metadata_value(&mut conn, "name").await.unwrap();
assert_eq!(res.unwrap(), "Major cities from Natural Earth data");
let res = mbt.get_metadata_value(&mut conn, "maxzoom").await.unwrap();
assert_eq!(res.unwrap(), "6");
let res = mbt.get_metadata_value(&mut conn, "nonexistent_key").await;
assert_eq!(res.unwrap(), None);
let res = mbt.get_metadata_value(&mut conn, "").await;
assert_eq!(res.unwrap(), None);
let res = mbt.get_metadata_value(&mut conn, "bounds").await?.unwrap();
assert_eq!(res, "-123.123590,-37.818085,174.763027,59.352706");
let res = mbt.get_metadata_value(&mut conn, "name").await?.unwrap();
assert_eq!(res, "Major cities from Natural Earth data");
let res = mbt.get_metadata_value(&mut conn, "maxzoom").await?.unwrap();
assert_eq!(res, "6");
let res = mbt.get_metadata_value(&mut conn, "nonexistent_key").await?;
assert_eq!(res, None);
let res = mbt.get_metadata_value(&mut conn, "").await?;
assert_eq!(res, None);
Ok(())
}
#[actix_rt::test]
async fn metadata_set_key() {
let (mut conn, mbt) = open("file:metadata_set_key_mem_db?mode=memory&cache=shared").await;
async fn metadata_set_key() -> MbtResult<()> {
let (mut conn, mbt) = open("file:metadata_set_key_mem_db?mode=memory&cache=shared").await?;
query("CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);")
.execute(&mut conn)
.await
.unwrap();
.await?;
mbt.set_metadata_value(&mut conn, "bounds", Some("0.0, 0.0, 0.0, 0.0".to_string()))
.await
.unwrap();
.await?;
assert_eq!(
mbt.get_metadata_value(&mut conn, "bounds")
.await
.unwrap()
.unwrap(),
mbt.get_metadata_value(&mut conn, "bounds").await?.unwrap(),
"0.0, 0.0, 0.0, 0.0"
);
@ -704,58 +722,53 @@ mod tests {
"bounds",
Some("-123.123590,-37.818085,174.763027,59.352706".to_string()),
)
.await
.unwrap();
.await?;
assert_eq!(
mbt.get_metadata_value(&mut conn, "bounds")
.await
.unwrap()
.unwrap(),
mbt.get_metadata_value(&mut conn, "bounds").await?.unwrap(),
"-123.123590,-37.818085,174.763027,59.352706"
);
mbt.set_metadata_value(&mut conn, "bounds", None)
.await
.unwrap();
assert_eq!(
mbt.get_metadata_value(&mut conn, "bounds").await.unwrap(),
None
);
mbt.set_metadata_value(&mut conn, "bounds", None).await?;
assert_eq!(mbt.get_metadata_value(&mut conn, "bounds").await?, None);
Ok(())
}
#[actix_rt::test]
async fn detect_type() {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
let res = mbt.detect_type(&mut conn).await.unwrap();
async fn detect_type() -> MbtResult<()> {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?;
let res = mbt.detect_type(&mut conn).await?;
assert_eq!(res, MbtType::Flat);
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await;
let res = mbt.detect_type(&mut conn).await.unwrap();
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await?;
let res = mbt.detect_type(&mut conn).await?;
assert_eq!(res, MbtType::FlatWithHash);
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await;
let res = mbt.detect_type(&mut conn).await.unwrap();
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await?;
let res = mbt.detect_type(&mut conn).await?;
assert_eq!(res, MbtType::Normalized);
let (mut conn, mbt) = open(":memory:").await;
let (mut conn, mbt) = open(":memory:").await?;
let res = mbt.detect_type(&mut conn).await;
assert!(matches!(res, Err(MbtError::InvalidDataFormat(_))));
Ok(())
}
#[actix_rt::test]
async fn validate_valid_file() {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await;
async fn validate_valid_file() -> MbtResult<()> {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await?;
mbt.check_integrity(&mut conn, IntegrityCheckType::Quick)
.await
.unwrap();
.await?;
Ok(())
}
#[actix_rt::test]
async fn validate_invalid_file() {
async fn validate_invalid_file() -> MbtResult<()> {
let (mut conn, mbt) =
open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await;
open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await?;
let result = mbt.check_agg_tiles_hashes(&mut conn).await;
assert!(matches!(result, Err(MbtError::AggHashMismatch(..))));
Ok(())
}
}

249
martin-mbtiles/src/mbtiles_queries.rs
View File

@ -7,42 +7,41 @@ where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
let sql = query!(
r#"SELECT (
-- Has a "map" table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'map'
AND type = 'table'
--
) AND (
-- "map" table's columns and their types are as expected:
-- 4 columns (zoom_level, tile_column, tile_row, tile_id).
-- The order is not important
SELECT COUNT(*) = 4
FROM pragma_table_info('map')
WHERE ((name = "zoom_level" AND type = "INTEGER")
OR (name = "tile_column" AND type = "INTEGER")
OR (name = "tile_row" AND type = "INTEGER")
OR (name = "tile_id" AND type = "TEXT"))
--
) AND (
-- Has a "images" table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'images'
AND type = 'table'
--
) AND (
-- "images" table's columns and their types are as expected:
-- 2 columns (tile_id, tile_data).
-- The order is not important
SELECT COUNT(*) = 2
FROM pragma_table_info('images')
WHERE ((name = "tile_id" AND type = "TEXT")
OR (name = "tile_data" AND type = "BLOB"))
--
) AS is_valid;
"#
"SELECT (
-- Has a 'map' table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'map'
AND type = 'table'
--
) AND (
-- 'map' table's columns and their types are as expected:
-- 4 columns (zoom_level, tile_column, tile_row, tile_id).
-- The order is not important
SELECT COUNT(*) = 4
FROM pragma_table_info('map')
WHERE ((name = 'zoom_level' AND type = 'INTEGER')
OR (name = 'tile_column' AND type = 'INTEGER')
OR (name = 'tile_row' AND type = 'INTEGER')
OR (name = 'tile_id' AND type = 'TEXT'))
--
) AND (
-- Has a 'images' table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'images'
AND type = 'table'
--
) AND (
-- 'images' table's columns and their types are as expected:
-- 2 columns (tile_id, tile_data).
-- The order is not important
SELECT COUNT(*) = 2
FROM pragma_table_info('images')
WHERE ((name = 'tile_id' AND type = 'TEXT')
OR (name = 'tile_data' AND type = 'BLOB'))
--
) AS is_valid;"
);
Ok(sql
@ -58,27 +57,26 @@ where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
let sql = query!(
r#"SELECT (
-- Has a "tiles_with_hash" table
"SELECT (
-- Has a 'tiles_with_hash' table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'tiles_with_hash'
AND type = 'table'
AND type = 'table'
--
) AND (
-- "tiles_with_hash" table's columns and their types are as expected:
-- 'tiles_with_hash' table's columns and their types are as expected:
-- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash).
-- The order is not important
SELECT COUNT(*) = 5
FROM pragma_table_info('tiles_with_hash')
WHERE ((name = "zoom_level" AND type = "INTEGER")
OR (name = "tile_column" AND type = "INTEGER")
OR (name = "tile_row" AND type = "INTEGER")
OR (name = "tile_data" AND type = "BLOB")
OR (name = "tile_hash" AND type = "TEXT"))
WHERE ((name = 'zoom_level' AND type = 'INTEGER')
OR (name = 'tile_column' AND type = 'INTEGER')
OR (name = 'tile_row' AND type = 'INTEGER')
OR (name = 'tile_data' AND type = 'BLOB')
OR (name = 'tile_hash' AND type = 'TEXT'))
--
) as is_valid;
"#
) as is_valid;"
);
Ok(sql
@ -94,26 +92,25 @@ where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
let sql = query!(
r#"SELECT (
-- Has a "tiles" table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'tiles'
AND type = 'table'
--
) AND (
-- "tiles" table's columns and their types are as expected:
-- 4 columns (zoom_level, tile_column, tile_row, tile_data).
-- The order is not important
SELECT COUNT(*) = 4
FROM pragma_table_info('tiles')
WHERE ((name = "zoom_level" AND type = "INTEGER")
OR (name = "tile_column" AND type = "INTEGER")
OR (name = "tile_row" AND type = "INTEGER")
OR (name = "tile_data" AND type = "BLOB"))
--
) as is_valid;
"#
"SELECT (
-- Has a 'tiles' table
SELECT COUNT(*) = 1
FROM sqlite_master
WHERE name = 'tiles'
AND type = 'table'
--
) AND (
-- 'tiles' table's columns and their types are as expected:
-- 4 columns (zoom_level, tile_column, tile_row, tile_data).
-- The order is not important
SELECT COUNT(*) = 4
FROM pragma_table_info('tiles')
WHERE ((name = 'zoom_level' AND type = 'INTEGER')
OR (name = 'tile_column' AND type = 'INTEGER')
OR (name = 'tile_row' AND type = 'INTEGER')
OR (name = 'tile_data' AND type = 'BLOB'))
--
) as is_valid;"
);
Ok(sql
@ -123,3 +120,121 @@ where
.unwrap_or_default()
== 1)
}
pub async fn create_metadata_table<T>(conn: &mut T) -> MbtResult<()>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
query(
"CREATE TABLE IF NOT EXISTS metadata (
name text NOT NULL PRIMARY KEY,
value text);",
)
.execute(&mut *conn)
.await?;
Ok(())
}
pub async fn create_flat_tables<T>(conn: &mut T) -> MbtResult<()>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
query(
"CREATE TABLE IF NOT EXISTS tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row));",
)
.execute(&mut *conn)
.await?;
Ok(())
}
pub async fn create_flat_with_hash_tables<T>(conn: &mut T) -> MbtResult<()>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
query(
"CREATE TABLE IF NOT EXISTS tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row));",
)
.execute(&mut *conn)
.await?;
query(
"CREATE VIEW IF NOT EXISTS tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;",
)
.execute(&mut *conn)
.await?;
Ok(())
}
pub async fn create_normalized_tables<T>(conn: &mut T) -> MbtResult<()>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
query(
"CREATE TABLE IF NOT EXISTS map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row));",
)
.execute(&mut *conn)
.await?;
query(
"CREATE TABLE IF NOT EXISTS images (
tile_data blob,
tile_id text NOT NULL PRIMARY KEY);",
)
.execute(&mut *conn)
.await?;
query(
"CREATE VIEW IF NOT EXISTS tiles AS
SELECT map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id;",
)
.execute(&mut *conn)
.await?;
Ok(())
}
pub async fn create_tiles_with_hash_view<T>(conn: &mut T) -> MbtResult<()>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
query(
"CREATE VIEW IF NOT EXISTS tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id",
)
.execute(&mut *conn)
.await?;
Ok(())
}

292
martin-mbtiles/src/tile_copier.rs
View File

@ -11,6 +11,10 @@ use sqlx::{query, Connection, Row, SqliteConnection};
use crate::errors::MbtResult;
use crate::mbtiles::MbtType::{Flat, FlatWithHash, Normalized};
use crate::mbtiles::{attach_hash_fn, MbtType};
use crate::mbtiles_queries::{
create_flat_tables, create_flat_with_hash_tables, create_metadata_table,
create_normalized_tables, create_tiles_with_hash_view,
};
use crate::{MbtError, Mbtiles};
#[derive(PartialEq, Eq, Default, Debug, Clone)]
@ -24,7 +28,7 @@ pub enum CopyDuplicateMode {
#[derive(Clone, Default, PartialEq, Eq, Debug)]
#[cfg_attr(feature = "cli", derive(Args))]
pub struct TileCopierOptions {
pub struct TileCopier {
/// MBTiles file to read from
src_file: PathBuf,
/// MBTiles file to write to
@ -85,13 +89,13 @@ impl clap::builder::TypedValueParser for HashSetValueParser {
}
#[derive(Clone, Debug)]
struct TileCopier {
struct TileCopierInt {
src_mbtiles: Mbtiles,
dst_mbtiles: Mbtiles,
options: TileCopierOptions,
options: TileCopier,
}
impl TileCopierOptions {
impl TileCopier {
#[must_use]
pub fn new(src_filepath: PathBuf, dst_filepath: PathBuf) -> Self {
Self {
@ -150,13 +154,13 @@ impl TileCopierOptions {
}
pub async fn run(self) -> MbtResult<SqliteConnection> {
TileCopier::new(self)?.run().await
TileCopierInt::new(self)?.run().await
}
}
impl TileCopier {
pub fn new(options: TileCopierOptions) -> MbtResult<Self> {
Ok(TileCopier {
impl TileCopierInt {
pub fn new(options: TileCopier) -> MbtResult<Self> {
Ok(TileCopierInt {
src_mbtiles: Mbtiles::new(&options.src_file)?,
dst_mbtiles: Mbtiles::new(&options.dst_file)?,
options,
@ -164,7 +168,8 @@ impl TileCopier {
}
pub async fn run(self) -> MbtResult<SqliteConnection> {
let src_type = open_and_detect_type(&self.src_mbtiles).await?;
// src file connection is not needed after this point, as it will be attached to the dst file
let src_type = self.src_mbtiles.open_and_detect_type().await?;
let mut conn = SqliteConnection::connect_with(
&SqliteConnectOptions::new()
@ -189,7 +194,7 @@ impl TileCopier {
return Err(MbtError::NonEmptyTargetFile(self.options.dst_file));
} else {
let dst_type = self.dst_mbtiles.detect_type(&mut conn).await?;
attach_source_db(&mut conn, self.src_mbtiles.filepath()).await?;
self.src_mbtiles.attach_to(&mut conn, "sourceDb").await?;
dst_type
};
@ -198,11 +203,8 @@ impl TileCopier {
let (select_from, query_args) = {
let select_from = if let Some(diff_file) = &self.options.diff_with_file {
let diff_with_mbtiles = Mbtiles::new(diff_file)?;
let diff_type = open_and_detect_type(&diff_with_mbtiles).await?;
let path = diff_with_mbtiles.filepath();
query!("ATTACH DATABASE ? AS newDb", path)
.execute(&mut conn)
.await?;
let diff_type = diff_with_mbtiles.open_and_detect_type().await?;
diff_with_mbtiles.attach_to(&mut conn, "newDb").await?;
Self::get_select_from_with_diff(dst_type, diff_type)
} else {
Self::get_select_from(dst_type, src_type).to_string()
@ -213,34 +215,40 @@ impl TileCopier {
(format!("{select_from} {options_sql}"), query_args)
};
let handle = conn.lock_handle().await?.as_raw_handle().as_ptr();
let rusqlite_conn = unsafe { rusqlite::Connection::from_handle(handle) }?;
match dst_type {
Flat => rusqlite_conn.execute(
&format!("INSERT {on_dupl} INTO tiles {select_from} {sql_cond}"),
params_from_iter(query_args),
)?,
FlatWithHash => rusqlite_conn.execute(
&format!("INSERT {on_dupl} INTO tiles_with_hash {select_from} {sql_cond}"),
params_from_iter(query_args),
)?,
Normalized => {
rusqlite_conn.execute(
&format!(
"INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id)
{
// Make sure not to execute any other queries while the handle is locked
let mut handle_lock = conn.lock_handle().await?;
let handle = handle_lock.as_raw_handle().as_ptr();
// SAFETY: this is safe as long as handle_lock is valid
let rusqlite_conn = unsafe { rusqlite::Connection::from_handle(handle) }?;
match dst_type {
Flat => rusqlite_conn.execute(
&format!("INSERT {on_dupl} INTO tiles {select_from} {sql_cond}"),
params_from_iter(query_args),
)?,
FlatWithHash => rusqlite_conn.execute(
&format!("INSERT {on_dupl} INTO tiles_with_hash {select_from} {sql_cond}"),
params_from_iter(query_args),
)?,
Normalized => {
rusqlite_conn.execute(
&format!(
"INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id)
SELECT zoom_level, tile_column, tile_row, hash as tile_id
FROM ({select_from} {sql_cond})"
),
params_from_iter(&query_args),
)?;
rusqlite_conn.execute(
&format!(
"INSERT OR IGNORE INTO images SELECT tile_data, hash FROM ({select_from})"
),
params_from_iter(query_args),
)?
}
};
),
params_from_iter(&query_args),
)?;
rusqlite_conn.execute(
&format!(
"INSERT OR IGNORE INTO images SELECT tile_data, hash FROM ({select_from})"
),
params_from_iter(query_args),
)?
}
};
}
if !self.options.skip_agg_tiles_hash {
self.dst_mbtiles.update_agg_tiles_hash(&mut conn).await?;
@ -258,7 +266,7 @@ impl TileCopier {
query!("PRAGMA page_size = 512").execute(&mut *conn).await?;
query!("VACUUM").execute(&mut *conn).await?;
attach_source_db(&mut *conn, self.src_mbtiles.filepath()).await?;
self.src_mbtiles.attach_to(&mut *conn, "sourceDb").await?;
if src == dst {
// DB objects must be created in a specific order: tables, views, triggers, indexes.
@ -281,27 +289,17 @@ impl TileCopier {
query(row.get(0)).execute(&mut *conn).await?;
}
} else {
create_metadata_table(&mut *conn).await?;
match dst {
Flat => self.create_flat_tables(&mut *conn).await?,
FlatWithHash => self.create_flat_with_hash_tables(&mut *conn).await?,
Normalized => self.create_normalized_tables(&mut *conn).await?,
Flat => create_flat_tables(&mut *conn).await?,
FlatWithHash => create_flat_with_hash_tables(&mut *conn).await?,
Normalized => create_normalized_tables(&mut *conn).await?,
};
};
if dst == Normalized {
query(
"CREATE VIEW tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id",
)
.execute(&mut *conn)
.await?;
// Some normalized mbtiles files might not have this view, so even if src == dst, it might not exist
create_tiles_with_hash_view(&mut *conn).await?;
}
query("INSERT INTO metadata SELECT * FROM sourceDb.metadata")
@ -311,64 +309,13 @@ impl TileCopier {
Ok(())
}
async fn create_flat_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
for statement in &[
"CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);",
"CREATE TABLE tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row));",
] {
query(statement).execute(&mut *conn).await?;
}
Ok(())
}
async fn create_flat_with_hash_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
for statement in &[
"CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);",
"CREATE TABLE tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row));",
"CREATE VIEW tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;",
] {
query(statement).execute(&mut *conn).await?;
}
Ok(())
}
async fn create_normalized_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> {
for statement in &[
"CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);",
"CREATE TABLE map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row));",
"CREATE TABLE images (tile_data blob, tile_id text NOT NULL PRIMARY KEY);",
"CREATE VIEW tiles AS
SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id;"] {
query(statement).execute(&mut *conn).await?;
}
Ok(())
}
fn get_on_duplicate_sql(&self, mbttype: MbtType) -> (String, String) {
/// Returns (ON DUPLICATE SQL, WHERE condition SQL)
fn get_on_duplicate_sql(&self, dst_type: MbtType) -> (String, String) {
match &self.options.on_duplicate {
CopyDuplicateMode::Override => ("OR REPLACE".to_string(), String::new()),
CopyDuplicateMode::Ignore => ("OR IGNORE".to_string(), String::new()),
CopyDuplicateMode::Abort => ("OR ABORT".to_string(), {
let (main_table, tile_identifier) = match mbttype {
let (main_table, tile_identifier) = match dst_type {
Flat => ("tiles", "tile_data"),
FlatWithHash => ("tiles_with_hash", "tile_data"),
Normalized => ("map", "tile_id"),
@ -418,12 +365,12 @@ impl TileCopier {
fn get_select_from(dst_type: MbtType, src_type: MbtType) -> &'static str {
if dst_type == Flat {
"SELECT * FROM sourceDb.tiles WHERE TRUE "
"SELECT * FROM sourceDb.tiles WHERE TRUE"
} else {
match src_type {
Flat => "SELECT zoom_level, tile_column, tile_row, tile_data, hex(md5(tile_data)) as hash FROM sourceDb.tiles WHERE TRUE ",
FlatWithHash => "SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM sourceDb.tiles_with_hash WHERE TRUE ",
Normalized => "SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM sourceDb.map JOIN sourceDb.images ON sourceDb.map.tile_id = sourceDb.images.tile_id WHERE TRUE "
Flat => "SELECT zoom_level, tile_column, tile_row, tile_data, hex(md5(tile_data)) as hash FROM sourceDb.tiles WHERE TRUE",
FlatWithHash => "SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM sourceDb.tiles_with_hash WHERE TRUE",
Normalized => "SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM sourceDb.map JOIN sourceDb.images ON sourceDb.map.tile_id = sourceDb.images.tile_id WHERE TRUE"
}
}
}
@ -459,34 +406,15 @@ impl TileCopier {
}
}
async fn attach_source_db(conn: &mut SqliteConnection, path: &str) -> MbtResult<()> {
query!("ATTACH DATABASE ? AS sourceDb", path)
.execute(&mut *conn)
.await?;
Ok(())
}
async fn open_and_detect_type(mbtiles: &Mbtiles) -> MbtResult<MbtType> {
let opt = SqliteConnectOptions::new()
.read_only(true)
.filename(mbtiles.filepath());
let mut conn = SqliteConnection::connect_with(&opt).await?;
mbtiles.detect_type(&mut conn).await
}
pub async fn apply_mbtiles_diff(src_file: PathBuf, diff_file: PathBuf) -> MbtResult<()> {
let src_mbtiles = Mbtiles::new(src_file)?;
let diff_mbtiles = Mbtiles::new(diff_file)?;
let src_type = open_and_detect_type(&src_mbtiles).await?;
let diff_type = open_and_detect_type(&diff_mbtiles).await?;
let diff_type = diff_mbtiles.open_and_detect_type().await?;
let mut conn = src_mbtiles.open_with_hashes(false).await?;
let path = diff_mbtiles.filepath();
query!("ATTACH DATABASE ? AS diffDb", path)
.execute(&mut conn)
.await?;
diff_mbtiles.attach_to(&mut conn, "diffDb").await?;
let src_type = src_mbtiles.detect_type(&mut conn).await?;
let select_from = if src_type == Flat {
"SELECT zoom_level, tile_column, tile_row, tile_data FROM diffDb.tiles"
} else {
@ -534,20 +462,6 @@ mod tests {
use super::*;
async fn attach_other_db(conn: &mut SqliteConnection, path: &str) -> MbtResult<()> {
query!("ATTACH DATABASE ? AS otherDb", path)
.execute(&mut *conn)
.await?;
Ok(())
}
async fn attach_src_db(conn: &mut SqliteConnection, path: &str) -> MbtResult<()> {
query!("ATTACH DATABASE ? AS srcDb", path)
.execute(&mut *conn)
.await?;
Ok(())
}
async fn get_one<T>(conn: &mut SqliteConnection, sql: &str) -> T
where
for<'r> T: Decode<'r, Sqlite> + Type<Sqlite>,
@ -561,15 +475,19 @@ mod tests {
dst_type: Option<MbtType>,
expected_dst_type: MbtType,
) -> MbtResult<()> {
let mut dst_conn = TileCopierOptions::new(src_filepath.clone(), dst_filepath.clone())
let mut dst_conn = TileCopier::new(src_filepath.clone(), dst_filepath.clone())
.dst_type(dst_type)
.run()
.await?;
attach_src_db(&mut dst_conn, src_filepath.to_str().unwrap()).await?;
Mbtiles::new(src_filepath)?
.attach_to(&mut dst_conn, "srcDb")
.await?;
assert_eq!(
open_and_detect_type(&Mbtiles::new(dst_filepath)?).await?,
Mbtiles::new(dst_filepath)?
.detect_type(&mut dst_conn)
.await?,
expected_dst_type
);
@ -584,7 +502,7 @@ mod tests {
}
async fn verify_copy_with_zoom_filter(
opts: TileCopierOptions,
opts: TileCopier,
expected_zoom_levels: u8,
) -> MbtResult<()> {
let mut dst_conn = opts.run().await?;
@ -678,7 +596,7 @@ mod tests {
async fn copy_with_min_max_zoom() -> MbtResult<()> {
let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared");
let opt = TileCopierOptions::new(src, dst)
let opt = TileCopier::new(src, dst)
.min_zoom(Some(2))
.max_zoom(Some(4));
verify_copy_with_zoom_filter(opt, 3).await
@ -688,7 +606,7 @@ mod tests {
async fn copy_with_zoom_levels() -> MbtResult<()> {
let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared");
let opt = TileCopierOptions::new(src, dst)
let opt = TileCopier::new(src, dst)
.min_zoom(Some(2))
.max_zoom(Some(4))
.zoom_levels(vec![1, 6]);
@ -703,8 +621,7 @@ mod tests {
let diff_file =
PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles");
let copy_opts =
TileCopierOptions::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone());
let copy_opts = TileCopier::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone());
let mut dst_conn = copy_opts.run().await?;
@ -752,9 +669,7 @@ mod tests {
"file:ignore_dst_type_when_copy_to_existing_mem_db?mode=memory&cache=shared",
);
let _dst_conn = TileCopierOptions::new(dst_file.clone(), dst.clone())
.run()
.await?;
let _dst_conn = TileCopier::new(dst_file.clone(), dst.clone()).run().await?;
verify_copy_all(src_file, dst, Some(Normalized), Flat).await
}
@ -765,7 +680,7 @@ mod tests {
let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let copy_opts =
TileCopierOptions::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort);
TileCopier::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort);
assert!(matches!(
copy_opts.run().await.unwrap_err(),
@ -782,16 +697,14 @@ mod tests {
let dst =
PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared");
let _dst_conn = TileCopierOptions::new(dst_file.clone(), dst.clone())
.run()
.await?;
let _dst_conn = TileCopier::new(dst_file.clone(), dst.clone()).run().await?;
let mut dst_conn = TileCopierOptions::new(src_file.clone(), dst.clone())
.run()
.await?;
let mut dst_conn = TileCopier::new(src_file.clone(), dst.clone()).run().await?;
// Verify the tiles in the destination file is a superset of the tiles in the source file
attach_other_db(&mut dst_conn, src_file.to_str().unwrap()).await?;
Mbtiles::new(src_file)?
.attach_to(&mut dst_conn, "otherDb")
.await?;
assert!(
query("SELECT * FROM otherDb.tiles EXCEPT SELECT * FROM tiles;")
.fetch_optional(&mut dst_conn)
@ -811,21 +724,19 @@ mod tests {
let dst =
PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared");
let _dst_conn = TileCopierOptions::new(dst_file.clone(), dst.clone())
.run()
.await?;
let _dst_conn = TileCopier::new(dst_file.clone(), dst.clone()).run().await?;
let mut dst_conn = TileCopierOptions::new(src_file.clone(), dst.clone())
let mut dst_conn = TileCopier::new(src_file.clone(), dst.clone())
.on_duplicate(CopyDuplicateMode::Ignore)
.run()
.await?;
// Verify the tiles in the destination file are the same as those in the source file except for those with duplicate (zoom_level, tile_column, tile_row)
attach_src_db(&mut dst_conn, src_file.to_str().unwrap()).await?;
let path = dst_file.to_str().unwrap();
query!("ATTACH DATABASE ? AS originalDb", path)
.execute(&mut dst_conn)
Mbtiles::new(src_file)?
.attach_to(&mut dst_conn, "srcDb")
.await?;
Mbtiles::new(dst_file)?
.attach_to(&mut dst_conn, "originalDb")
.await?;
// Create a temporary table with all the tiles in the original database and
@ -839,8 +750,7 @@ mod tests {
FULL OUTER JOIN srcDb.tiles as t2
ON t1.zoom_level = t2.zoom_level AND t1.tile_column = t2.tile_column AND t1.tile_row = t2.tile_row")
.execute(&mut dst_conn)
.await
?;
.await?;
// Ensure all entries in expected_tiles are in tiles and vice versa
assert!(query(
@ -861,17 +771,16 @@ mod tests {
let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared");
let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone())
.run()
.await?;
let mut src_conn = TileCopier::new(src_file.clone(), src.clone()).run().await?;
// Apply diff to the src data in in-memory DB
let diff_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles");
apply_mbtiles_diff(src, diff_file).await?;
// Verify the data is the same as the file the diff was generated from
let path = "../tests/fixtures/mbtiles/world_cities_modified.mbtiles";
attach_other_db(&mut src_conn, path).await?;
Mbtiles::new("../tests/fixtures/mbtiles/world_cities_modified.mbtiles")?
.attach_to(&mut src_conn, "otherDb")
.await?;
assert!(
query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")
@ -889,17 +798,16 @@ mod tests {
let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles");
let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared");
let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone())
.run()
.await?;
let mut src_conn = TileCopier::new(src_file.clone(), src.clone()).run().await?;
// Apply diff to the src data in in-memory DB
let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles");
apply_mbtiles_diff(src, diff_file).await?;
// Verify the data is the same as the file the diff was generated from
let path = "../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles";
attach_other_db(&mut src_conn, path).await?;
Mbtiles::new("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles")?
.attach_to(&mut src_conn, "otherDb")
.await?;
assert!(
query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")