Feature: mbtiles copy --bbox ... to copy tiles within a bbox only (#1060)

Allow users to copy tiles between mbtiles files that are only within a
bounding box. Multiple `--bbox` params are allowed.
This commit is contained in:
Yuri Astrakhan 2023-12-15 23:13:57 -05:00 committed by GitHub
parent 5a0f4c2d11
commit bb802c5688
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 678 additions and 53 deletions

1
Cargo.lock generated
View File

@ -1919,6 +1919,7 @@ dependencies = [
"env_logger",
"futures",
"insta",
"itertools 0.12.0",
"log",
"martin-tile-utils",
"pretty_assertions",

View File

@ -264,6 +264,10 @@ fmt-md:
fmt2:
cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate
# Run cargo check
check:
cargo check --workspace --all-targets --bins --tests --lib --benches
# Run cargo clippy
clippy:
cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings

View File

@ -11,7 +11,7 @@ use tile_grid::{tms, Tms, Xyz};
pub const EARTH_CIRCUMFERENCE: f64 = 40_075_016.685_578_5;
pub const EARTH_RADIUS: f64 = EARTH_CIRCUMFERENCE / 2.0 / PI;
pub const MAX_ZOOM: u8 = 30;
pub const MAX_ZOOM: u8 = 24;
use std::sync::OnceLock;
fn web_merc() -> &'static Tms {
@ -209,6 +209,8 @@ pub fn tile_index(lon: f64, lat: f64, zoom: u8) -> (u32, u32) {
#[must_use]
pub fn xyz_to_bbox(zoom: u8, min_x: u32, min_y: u32, max_x: u32, max_y: u32) -> [f64; 4] {
assert!(zoom <= MAX_ZOOM, "zoom {zoom} must be <= {MAX_ZOOM}");
assert!(min_x <= max_x, "min_x {min_x} must be <= max_x {max_x}");
assert!(min_y <= max_y, "min_y {min_y} must be <= max_y {max_y}");
let left_top_bounds = web_merc().xy_bounds(&Xyz::new(u64::from(min_x), u64::from(min_y), zoom));
let right_bottom_bounds =
web_merc().xy_bounds(&Xyz::new(u64::from(max_x), u64::from(max_y), zoom));

View File

@ -6,7 +6,7 @@ use std::path::PathBuf;
use std::sync::OnceLock;
use bit_set::BitSet;
use itertools::Itertools;
use itertools::Itertools as _;
use log::{debug, info, warn};
use pbf_font_tools::freetype::{Face, Library};
use pbf_font_tools::protobuf::Message;
@ -335,7 +335,6 @@ fn parse_font(
} else {
format!("{s:02X}-{e:02X}")
})
.collect::<Vec<_>>()
.join(", "),
);

View File

@ -2,7 +2,7 @@ use std::cmp::Ordering;
use std::collections::HashSet;
use futures::future::join_all;
use itertools::Itertools;
use itertools::Itertools as _;
use log::{debug, error, info, warn};
use crate::args::BoundsCalcType;

View File

@ -1,4 +1,4 @@
use std::fmt::Write;
use std::fmt::Write as _;
use std::iter::zip;
use log::{debug, warn};

View File

@ -1,6 +1,7 @@
use std::collections::{BTreeMap, HashMap};
use deadpool_postgres::tokio_postgres::types::Json;
use itertools::Itertools as _;
use log::{error, info, warn};
use postgis::{ewkb, LineString, Point, Polygon};
use tilejson::{Bounds, TileJSON};
@ -104,7 +105,7 @@ fn find_info_kv<'a, T>(
match find_kv_ignore_case(map, key) {
Ok(None) => {
warn!("Unable to configure source {id} because {info} '{key}' was not found. Possible values are: {}",
map.keys().map(String::as_str).collect::<Vec<_>>().join(", "));
map.keys().map(String::as_str).join(", "));
None
}
Ok(Some(result)) => {

View File

@ -1,5 +1,5 @@
use std::error::Error;
use std::fmt::Write;
use std::fmt::Write as _;
use std::io;
use std::path::PathBuf;

View File

@ -1,6 +1,6 @@
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::fmt::Write;
use std::fmt::Write as _;
use std::sync::{Arc, Mutex};
use log::warn;

View File

@ -19,10 +19,11 @@ cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio"
[dependencies]
enum-display.workspace = true
futures.workspace = true
itertools.workspace = true
log.workspace = true
martin-tile-utils.workspace = true
serde_json.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_with.workspace = true
size_format.workspace = true
sqlite-hashes.workspace = true

View File

@ -4,7 +4,7 @@ use clap::{Parser, Subcommand};
use log::error;
use mbtiles::{apply_patch, AggHashType, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier};
#[derive(Parser, PartialEq, Eq, Debug)]
#[derive(Parser, PartialEq, Debug)]
#[command(
version,
name = "mbtiles",
@ -19,7 +19,7 @@ pub struct Args {
command: Commands,
}
#[derive(Subcommand, PartialEq, Eq, Debug)]
#[derive(Subcommand, PartialEq, Debug)]
enum Commands {
/// Show MBTiles file summary statistics
#[command(name = "summary", alias = "info")]

View File

@ -1,13 +1,16 @@
use std::collections::HashSet;
use std::fmt::Write as _;
use std::path::PathBuf;
#[cfg(feature = "cli")]
use clap::{Args, ValueEnum};
use enum_display::EnumDisplay;
use log::{debug, info};
use itertools::Itertools as _;
use log::{debug, info, trace};
use martin_tile_utils::{bbox_to_xyz, MAX_ZOOM};
use serde::{Deserialize, Serialize};
use sqlite_hashes::rusqlite::{params_from_iter, Connection};
use sqlite_hashes::rusqlite::Connection;
use sqlx::{query, Executor as _, Row, SqliteConnection};
use tilejson::Bounds;
use crate::errors::MbtResult;
use crate::queries::{
@ -15,7 +18,7 @@ use crate::queries::{
};
use crate::MbtType::{Flat, FlatWithHash, Normalized};
use crate::{
reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH,
invert_y_value, reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH,
AGG_TILES_HASH_IN_DIFF,
};
@ -39,7 +42,7 @@ impl CopyDuplicateMode {
}
}
#[derive(Clone, Default, PartialEq, Eq, Debug)]
#[derive(Clone, Default, PartialEq, Debug)]
#[cfg_attr(feature = "cli", derive(Args))]
pub struct MbtilesCopier {
/// MBTiles file to read from
@ -73,6 +76,9 @@ pub struct MbtilesCopier {
/// List of zoom levels to copy
#[cfg_attr(feature = "cli", arg(long, value_delimiter = ','))]
pub zoom_levels: Vec<u8>,
/// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times.
#[cfg_attr(feature = "cli", arg(long))]
pub bbox: Vec<Bounds>,
/// Compare source file with this file, and only copy non-identical tiles to destination.
/// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file.
#[cfg_attr(feature = "cli", arg(long, conflicts_with("apply_patch")))]
@ -105,6 +111,7 @@ impl MbtilesCopier {
min_zoom: None,
max_zoom: None,
zoom_levels: Vec::default(),
bbox: vec![],
diff_with_file: None,
apply_patch: None,
skip_agg_tiles_hash: false,
@ -216,7 +223,7 @@ impl MbtileCopierInt {
Self::get_select_from(src_type, dst_type).to_string()
};
let (where_clause, query_args) = self.get_where_clause();
let where_clause = self.get_where_clause();
let select_from = format!("{select_from} {where_clause}");
let on_dupl = on_duplicate.to_sql();
let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type);
@ -232,14 +239,7 @@ impl MbtileCopierInt {
// SAFETY: this is safe as long as handle_lock is valid. We will drop the lock.
let rusqlite_conn = unsafe { Connection::from_handle(handle) }?;
Self::copy_tiles(
&rusqlite_conn,
dst_type,
&query_args,
on_dupl,
&select_from,
&sql_cond,
)?;
Self::copy_tiles(&rusqlite_conn, dst_type, on_dupl, &select_from, &sql_cond)?;
self.copy_metadata(&rusqlite_conn, &dif, on_dupl)?;
}
@ -316,7 +316,6 @@ impl MbtileCopierInt {
fn copy_tiles(
rusqlite_conn: &Connection,
dst_type: MbtType,
query_args: &Vec<u8>,
on_dupl: &str,
select_from: &str,
sql_cond: &str,
@ -346,8 +345,8 @@ impl MbtileCopierInt {
SELECT tile_hash as tile_id, tile_data
FROM ({select_from})"
);
debug!("Copying to {dst_type} with {sql} {query_args:?}");
rusqlite_conn.execute(&sql, params_from_iter(query_args))?;
debug!("Copying to {dst_type} with {sql}");
rusqlite_conn.execute(&sql, [])?;
format!(
"
@ -359,8 +358,8 @@ impl MbtileCopierInt {
}
};
debug!("Copying to {dst_type} with {sql} {query_args:?}");
rusqlite_conn.execute(&sql, params_from_iter(query_args))?;
debug!("Copying to {dst_type} with {sql}");
rusqlite_conn.execute(&sql, [])?;
Ok(())
}
@ -585,32 +584,49 @@ impl MbtileCopierInt {
}
}
fn get_where_clause(&self) -> (String, Vec<u8>) {
let mut query_args = vec![];
let sql = if !&self.options.zoom_levels.is_empty() {
let zooms: HashSet<u8> = self.options.zoom_levels.iter().copied().collect();
for z in &zooms {
query_args.push(*z);
}
format!(" AND zoom_level IN ({})", vec!["?"; zooms.len()].join(","))
/// Format SQL WHERE clause and return it along with the query arguments.
/// Note that there is no risk of SQL injection here, as the arguments are integers.
fn get_where_clause(&self) -> String {
let mut sql = if !&self.options.zoom_levels.is_empty() {
let zooms = self.options.zoom_levels.iter().join(",");
format!(" AND zoom_level IN ({zooms})")
} else if let Some(min_zoom) = self.options.min_zoom {
if let Some(max_zoom) = self.options.max_zoom {
query_args.push(min_zoom);
query_args.push(max_zoom);
" AND zoom_level BETWEEN ? AND ?".to_string()
format!(" AND zoom_level BETWEEN {min_zoom} AND {max_zoom}")
} else {
query_args.push(min_zoom);
" AND zoom_level >= ?".to_string()
format!(" AND zoom_level >= {min_zoom}")
}
} else if let Some(max_zoom) = self.options.max_zoom {
query_args.push(max_zoom);
" AND zoom_level <= ?".to_string()
format!(" AND zoom_level <= {max_zoom}")
} else {
String::new()
};
(sql, query_args)
if !self.options.bbox.is_empty() {
sql.push_str(" AND (\n");
for (idx, bbox) in self.options.bbox.iter().enumerate() {
// Use maximum zoom value for easy filtering,
// converting it on the fly to the actual zoom level
let (min_x, min_y, max_x, max_y) =
bbox_to_xyz(bbox.left, bbox.bottom, bbox.right, bbox.top, MAX_ZOOM);
trace!("Bounding box {bbox} converted to {min_x},{min_y},{max_x},{max_y} at zoom {MAX_ZOOM}");
let (min_y, max_y) = (
invert_y_value(MAX_ZOOM, max_y),
invert_y_value(MAX_ZOOM, min_y),
);
if idx > 0 {
sql.push_str(" OR\n");
}
writeln!(
sql,
"((tile_column * (1 << ({MAX_ZOOM} - zoom_level))) BETWEEN {min_x} AND {max_x} AND (tile_row * (1 << ({MAX_ZOOM} - zoom_level))) BETWEEN {min_y} AND {max_y})",
).unwrap();
}
sql.push(')');
}
sql
}
}

View File

@ -4,11 +4,15 @@ use std::str::from_utf8;
use ctor::ctor;
use insta::{allow_duplicates, assert_display_snapshot};
use itertools::Itertools as _;
use log::info;
use martin_tile_utils::xyz_to_bbox;
use mbtiles::AggHashType::Verify;
use mbtiles::IntegrityCheckType::Off;
use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized};
use mbtiles::{apply_patch, init_mbtiles_schema, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier};
use mbtiles::{
apply_patch, init_mbtiles_schema, invert_y_value, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier,
};
use pretty_assertions::assert_eq as pretty_assert_eq;
use rstest::{fixture, rstest};
use serde::Serialize;
@ -255,6 +259,20 @@ async fn convert(
let z6only = dump(&mut opt.run().await?).await?;
assert_snapshot!(z6only, "v1__z6__{frm}-{to}");
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
// Filter (0, 0, 2, 2) in mbtiles coordinates, which is (0, 2^5-1-2, 2, 2^5-1-0) = (0, 29, 2, 31) in XYZ coordinates, and slightly decrease it
let mut bbox = xyz_to_bbox(5, 0, invert_y_value(5, 2), 2, invert_y_value(5, 0));
bbox[0] += 180.0 * 0.1 / f64::from(1 << 5);
bbox[1] += 90.0 * 0.1 / f64::from(1 << 5);
bbox[2] -= 180.0 * 0.1 / f64::from(1 << 5);
bbox[3] -= 90.0 * 0.1 / f64::from(1 << 5);
opt.bbox.push(bbox.into());
let dmp = dump(&mut opt.run().await?).await?;
assert_snapshot!(dmp, "v1__bbox__{frm}-{to}");
let mut opt = copier(&frm_mbt, &mem);
opt.dst_type_cli = Some(dst_type);
opt.min_zoom = Some(6);
@ -370,11 +388,16 @@ async fn patch_on_copy(
#[actix_rt::test]
#[ignore]
async fn test_one() {
let db = Databases::default();
// Test convert
convert(Flat, Flat, &db).await.unwrap();
// Test diff patch copy
let src_type = FlatWithHash;
let dif_type = FlatWithHash;
// let dst_type = Some(FlatWithHash);
let dst_type = None;
let db = databases();
diff_and_patch(src_type, dif_type, dst_type, &db)
.await
@ -465,7 +488,6 @@ async fn dump(conn: &mut SqliteConnection) -> MbtResult<Vec<SqliteEntry>> {
})
.unwrap_or("NULL".to_string())
})
.collect::<Vec<_>>()
.join(", ");
format!("( {val} )")
})

View File

@ -0,0 +1,42 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles'
sql = '''
CREATE TABLE tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, blob(edit-v1) )',
'( 5, 1, 2, blob() )',
'( 5, 2, 2, blob(remove) )',
'( 6, 1, 4, blob(edit-v1) )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles'

View File

@ -0,0 +1,50 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE TABLE tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles_with_hash'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash'''

View File

@ -0,0 +1,85 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'images'
sql = '''
CREATE TABLE images (
tile_id text NOT NULL PRIMARY KEY,
tile_data blob)'''
values = [
'( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )',
'( "D41D8CD98F00B204E9800998ECF8427E", blob() )',
'( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )',
]
[[]]
type = 'table'
tbl_name = 'map'
sql = '''
CREATE TABLE map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
]
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'index'
tbl_name = 'images'
[[]]
type = 'index'
tbl_name = 'map'
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id'''
[[]]
type = 'view'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE VIEW tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id'''

View File

@ -0,0 +1,42 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles'
sql = '''
CREATE TABLE tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, blob(edit-v1) )',
'( 5, 1, 2, blob() )',
'( 5, 2, 2, blob(remove) )',
'( 6, 1, 4, blob(edit-v1) )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles'

View File

@ -0,0 +1,50 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE TABLE tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles_with_hash'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash'''

View File

@ -0,0 +1,85 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'images'
sql = '''
CREATE TABLE images (
tile_id text NOT NULL PRIMARY KEY,
tile_data blob)'''
values = [
'( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )',
'( "D41D8CD98F00B204E9800998ECF8427E", blob() )',
'( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )',
]
[[]]
type = 'table'
tbl_name = 'map'
sql = '''
CREATE TABLE map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
]
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'index'
tbl_name = 'images'
[[]]
type = 'index'
tbl_name = 'map'
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id'''
[[]]
type = 'view'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE VIEW tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id'''

View File

@ -0,0 +1,42 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles'
sql = '''
CREATE TABLE tiles (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, blob(edit-v1) )',
'( 5, 1, 2, blob() )',
'( 5, 2, 2, blob(remove) )',
'( 6, 1, 4, blob(edit-v1) )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles'

View File

@ -0,0 +1,50 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'table'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE TABLE tiles_with_hash (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_data blob,
tile_hash text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )',
]
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'index'
tbl_name = 'tiles_with_hash'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash'''

View File

@ -0,0 +1,85 @@
---
source: mbtiles/tests/copy.rs
expression: actual_value
---
[[]]
type = 'table'
tbl_name = 'images'
sql = '''
CREATE TABLE images (
tile_id text NOT NULL PRIMARY KEY,
tile_data blob)'''
values = [
'( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )',
'( "D41D8CD98F00B204E9800998ECF8427E", blob() )',
'( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )',
]
[[]]
type = 'table'
tbl_name = 'map'
sql = '''
CREATE TABLE map (
zoom_level integer NOT NULL,
tile_column integer NOT NULL,
tile_row integer NOT NULL,
tile_id text,
PRIMARY KEY(zoom_level, tile_column, tile_row))'''
values = [
'( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
'( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )',
'( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )',
'( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )',
]
[[]]
type = 'table'
tbl_name = 'metadata'
sql = '''
CREATE TABLE metadata (
name text NOT NULL PRIMARY KEY,
value text)'''
values = [
'( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )',
'( "md-edit", "value - v1" )',
'( "md-remove", "value - remove" )',
'( "md-same", "value - same" )',
]
[[]]
type = 'index'
tbl_name = 'images'
[[]]
type = 'index'
tbl_name = 'map'
[[]]
type = 'index'
tbl_name = 'metadata'
[[]]
type = 'view'
tbl_name = 'tiles'
sql = '''
CREATE VIEW tiles AS
SELECT map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data
FROM map
JOIN images ON images.tile_id = map.tile_id'''
[[]]
type = 'view'
tbl_name = 'tiles_with_hash'
sql = '''
CREATE VIEW tiles_with_hash AS
SELECT
map.zoom_level AS zoom_level,
map.tile_column AS tile_column,
map.tile_row AS tile_row,
images.tile_data AS tile_data,
images.tile_id AS tile_hash
FROM map
JOIN images ON images.tile_id = map.tile_id'''

View File

@ -1,4 +1,7 @@
use martin_tile_utils::MAX_ZOOM;
#![allow(clippy::unreadable_literal)]
use insta::assert_snapshot;
use martin_tile_utils::{bbox_to_xyz, MAX_ZOOM};
use mbtiles::MbtError::InvalidTileIndex;
use mbtiles::{create_metadata_table, Mbtiles};
use rstest::rstest;
@ -92,13 +95,16 @@ async fn tile_coordinate(#[case] prefix: &str, #[case] suffix: &str) {
ok!("1, {prefix} 1 {suffix}");
ok!("2, {prefix} 3 {suffix}");
ok!("3, {prefix} 7 {suffix}");
ok!("30, {prefix} 0 {suffix}");
ok!("30, {prefix} 1073741823 {suffix}");
ok!("24, {prefix} 0 {suffix}");
ok!("24, {prefix} 16777215 {suffix}");
// ok!("30, {prefix} 0 {suffix}");
// ok!("30, {prefix} 1073741823 {suffix}");
err!("0, {prefix} 1 {suffix}");
err!("1, {prefix} 2 {suffix}");
err!("2, {prefix} 4 {suffix}");
err!("3, {prefix} 8 {suffix}");
err!("24, {prefix} 16777216 {suffix}");
err!("30, {prefix} 1073741824 {suffix}");
err!("{MAX_ZOOM}, {prefix} 1073741824 {suffix}");
err!("{}, {prefix} 0 {suffix}", MAX_ZOOM + 1); // unsupported zoom
@ -117,3 +123,45 @@ async fn tile_data() {
err!("0, 0, 0, CAST('abc' AS TEXT)");
err!("0, 0, 0, CAST(123 AS TEXT)");
}
#[test]
fn test_box() {
fn tst(left: f64, bottom: f64, right: f64, top: f64, zoom: u8) -> String {
let (x0, y0, x1, y1) = bbox_to_xyz(left, bottom, right, top, zoom);
format!("({x0}, {y0}, {x1}, {y1})")
}
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 0), @"(0, 0, 0, 0)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 1), @"(0, 1, 0, 1)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 2), @"(0, 3, 0, 3)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 3), @"(0, 7, 0, 7)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 4), @"(0, 14, 1, 15)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 5), @"(0, 29, 2, 31)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 6), @"(0, 58, 5, 63)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 7), @"(0, 116, 11, 126)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 8), @"(0, 233, 23, 253)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 9), @"(0, 466, 47, 507)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 10), @"(1, 933, 94, 1014)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 11), @"(3, 1866, 188, 2029)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 12), @"(6, 3732, 377, 4059)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 13), @"(12, 7465, 755, 8119)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 14), @"(25, 14931, 1510, 16239)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 15), @"(51, 29863, 3020, 32479)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 16), @"(102, 59727, 6041, 64958)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 17), @"(204, 119455, 12083, 129917)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 18), @"(409, 238911, 24166, 259834)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 19), @"(819, 477823, 48332, 519669)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 20), @"(1638, 955647, 96665, 1039339)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 21), @"(3276, 1911295, 193331, 2078678)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 22), @"(6553, 3822590, 386662, 4157356)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 23), @"(13107, 7645181, 773324, 8314713)");
assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 24), @"(26214, 15290363, 1546649, 16629427)");
// All these are incorrect
// assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 25), @"(33554431, 33554431, 33554431, 33554431)");
// assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 26), @"(67108863, 67108863, 67108863, 67108863)");
// assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 27), @"(134217727, 134217727, 134217727, 134217727)");
// assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 28), @"(268435455, 268435455, 268435455, 268435455)");
// assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 29), @"(536870911, 536870911, 536870911, 536870911)");
// assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 30), @"(1073741823, 1073741823, 1073741823, 1073741823)");
}