diff --git a/Cargo.lock b/Cargo.lock index c939d7e3..e789b0ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" [[package]] name = "approx" @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", @@ -1025,9 +1025,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -1707,9 +1707,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2010,7 +2010,7 @@ dependencies = [ [[package]] name = "martin" -version = "0.12.0" +version = "0.13.0" dependencies = [ "actix-cors", "actix-http", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" dependencies = [ "approx", "insta", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix-rt", "anyhow", @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" dependencies = [ "unicode-ident", ] @@ -4004,18 +4004,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" +checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" +checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19" dependencies = [ "proc-macro2", "quote", @@ -4674,11 +4674,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] diff --git a/debian/config.yaml b/debian/config.yaml index f44db6ad..30b9ad27 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -7,6 +7,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 512 + # see https://maplibre.org/martin/config-file.html # postgres: @@ -17,7 +20,6 @@ worker_processes: 8 # auto_bounds: skip # pmtiles: -# dir_cache_size_mb: 100 # paths: # - /dir-path # - /path/to/pmtiles.pmtiles diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 19cfec94..542677de 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -24,6 +24,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 1024 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string. You can use env vars too, for example: @@ -155,8 +158,6 @@ postgres: # Publish PMTiles files from local disk or proxy to a web server pmtiles: - # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] - dir_cache_size_mb: 100 paths: # scan this whole dir, matching all *.pmtiles files - /dir-path diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index 1f689956..90d39ce7 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server." keywords = ["maps", "tiles", "mvt", "tileserver"] diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index 029949eb..9df4fd49 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -86,7 +86,7 @@ impl Display for Format { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub enum Encoding { /// Data is not compressed, but it can be Uncompressed = 0b0000_0000, diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 38405a0f..9e81bec8 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -3,7 +3,7 @@ lints.workspace = true [package] name = "martin" # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin -version = "0.12.0" +version = "0.13.0" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"] @@ -62,7 +62,7 @@ harness = false default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] fonts = ["dep:bit-set", "dep:pbf_font_tools"] mbtiles = [] -pmtiles = ["dep:moka"] +pmtiles = [] postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] sprites = ["dep:spreet"] bless-tests = [] @@ -85,7 +85,7 @@ json-patch = { workspace = true, optional = true } log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true -moka = { workspace = true, optional = true } +moka.workspace = true num_cpus.workspace = true pbf_font_tools = { workspace = true, optional = true } pmtiles.workspace = true diff --git a/martin/benches/bench.rs b/martin/benches/bench.rs index 77363284..27f4904b 100644 --- a/martin/benches/bench.rs +++ b/martin/benches/bench.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use criterion::async_executor::FuturesExecutor; use criterion::{criterion_group, criterion_main, Criterion}; -use martin::srv::get_tile_response; +use martin::srv::DynTileSource; use martin::{ CatalogSourceEntry, MartinResult, Source, TileCoord, TileData, TileSources, UrlQuery, }; @@ -58,7 +58,8 @@ impl Source for NullSource { } async fn process_tile(sources: &TileSources) { - get_tile_response(sources, TileCoord { z: 0, x: 0, y: 0 }, "null", "", None) + let src = DynTileSource::new(sources, "null", Some(0), "", None, None).unwrap(); + src.get_http_response(TileCoord { z: 0, x: 0, y: 0 }) .await .unwrap(); } diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 2fc83c2d..7fc81d1b 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -43,6 +43,9 @@ pub struct MetaArgs { /// By default, only print if sources are auto-detected. #[arg(long)] pub save_config: Option, + /// Main cache size (in MB) + #[arg(short = 'C', long)] + pub cache_size: Option, /// **Deprecated** Scan for new sources on sources list requests #[arg(short, long, hide = true)] pub watch: bool, @@ -74,6 +77,10 @@ impl Args { return Err(ConfigAndConnectionsError(self.meta.connection)); } + if self.meta.cache_size.is_some() { + config.cache_size_mb = self.meta.cache_size; + } + self.srv.merge_into_config(&mut config.srv); #[allow(unused_mut)] diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 405d16ba..feecbfc9 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -12,10 +12,10 @@ use futures::stream::{self, StreamExt}; use futures::TryStreamExt; use log::{debug, error, info, log_enabled}; use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs}; -use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS}; +use martin::srv::{merge_tilejson, DynTileSource}; use martin::{ - append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source, - TileCoord, TileData, TileRect, + append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileCoord, + TileData, TileRect, }; use martin_tile_utils::{bbox_to_xyz, TileInfo}; use mbtiles::sqlx::SqliteConnection; @@ -144,7 +144,8 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; @@ -274,9 +275,18 @@ fn iterate_tiles(tiles: Vec) -> impl Iterator { async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> { let output_file = &args.output_file; let concurrency = args.concurrency.unwrap_or(1); - let (sources, _use_url_query, info) = state.tiles.get_sources(args.source.as_str(), None)?; - let sources = sources.as_slice(); - let tile_info = sources.first().unwrap().get_tile_info(); + + let src = DynTileSource::new( + &state.tiles, + args.source.as_str(), + None, + args.url_query.as_deref().unwrap_or_default(), + Some(parse_encoding(args.encoding.as_str())?), + None, + )?; + // parallel async below uses move, so we must only use copyable types + let src = &src; + let (tx, mut rx) = channel::(500); let tiles = compute_tile_ranges(&args); let mbt = Mbtiles::new(output_file)?; @@ -288,30 +298,26 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> } else { CopyDuplicateMode::Override }; - let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?; - let query = args.url_query.as_deref(); - let req = TestRequest::default() - .insert_header((ACCEPT_ENCODING, args.encoding.as_str())) - .finish(); - let accept_encoding = AcceptEncoding::parse(&req)?; - let encodings = Some(&accept_encoding); + let mbt_type = init_schema(&mbt, &mut conn, src.sources.as_slice(), src.info, &args).await?; let progress = Progress::new(&tiles); info!( - "Copying {} {tile_info} tiles from {} to {}", + "Copying {} {} tiles from {} to {}", progress.total, + src.info, args.source, args.output_file.display() ); try_join!( + // Note: for some reason, tests hang here without the `move` keyword async move { stream::iter(iterate_tiles(tiles)) .map(MartinResult::Ok) .try_for_each_concurrent(concurrency, |xyz| { let tx = tx.clone(); async move { - let tile = get_tile_content(sources, info, xyz, query, encodings).await?; + let tile = src.get_tile_content(xyz).await?; let data = tile.data; tx.send(TileXyz { xyz, data }) .await @@ -375,6 +381,13 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> Ok(()) } +fn parse_encoding(encoding: &str) -> MartinCpResult { + let req = TestRequest::default() + .insert_header((ACCEPT_ENCODING, encoding)) + .finish(); + Ok(AcceptEncoding::parse(&req)?) +} + async fn init_schema( mbt: &Mbtiles, conn: &mut SqliteConnection, diff --git a/martin/src/bin/martin.rs b/martin/src/bin/martin.rs index 5765e66e..054970e4 100644 --- a/martin/src/bin/martin.rs +++ b/martin/src/bin/martin.rs @@ -4,8 +4,8 @@ use actix_web::dev::Server; use clap::Parser; use log::{error, info, log_enabled}; use martin::args::{Args, OsEnv}; -use martin::srv::{new_server, RESERVED_KEYWORDS}; -use martin::{read_config, Config, IdResolver, MartinResult}; +use martin::srv::new_server; +use martin::{read_config, Config, MartinResult}; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -24,7 +24,7 @@ async fn start(args: Args) -> MartinResult { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; diff --git a/martin/src/config.rs b/martin/src/config.rs index f311d5e6..8fed9347 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -18,13 +18,15 @@ use crate::fonts::FontSources; use crate::source::{TileInfoSources, TileSources}; #[cfg(feature = "sprites")] use crate::sprites::{SpriteConfig, SpriteSources}; -use crate::srv::SrvConfig; +use crate::srv::{SrvConfig, RESERVED_KEYWORDS}; +use crate::utils::{CacheValue, MainCache, OptMainCache}; use crate::MartinError::{ConfigLoadError, ConfigParseError, ConfigWriteError, NoSources}; use crate::{IdResolver, MartinResult, OptOneMany}; pub type UnrecognizedValues = HashMap; pub struct ServerState { + pub cache: OptMainCache, pub tiles: TileSources, #[cfg(feature = "sprites")] pub sprites: SpriteSources, @@ -32,8 +34,11 @@ pub struct ServerState { pub fonts: FontSources, } +#[serde_with::skip_serializing_none] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Config { + pub cache_size_mb: Option, + #[serde(flatten)] pub srv: SrvConfig, @@ -107,19 +112,43 @@ impl Config { } } - pub async fn resolve(&mut self, idr: IdResolver) -> MartinResult { + pub async fn resolve(&mut self) -> MartinResult { + let resolver = IdResolver::new(RESERVED_KEYWORDS); + let cache_size = self.cache_size_mb.unwrap_or(512) * 1024 * 1024; + let cache = if cache_size > 0 { + info!("Initializing main cache with maximum size {cache_size}B"); + Some( + MainCache::builder() + .weigher(|_key, value: &CacheValue| -> u32 { + match value { + CacheValue::Tile(v) => v.len().try_into().unwrap_or(u32::MAX), + CacheValue::PmtDirectory(v) => { + v.get_approx_byte_size().try_into().unwrap_or(u32::MAX) + } + } + }) + .max_capacity(cache_size) + .build(), + ) + } else { + info!("Caching is disabled"); + None + }; + Ok(ServerState { - tiles: self.resolve_tile_sources(idr).await?, + tiles: self.resolve_tile_sources(&resolver, cache.clone()).await?, #[cfg(feature = "sprites")] sprites: SpriteSources::resolve(&mut self.sprites)?, #[cfg(feature = "fonts")] fonts: FontSources::resolve(&mut self.fonts)?, + cache, }) } async fn resolve_tile_sources( &mut self, - #[allow(unused_variables)] idr: IdResolver, + #[allow(unused_variables)] idr: &IdResolver, + #[allow(unused_variables)] cache: OptMainCache, ) -> MartinResult { #[allow(unused_mut)] let mut sources: Vec>>>> = @@ -133,14 +162,14 @@ impl Config { #[cfg(feature = "pmtiles")] if !self.pmtiles.is_empty() { let cfg = &mut self.pmtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "pmtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "pmtiles"); sources.push(Box::pin(val)); } #[cfg(feature = "mbtiles")] if !self.mbtiles.is_empty() { let cfg = &mut self.mbtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "mbtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "mbtiles"); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 636ec3f1..3692f75b 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -14,7 +14,7 @@ use crate::file_config::FileError::{ InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError, }; use crate::source::{Source, TileInfoSources}; -use crate::utils::{IdResolver, OptOneMany}; +use crate::utils::{IdResolver, OptMainCache, OptOneMany}; use crate::MartinResult; use crate::OptOneMany::{Many, One}; @@ -48,7 +48,7 @@ pub enum FileError { } pub trait ConfigExtras: Clone + Debug + Default + PartialEq + Send { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, _cache: OptMainCache) -> FileResult<()> { Ok(()) } @@ -127,7 +127,10 @@ impl FileConfigEnum { } } - pub fn extract_file_config(&mut self) -> FileResult>> { + pub fn extract_file_config( + &mut self, + cache: OptMainCache, + ) -> FileResult>> { let mut res = match self { FileConfigEnum::None => return Ok(None), FileConfigEnum::Path(path) => FileConfig { @@ -140,7 +143,7 @@ impl FileConfigEnum { }, FileConfigEnum::Config(cfg) => mem::take(cfg), }; - res.custom.init_parsing()?; + res.custom.init_parsing(cache)?; Ok(Some(res)) } @@ -218,20 +221,22 @@ pub struct FileConfigSource { pub async fn resolve_files( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> MartinResult { - resolve_int(config, idr, extension) + resolve_int(config, idr, cache, extension) .map_err(crate::MartinError::from) .await } async fn resolve_int( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(cache)? else { return Ok(TileInfoSources::default()); }; diff --git a/martin/src/lib.rs b/martin/src/lib.rs index c0c9d472..ef4aad9f 100644 --- a/martin/src/lib.rs +++ b/martin/src/lib.rs @@ -10,7 +10,7 @@ pub use source::{CatalogSourceEntry, Source, Tile, TileData, TileSources, UrlQue mod utils; pub use utils::{ append_rect, decode_brotli, decode_gzip, IdResolver, MartinError, MartinResult, OptBoolObj, - OptOneMany, TileCoord, TileRect, + OptOneMany, TileCoord, TileRect, NO_MAIN_CACHE, }; pub mod args; diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index ec651453..3a9ddf0d 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -9,7 +9,6 @@ use std::sync::Arc; use async_trait::async_trait; use log::{trace, warn}; use martin_tile_utils::{Encoding, Format, TileInfo}; -use moka::future::Cache; use pmtiles::async_reader::AsyncPmTilesReader; use pmtiles::cache::{DirCacheResult, DirectoryCache}; use pmtiles::http::HttpBackend; @@ -24,20 +23,20 @@ use crate::config::UnrecognizedValues; use crate::file_config::FileError::{InvalidMetadata, InvalidUrlMetadata, IoError}; use crate::file_config::{ConfigExtras, FileError, FileResult, SourceConfigExtras}; use crate::source::UrlQuery; +use crate::utils::cache::get_cached_value; +use crate::utils::{CacheKey, CacheValue, OptMainCache}; use crate::{MartinResult, Source, TileCoord, TileData}; -type PmtCacheObject = Cache<(usize, usize), Directory>; - #[derive(Clone, Debug)] pub struct PmtCache { id: usize, - /// (id, offset) -> Directory, or None to disable caching - cache: Option, + /// Storing (id, offset) -> Directory, or None to disable caching + cache: OptMainCache, } impl PmtCache { #[must_use] - pub fn new(id: usize, cache: Option) -> Self { + pub fn new(id: usize, cache: OptMainCache) -> Self { Self { id, cache } } } @@ -45,17 +44,23 @@ impl PmtCache { #[async_trait] impl DirectoryCache for PmtCache { async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { - if let Some(cache) = &self.cache { - if let Some(dir) = cache.get(&(self.id, offset)).await { - return dir.find_tile_id(tile_id).into(); - } + if let Some(dir) = get_cached_value!(&self.cache, CacheValue::PmtDirectory, { + CacheKey::PmtDirectory(self.id, offset) + }) { + dir.find_tile_id(tile_id).into() + } else { + DirCacheResult::NotCached } - DirCacheResult::NotCached } async fn insert_dir(&self, offset: usize, directory: Directory) { if let Some(cache) = &self.cache { - cache.insert((self.id, offset), directory).await; + cache + .insert( + CacheKey::PmtDirectory(self.id, offset), + CacheValue::PmtDirectory(directory), + ) + .await; } } } @@ -63,8 +68,6 @@ impl DirectoryCache for PmtCache { #[serde_with::skip_serializing_none] #[derive(Debug, Default, Serialize, Deserialize)] pub struct PmtConfig { - pub dir_cache_size_mb: Option, - #[serde(flatten)] pub unrecognized: UnrecognizedValues, @@ -78,12 +81,12 @@ pub struct PmtConfig { pub next_cache_id: AtomicUsize, #[serde(skip)] - pub cache: Option, + pub cache: OptMainCache, } impl PartialEq for PmtConfig { fn eq(&self, other: &Self) -> bool { - self.dir_cache_size_mb == other.dir_cache_size_mb && self.unrecognized == other.unrecognized + self.unrecognized == other.unrecognized } } @@ -91,7 +94,6 @@ impl Clone for PmtConfig { fn clone(&self) -> Self { // State is not shared between clones, only the serialized config Self { - dir_cache_size_mb: self.dir_cache_size_mb, unrecognized: self.unrecognized.clone(), ..Default::default() } @@ -107,24 +109,17 @@ impl PmtConfig { } impl ConfigExtras for PmtConfig { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, cache: OptMainCache) -> FileResult<()> { assert!(self.client.is_none()); assert!(self.cache.is_none()); self.client = Some(Client::new()); + self.cache = cache; - // Allow cache size to be disabled with 0 - let dir_cache_size = self.dir_cache_size_mb.unwrap_or(32) * 1024 * 1024; - if dir_cache_size > 0 { - self.cache = Some( - Cache::builder() - .weigher(|_key, value: &Directory| -> u32 { - value.get_approx_byte_size().try_into().unwrap_or(u32::MAX) - }) - .max_capacity(dir_cache_size) - .build(), - ); + if self.unrecognized.contains_key("dir_cache_size_mb") { + warn!("dir_cache_size_mb is no longer used. Instead, use cache_size_mb param in the root of the config file."); } + Ok(()) } diff --git a/martin/src/source.rs b/martin/src/source.rs index 01c2bc2b..ba04293b 100644 --- a/martin/src/source.rs +++ b/martin/src/source.rs @@ -167,6 +167,7 @@ mod tests { } } +#[derive(Debug, Clone)] pub struct Tile { pub data: TileData, pub info: TileInfo, diff --git a/martin/src/sprites/mod.rs b/martin/src/sprites/mod.rs index fa81f179..7a0fc8cc 100644 --- a/martin/src/sprites/mod.rs +++ b/martin/src/sprites/mod.rs @@ -77,7 +77,7 @@ pub struct SpriteSources(HashMap); impl SpriteSources { pub fn resolve(config: &mut FileConfigEnum) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(None)? else { return Ok(Self::default()); }; diff --git a/martin/src/srv/fonts.rs b/martin/src/srv/fonts.rs old mode 100755 new mode 100644 diff --git a/martin/src/srv/mod.rs b/martin/src/srv/mod.rs index ea94de2e..b9ad026f 100644 --- a/martin/src/srv/mod.rs +++ b/martin/src/srv/mod.rs @@ -1,14 +1,14 @@ mod config; pub use config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT}; +#[cfg(feature = "fonts")] +mod fonts; + mod server; pub use server::{new_server, router, Catalog, RESERVED_KEYWORDS}; mod tiles; -pub use tiles::{get_tile_content, get_tile_response, TileRequest}; - -#[cfg(feature = "fonts")] -mod fonts; +pub use tiles::{DynTileSource, TileRequest}; mod tiles_info; pub use tiles_info::{merge_tilejson, SourceIDsRequest}; diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index 377638b6..4fe0e4d9 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -112,7 +112,9 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server .allow_any_origin() .allowed_methods(vec!["GET"]); - let app = App::new().app_data(Data::new(state.tiles.clone())); + let app = App::new() + .app_data(Data::new(state.tiles.clone())) + .app_data(Data::new(state.cache.clone())); #[cfg(feature = "sprites")] let app = app.app_data(Data::new(state.sprites.clone())); diff --git a/martin/src/srv/sprites.rs b/martin/src/srv/sprites.rs index c3275bbe..55a61a81 100644 --- a/martin/src/srv/sprites.rs +++ b/martin/src/srv/sprites.rs @@ -4,28 +4,18 @@ use actix_web::error::ErrorNotFound; use actix_web::http::header::ContentType; use actix_web::web::{Data, Path}; use actix_web::{middleware, route, HttpResponse, Result as ActixResult}; +use spreet::Spritesheet; use crate::sprites::{SpriteError, SpriteSources}; use crate::srv::server::map_internal_error; use crate::srv::SourceIDsRequest; -pub fn map_sprite_error(e: SpriteError) -> actix_web::Error { - use SpriteError::SpriteNotFound; - match e { - SpriteNotFound(_) => ErrorNotFound(e.to_string()), - _ => map_internal_error(e), - } -} - #[route("/sprite/{source_ids}.png", method = "GET", method = "HEAD")] async fn get_sprite_png( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites - .get_sprites(&path.source_ids) - .await - .map_err(map_sprite_error)?; + let sheet = get_sprite(&path, &sprites).await?; Ok(HttpResponse::Ok() .content_type(ContentType::png()) .body(sheet.encode_png().map_err(map_internal_error)?)) @@ -41,9 +31,16 @@ async fn get_sprite_json( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites - .get_sprites(&path.source_ids) - .await - .map_err(map_sprite_error)?; + let sheet = get_sprite(&path, &sprites).await?; Ok(HttpResponse::Ok().json(sheet.get_index())) } + +async fn get_sprite(path: &SourceIDsRequest, sprites: &SpriteSources) -> ActixResult { + sprites + .get_sprites(&path.source_ids) + .await + .map_err(|e| match e { + SpriteError::SpriteNotFound(_) => ErrorNotFound(e.to_string()), + _ => map_internal_error(e), + }) +} diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs old mode 100644 new mode 100755 index 32463a6f..f2582996 --- a/martin/src/srv/tiles.rs +++ b/martin/src/srv/tiles.rs @@ -6,13 +6,18 @@ use actix_web::http::header::{ use actix_web::web::{Data, Path, Query}; use actix_web::{route, HttpMessage, HttpRequest, HttpResponse, Result as ActixResult}; use futures::future::try_join_all; +use log::trace; use martin_tile_utils::{Encoding, Format, TileInfo}; use serde::Deserialize; use crate::source::{Source, TileSources, UrlQuery}; use crate::srv::server::map_internal_error; -use crate::utils::{decode_brotli, decode_gzip, encode_brotli, encode_gzip}; -use crate::{Tile, TileCoord}; +use crate::utils::cache::get_or_insert_cached_value; +use crate::utils::{ + decode_brotli, decode_gzip, encode_brotli, encode_gzip, CacheKey, CacheValue, MainCache, + OptMainCache, +}; +use crate::{Tile, TileCoord, TileData}; static SUPPORTED_ENCODINGS: &[HeaderEnc] = &[ HeaderEnc::brotli(), @@ -33,125 +38,165 @@ async fn get_tile( req: HttpRequest, path: Path, sources: Data, + cache: Data, ) -> ActixResult { - let xyz = TileCoord { + let src = DynTileSource::new( + sources.as_ref(), + &path.source_ids, + Some(path.z), + req.query_string(), + req.get_header::(), + cache.as_ref().as_ref(), + )?; + + src.get_http_response(TileCoord { z: path.z, x: path.x, y: path.y, - }; - - let source_ids = &path.source_ids; - let query = req.query_string(); - let encodings = req.get_header::(); - - get_tile_response(sources.as_ref(), xyz, source_ids, query, encodings).await -} - -pub async fn get_tile_response( - sources: &TileSources, - xyz: TileCoord, - source_ids: &str, - query: &str, - encodings: Option, -) -> ActixResult { - let (sources, use_url_query, info) = sources.get_sources(source_ids, Some(xyz.z))?; - - let query = use_url_query.then_some(query); - let tile = get_tile_content(sources.as_slice(), info, xyz, query, encodings.as_ref()).await?; - - Ok(if tile.data.is_empty() { - HttpResponse::NoContent().finish() - } else { - let mut response = HttpResponse::Ok(); - response.content_type(tile.info.format.content_type()); - if let Some(val) = tile.info.encoding.content_encoding() { - response.insert_header((CONTENT_ENCODING, val)); - } - response.body(tile.data) }) + .await } -pub async fn get_tile_content( - sources: &[&dyn Source], - info: TileInfo, - xyz: TileCoord, - query: Option<&str>, - encodings: Option<&AcceptEncoding>, -) -> ActixResult { - if sources.is_empty() { - return Err(ErrorNotFound("No valid sources found")); - } - let query_str = query.filter(|v| !v.is_empty()); - let query = match query_str { - Some(v) => Some(Query::::from_query(v)?.into_inner()), - None => None, - }; +pub struct DynTileSource<'a> { + pub sources: Vec<&'a dyn Source>, + pub info: TileInfo, + pub query_str: Option<&'a str>, + pub query_obj: Option, + pub encodings: Option, + pub cache: Option<&'a MainCache>, +} - let mut tiles = try_join_all(sources.iter().map(|s| s.get_tile(xyz, query.as_ref()))) +impl<'a> DynTileSource<'a> { + pub fn new( + sources: &'a TileSources, + source_ids: &str, + zoom: Option, + query: &'a str, + encodings: Option, + cache: Option<&'a MainCache>, + ) -> ActixResult { + let (sources, use_url_query, info) = sources.get_sources(source_ids, zoom)?; + + if sources.is_empty() { + return Err(ErrorNotFound("No valid sources found")); + } + + let mut query_obj = None; + let mut query_str = None; + if use_url_query && !query.is_empty() { + query_obj = Some(Query::::from_query(query)?.into_inner()); + query_str = Some(query); + } + + Ok(Self { + sources, + info, + query_str, + query_obj, + encodings, + cache, + }) + } + + pub async fn get_http_response(&self, xyz: TileCoord) -> ActixResult { + let tile = self.get_tile_content(xyz).await?; + + Ok(if tile.data.is_empty() { + HttpResponse::NoContent().finish() + } else { + let mut response = HttpResponse::Ok(); + response.content_type(tile.info.format.content_type()); + if let Some(val) = tile.info.encoding.content_encoding() { + response.insert_header((CONTENT_ENCODING, val)); + } + response.body(tile.data) + }) + } + + pub async fn get_tile_content(&self, xyz: TileCoord) -> ActixResult { + let mut tiles = try_join_all(self.sources.iter().map(|s| async { + get_or_insert_cached_value!( + self.cache, + CacheValue::Tile, + s.get_tile(xyz, self.query_obj.as_ref()), + { + let id = s.get_id().to_owned(); + if let Some(query_str) = self.query_str { + CacheKey::TileWithQuery(id, xyz, query_str.to_owned()) + } else { + CacheKey::Tile(id, xyz) + } + } + ) + })) .await .map_err(map_internal_error)?; - let mut layer_count = 0; - let mut last_non_empty_layer = 0; - for (idx, tile) in tiles.iter().enumerate() { - if !tile.is_empty() { - layer_count += 1; - last_non_empty_layer = idx; + let mut layer_count = 0; + let mut last_non_empty_layer = 0; + for (idx, tile) in tiles.iter().enumerate() { + if !tile.is_empty() { + layer_count += 1; + last_non_empty_layer = idx; + } } + + // Minor optimization to prevent concatenation if there are less than 2 tiles + let data = match layer_count { + 1 => tiles.swap_remove(last_non_empty_layer), + 0 => return Ok(Tile::new(Vec::new(), self.info)), + _ => { + // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level + // TODO: can zlib, brotli, or zstd be concatenated? + // TODO: implement decompression step for other concatenate-able formats + let can_join = self.info.format == Format::Mvt + && (self.info.encoding == Encoding::Uncompressed + || self.info.encoding == Encoding::Gzip); + if !can_join { + return Err(ErrorBadRequest(format!( + "Can't merge {} tiles. Make sure there is only one non-empty tile source at zoom level {}", + self.info, + xyz.z + )))?; + } + tiles.concat() + } + }; + + // decide if (re-)encoding of the tile data is needed, and recompress if so + self.recompress(data) } - // Minor optimization to prevent concatenation if there are less than 2 tiles - let data = match layer_count { - 1 => tiles.swap_remove(last_non_empty_layer), - 0 => return Ok(Tile::new(Vec::new(), info)), - _ => { - // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level - // TODO: can zlib, brotli, or zstd be concatenated? - // TODO: implement decompression step for other concatenate-able formats - let can_join = info.format == Format::Mvt - && (info.encoding == Encoding::Uncompressed || info.encoding == Encoding::Gzip); - if !can_join { - return Err(ErrorBadRequest(format!( - "Can't merge {info} tiles. Make sure there is only one non-empty tile source at zoom level {}", - xyz.z - )))?; - } - tiles.concat() - } - }; - - // decide if (re-)encoding of the tile data is needed, and recompress if so - let tile = recompress(Tile::new(data, info), encodings)?; - - Ok(tile) -} - -fn recompress(mut tile: Tile, accept_enc: Option<&AcceptEncoding>) -> ActixResult { - if let Some(accept_enc) = accept_enc { - if tile.info.encoding.is_encoded() { - // already compressed, see if we can send it as is, or need to re-compress - if !accept_enc.iter().any(|e| { - if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { - to_encoding(enc) == Some(tile.info.encoding) - } else { - false + fn recompress(&self, tile: TileData) -> ActixResult { + let mut tile = Tile::new(tile, self.info); + if let Some(accept_enc) = &self.encodings { + if self.info.encoding.is_encoded() { + // already compressed, see if we can send it as is, or need to re-compress + if !accept_enc.iter().any(|e| { + if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { + to_encoding(enc) == Some(tile.info.encoding) + } else { + false + } + }) { + // need to re-compress the tile - uncompress it first + tile = decode(tile)?; } - }) { - // need to re-compress the tile - uncompress it first - tile = decode(tile)?; } - } - if tile.info.encoding == Encoding::Uncompressed { - // only apply compression if the content supports it - if let Some(HeaderEnc::Known(enc)) = accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) { - // (re-)compress the tile into the preferred encoding - tile = encode(tile, enc)?; + if tile.info.encoding == Encoding::Uncompressed { + // only apply compression if the content supports it + if let Some(HeaderEnc::Known(enc)) = + accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) + { + // (re-)compress the tile into the preferred encoding + tile = encode(tile, enc)?; + } } + Ok(tile) + } else { + // no accepted-encoding header, decode the tile if compressed + decode(tile) } - Ok(tile) - } else { - // no accepted-encoding header, decode the tile if compressed - decode(tile) } } @@ -189,7 +234,7 @@ fn decode(tile: Tile) -> ActixResult { }) } -fn to_encoding(val: ContentEncoding) -> Option { +pub fn to_encoding(val: ContentEncoding) -> Option { Some(match val { ContentEncoding::Identity => Encoding::Uncompressed, ContentEncoding::Gzip => Encoding::Gzip, @@ -233,15 +278,9 @@ mod tests { ("empty,non-empty", vec![1_u8, 2, 3]), ("empty,non-empty,empty", vec![1_u8, 2, 3]), ] { - let (src, _, info) = sources.get_sources(source_id, None).unwrap(); + let src = DynTileSource::new(&sources, source_id, None, "", None, None).unwrap(); let xyz = TileCoord { z: 0, x: 0, y: 0 }; - assert_eq!( - expected, - &get_tile_content(src.as_slice(), info, xyz, None, None) - .await - .unwrap() - .data - ); + assert_eq!(expected, &src.get_tile_content(xyz).await.unwrap().data); } } } diff --git a/martin/src/srv/tiles_info.rs b/martin/src/srv/tiles_info.rs old mode 100644 new mode 100755 diff --git a/martin/src/utils/cache.rs b/martin/src/utils/cache.rs new file mode 100755 index 00000000..ff442156 --- /dev/null +++ b/martin/src/utils/cache.rs @@ -0,0 +1,91 @@ +use moka::future::Cache; +use pmtiles::Directory; + +use crate::{TileCoord, TileData}; + +pub type MainCache = Cache; +pub type OptMainCache = Option; +pub const NO_MAIN_CACHE: OptMainCache = None; + +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum CacheKey { + /// (pmtiles_id, offset) + PmtDirectory(usize, usize), + /// (source_id, xyz) + Tile(String, TileCoord), + /// (source_id, xyz, url_query) + TileWithQuery(String, TileCoord, String), +} + +#[derive(Debug, Clone)] +pub enum CacheValue { + Tile(TileData), + PmtDirectory(Directory), +} + +macro_rules! trace_cache { + ($typ: literal, $cache: expr, $key: expr) => { + trace!( + "Cache {} for {:?} in {:?} that has {} entries taking up {} space", + $typ, + $key, + $cache.name(), + $cache.entry_count(), + $cache.weighted_size(), + ); + }; +} + +macro_rules! from_cache_value { + ($value_type: path, $data: expr, $key: expr) => { + if let $value_type(data) = $data { + data + } else { + panic!("Unexpected value type {:?} for key {:?} cache", $data, $key) + } + }; +} +#[cfg(feature = "pmtiles")] +macro_rules! get_cached_value { + ($cache: expr, $value_type: path, $make_key: expr) => { + if let Some(cache) = $cache { + let key = $make_key; + if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + Some($crate::utils::cache::from_cache_value!( + $value_type, + data, + key + )) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + None + } + } else { + None + } + }; +} + +macro_rules! get_or_insert_cached_value { + ($cache: expr, $value_type: path, $make_item:expr, $make_key: expr) => {{ + if let Some(cache) = $cache { + let key = $make_key; + Ok(if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + $crate::utils::cache::from_cache_value!($value_type, data, key) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + let data = $make_item.await?; + cache.insert(key, $value_type(data.clone())).await; + data + }) + } else { + $make_item.await + } + }}; +} + +#[cfg(feature = "pmtiles")] +pub(crate) use get_cached_value; +pub(crate) use {from_cache_value, get_or_insert_cached_value, trace_cache}; diff --git a/martin/src/utils/mod.rs b/martin/src/utils/mod.rs index e0444cde..306da7e5 100644 --- a/martin/src/utils/mod.rs +++ b/martin/src/utils/mod.rs @@ -1,3 +1,6 @@ +pub(crate) mod cache; +pub use cache::{CacheKey, CacheValue, MainCache, OptMainCache, NO_MAIN_CACHE}; + mod cfg_containers; pub use cfg_containers::{OptBoolObj, OptOneMany}; diff --git a/martin/src/utils/xyz.rs b/martin/src/utils/xyz.rs index 421ec6df..1a968207 100644 --- a/martin/src/utils/xyz.rs +++ b/martin/src/utils/xyz.rs @@ -1,6 +1,6 @@ use std::fmt::{Display, Formatter}; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct TileCoord { pub z: u8, pub x: u32, diff --git a/martin/tests/mb_server_test.rs b/martin/tests/mb_server_test.rs index b20b64f5..b38d490b 100644 --- a/martin/tests/mb_server_test.rs +++ b/martin/tests/mb_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index ebbc5114..55b7f37a 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -26,6 +26,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) @@ -1086,6 +1087,7 @@ tables: .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pmt_server_test.rs b/martin/tests/pmt_server_test.rs index 5b1a4d19..9a6e7ce4 100644 --- a/martin/tests/pmt_server_test.rs +++ b/martin/tests/pmt_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs index 334b1179..02116664 100644 --- a/martin/tests/utils/pg_utils.rs +++ b/martin/tests/utils/pg_utils.rs @@ -1,6 +1,6 @@ use indoc::formatdoc; pub use martin::args::Env; -use martin::{Config, IdResolver, ServerState, Source}; +use martin::{Config, ServerState, Source}; use crate::mock_cfg; @@ -22,7 +22,7 @@ pub fn mock_pgcfg(yaml: &str) -> Config { #[allow(dead_code)] pub async fn mock_sources(mut config: Config) -> MockSource { - let res = config.resolve(IdResolver::default()).await; + let res = config.resolve().await; let res = res.unwrap_or_else(|e| panic!("Failed to resolve config {config:?}: {e}")); (res, config) } diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index f3021360..1f22df56 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/tests/config.yaml b/tests/config.yaml index 7a3f2848..70b08a05 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -8,6 +8,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 8 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string @@ -166,7 +169,6 @@ postgres: pmtiles: - dir_cache_size_mb: 100 paths: - http://localhost:5412/webp2.pmtiles sources: diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 47379599..308490c1 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -1,3 +1,4 @@ +cache_size_mb: 8 keep_alive: 75 listen_addresses: localhost:3111 worker_processes: 1 @@ -165,7 +166,6 @@ pmtiles: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles pmt2: http://localhost:5412/webp2.pmtiles webp2: http://localhost:5412/webp2.pmtiles - dir_cache_size_mb: 100 sprites: paths: tests/fixtures/sprites/src1 sources: