From 3dc54d7f9eaac777f3d650cdb285184e51845725 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Dec 2023 22:48:23 -0500 Subject: [PATCH] Implement tile caching (#1105) Add a top level config parameter -- the size of cache memory (in MB) to use for caching tiles and PMT directories, defaulting to 512, and 0 to disable. This also removes the `pmtiles.dir_cache_size_mb` parameter (it will be ignored, but will give a warning) ``` cache_size_mb: 512 ``` The new cache will contain all tiles as provided by the source. So if PostgreSQL returns a non-compressed tile, the cache will contain the uncompressed variant, and will be compressed for each response. This will be fixed in the later releases. Note that fonts and sprites are not cached at this time, and are still a TODO. --- Cargo.lock | 40 ++-- debian/config.yaml | 4 +- docs/src/config-file.md | 5 +- martin-tile-utils/Cargo.toml | 2 +- martin-tile-utils/src/lib.rs | 2 +- martin/Cargo.toml | 6 +- martin/benches/bench.rs | 5 +- martin/src/args/root.rs | 7 + martin/src/bin/martin-cp.rs | 45 ++-- martin/src/bin/martin.rs | 6 +- martin/src/config.rs | 41 +++- martin/src/file_config.rs | 21 +- martin/src/lib.rs | 2 +- martin/src/pmtiles/mod.rs | 53 ++--- martin/src/source.rs | 1 + martin/src/sprites/mod.rs | 2 +- martin/src/srv/fonts.rs | 0 martin/src/srv/mod.rs | 8 +- martin/src/srv/server.rs | 4 +- martin/src/srv/sprites.rs | 29 +-- martin/src/srv/tiles.rs | 263 ++++++++++++--------- martin/src/srv/tiles_info.rs | 0 martin/src/utils/cache.rs | 91 +++++++ martin/src/utils/mod.rs | 3 + martin/src/utils/xyz.rs | 2 +- martin/tests/mb_server_test.rs | 1 + martin/tests/pg_server_test.rs | 2 + martin/tests/pmt_server_test.rs | 1 + martin/tests/utils/pg_utils.rs | 4 +- mbtiles/Cargo.toml | 2 +- tests/config.yaml | 4 +- tests/expected/configured/save_config.yaml | 2 +- 32 files changed, 425 insertions(+), 233 deletions(-) mode change 100755 => 100644 martin/src/srv/fonts.rs mode change 100644 => 100755 martin/src/srv/tiles.rs mode change 100644 => 100755 martin/src/srv/tiles_info.rs create mode 100755 martin/src/utils/cache.rs diff --git a/Cargo.lock b/Cargo.lock index c939d7e3..e789b0ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" [[package]] name = "approx" @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", @@ -1025,9 +1025,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -1707,9 +1707,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2010,7 +2010,7 @@ dependencies = [ [[package]] name = "martin" -version = "0.12.0" +version = "0.13.0" dependencies = [ "actix-cors", "actix-http", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" dependencies = [ "approx", "insta", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix-rt", "anyhow", @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" dependencies = [ "unicode-ident", ] @@ -4004,18 +4004,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" +checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" +checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19" dependencies = [ "proc-macro2", "quote", @@ -4674,11 +4674,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] diff --git a/debian/config.yaml b/debian/config.yaml index f44db6ad..30b9ad27 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -7,6 +7,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 512 + # see https://maplibre.org/martin/config-file.html # postgres: @@ -17,7 +20,6 @@ worker_processes: 8 # auto_bounds: skip # pmtiles: -# dir_cache_size_mb: 100 # paths: # - /dir-path # - /path/to/pmtiles.pmtiles diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 19cfec94..542677de 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -24,6 +24,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 1024 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string. You can use env vars too, for example: @@ -155,8 +158,6 @@ postgres: # Publish PMTiles files from local disk or proxy to a web server pmtiles: - # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] - dir_cache_size_mb: 100 paths: # scan this whole dir, matching all *.pmtiles files - /dir-path diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index 1f689956..90d39ce7 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server." keywords = ["maps", "tiles", "mvt", "tileserver"] diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index 029949eb..9df4fd49 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -86,7 +86,7 @@ impl Display for Format { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub enum Encoding { /// Data is not compressed, but it can be Uncompressed = 0b0000_0000, diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 38405a0f..9e81bec8 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -3,7 +3,7 @@ lints.workspace = true [package] name = "martin" # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin -version = "0.12.0" +version = "0.13.0" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"] @@ -62,7 +62,7 @@ harness = false default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] fonts = ["dep:bit-set", "dep:pbf_font_tools"] mbtiles = [] -pmtiles = ["dep:moka"] +pmtiles = [] postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] sprites = ["dep:spreet"] bless-tests = [] @@ -85,7 +85,7 @@ json-patch = { workspace = true, optional = true } log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true -moka = { workspace = true, optional = true } +moka.workspace = true num_cpus.workspace = true pbf_font_tools = { workspace = true, optional = true } pmtiles.workspace = true diff --git a/martin/benches/bench.rs b/martin/benches/bench.rs index 77363284..27f4904b 100644 --- a/martin/benches/bench.rs +++ b/martin/benches/bench.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use criterion::async_executor::FuturesExecutor; use criterion::{criterion_group, criterion_main, Criterion}; -use martin::srv::get_tile_response; +use martin::srv::DynTileSource; use martin::{ CatalogSourceEntry, MartinResult, Source, TileCoord, TileData, TileSources, UrlQuery, }; @@ -58,7 +58,8 @@ impl Source for NullSource { } async fn process_tile(sources: &TileSources) { - get_tile_response(sources, TileCoord { z: 0, x: 0, y: 0 }, "null", "", None) + let src = DynTileSource::new(sources, "null", Some(0), "", None, None).unwrap(); + src.get_http_response(TileCoord { z: 0, x: 0, y: 0 }) .await .unwrap(); } diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 2fc83c2d..7fc81d1b 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -43,6 +43,9 @@ pub struct MetaArgs { /// By default, only print if sources are auto-detected. #[arg(long)] pub save_config: Option, + /// Main cache size (in MB) + #[arg(short = 'C', long)] + pub cache_size: Option, /// **Deprecated** Scan for new sources on sources list requests #[arg(short, long, hide = true)] pub watch: bool, @@ -74,6 +77,10 @@ impl Args { return Err(ConfigAndConnectionsError(self.meta.connection)); } + if self.meta.cache_size.is_some() { + config.cache_size_mb = self.meta.cache_size; + } + self.srv.merge_into_config(&mut config.srv); #[allow(unused_mut)] diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 405d16ba..feecbfc9 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -12,10 +12,10 @@ use futures::stream::{self, StreamExt}; use futures::TryStreamExt; use log::{debug, error, info, log_enabled}; use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs}; -use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS}; +use martin::srv::{merge_tilejson, DynTileSource}; use martin::{ - append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source, - TileCoord, TileData, TileRect, + append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileCoord, + TileData, TileRect, }; use martin_tile_utils::{bbox_to_xyz, TileInfo}; use mbtiles::sqlx::SqliteConnection; @@ -144,7 +144,8 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; @@ -274,9 +275,18 @@ fn iterate_tiles(tiles: Vec) -> impl Iterator { async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> { let output_file = &args.output_file; let concurrency = args.concurrency.unwrap_or(1); - let (sources, _use_url_query, info) = state.tiles.get_sources(args.source.as_str(), None)?; - let sources = sources.as_slice(); - let tile_info = sources.first().unwrap().get_tile_info(); + + let src = DynTileSource::new( + &state.tiles, + args.source.as_str(), + None, + args.url_query.as_deref().unwrap_or_default(), + Some(parse_encoding(args.encoding.as_str())?), + None, + )?; + // parallel async below uses move, so we must only use copyable types + let src = &src; + let (tx, mut rx) = channel::(500); let tiles = compute_tile_ranges(&args); let mbt = Mbtiles::new(output_file)?; @@ -288,30 +298,26 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> } else { CopyDuplicateMode::Override }; - let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?; - let query = args.url_query.as_deref(); - let req = TestRequest::default() - .insert_header((ACCEPT_ENCODING, args.encoding.as_str())) - .finish(); - let accept_encoding = AcceptEncoding::parse(&req)?; - let encodings = Some(&accept_encoding); + let mbt_type = init_schema(&mbt, &mut conn, src.sources.as_slice(), src.info, &args).await?; let progress = Progress::new(&tiles); info!( - "Copying {} {tile_info} tiles from {} to {}", + "Copying {} {} tiles from {} to {}", progress.total, + src.info, args.source, args.output_file.display() ); try_join!( + // Note: for some reason, tests hang here without the `move` keyword async move { stream::iter(iterate_tiles(tiles)) .map(MartinResult::Ok) .try_for_each_concurrent(concurrency, |xyz| { let tx = tx.clone(); async move { - let tile = get_tile_content(sources, info, xyz, query, encodings).await?; + let tile = src.get_tile_content(xyz).await?; let data = tile.data; tx.send(TileXyz { xyz, data }) .await @@ -375,6 +381,13 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> Ok(()) } +fn parse_encoding(encoding: &str) -> MartinCpResult { + let req = TestRequest::default() + .insert_header((ACCEPT_ENCODING, encoding)) + .finish(); + Ok(AcceptEncoding::parse(&req)?) +} + async fn init_schema( mbt: &Mbtiles, conn: &mut SqliteConnection, diff --git a/martin/src/bin/martin.rs b/martin/src/bin/martin.rs index 5765e66e..054970e4 100644 --- a/martin/src/bin/martin.rs +++ b/martin/src/bin/martin.rs @@ -4,8 +4,8 @@ use actix_web::dev::Server; use clap::Parser; use log::{error, info, log_enabled}; use martin::args::{Args, OsEnv}; -use martin::srv::{new_server, RESERVED_KEYWORDS}; -use martin::{read_config, Config, IdResolver, MartinResult}; +use martin::srv::new_server; +use martin::{read_config, Config, MartinResult}; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -24,7 +24,7 @@ async fn start(args: Args) -> MartinResult { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; diff --git a/martin/src/config.rs b/martin/src/config.rs index f311d5e6..8fed9347 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -18,13 +18,15 @@ use crate::fonts::FontSources; use crate::source::{TileInfoSources, TileSources}; #[cfg(feature = "sprites")] use crate::sprites::{SpriteConfig, SpriteSources}; -use crate::srv::SrvConfig; +use crate::srv::{SrvConfig, RESERVED_KEYWORDS}; +use crate::utils::{CacheValue, MainCache, OptMainCache}; use crate::MartinError::{ConfigLoadError, ConfigParseError, ConfigWriteError, NoSources}; use crate::{IdResolver, MartinResult, OptOneMany}; pub type UnrecognizedValues = HashMap; pub struct ServerState { + pub cache: OptMainCache, pub tiles: TileSources, #[cfg(feature = "sprites")] pub sprites: SpriteSources, @@ -32,8 +34,11 @@ pub struct ServerState { pub fonts: FontSources, } +#[serde_with::skip_serializing_none] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Config { + pub cache_size_mb: Option, + #[serde(flatten)] pub srv: SrvConfig, @@ -107,19 +112,43 @@ impl Config { } } - pub async fn resolve(&mut self, idr: IdResolver) -> MartinResult { + pub async fn resolve(&mut self) -> MartinResult { + let resolver = IdResolver::new(RESERVED_KEYWORDS); + let cache_size = self.cache_size_mb.unwrap_or(512) * 1024 * 1024; + let cache = if cache_size > 0 { + info!("Initializing main cache with maximum size {cache_size}B"); + Some( + MainCache::builder() + .weigher(|_key, value: &CacheValue| -> u32 { + match value { + CacheValue::Tile(v) => v.len().try_into().unwrap_or(u32::MAX), + CacheValue::PmtDirectory(v) => { + v.get_approx_byte_size().try_into().unwrap_or(u32::MAX) + } + } + }) + .max_capacity(cache_size) + .build(), + ) + } else { + info!("Caching is disabled"); + None + }; + Ok(ServerState { - tiles: self.resolve_tile_sources(idr).await?, + tiles: self.resolve_tile_sources(&resolver, cache.clone()).await?, #[cfg(feature = "sprites")] sprites: SpriteSources::resolve(&mut self.sprites)?, #[cfg(feature = "fonts")] fonts: FontSources::resolve(&mut self.fonts)?, + cache, }) } async fn resolve_tile_sources( &mut self, - #[allow(unused_variables)] idr: IdResolver, + #[allow(unused_variables)] idr: &IdResolver, + #[allow(unused_variables)] cache: OptMainCache, ) -> MartinResult { #[allow(unused_mut)] let mut sources: Vec>>>> = @@ -133,14 +162,14 @@ impl Config { #[cfg(feature = "pmtiles")] if !self.pmtiles.is_empty() { let cfg = &mut self.pmtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "pmtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "pmtiles"); sources.push(Box::pin(val)); } #[cfg(feature = "mbtiles")] if !self.mbtiles.is_empty() { let cfg = &mut self.mbtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "mbtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "mbtiles"); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 636ec3f1..3692f75b 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -14,7 +14,7 @@ use crate::file_config::FileError::{ InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError, }; use crate::source::{Source, TileInfoSources}; -use crate::utils::{IdResolver, OptOneMany}; +use crate::utils::{IdResolver, OptMainCache, OptOneMany}; use crate::MartinResult; use crate::OptOneMany::{Many, One}; @@ -48,7 +48,7 @@ pub enum FileError { } pub trait ConfigExtras: Clone + Debug + Default + PartialEq + Send { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, _cache: OptMainCache) -> FileResult<()> { Ok(()) } @@ -127,7 +127,10 @@ impl FileConfigEnum { } } - pub fn extract_file_config(&mut self) -> FileResult>> { + pub fn extract_file_config( + &mut self, + cache: OptMainCache, + ) -> FileResult>> { let mut res = match self { FileConfigEnum::None => return Ok(None), FileConfigEnum::Path(path) => FileConfig { @@ -140,7 +143,7 @@ impl FileConfigEnum { }, FileConfigEnum::Config(cfg) => mem::take(cfg), }; - res.custom.init_parsing()?; + res.custom.init_parsing(cache)?; Ok(Some(res)) } @@ -218,20 +221,22 @@ pub struct FileConfigSource { pub async fn resolve_files( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> MartinResult { - resolve_int(config, idr, extension) + resolve_int(config, idr, cache, extension) .map_err(crate::MartinError::from) .await } async fn resolve_int( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(cache)? else { return Ok(TileInfoSources::default()); }; diff --git a/martin/src/lib.rs b/martin/src/lib.rs index c0c9d472..ef4aad9f 100644 --- a/martin/src/lib.rs +++ b/martin/src/lib.rs @@ -10,7 +10,7 @@ pub use source::{CatalogSourceEntry, Source, Tile, TileData, TileSources, UrlQue mod utils; pub use utils::{ append_rect, decode_brotli, decode_gzip, IdResolver, MartinError, MartinResult, OptBoolObj, - OptOneMany, TileCoord, TileRect, + OptOneMany, TileCoord, TileRect, NO_MAIN_CACHE, }; pub mod args; diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index ec651453..3a9ddf0d 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -9,7 +9,6 @@ use std::sync::Arc; use async_trait::async_trait; use log::{trace, warn}; use martin_tile_utils::{Encoding, Format, TileInfo}; -use moka::future::Cache; use pmtiles::async_reader::AsyncPmTilesReader; use pmtiles::cache::{DirCacheResult, DirectoryCache}; use pmtiles::http::HttpBackend; @@ -24,20 +23,20 @@ use crate::config::UnrecognizedValues; use crate::file_config::FileError::{InvalidMetadata, InvalidUrlMetadata, IoError}; use crate::file_config::{ConfigExtras, FileError, FileResult, SourceConfigExtras}; use crate::source::UrlQuery; +use crate::utils::cache::get_cached_value; +use crate::utils::{CacheKey, CacheValue, OptMainCache}; use crate::{MartinResult, Source, TileCoord, TileData}; -type PmtCacheObject = Cache<(usize, usize), Directory>; - #[derive(Clone, Debug)] pub struct PmtCache { id: usize, - /// (id, offset) -> Directory, or None to disable caching - cache: Option, + /// Storing (id, offset) -> Directory, or None to disable caching + cache: OptMainCache, } impl PmtCache { #[must_use] - pub fn new(id: usize, cache: Option) -> Self { + pub fn new(id: usize, cache: OptMainCache) -> Self { Self { id, cache } } } @@ -45,17 +44,23 @@ impl PmtCache { #[async_trait] impl DirectoryCache for PmtCache { async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { - if let Some(cache) = &self.cache { - if let Some(dir) = cache.get(&(self.id, offset)).await { - return dir.find_tile_id(tile_id).into(); - } + if let Some(dir) = get_cached_value!(&self.cache, CacheValue::PmtDirectory, { + CacheKey::PmtDirectory(self.id, offset) + }) { + dir.find_tile_id(tile_id).into() + } else { + DirCacheResult::NotCached } - DirCacheResult::NotCached } async fn insert_dir(&self, offset: usize, directory: Directory) { if let Some(cache) = &self.cache { - cache.insert((self.id, offset), directory).await; + cache + .insert( + CacheKey::PmtDirectory(self.id, offset), + CacheValue::PmtDirectory(directory), + ) + .await; } } } @@ -63,8 +68,6 @@ impl DirectoryCache for PmtCache { #[serde_with::skip_serializing_none] #[derive(Debug, Default, Serialize, Deserialize)] pub struct PmtConfig { - pub dir_cache_size_mb: Option, - #[serde(flatten)] pub unrecognized: UnrecognizedValues, @@ -78,12 +81,12 @@ pub struct PmtConfig { pub next_cache_id: AtomicUsize, #[serde(skip)] - pub cache: Option, + pub cache: OptMainCache, } impl PartialEq for PmtConfig { fn eq(&self, other: &Self) -> bool { - self.dir_cache_size_mb == other.dir_cache_size_mb && self.unrecognized == other.unrecognized + self.unrecognized == other.unrecognized } } @@ -91,7 +94,6 @@ impl Clone for PmtConfig { fn clone(&self) -> Self { // State is not shared between clones, only the serialized config Self { - dir_cache_size_mb: self.dir_cache_size_mb, unrecognized: self.unrecognized.clone(), ..Default::default() } @@ -107,24 +109,17 @@ impl PmtConfig { } impl ConfigExtras for PmtConfig { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, cache: OptMainCache) -> FileResult<()> { assert!(self.client.is_none()); assert!(self.cache.is_none()); self.client = Some(Client::new()); + self.cache = cache; - // Allow cache size to be disabled with 0 - let dir_cache_size = self.dir_cache_size_mb.unwrap_or(32) * 1024 * 1024; - if dir_cache_size > 0 { - self.cache = Some( - Cache::builder() - .weigher(|_key, value: &Directory| -> u32 { - value.get_approx_byte_size().try_into().unwrap_or(u32::MAX) - }) - .max_capacity(dir_cache_size) - .build(), - ); + if self.unrecognized.contains_key("dir_cache_size_mb") { + warn!("dir_cache_size_mb is no longer used. Instead, use cache_size_mb param in the root of the config file."); } + Ok(()) } diff --git a/martin/src/source.rs b/martin/src/source.rs index 01c2bc2b..ba04293b 100644 --- a/martin/src/source.rs +++ b/martin/src/source.rs @@ -167,6 +167,7 @@ mod tests { } } +#[derive(Debug, Clone)] pub struct Tile { pub data: TileData, pub info: TileInfo, diff --git a/martin/src/sprites/mod.rs b/martin/src/sprites/mod.rs index fa81f179..7a0fc8cc 100644 --- a/martin/src/sprites/mod.rs +++ b/martin/src/sprites/mod.rs @@ -77,7 +77,7 @@ pub struct SpriteSources(HashMap); impl SpriteSources { pub fn resolve(config: &mut FileConfigEnum) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(None)? else { return Ok(Self::default()); }; diff --git a/martin/src/srv/fonts.rs b/martin/src/srv/fonts.rs old mode 100755 new mode 100644 diff --git a/martin/src/srv/mod.rs b/martin/src/srv/mod.rs index ea94de2e..b9ad026f 100644 --- a/martin/src/srv/mod.rs +++ b/martin/src/srv/mod.rs @@ -1,14 +1,14 @@ mod config; pub use config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT}; +#[cfg(feature = "fonts")] +mod fonts; + mod server; pub use server::{new_server, router, Catalog, RESERVED_KEYWORDS}; mod tiles; -pub use tiles::{get_tile_content, get_tile_response, TileRequest}; - -#[cfg(feature = "fonts")] -mod fonts; +pub use tiles::{DynTileSource, TileRequest}; mod tiles_info; pub use tiles_info::{merge_tilejson, SourceIDsRequest}; diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index 377638b6..4fe0e4d9 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -112,7 +112,9 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server .allow_any_origin() .allowed_methods(vec!["GET"]); - let app = App::new().app_data(Data::new(state.tiles.clone())); + let app = App::new() + .app_data(Data::new(state.tiles.clone())) + .app_data(Data::new(state.cache.clone())); #[cfg(feature = "sprites")] let app = app.app_data(Data::new(state.sprites.clone())); diff --git a/martin/src/srv/sprites.rs b/martin/src/srv/sprites.rs index c3275bbe..55a61a81 100644 --- a/martin/src/srv/sprites.rs +++ b/martin/src/srv/sprites.rs @@ -4,28 +4,18 @@ use actix_web::error::ErrorNotFound; use actix_web::http::header::ContentType; use actix_web::web::{Data, Path}; use actix_web::{middleware, route, HttpResponse, Result as ActixResult}; +use spreet::Spritesheet; use crate::sprites::{SpriteError, SpriteSources}; use crate::srv::server::map_internal_error; use crate::srv::SourceIDsRequest; -pub fn map_sprite_error(e: SpriteError) -> actix_web::Error { - use SpriteError::SpriteNotFound; - match e { - SpriteNotFound(_) => ErrorNotFound(e.to_string()), - _ => map_internal_error(e), - } -} - #[route("/sprite/{source_ids}.png", method = "GET", method = "HEAD")] async fn get_sprite_png( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites - .get_sprites(&path.source_ids) - .await - .map_err(map_sprite_error)?; + let sheet = get_sprite(&path, &sprites).await?; Ok(HttpResponse::Ok() .content_type(ContentType::png()) .body(sheet.encode_png().map_err(map_internal_error)?)) @@ -41,9 +31,16 @@ async fn get_sprite_json( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites - .get_sprites(&path.source_ids) - .await - .map_err(map_sprite_error)?; + let sheet = get_sprite(&path, &sprites).await?; Ok(HttpResponse::Ok().json(sheet.get_index())) } + +async fn get_sprite(path: &SourceIDsRequest, sprites: &SpriteSources) -> ActixResult { + sprites + .get_sprites(&path.source_ids) + .await + .map_err(|e| match e { + SpriteError::SpriteNotFound(_) => ErrorNotFound(e.to_string()), + _ => map_internal_error(e), + }) +} diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs old mode 100644 new mode 100755 index 32463a6f..f2582996 --- a/martin/src/srv/tiles.rs +++ b/martin/src/srv/tiles.rs @@ -6,13 +6,18 @@ use actix_web::http::header::{ use actix_web::web::{Data, Path, Query}; use actix_web::{route, HttpMessage, HttpRequest, HttpResponse, Result as ActixResult}; use futures::future::try_join_all; +use log::trace; use martin_tile_utils::{Encoding, Format, TileInfo}; use serde::Deserialize; use crate::source::{Source, TileSources, UrlQuery}; use crate::srv::server::map_internal_error; -use crate::utils::{decode_brotli, decode_gzip, encode_brotli, encode_gzip}; -use crate::{Tile, TileCoord}; +use crate::utils::cache::get_or_insert_cached_value; +use crate::utils::{ + decode_brotli, decode_gzip, encode_brotli, encode_gzip, CacheKey, CacheValue, MainCache, + OptMainCache, +}; +use crate::{Tile, TileCoord, TileData}; static SUPPORTED_ENCODINGS: &[HeaderEnc] = &[ HeaderEnc::brotli(), @@ -33,125 +38,165 @@ async fn get_tile( req: HttpRequest, path: Path, sources: Data, + cache: Data, ) -> ActixResult { - let xyz = TileCoord { + let src = DynTileSource::new( + sources.as_ref(), + &path.source_ids, + Some(path.z), + req.query_string(), + req.get_header::(), + cache.as_ref().as_ref(), + )?; + + src.get_http_response(TileCoord { z: path.z, x: path.x, y: path.y, - }; - - let source_ids = &path.source_ids; - let query = req.query_string(); - let encodings = req.get_header::(); - - get_tile_response(sources.as_ref(), xyz, source_ids, query, encodings).await -} - -pub async fn get_tile_response( - sources: &TileSources, - xyz: TileCoord, - source_ids: &str, - query: &str, - encodings: Option, -) -> ActixResult { - let (sources, use_url_query, info) = sources.get_sources(source_ids, Some(xyz.z))?; - - let query = use_url_query.then_some(query); - let tile = get_tile_content(sources.as_slice(), info, xyz, query, encodings.as_ref()).await?; - - Ok(if tile.data.is_empty() { - HttpResponse::NoContent().finish() - } else { - let mut response = HttpResponse::Ok(); - response.content_type(tile.info.format.content_type()); - if let Some(val) = tile.info.encoding.content_encoding() { - response.insert_header((CONTENT_ENCODING, val)); - } - response.body(tile.data) }) + .await } -pub async fn get_tile_content( - sources: &[&dyn Source], - info: TileInfo, - xyz: TileCoord, - query: Option<&str>, - encodings: Option<&AcceptEncoding>, -) -> ActixResult { - if sources.is_empty() { - return Err(ErrorNotFound("No valid sources found")); - } - let query_str = query.filter(|v| !v.is_empty()); - let query = match query_str { - Some(v) => Some(Query::::from_query(v)?.into_inner()), - None => None, - }; +pub struct DynTileSource<'a> { + pub sources: Vec<&'a dyn Source>, + pub info: TileInfo, + pub query_str: Option<&'a str>, + pub query_obj: Option, + pub encodings: Option, + pub cache: Option<&'a MainCache>, +} - let mut tiles = try_join_all(sources.iter().map(|s| s.get_tile(xyz, query.as_ref()))) +impl<'a> DynTileSource<'a> { + pub fn new( + sources: &'a TileSources, + source_ids: &str, + zoom: Option, + query: &'a str, + encodings: Option, + cache: Option<&'a MainCache>, + ) -> ActixResult { + let (sources, use_url_query, info) = sources.get_sources(source_ids, zoom)?; + + if sources.is_empty() { + return Err(ErrorNotFound("No valid sources found")); + } + + let mut query_obj = None; + let mut query_str = None; + if use_url_query && !query.is_empty() { + query_obj = Some(Query::::from_query(query)?.into_inner()); + query_str = Some(query); + } + + Ok(Self { + sources, + info, + query_str, + query_obj, + encodings, + cache, + }) + } + + pub async fn get_http_response(&self, xyz: TileCoord) -> ActixResult { + let tile = self.get_tile_content(xyz).await?; + + Ok(if tile.data.is_empty() { + HttpResponse::NoContent().finish() + } else { + let mut response = HttpResponse::Ok(); + response.content_type(tile.info.format.content_type()); + if let Some(val) = tile.info.encoding.content_encoding() { + response.insert_header((CONTENT_ENCODING, val)); + } + response.body(tile.data) + }) + } + + pub async fn get_tile_content(&self, xyz: TileCoord) -> ActixResult { + let mut tiles = try_join_all(self.sources.iter().map(|s| async { + get_or_insert_cached_value!( + self.cache, + CacheValue::Tile, + s.get_tile(xyz, self.query_obj.as_ref()), + { + let id = s.get_id().to_owned(); + if let Some(query_str) = self.query_str { + CacheKey::TileWithQuery(id, xyz, query_str.to_owned()) + } else { + CacheKey::Tile(id, xyz) + } + } + ) + })) .await .map_err(map_internal_error)?; - let mut layer_count = 0; - let mut last_non_empty_layer = 0; - for (idx, tile) in tiles.iter().enumerate() { - if !tile.is_empty() { - layer_count += 1; - last_non_empty_layer = idx; + let mut layer_count = 0; + let mut last_non_empty_layer = 0; + for (idx, tile) in tiles.iter().enumerate() { + if !tile.is_empty() { + layer_count += 1; + last_non_empty_layer = idx; + } } + + // Minor optimization to prevent concatenation if there are less than 2 tiles + let data = match layer_count { + 1 => tiles.swap_remove(last_non_empty_layer), + 0 => return Ok(Tile::new(Vec::new(), self.info)), + _ => { + // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level + // TODO: can zlib, brotli, or zstd be concatenated? + // TODO: implement decompression step for other concatenate-able formats + let can_join = self.info.format == Format::Mvt + && (self.info.encoding == Encoding::Uncompressed + || self.info.encoding == Encoding::Gzip); + if !can_join { + return Err(ErrorBadRequest(format!( + "Can't merge {} tiles. Make sure there is only one non-empty tile source at zoom level {}", + self.info, + xyz.z + )))?; + } + tiles.concat() + } + }; + + // decide if (re-)encoding of the tile data is needed, and recompress if so + self.recompress(data) } - // Minor optimization to prevent concatenation if there are less than 2 tiles - let data = match layer_count { - 1 => tiles.swap_remove(last_non_empty_layer), - 0 => return Ok(Tile::new(Vec::new(), info)), - _ => { - // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level - // TODO: can zlib, brotli, or zstd be concatenated? - // TODO: implement decompression step for other concatenate-able formats - let can_join = info.format == Format::Mvt - && (info.encoding == Encoding::Uncompressed || info.encoding == Encoding::Gzip); - if !can_join { - return Err(ErrorBadRequest(format!( - "Can't merge {info} tiles. Make sure there is only one non-empty tile source at zoom level {}", - xyz.z - )))?; - } - tiles.concat() - } - }; - - // decide if (re-)encoding of the tile data is needed, and recompress if so - let tile = recompress(Tile::new(data, info), encodings)?; - - Ok(tile) -} - -fn recompress(mut tile: Tile, accept_enc: Option<&AcceptEncoding>) -> ActixResult { - if let Some(accept_enc) = accept_enc { - if tile.info.encoding.is_encoded() { - // already compressed, see if we can send it as is, or need to re-compress - if !accept_enc.iter().any(|e| { - if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { - to_encoding(enc) == Some(tile.info.encoding) - } else { - false + fn recompress(&self, tile: TileData) -> ActixResult { + let mut tile = Tile::new(tile, self.info); + if let Some(accept_enc) = &self.encodings { + if self.info.encoding.is_encoded() { + // already compressed, see if we can send it as is, or need to re-compress + if !accept_enc.iter().any(|e| { + if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { + to_encoding(enc) == Some(tile.info.encoding) + } else { + false + } + }) { + // need to re-compress the tile - uncompress it first + tile = decode(tile)?; } - }) { - // need to re-compress the tile - uncompress it first - tile = decode(tile)?; } - } - if tile.info.encoding == Encoding::Uncompressed { - // only apply compression if the content supports it - if let Some(HeaderEnc::Known(enc)) = accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) { - // (re-)compress the tile into the preferred encoding - tile = encode(tile, enc)?; + if tile.info.encoding == Encoding::Uncompressed { + // only apply compression if the content supports it + if let Some(HeaderEnc::Known(enc)) = + accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) + { + // (re-)compress the tile into the preferred encoding + tile = encode(tile, enc)?; + } } + Ok(tile) + } else { + // no accepted-encoding header, decode the tile if compressed + decode(tile) } - Ok(tile) - } else { - // no accepted-encoding header, decode the tile if compressed - decode(tile) } } @@ -189,7 +234,7 @@ fn decode(tile: Tile) -> ActixResult { }) } -fn to_encoding(val: ContentEncoding) -> Option { +pub fn to_encoding(val: ContentEncoding) -> Option { Some(match val { ContentEncoding::Identity => Encoding::Uncompressed, ContentEncoding::Gzip => Encoding::Gzip, @@ -233,15 +278,9 @@ mod tests { ("empty,non-empty", vec![1_u8, 2, 3]), ("empty,non-empty,empty", vec![1_u8, 2, 3]), ] { - let (src, _, info) = sources.get_sources(source_id, None).unwrap(); + let src = DynTileSource::new(&sources, source_id, None, "", None, None).unwrap(); let xyz = TileCoord { z: 0, x: 0, y: 0 }; - assert_eq!( - expected, - &get_tile_content(src.as_slice(), info, xyz, None, None) - .await - .unwrap() - .data - ); + assert_eq!(expected, &src.get_tile_content(xyz).await.unwrap().data); } } } diff --git a/martin/src/srv/tiles_info.rs b/martin/src/srv/tiles_info.rs old mode 100644 new mode 100755 diff --git a/martin/src/utils/cache.rs b/martin/src/utils/cache.rs new file mode 100755 index 00000000..ff442156 --- /dev/null +++ b/martin/src/utils/cache.rs @@ -0,0 +1,91 @@ +use moka::future::Cache; +use pmtiles::Directory; + +use crate::{TileCoord, TileData}; + +pub type MainCache = Cache; +pub type OptMainCache = Option; +pub const NO_MAIN_CACHE: OptMainCache = None; + +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum CacheKey { + /// (pmtiles_id, offset) + PmtDirectory(usize, usize), + /// (source_id, xyz) + Tile(String, TileCoord), + /// (source_id, xyz, url_query) + TileWithQuery(String, TileCoord, String), +} + +#[derive(Debug, Clone)] +pub enum CacheValue { + Tile(TileData), + PmtDirectory(Directory), +} + +macro_rules! trace_cache { + ($typ: literal, $cache: expr, $key: expr) => { + trace!( + "Cache {} for {:?} in {:?} that has {} entries taking up {} space", + $typ, + $key, + $cache.name(), + $cache.entry_count(), + $cache.weighted_size(), + ); + }; +} + +macro_rules! from_cache_value { + ($value_type: path, $data: expr, $key: expr) => { + if let $value_type(data) = $data { + data + } else { + panic!("Unexpected value type {:?} for key {:?} cache", $data, $key) + } + }; +} +#[cfg(feature = "pmtiles")] +macro_rules! get_cached_value { + ($cache: expr, $value_type: path, $make_key: expr) => { + if let Some(cache) = $cache { + let key = $make_key; + if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + Some($crate::utils::cache::from_cache_value!( + $value_type, + data, + key + )) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + None + } + } else { + None + } + }; +} + +macro_rules! get_or_insert_cached_value { + ($cache: expr, $value_type: path, $make_item:expr, $make_key: expr) => {{ + if let Some(cache) = $cache { + let key = $make_key; + Ok(if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + $crate::utils::cache::from_cache_value!($value_type, data, key) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + let data = $make_item.await?; + cache.insert(key, $value_type(data.clone())).await; + data + }) + } else { + $make_item.await + } + }}; +} + +#[cfg(feature = "pmtiles")] +pub(crate) use get_cached_value; +pub(crate) use {from_cache_value, get_or_insert_cached_value, trace_cache}; diff --git a/martin/src/utils/mod.rs b/martin/src/utils/mod.rs index e0444cde..306da7e5 100644 --- a/martin/src/utils/mod.rs +++ b/martin/src/utils/mod.rs @@ -1,3 +1,6 @@ +pub(crate) mod cache; +pub use cache::{CacheKey, CacheValue, MainCache, OptMainCache, NO_MAIN_CACHE}; + mod cfg_containers; pub use cfg_containers::{OptBoolObj, OptOneMany}; diff --git a/martin/src/utils/xyz.rs b/martin/src/utils/xyz.rs index 421ec6df..1a968207 100644 --- a/martin/src/utils/xyz.rs +++ b/martin/src/utils/xyz.rs @@ -1,6 +1,6 @@ use std::fmt::{Display, Formatter}; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct TileCoord { pub z: u8, pub x: u32, diff --git a/martin/tests/mb_server_test.rs b/martin/tests/mb_server_test.rs index b20b64f5..b38d490b 100644 --- a/martin/tests/mb_server_test.rs +++ b/martin/tests/mb_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index ebbc5114..55b7f37a 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -26,6 +26,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) @@ -1086,6 +1087,7 @@ tables: .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pmt_server_test.rs b/martin/tests/pmt_server_test.rs index 5b1a4d19..9a6e7ce4 100644 --- a/martin/tests/pmt_server_test.rs +++ b/martin/tests/pmt_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs index 334b1179..02116664 100644 --- a/martin/tests/utils/pg_utils.rs +++ b/martin/tests/utils/pg_utils.rs @@ -1,6 +1,6 @@ use indoc::formatdoc; pub use martin::args::Env; -use martin::{Config, IdResolver, ServerState, Source}; +use martin::{Config, ServerState, Source}; use crate::mock_cfg; @@ -22,7 +22,7 @@ pub fn mock_pgcfg(yaml: &str) -> Config { #[allow(dead_code)] pub async fn mock_sources(mut config: Config) -> MockSource { - let res = config.resolve(IdResolver::default()).await; + let res = config.resolve().await; let res = res.unwrap_or_else(|e| panic!("Failed to resolve config {config:?}: {e}")); (res, config) } diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index f3021360..1f22df56 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/tests/config.yaml b/tests/config.yaml index 7a3f2848..70b08a05 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -8,6 +8,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 8 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string @@ -166,7 +169,6 @@ postgres: pmtiles: - dir_cache_size_mb: 100 paths: - http://localhost:5412/webp2.pmtiles sources: diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 47379599..308490c1 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -1,3 +1,4 @@ +cache_size_mb: 8 keep_alive: 75 listen_addresses: localhost:3111 worker_processes: 1 @@ -165,7 +166,6 @@ pmtiles: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles pmt2: http://localhost:5412/webp2.pmtiles webp2: http://localhost:5412/webp2.pmtiles - dir_cache_size_mb: 100 sprites: paths: tests/fixtures/sprites/src1 sources: