diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a770564..cb1e5be8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,8 @@ jobs: tool: cross - name: Checkout sources uses: actions/checkout@v4 + with: + set-safe-directory: false - uses: Swatinem/rust-cache@v2 if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' - name: Init database @@ -162,6 +164,13 @@ jobs: mkdir -p target_releases/linux/amd64 mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ + - name: Start HTTP Server + run: | + docker run --rm -d \ + --name fileserver \ + -p 5412:80 \ + -v ${{ github.workspace }}/tests/fixtures/pmtiles2:/usr/share/nginx/html \ + nginx:alpine - name: Build linux/arm64 Docker image uses: docker/build-push-action@v5 # https://github.com/docker/build-push-action @@ -229,6 +238,9 @@ jobs: tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} platforms: linux/amd64,linux/arm64 + - name: Stop HTTP Server + if: always() + run: docker stop fileserver build: name: Build ${{ matrix.target }} @@ -302,6 +314,61 @@ jobs: uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' + - name: Install Docker (MacOS) + if: runner.os == 'macos' + run: brew install docker && colima start + - name: Start HTTP Server (with Docker) + if: runner.os != 'windows' + run: | + docker run --rm -d \ + --name fileserver \ + -p 5412:80 \ + -v ${{ github.workspace }}/tests/fixtures/pmtiles2:/usr/share/nginx/html \ + nginx:alpine + - name: Start HTTP Server (Windows, no Docker) + if: runner.os == 'windows' + shell: pwsh + run: | + $nginxConf = @" + worker_processes 1; + events { + worker_connections 1024; + } + http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + server { + listen 5412; + server_name localhost; + location / { + root $($PWD.Path)/tests/fixtures/pmtiles2; + index index.html index.htm; + } + } + } + "@ + + # Replace the default Nginx configuration file + echo "$nginxConf" + Set-Content -Path "C:\tools\nginx-1.25.3\conf\nginx.conf" -Value $nginxConf + Get-Content -Path "C:\tools\nginx-1.25.3\conf\nginx.conf" + + # Start Nginx + #Get-Service -ErrorAction SilentlyContinue + #Get-CimInstance -ClassName Win32_Service + Set-Service nginx -StartupType manual + Start-Service nginx + #Start-Process -FilePath "C:\tools\nginx-1.25.3\nginx.exe" + dir C:\tools\nginx-1.25.3\logs\ + Start-Sleep -Seconds 5 + netstat -a + + # Print Nginx Error Logs (on Windows systems) + #nginx -t + Get-Content -Path "C:\tools\nginx-1.25.3\logs\error.log" + dir D:\a\martin\martin\ - name: Start postgres uses: nyurik/action-setup-postgis@v1.1 id: pg @@ -355,7 +422,10 @@ jobs: tests/test.sh env: DATABASE_URL: ${{ steps.pg.outputs.connection-uri }} - - name: Save test output on failure + - name: Stop HTTP Server (with Docker) + if: runner.os != 'windows' + run: docker stop fileserver + - name: Save test output (on error) if: failure() uses: actions/upload-artifact@v3 with: @@ -422,6 +492,13 @@ jobs: uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' + - name: Start HTTP Server + run: | + docker run --rm -d \ + --name fileserver \ + -p 5412:80 \ + -v ${{ github.workspace }}/tests/fixtures/pmtiles2:/usr/share/nginx/html \ + nginx:alpine - name: Init database run: tests/fixtures/initdb.sh env: @@ -483,7 +560,10 @@ jobs: cargo clean env: DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }} - - name: On error, save test output + - name: Stop HTTP Server + if: always() + run: docker stop fileserver + - name: Save test output (on error) if: failure() uses: actions/upload-artifact@v3 with: diff --git a/Cargo.lock b/Cargo.lock index 9f3f0460..df3123c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -367,6 +367,15 @@ dependencies = [ "zstd-safe 7.0.0", ] +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener", +] + [[package]] name = "async-recursion" version = "1.0.5" @@ -519,6 +528,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "bytecount" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" + [[package]] name = "bytemuck" version = "1.14.0" @@ -546,12 +561,43 @@ dependencies = [ "bytes", ] +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + [[package]] name = "cargo-husky" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" +[[package]] +name = "cargo-platform" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cast" version = "0.3.0" @@ -1092,6 +1138,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -1554,6 +1609,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.8.0" @@ -1572,6 +1638,44 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1661,6 +1765,12 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "is-terminal" version = "0.4.9" @@ -1842,6 +1952,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "martin" version = "0.11.6" @@ -1868,6 +1987,7 @@ dependencies = [ "log", "martin-tile-utils", "mbtiles", + "moka", "num_cpus", "pbf_font_tools", "pmtiles", @@ -1875,6 +1995,7 @@ dependencies = [ "postgres", "postgres-protocol", "regex", + "reqwest", "rustls", "rustls-native-certs", "rustls-pemfile", @@ -1889,6 +2010,7 @@ dependencies = [ "tilejson", "tokio", "tokio-postgres-rustls", + "url", ] [[package]] @@ -2005,6 +2127,30 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "moka" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8017ec3548ffe7d4cef7ac0e12b044c01164a74c0f3119420faeaf13490ad8b" +dependencies = [ + "async-lock", + "async-trait", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "futures-util", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "multimap" version = "0.9.1" @@ -2362,6 +2508,7 @@ dependencies = [ "bytes", "fmmap", "hilbert_2d", + "reqwest", "serde", "serde_json", "thiserror", @@ -2573,6 +2720,33 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.33" @@ -2618,6 +2792,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "rayon" version = "1.8.0" @@ -2694,6 +2877,46 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" +[[package]] +name = "reqwest" +version = "0.11.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "resvg" version = "0.36.0" @@ -2992,6 +3215,9 @@ name = "semver" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -3177,6 +3403,21 @@ dependencies = [ "num", ] +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.9" @@ -3571,6 +3812,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -3825,6 +4093,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.40" @@ -3866,6 +4140,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "triomphe" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "ttf-parser" version = "0.19.2" @@ -3884,6 +4170,15 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.14" @@ -4060,6 +4355,9 @@ name = "uuid" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "getrandom", +] [[package]] name = "varint-rs" @@ -4089,6 +4387,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -4120,6 +4427,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.89" @@ -4425,6 +4744,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 6136a223..ff918e8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,14 +50,16 @@ json-patch = "1.2" log = "0.4" martin-tile-utils = { path = "./martin-tile-utils", version = "0.3.0" } mbtiles = { path = "./mbtiles", version = "0.8.0" } +moka = { version = "0.12", features = ["future"] } num_cpus = "1" pbf_font_tools = { version = "2.5.0", features = ["freetype"] } -pmtiles = { version = "0.5", features = ["mmap-async-tokio", "tilejson"] } +pmtiles = { version = "0.5", features = ["http-async", "mmap-async-tokio", "tilejson"] } postgis = "0.9" postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] } postgres-protocol = "0.6" pretty_assertions = "1" regex = "1" +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-native-roots"] } rstest = "0.18" rustls = { version = "0.21", features = ["dangerous_configuration"] } rustls-native-certs = "0.6" @@ -77,6 +79,7 @@ tile-grid = "0.5" tilejson = "0.4" tokio = { version = "1", features = ["macros"] } tokio-postgres-rustls = "0.10" +url = "2.5" [profile.dev.package] # See https://github.com/launchbadge/sqlx#compile-time-verification diff --git a/README.md b/README.md index d65ebf78..81fb3b42 100755 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![CI build](https://github.com/maplibre/martin/actions/workflows/ci.yml/badge.svg)](https://github.com/maplibre/martin/actions) [![](https://img.shields.io/badge/Slack-%23maplibre--martin-2EB67D?logo=slack)](https://slack.openstreetmap.us/) -Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new), and [MBTile](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust). +Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) (local or remote), and [MBTile](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust). Additionally, there are [several tools](https://maplibre.org/martin/tools.html) for generating tiles in bulk from any Martin-supported sources (similar to `tilelive-copy`), copying tiles between MBTiles files, creating deltas (patches) and applying them, and validating MBTiles files. diff --git a/debian/config.yaml b/debian/config.yaml index 4fad0cee..f449ba47 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -20,8 +20,10 @@ worker_processes: 8 # paths: # - /dir-path # - /path/to/pmtiles.pmtiles +# - http://example.org/pmtiles.pmtiles # sources: # pm-src1: /path/to/pmtiles1.pmtiles +# pm-web1: http://example.org/pmtiles1.pmtiles # mbtiles: # paths: diff --git a/docker-compose.yml b/docker-compose.yml index 673b4d8d..a29dd5c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,14 @@ version: '3' services: + fileserver: + image: nginx:alpine + restart: unless-stopped + ports: + - '5412:80' + volumes: + - ./tests/fixtures/pmtiles2:/usr/share/nginx/html + db-is-ready: # This should match the version of postgres used in the CI workflow image: postgis/postgis:14-3.3-alpine diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 5e7257ab..008b1a95 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -153,16 +153,20 @@ postgres: # Values may be integers or floating point numbers. bounds: [-180.0, -90.0, 180.0, 90.0] -# Publish PMTiles files +# Publish PMTiles files from local disk or proxy to a web server pmtiles: paths: # scan this whole dir, matching all *.pmtiles files - /dir-path # specific pmtiles file will be published as a pmt source (filename without extension) - /path/to/pmt.pmtiles + # A web server with a PMTiles file that supports range requests + - https://example.org/path/tiles.pmtiles sources: # named source matching source name to a single file pm-src1: /path/to/pmt.pmtiles + # A named source to a web server with a PMTiles file that supports range requests + pm-web2: https://example.org/path/tiles.pmtiles # Publish MBTiles files mbtiles: diff --git a/docs/src/installation.md b/docs/src/installation.md index 1fa00d33..706bda63 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -36,7 +36,7 @@ brew install martin ### Docker -Martin is also available as a [Docker image](https://ghcr.io/maplibre/martin). You could either share a configuration file from the host with the container via the `-v` param, or you can let Martin auto-discover all sources e.g. by passing `DATABASE_URL` or specifying the .mbtiles/.pmtiles files. +Martin is also available as a [Docker image](https://ghcr.io/maplibre/martin). You could either share a configuration file from the host with the container via the `-v` param, or you can let Martin auto-discover all sources e.g. by passing `DATABASE_URL` or specifying the .mbtiles/.pmtiles files or URLs to .pmtiles. ```shell export PGPASSWORD=postgres # secret! diff --git a/docs/src/introduction.md b/docs/src/introduction.md index 97588910..eb0095a3 100644 --- a/docs/src/introduction.md +++ b/docs/src/introduction.md @@ -1,6 +1,6 @@ ![Martin](https://raw.githubusercontent.com/maplibre/martin/main/logo.png) -Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTiles](https://protomaps.com/blog/pmtiles-v3-whats-new), and [MBTiles](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust). +Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTiles](https://protomaps.com/blog/pmtiles-v3-whats-new) (local or remote), and [MBTiles](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust). See also [Martin demo site](https://martin.maplibre.org/) diff --git a/docs/src/run-with-docker.md b/docs/src/run-with-docker.md index 3f4714f5..75ae7e48 100644 --- a/docs/src/run-with-docker.md +++ b/docs/src/run-with-docker.md @@ -7,7 +7,7 @@ You can use official Docker image [`ghcr.io/maplibre/martin`](https://ghcr.io/ma ```shell docker run \ -p 3000:3000 \ - -e DATABASE_URL=postgresql://postgres@postgres.example.com/db \ + -e DATABASE_URL=postgresql://postgres@postgres.example.org/db \ ghcr.io/maplibre/martin ``` diff --git a/docs/src/sources-files.md b/docs/src/sources-files.md index 40ec6b5e..658453c2 100644 --- a/docs/src/sources-files.md +++ b/docs/src/sources-files.md @@ -1,9 +1,9 @@ ## MBTiles and PMTiles File Sources -Martin can serve any type of tiles from [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) and [MBTile](https://github.com/mapbox/mbtiles-spec) files. To serve a file from CLI, simply put the path to the file or the directory with `*.mbtiles` or `*.pmtiles` files. For example: +Martin can serve any type of tiles from [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) and [MBTile](https://github.com/mapbox/mbtiles-spec) files. To serve a file from CLI, simply put the path to the file or the directory with `*.mbtiles` or `*.pmtiles` files. A path to PMTiles file may be a URL. For example: ```shell -martin /path/to/mbtiles/file.mbtiles /path/to/directory +martin /path/to/mbtiles/file.mbtiles /path/to/directory https://example.org/path/tiles.pmtiles ``` You may also want to generate a [config file](config-file.md) using the `--save-config my-config.yaml`, and later edit it and use it with `--config my-config.yaml` option. diff --git a/justfile b/justfile index 360620ed..71f0c687 100644 --- a/justfile +++ b/justfile @@ -67,7 +67,7 @@ start-legacy: (docker-up "db-legacy") docker-is-ready # Start a specific test database, e.g. db or db-legacy [private] -docker-up name: +docker-up name: start-pmtiles-server docker-compose up -d {{ name }} # Wait for the test database to be ready @@ -88,6 +88,10 @@ restart: stop: docker-compose down --remove-orphans +# Start test server for testing HTTP pmtiles +start-pmtiles-server: + docker-compose up -d fileserver + # Run benchmark tests bench: start cargo bench diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 2538b985..790abec1 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -80,6 +80,7 @@ json-patch.workspace = true log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true +moka.workspace = true num_cpus.workspace = true pbf_font_tools.workspace = true pmtiles.workspace = true @@ -87,6 +88,7 @@ postgis.workspace = true postgres-protocol.workspace = true postgres.workspace = true regex.workspace = true +reqwest.workspace = true rustls-native-certs.workspace = true rustls-pemfile.workspace = true rustls.workspace = true @@ -101,6 +103,7 @@ thiserror.workspace = true tilejson.workspace = true tokio = { workspace = true, features = ["io-std"] } tokio-postgres-rustls.workspace = true +url.workspace = true [dev-dependencies] cargo-husky.workspace = true diff --git a/martin/benches/bench.rs b/martin/benches/bench.rs index 55cc82ba..53ddff47 100644 --- a/martin/benches/bench.rs +++ b/martin/benches/bench.rs @@ -16,7 +16,7 @@ struct NullSource { impl NullSource { fn new() -> Self { Self { - tilejson: tilejson! { "https://example.com/".to_string() }, + tilejson: tilejson! { "https://example.org/".to_string() }, } } } diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index e713b72e..90c81133 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use clap::Parser; use log::warn; +use url::Url; use crate::args::connections::Arguments; use crate::args::environment::Env; @@ -89,11 +90,11 @@ impl Args { } if !cli_strings.is_empty() { - config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles"); + config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles", true); } if !cli_strings.is_empty() { - config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles"); + config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles", false); } if !self.extras.sprite.is_empty() { @@ -108,10 +109,29 @@ impl Args { } } -pub fn parse_file_args(cli_strings: &mut Arguments, extension: &str) -> FileConfigEnum { - let paths = cli_strings.process(|v| match PathBuf::try_from(v) { +fn is_url(s: &str, extension: &str) -> bool { + if s.starts_with("http") { + if let Ok(url) = Url::parse(s) { + if url.scheme() == "http" || url.scheme() == "https" { + if let Some(ext) = url.path().rsplit('.').next() { + return ext == extension; + } + } + } + } + false +} + +pub fn parse_file_args( + cli_strings: &mut Arguments, + extension: &str, + allow_url: bool, +) -> FileConfigEnum { + let paths = cli_strings.process(|s| match PathBuf::try_from(s) { Ok(v) => { - if v.is_dir() { + if allow_url && is_url(s, extension) { + Take(v) + } else if v.is_dir() { Share(v) } else if v.is_file() && v.extension().map_or(false, |e| e == extension) { Take(v) diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 55ebe092..1c67545d 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -71,7 +71,7 @@ pub struct CopyArgs { pub url_query: Option, /// Optional accepted encoding parameter as if the browser sent it in the HTTP request. /// If set to multiple values like `gzip,br`, martin-cp will use the first encoding, - /// or re-encode if the tile is already encoded and that encoding is not listed. + /// or re-encode if the tile is already encoded and that encoding is not listed. /// Use `identity` to disable compression. Ignored for non-encodable tiles like PNG and JPEG. #[arg(long, alias = "encodings", default_value = "gzip")] pub encoding: String, diff --git a/martin/src/config.rs b/martin/src/config.rs index d06b1621..37950003 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -11,11 +11,11 @@ use log::info; use serde::{Deserialize, Serialize}; use subst::VariableMap; -use crate::file_config::{resolve_files, FileConfigEnum}; +use crate::file_config::{resolve_files, resolve_files_urls, FileConfigEnum}; use crate::fonts::FontSources; use crate::mbtiles::MbtSource; use crate::pg::PgConfig; -use crate::pmtiles::PmtSource; +use crate::pmtiles::{PmtFileSource, PmtHttpSource}; use crate::source::{TileInfoSources, TileSources}; use crate::sprites::SpriteSources; use crate::srv::SrvConfig; @@ -92,7 +92,8 @@ impl Config { } async fn resolve_tile_sources(&mut self, idr: IdResolver) -> MartinResult { - let new_pmt_src = &mut PmtSource::new_box; + let new_pmt_src = &mut PmtFileSource::new_box; + let new_pmt_url_src = &mut PmtHttpSource::new_url_box; let new_mbt_src = &mut MbtSource::new_box; let mut sources: Vec>>>> = Vec::new(); @@ -102,12 +103,14 @@ impl Config { } if !self.pmtiles.is_empty() { - let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", new_pmt_src); + let cfg = &mut self.pmtiles; + let val = resolve_files_urls(cfg, idr.clone(), "pmtiles", new_pmt_src, new_pmt_url_src); sources.push(Box::pin(val)); } if !self.mbtiles.is_empty() { - let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", new_mbt_src); + let cfg = &mut self.mbtiles; + let val = resolve_files(cfg, idr.clone(), "mbtiles", new_mbt_src); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 161ab3a5..8fe96653 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -1,14 +1,17 @@ use std::collections::{BTreeMap, HashSet}; use std::future::Future; use std::mem; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use futures::TryFutureExt; use log::{info, warn}; use serde::{Deserialize, Serialize}; +use url::Url; use crate::config::{copy_unrecognized_config, UnrecognizedValues}; -use crate::file_config::FileError::{InvalidFilePath, InvalidSourceFilePath, IoError}; +use crate::file_config::FileError::{ + InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError, +}; use crate::source::{Source, TileInfoSources}; use crate::utils::{IdResolver, OptOneMany}; use crate::MartinResult; @@ -24,14 +27,23 @@ pub enum FileError { #[error("Source path is not a file: {}", .0.display())] InvalidFilePath(PathBuf), + #[error("Error {0} while parsing URL {1}")] + InvalidSourceUrl(url::ParseError, String), + #[error("Source {0} uses bad file {}", .1.display())] InvalidSourceFilePath(String, PathBuf), #[error(r"Unable to parse metadata in file {}: {0}", .1.display())] InvalidMetadata(String, PathBuf), + #[error(r"Unable to parse metadata in file {1}: {0}")] + InvalidUrlMetadata(String, Url), + #[error(r#"Unable to aquire connection to file: {0}"#)] AquireConnError(String), + + #[error(r#"PMTiles error {0} processing {1}"#)] + PmtError(pmtiles::PmtError, String), } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -169,6 +181,10 @@ pub struct FileConfigSource { pub path: PathBuf, } +async fn dummy_resolver(_id: String, _url: Url) -> FileResult> { + unreachable!() +} + pub async fn resolve_files( config: &mut FileConfigEnum, idr: IdResolver, @@ -178,19 +194,39 @@ pub async fn resolve_files( where Fut: Future, FileError>>, { - resolve_int(config, idr, extension, new_source) + let dummy = &mut dummy_resolver; + resolve_int(config, idr, extension, false, new_source, dummy) .map_err(crate::MartinError::from) .await } -async fn resolve_int( +pub async fn resolve_files_urls( config: &mut FileConfigEnum, idr: IdResolver, extension: &str, - new_source: &mut impl FnMut(String, PathBuf) -> Fut, + new_source: &mut impl FnMut(String, PathBuf) -> Fut1, + new_url_source: &mut impl FnMut(String, Url) -> Fut2, +) -> MartinResult +where + Fut1: Future, FileError>>, + Fut2: Future, FileError>>, +{ + resolve_int(config, idr, extension, true, new_source, new_url_source) + .map_err(crate::MartinError::from) + .await +} + +async fn resolve_int( + config: &mut FileConfigEnum, + idr: IdResolver, + extension: &str, + parse_urls: bool, + new_source: &mut impl FnMut(String, PathBuf) -> Fut1, + new_url_source: &mut impl FnMut(String, Url) -> Fut2, ) -> FileResult where - Fut: Future, FileError>>, + Fut1: Future, FileError>>, + Fut2: Future, FileError>>, { let Some(cfg) = config.extract_file_config() else { return Ok(TileInfoSources::default()); @@ -203,67 +239,76 @@ where if let Some(sources) = cfg.sources { for (id, source) in sources { - let can = source.abs_path()?; - if !can.is_file() { - // todo: maybe warn instead? - return Err(InvalidSourceFilePath(id.to_string(), can)); + if let Some(url) = parse_url(parse_urls, source.get_path())? { + let dup = !files.insert(source.get_path().clone()); + let dup = if dup { "duplicate " } else { "" }; + let id = idr.resolve(&id, url.to_string()); + configs.insert(id.clone(), source); + results.push(new_url_source(id.clone(), url.clone()).await?); + info!("Configured {dup}source {id} from {}", sanitize_url(&url)); + } else { + let can = source.abs_path()?; + if !can.is_file() { + // todo: maybe warn instead? + return Err(InvalidSourceFilePath(id.to_string(), can)); + } + + let dup = !files.insert(can.clone()); + let dup = if dup { "duplicate " } else { "" }; + let id = idr.resolve(&id, can.to_string_lossy().to_string()); + info!("Configured {dup}source {id} from {}", can.display()); + configs.insert(id.clone(), source.clone()); + results.push(new_source(id, source.into_path()).await?); } - - let dup = !files.insert(can.clone()); - let dup = if dup { "duplicate " } else { "" }; - let id = idr.resolve(&id, can.to_string_lossy().to_string()); - info!("Configured {dup}source {id} from {}", can.display()); - configs.insert(id.clone(), source.clone()); - - let path = match source { - FileConfigSrc::Obj(pmt) => pmt.path, - FileConfigSrc::Path(path) => path, - }; - results.push(new_source(id, path).await?); } } for path in cfg.paths { - let is_dir = path.is_dir(); - let dir_files = if is_dir { - // directories will be kept in the config just in case there are new files - directories.push(path.clone()); - path.read_dir() - .map_err(|e| IoError(e, path.clone()))? - .filter_map(Result::ok) - .filter(|f| { - f.path().extension().filter(|e| *e == extension).is_some() && f.path().is_file() + if let Some(url) = parse_url(parse_urls, &path)? { + let id = url + .path_segments() + .and_then(Iterator::last) + .and_then(|s| { + // Strip extension and trailing dot, or keep the original string + s.strip_suffix(extension) + .and_then(|s| s.strip_suffix('.')) + .or(Some(s)) }) - .map(|f| f.path()) - .collect() - } else if path.is_file() { - vec![path] - } else { - return Err(InvalidFilePath(path.canonicalize().unwrap_or(path))); - }; - for path in dir_files { - let can = path.canonicalize().map_err(|e| IoError(e, path.clone()))?; - if files.contains(&can) { - if !is_dir { - warn!("Ignoring duplicate MBTiles path: {}", can.display()); - } - continue; - } - let id = path.file_stem().map_or_else( - || "_unknown".to_string(), - |s| s.to_string_lossy().to_string(), - ); - let source = FileConfigSrc::Path(path); - let id = idr.resolve(&id, can.to_string_lossy().to_string()); - info!("Configured source {id} from {}", can.display()); - files.insert(can); - configs.insert(id.clone(), source.clone()); + .unwrap_or("pmt_web_source"); - let path = match source { - FileConfigSrc::Obj(pmt) => pmt.path, - FileConfigSrc::Path(path) => path, + let id = idr.resolve(id, url.to_string()); + configs.insert(id.clone(), FileConfigSrc::Path(path)); + results.push(new_url_source(id.clone(), url.clone()).await?); + info!("Configured source {id} from URL {}", sanitize_url(&url)); + } else { + let is_dir = path.is_dir(); + let dir_files = if is_dir { + // directories will be kept in the config just in case there are new files + directories.push(path.clone()); + dir_to_paths(&path, extension)? + } else if path.is_file() { + vec![path] + } else { + return Err(InvalidFilePath(path.canonicalize().unwrap_or(path))); }; - results.push(new_source(id, path).await?); + for path in dir_files { + let can = path.canonicalize().map_err(|e| IoError(e, path.clone()))?; + if files.contains(&can) { + if !is_dir { + warn!("Ignoring duplicate MBTiles path: {}", can.display()); + } + continue; + } + let id = path.file_stem().map_or_else( + || "_unknown".to_string(), + |s| s.to_string_lossy().to_string(), + ); + let id = idr.resolve(&id, can.to_string_lossy().to_string()); + info!("Configured source {id} from {}", can.display()); + files.insert(can); + configs.insert(id.clone(), FileConfigSrc::Path(path.clone())); + results.push(new_source(id, path).await?); + } } } @@ -272,6 +317,41 @@ where Ok(results) } +fn dir_to_paths(path: &Path, extension: &str) -> Result, FileError> { + Ok(path + .read_dir() + .map_err(|e| IoError(e, path.to_path_buf()))? + .filter_map(Result::ok) + .filter(|f| { + f.path().extension().filter(|e| *e == extension).is_some() && f.path().is_file() + }) + .map(|f| f.path()) + .collect()) +} + +fn sanitize_url(url: &Url) -> String { + let mut result = format!("{}://", url.scheme()); + if let Some(host) = url.host_str() { + result.push_str(host); + } + if let Some(port) = url.port() { + result.push(':'); + result.push_str(&port.to_string()); + } + result.push_str(url.path()); + result +} + +fn parse_url(is_enabled: bool, path: &Path) -> Result, FileError> { + if !is_enabled { + return Ok(None); + } + path.to_str() + .filter(|v| v.starts_with("http://") || v.starts_with("https://")) + .map(|v| Url::parse(v).map_err(|e| InvalidSourceUrl(e, v.to_string()))) + .transpose() +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -287,10 +367,14 @@ mod tests { paths: - /dir-path - /path/to/file2.ext + - http://example.org/file.ext sources: pm-src1: /tmp/file.ext pm-src2: path: /tmp/file.ext + pm-src3: https://example.org/file3.ext + pm-src4: + path: https://example.org/file4.ext "}) .unwrap(); let res = cfg.finalize("").unwrap(); @@ -304,6 +388,7 @@ mod tests { vec![ PathBuf::from("/dir-path"), PathBuf::from("/path/to/file2.ext"), + PathBuf::from("http://example.org/file.ext"), ] ); assert_eq!( @@ -318,7 +403,17 @@ mod tests { FileConfigSrc::Obj(FileConfigSource { path: PathBuf::from("/tmp/file.ext"), }) - ) + ), + ( + "pm-src3".to_string(), + FileConfigSrc::Path(PathBuf::from("https://example.org/file3.ext")) + ), + ( + "pm-src4".to_string(), + FileConfigSrc::Obj(FileConfigSource { + path: PathBuf::from("https://example.org/file4.ext"), + }) + ), ])) ); } diff --git a/martin/src/pmtiles/file_pmtiles.rs b/martin/src/pmtiles/file_pmtiles.rs new file mode 100644 index 00000000..42d833f7 --- /dev/null +++ b/martin/src/pmtiles/file_pmtiles.rs @@ -0,0 +1,59 @@ +use std::fmt::{Debug, Display, Formatter}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use log::{trace, warn}; +use martin_tile_utils::{Encoding, Format, TileInfo}; +use pmtiles::async_reader::AsyncPmTilesReader; +use pmtiles::cache::NoCache; +use pmtiles::mmap::MmapBackend; +use pmtiles::{Compression, TileType}; +use tilejson::TileJSON; + +use crate::file_config::FileError::{InvalidMetadata, IoError}; +use crate::file_config::{FileError, FileResult}; +use crate::pmtiles::impl_pmtiles_source; +use crate::source::{Source, UrlQuery}; +use crate::{MartinResult, TileCoord, TileData}; + +impl_pmtiles_source!(PmtFileSource, MmapBackend, NoCache, PathBuf); + +impl PmtFileSource { + pub async fn new_box(id: String, path: PathBuf) -> FileResult> { + Ok(Box::new(PmtFileSource::new(id, path).await?)) + } + + async fn new(id: String, path: PathBuf) -> FileResult { + let backend = MmapBackend::try_from(path.as_path()) + .await + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("{e:?}: Cannot open file {}", path.display()), + ) + }) + .map_err(|e| IoError(e, path.clone()))?; + + let reader = AsyncPmTilesReader::try_from_source(backend).await; + let reader = reader + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("{e:?}: Cannot open file {}", path.display()), + ) + }) + .map_err(|e| IoError(e, path.clone()))?; + + Self::new_int(id, path, reader).await + } + + fn display_path(path: &Path) -> impl Display + '_ { + path.display() + } + + fn metadata_err(message: String, path: PathBuf) -> FileError { + InvalidMetadata(message, path) + } +} diff --git a/martin/src/pmtiles/http_pmtiles.rs b/martin/src/pmtiles/http_pmtiles.rs new file mode 100644 index 00000000..812a220a --- /dev/null +++ b/martin/src/pmtiles/http_pmtiles.rs @@ -0,0 +1,69 @@ +use std::fmt::{Debug, Display, Formatter}; +use std::sync::Arc; + +use async_trait::async_trait; +use log::{trace, warn}; +use martin_tile_utils::{Encoding, Format, TileInfo}; +use moka::future::Cache; +use pmtiles::async_reader::AsyncPmTilesReader; +use pmtiles::cache::{DirCacheResult, DirectoryCache}; +use pmtiles::http::HttpBackend; +use pmtiles::{Compression, Directory, TileType}; +use reqwest::Client; +use tilejson::TileJSON; +use url::Url; + +use crate::file_config::FileError::InvalidUrlMetadata; +use crate::file_config::{FileError, FileResult}; +use crate::pmtiles::impl_pmtiles_source; +use crate::source::{Source, UrlQuery}; +use crate::{MartinResult, TileCoord, TileData}; + +struct PmtCache(Cache); + +impl PmtCache { + fn new() -> Self { + Self(Cache::new(10_000)) + } +} + +#[async_trait] +impl DirectoryCache for PmtCache { + async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { + match self.0.get(&offset).await { + Some(dir) => dir.find_tile_id(tile_id).into(), + None => DirCacheResult::NotCached, + } + } + + async fn insert_dir(&self, offset: usize, directory: Directory) { + self.0.insert(offset, directory).await; + } +} + +impl_pmtiles_source!(PmtHttpSource, HttpBackend, PmtCache, Url); + +impl PmtHttpSource { + pub async fn new_url_box(id: String, url: Url) -> FileResult> { + let client = Client::new(); + let cache = PmtCache::new(); + Ok(Box::new( + PmtHttpSource::new_url(client, cache, id, url).await?, + )) + } + + async fn new_url(client: Client, cache: PmtCache, id: String, url: Url) -> FileResult { + let reader = AsyncPmTilesReader::new_with_cached_url(cache, client, url.clone()).await; + let reader = reader.map_err(|e| FileError::PmtError(e, url.to_string()))?; + + Self::new_int(id, url, reader).await + } + + fn display_path(path: &Url) -> impl Display + '_ { + path + } + + fn metadata_err(message: String, path: Url) -> FileError { + InvalidUrlMetadata(message, path) + } +} diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index 0513a21f..cf3462b4 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -1,155 +1,137 @@ -use std::fmt::{Debug, Formatter}; -use std::io; -use std::path::PathBuf; -use std::sync::Arc; +mod file_pmtiles; +mod http_pmtiles; -use async_trait::async_trait; -use log::{trace, warn}; -use martin_tile_utils::{Encoding, Format, TileInfo}; -use pmtiles::async_reader::AsyncPmTilesReader; -use pmtiles::mmap::MmapBackend; -use pmtiles::{Compression, TileType}; -use tilejson::TileJSON; +pub use file_pmtiles::PmtFileSource; +pub use http_pmtiles::PmtHttpSource; -use crate::file_config::FileError::{InvalidMetadata, IoError}; -use crate::file_config::FileResult; -use crate::source::{Source, TileData, UrlQuery}; -use crate::{MartinResult, TileCoord}; - -#[derive(Clone)] -pub struct PmtSource { - id: String, - path: PathBuf, - pmtiles: Arc>, - tilejson: TileJSON, - tile_info: TileInfo, -} - -impl Debug for PmtSource { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "PmtSource {{ id: {}, path: {:?} }}", self.id, self.path) - } -} - -impl PmtSource { - pub async fn new_box(id: String, path: PathBuf) -> FileResult> { - Ok(Box::new(PmtSource::new(id, path).await?)) - } - - async fn new(id: String, path: PathBuf) -> FileResult { - let backend = MmapBackend::try_from(path.as_path()) - .await - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) - .map_err(|e| IoError(e, path.clone()))?; - - let reader = AsyncPmTilesReader::try_from_source(backend).await; - let reader = reader - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) - .map_err(|e| IoError(e, path.clone()))?; - let hdr = &reader.get_header(); - - if hdr.tile_type != TileType::Mvt && hdr.tile_compression != Compression::None { - return Err(InvalidMetadata( - format!( - "Format {:?} and compression {:?} are not yet supported", - hdr.tile_type, hdr.tile_compression - ), - path, - )); +macro_rules! impl_pmtiles_source { + ($name: ident, $backend: ty, $cache: ty, $path: ty) => { + #[derive(Clone)] + pub struct $name { + id: String, + path: $path, + pmtiles: Arc>, + tilejson: TileJSON, + tile_info: TileInfo, } - let format = match hdr.tile_type { - TileType::Mvt => TileInfo::new( - Format::Mvt, - match hdr.tile_compression { - Compression::None => Encoding::Uncompressed, - Compression::Unknown => { - warn!( - "MVT tiles have unknown compression in file {}", - path.display() - ); - Encoding::Uncompressed - } - Compression::Gzip => Encoding::Gzip, - Compression::Brotli => Encoding::Brotli, - Compression::Zstd => Encoding::Zstd, - }, - ), - TileType::Png => Format::Png.into(), - TileType::Jpeg => Format::Jpeg.into(), - TileType::Webp => Format::Webp.into(), - TileType::Unknown => { - return Err(InvalidMetadata( - "Unknown tile type".to_string(), - path.clone(), - )) + impl Debug for $name { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} {{ id: {}, path: {:?} }}", + stringify!($name), + self.id, + self.path + ) } - }; - - let tilejson = reader.parse_tilejson(Vec::new()).await.unwrap_or_else(|e| { - warn!("{e:?}: Unable to parse metadata for {}", path.display()); - hdr.get_tilejson(Vec::new()) - }); - - Ok(Self { - id, - path, - pmtiles: Arc::new(reader), - tilejson, - tile_info: format, - }) - } -} - -#[async_trait] -impl Source for PmtSource { - fn get_id(&self) -> &str { - &self.id - } - - fn get_tilejson(&self) -> &TileJSON { - &self.tilejson - } - - fn get_tile_info(&self) -> TileInfo { - self.tile_info - } - - fn clone_source(&self) -> Box { - Box::new(self.clone()) - } - - async fn get_tile( - &self, - xyz: &TileCoord, - _url_query: &Option, - ) -> MartinResult { - // TODO: optimize to return Bytes - if let Some(t) = self - .pmtiles - .get_tile(xyz.z, u64::from(xyz.x), u64::from(xyz.y)) - .await - { - Ok(t.to_vec()) - } else { - trace!( - "Couldn't find tile data in {}/{}/{} of {}", - xyz.z, - xyz.x, - xyz.y, - &self.id - ); - Ok(Vec::new()) } - } + + impl $name { + async fn new_int( + id: String, + path: $path, + reader: AsyncPmTilesReader<$backend, $cache>, + ) -> FileResult { + let hdr = &reader.get_header(); + + if hdr.tile_type != TileType::Mvt && hdr.tile_compression != Compression::None { + return Err(Self::metadata_err( + format!( + "Format {:?} and compression {:?} are not yet supported", + hdr.tile_type, hdr.tile_compression + ), + path, + )); + } + + let format = match hdr.tile_type { + TileType::Mvt => TileInfo::new( + Format::Mvt, + match hdr.tile_compression { + Compression::None => Encoding::Uncompressed, + Compression::Unknown => { + warn!( + "MVT tiles have unknown compression in file {}", + Self::display_path(&path) + ); + Encoding::Uncompressed + } + Compression::Gzip => Encoding::Gzip, + Compression::Brotli => Encoding::Brotli, + Compression::Zstd => Encoding::Zstd, + }, + ), + // All these assume uncompressed data (validated above) + TileType::Png => Format::Png.into(), + TileType::Jpeg => Format::Jpeg.into(), + TileType::Webp => Format::Webp.into(), + TileType::Unknown => { + return Err(Self::metadata_err("Unknown tile type".to_string(), path)) + } + }; + + let tilejson = reader.parse_tilejson(Vec::new()).await.unwrap_or_else(|e| { + warn!( + "{e:?}: Unable to parse metadata for {}", + Self::display_path(&path) + ); + hdr.get_tilejson(Vec::new()) + }); + + Ok(Self { + id, + path, + pmtiles: Arc::new(reader), + tilejson, + tile_info: format, + }) + } + } + + #[async_trait] + impl Source for $name { + fn get_id(&self) -> &str { + &self.id + } + + fn get_tilejson(&self) -> &TileJSON { + &self.tilejson + } + + fn get_tile_info(&self) -> TileInfo { + self.tile_info + } + + fn clone_source(&self) -> Box { + Box::new(self.clone()) + } + + async fn get_tile( + &self, + xyz: &TileCoord, + _url_query: &Option, + ) -> MartinResult { + // TODO: optimize to return Bytes + if let Some(t) = self + .pmtiles + .get_tile(xyz.z, u64::from(xyz.x), u64::from(xyz.y)) + .await + { + Ok(t.to_vec()) + } else { + trace!( + "Couldn't find tile data in {}/{}/{} of {}", + xyz.z, + xyz.x, + xyz.y, + &self.id + ); + Ok(Vec::new()) + } + } + } + }; } + +pub(crate) use impl_pmtiles_source; diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 0e33b960..f17dc5f3 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -52,7 +52,7 @@ const TILES_V2: &str = " -- , (6, 2, 6, cast('1-keep-1-rm' as blob)) -- this row is removed , (5, 3, 7, cast('new' as blob)) -- this row is added, dup value , (5, 3, 8, cast('new' as blob)) -- this row is added, dup value - + -- Expected delta: -- 5/1/1 edit -- 5/1/2 edit diff --git a/tests/config.yaml b/tests/config.yaml index 29577297..08d616b9 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -166,9 +166,11 @@ postgres: pmtiles: + paths: + - http://localhost:5412/webp2.pmtiles sources: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles - pmt2: tests/fixtures/pmtiles2/webp2.pmtiles + pmt2: http://localhost:5412/webp2.pmtiles sprites: paths: tests/fixtures/sprites/src1 diff --git a/tests/expected/auto/save_config.yaml b/tests/expected/auto/save_config.yaml index c2317e9c..20acee04 100644 --- a/tests/expected/auto/save_config.yaml +++ b/tests/expected/auto/save_config.yaml @@ -191,16 +191,14 @@ pmtiles: paths: - tests/fixtures/mbtiles - tests/fixtures/pmtiles - - tests/fixtures/pmtiles2 sources: png: tests/fixtures/pmtiles/png.pmtiles stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles - webp2: tests/fixtures/pmtiles2/webp2.pmtiles + webp2: http://localhost:5412/webp2.pmtiles mbtiles: paths: - tests/fixtures/mbtiles - tests/fixtures/pmtiles - - tests/fixtures/pmtiles2 sources: geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles diff --git a/tests/expected/configured/catalog_cfg.json b/tests/expected/configured/catalog_cfg.json index 964f26ad..06bdba89 100644 --- a/tests/expected/configured/catalog_cfg.json +++ b/tests/expected/configured/catalog_cfg.json @@ -44,6 +44,10 @@ }, "table_source": { "content_type": "application/x-protobuf" + }, + "webp2": { + "content_type": "image/webp", + "name": "ne2sr" } }, "sprites": { diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 6f1cb961..c0e206db 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -163,7 +163,8 @@ postgres: pmtiles: sources: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles - pmt2: tests/fixtures/pmtiles2/webp2.pmtiles + pmt2: http://localhost:5412/webp2.pmtiles + webp2: http://localhost:5412/webp2.pmtiles sprites: paths: tests/fixtures/sprites/src1 sources: diff --git a/tests/test.sh b/tests/test.sh index 26c5f024..fb067f6f 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -9,6 +9,7 @@ CURL=${CURL:-curl --silent --show-error --fail --compressed} MARTIN_BUILD_ALL="${MARTIN_BUILD_ALL:-cargo build}" +STATICS_URL="${STATICS_URL:-http://localhost:5412}" MARTIN_PORT="${MARTIN_PORT:-3111}" MARTIN_URL="http://localhost:${MARTIN_PORT}" MARTIN_ARGS="${MARTIN_ARGS:---listen-addresses localhost:${MARTIN_PORT}}" @@ -199,6 +200,10 @@ if [[ "$MARTIN_BUILD_ALL" != "-" ]]; then fi +echo "------------------------------------------------------------------------------------------------------------------------" +echo "Check HTTP server is running" +$CURL --head "$STATICS_URL/webp2.pmtiles" + echo "------------------------------------------------------------------------------------------------------------------------" echo "Test auto configured Martin" @@ -207,7 +212,7 @@ LOG_FILE="${LOG_DIR}/${TEST_NAME}.txt" TEST_OUT_DIR="${TEST_OUT_BASE_DIR}/${TEST_NAME}" mkdir -p "$TEST_OUT_DIR" -ARG=(--default-srid 900913 --auto-bounds calc --save-config "${TEST_OUT_DIR}/save_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2 --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts) +ARG=(--default-srid 900913 --auto-bounds calc --save-config "${TEST_OUT_DIR}/save_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles "$STATICS_URL/webp2.pmtiles" --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts) export DATABASE_URL="$MARTIN_DATABASE_URL" set -x