Implement http pmtiles (#991)

PMTiles is a web-optimized format, allowing the actual file to be read with HTTP range requests. Supporting this use case instantly allows Martin to function as a lambda executable accessing PMTiles, but without any significant investment into devops or hosting large file. PMTiles config now also allows `http` and `https` protocol. ``` # Publish PMTiles files pmtiles: paths: # specific pmtiles file will be published as mypmtiles source # (use last portion of the URL without extension) - http://example.org/path/to/mypmtiles.pmtiles sources: # named source matching source name to a single file pm-src1: https://example.org/path/to/some_pmtiles.pmtiles ``` fixes #884 --------- Co-authored-by: Kyle Slugg-Urbino <35903887+kyleslugg@users.noreply.github.com>
2024-12-18 12:21:56 +03:00 · 2023-12-22 01:01:50 -05:00 · 2023-12-22 01:01:50 -05:00 · 1a8e7c89a4
commit 1a8e7c89a4
parent 30491ae353
27 changed files with 909 additions and 238 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -115,6 +115,8 @@ jobs:
          tool: cross
      - name: Checkout sources
        uses: actions/checkout@v4
+        with:
+          set-safe-directory: false
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
      - name: Init database
@ -162,6 +164,13 @@ jobs:
          mkdir -p target_releases/linux/amd64
          mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/

+      - name: Start HTTP Server
+        run: |
+          docker run --rm -d \
+          --name fileserver \
+          -p 5412:80 \
+          -v ${{ github.workspace }}/tests/fixtures/pmtiles2:/usr/share/nginx/html \
+          nginx:alpine
      - name: Build linux/arm64 Docker image
        uses: docker/build-push-action@v5
        # https://github.com/docker/build-push-action
@ -229,6 +238,9 @@ jobs:
          tags: ${{ steps.docker_meta.outputs.tags }}
          labels: ${{ steps.docker_meta.outputs.labels }}
          platforms: linux/amd64,linux/arm64
+      - name: Stop HTTP Server
+        if: always()
+        run: docker stop fileserver

  build:
    name: Build ${{ matrix.target }}
@ -302,6 +314,61 @@ jobs:
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
+      - name: Install Docker (MacOS)
+        if: runner.os == 'macos'
+        run: brew install docker && colima start
+      - name: Start HTTP Server (with Docker)
+        if: runner.os != 'windows'
+        run: |
+          docker run --rm -d \
+            --name fileserver \
+            -p 5412:80 \
+            -v ${{ github.workspace }}/tests/fixtures/pmtiles2:/usr/share/nginx/html \
+            nginx:alpine
+      - name: Start HTTP Server (Windows, no Docker)
+        if: runner.os == 'windows'
+        shell: pwsh
+        run: |
+          $nginxConf = @"
+          worker_processes     1;
+          events {
+            worker_connections 1024;
+          }
+          http {
+            include            mime.types;
+            default_type       application/octet-stream;
+            sendfile           on;
+            keepalive_timeout  65;
+            server {
+              listen       5412;
+              server_name  localhost;
+              location / {
+                root       $($PWD.Path)/tests/fixtures/pmtiles2;
+                index      index.html index.htm;
+              }
+            }
+          }
+          "@
+
+          # Replace the default Nginx configuration file
+          echo "$nginxConf"
+          Set-Content -Path "C:\tools\nginx-1.25.3\conf\nginx.conf" -Value $nginxConf
+          Get-Content -Path "C:\tools\nginx-1.25.3\conf\nginx.conf"
+
+          # Start Nginx
+          #Get-Service -ErrorAction SilentlyContinue
+          #Get-CimInstance -ClassName Win32_Service
+          Set-Service nginx -StartupType manual
+          Start-Service nginx
+          #Start-Process -FilePath "C:\tools\nginx-1.25.3\nginx.exe"
+          dir C:\tools\nginx-1.25.3\logs\
+          Start-Sleep -Seconds 5
+          netstat -a
+
+          # Print Nginx Error Logs (on Windows systems)
+          #nginx -t
+          Get-Content -Path "C:\tools\nginx-1.25.3\logs\error.log"
+          dir D:\a\martin\martin\
      - name: Start postgres
        uses: nyurik/action-setup-postgis@v1.1
        id: pg
@ -355,7 +422,10 @@ jobs:
          tests/test.sh
        env:
          DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
-      - name: Save test output on failure
+      - name: Stop HTTP Server (with Docker)
+        if: runner.os != 'windows'
+        run: docker stop fileserver
+      - name: Save test output (on error)
        if: failure()
        uses: actions/upload-artifact@v3
        with:
@ -422,6 +492,13 @@ jobs:
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
+      - name: Start HTTP Server
+        run: |
+          docker run --rm -d \
+          --name fileserver \
+          -p 5412:80 \
+          -v ${{ github.workspace }}/tests/fixtures/pmtiles2:/usr/share/nginx/html \
+          nginx:alpine
      - name: Init database
        run: tests/fixtures/initdb.sh
        env:
@ -483,7 +560,10 @@ jobs:
          cargo clean
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
-      - name: On error, save test output
+      - name: Stop HTTP Server
+        if: always()
+        run: docker stop fileserver
+      - name: Save test output (on error)
        if: failure()
        uses: actions/upload-artifact@v3
        with:
--- a/Cargo.lock
+++ b/Cargo.lock
@ -367,6 +367,15 @@ dependencies = [
 "zstd-safe 7.0.0",
 ]

+[[package]]
+name = "async-lock"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b"
+dependencies = [
+ "event-listener",
+]
+
 [[package]]
 name = "async-recursion"
 version = "1.0.5"
@ -519,6 +528,12 @@ version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"

+[[package]]
+name = "bytecount"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205"
+
 [[package]]
 name = "bytemuck"
 version = "1.14.0"
@ -546,12 +561,43 @@ dependencies = [
 "bytes",
 ]

+[[package]]
+name = "camino"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "cargo-husky"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad"

+[[package]]
+name = "cargo-platform"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "cast"
 version = "0.3.0"
@ -1092,6 +1138,15 @@ dependencies = [
 "windows-sys 0.52.0",
 ]

+[[package]]
+name = "error-chain"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "etcetera"
 version = "0.8.0"
@ -1554,6 +1609,17 @@ dependencies = [
 "itoa",
 ]

+[[package]]
+name = "http-body"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "httparse"
 version = "1.8.0"
@ -1572,6 +1638,44 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"

+[[package]]
+name = "hyper"
+version = "0.14.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
+dependencies = [
+ "futures-util",
+ "http",
+ "hyper",
+ "rustls",
+ "tokio",
+ "tokio-rustls",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.58"
@ -1661,6 +1765,12 @@ dependencies = [
 "yaml-rust",
 ]

+[[package]]
+name = "ipnet"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+
 [[package]]
 name = "is-terminal"
 version = "0.4.9"
@ -1842,6 +1952,15 @@ version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"

+[[package]]
+name = "mach2"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "martin"
 version = "0.11.6"
@ -1868,6 +1987,7 @@ dependencies = [
 "log",
 "martin-tile-utils",
 "mbtiles",
+ "moka",
 "num_cpus",
 "pbf_font_tools",
 "pmtiles",
@ -1875,6 +1995,7 @@ dependencies = [
 "postgres",
 "postgres-protocol",
 "regex",
+ "reqwest",
 "rustls",
 "rustls-native-certs",
 "rustls-pemfile",
@ -1889,6 +2010,7 @@ dependencies = [
 "tilejson",
 "tokio",
 "tokio-postgres-rustls",
+ "url",
 ]

 [[package]]
@ -2005,6 +2127,30 @@ dependencies = [
 "windows-sys 0.48.0",
 ]

+[[package]]
+name = "moka"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8017ec3548ffe7d4cef7ac0e12b044c01164a74c0f3119420faeaf13490ad8b"
+dependencies = [
+ "async-lock",
+ "async-trait",
+ "crossbeam-channel",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "futures-util",
+ "once_cell",
+ "parking_lot",
+ "quanta",
+ "rustc_version",
+ "skeptic",
+ "smallvec",
+ "tagptr",
+ "thiserror",
+ "triomphe",
+ "uuid",
+]
+
 [[package]]
 name = "multimap"
 version = "0.9.1"
@ -2362,6 +2508,7 @@ dependencies = [
 "bytes",
 "fmmap",
 "hilbert_2d",
+ "reqwest",
 "serde",
 "serde_json",
 "thiserror",
@ -2573,6 +2720,33 @@ version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804"

+[[package]]
+name = "pulldown-cmark"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998"
+dependencies = [
+ "bitflags 1.3.2",
+ "memchr",
+ "unicase",
+]
+
+[[package]]
+name = "quanta"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "mach2",
+ "once_cell",
+ "raw-cpuid",
+ "wasi",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.33"
@ -2618,6 +2792,15 @@ dependencies = [
 "getrandom",
 ]

+[[package]]
+name = "raw-cpuid"
+version = "10.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "rayon"
 version = "1.8.0"
@ -2694,6 +2877,46 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca"

+[[package]]
+name = "reqwest"
+version = "0.11.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41"
+dependencies = [
+ "base64",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-rustls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pemfile",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "system-configuration",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "resvg"
 version = "0.36.0"
@ -2992,6 +3215,9 @@ name = "semver"
 version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+dependencies = [
+ "serde",
+]

 [[package]]
 name = "serde"
@ -3177,6 +3403,21 @@ dependencies = [
 "num",
 ]

+[[package]]
+name = "skeptic"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
+dependencies = [
+ "bytecount",
+ "cargo_metadata",
+ "error-chain",
+ "glob",
+ "pulldown-cmark",
+ "tempfile",
+ "walkdir",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@ -3571,6 +3812,33 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "tagptr"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
+
 [[package]]
 name = "tap"
 version = "1.0.1"
@ -3825,6 +4093,12 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
 [[package]]
 name = "tracing"
 version = "0.1.40"
@ -3866,6 +4140,18 @@ dependencies = [
 "serde_json",
 ]

+[[package]]
+name = "triomphe"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3"
+
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
 [[package]]
 name = "ttf-parser"
 version = "0.19.2"
@ -3884,6 +4170,15 @@ version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"

+[[package]]
+name = "unicase"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.14"
@ -4060,6 +4355,9 @@ name = "uuid"
 version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560"
+dependencies = [
+ "getrandom",
+]

 [[package]]
 name = "varint-rs"
@ -4089,6 +4387,15 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@ -4120,6 +4427,18 @@ dependencies = [
 "wasm-bindgen-shared",
 ]

+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.89"
@ -4425,6 +4744,16 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"

+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "wyz"
 version = "0.5.1"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -50,14 +50,16 @@ json-patch = "1.2"
 log = "0.4"
 martin-tile-utils = { path = "./martin-tile-utils", version = "0.3.0" }
 mbtiles = { path = "./mbtiles", version = "0.8.0" }
+moka = { version = "0.12", features = ["future"] }
 num_cpus = "1"
 pbf_font_tools = { version = "2.5.0", features = ["freetype"] }
-pmtiles = { version = "0.5", features = ["mmap-async-tokio", "tilejson"] }
+pmtiles = { version = "0.5", features = ["http-async", "mmap-async-tokio", "tilejson"] }
 postgis = "0.9"
 postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] }
 postgres-protocol = "0.6"
 pretty_assertions = "1"
 regex = "1"
+reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-native-roots"] }
 rstest = "0.18"
 rustls = { version = "0.21", features = ["dangerous_configuration"] }
 rustls-native-certs = "0.6"
@ -77,6 +79,7 @@ tile-grid = "0.5"
 tilejson = "0.4"
 tokio = { version = "1", features = ["macros"] }
 tokio-postgres-rustls = "0.10"
+url = "2.5"

 [profile.dev.package]
 # See https://github.com/launchbadge/sqlx#compile-time-verification
--- a/README.md
+++ b/README.md
@ -9,7 +9,7 @@
 [![CI build](https://github.com/maplibre/martin/actions/workflows/ci.yml/badge.svg)](https://github.com/maplibre/martin/actions)
 [![](https://img.shields.io/badge/Slack-%23maplibre--martin-2EB67D?logo=slack)](https://slack.openstreetmap.us/)

-Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new), and [MBTile](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust).
+Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) (local or remote), and [MBTile](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust).

 Additionally, there are [several tools](https://maplibre.org/martin/tools.html) for generating tiles in bulk from any Martin-supported sources (similar to `tilelive-copy`), copying tiles between MBTiles files, creating deltas (patches) and applying them, and validating MBTiles files.

--- a/debian/config.yaml
+++ b/debian/config.yaml
@ -20,8 +20,10 @@ worker_processes: 8
 #   paths:
 #     - /dir-path
 #     - /path/to/pmtiles.pmtiles
+#     - http://example.org/pmtiles.pmtiles
 #   sources:
 #     pm-src1: /path/to/pmtiles1.pmtiles
+#     pm-web1: http://example.org/pmtiles1.pmtiles

 # mbtiles:
 #   paths:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,6 +1,14 @@
 version: '3'

 services:
+  fileserver:
+    image: nginx:alpine
+    restart: unless-stopped
+    ports:
+      - '5412:80'
+    volumes:
+      - ./tests/fixtures/pmtiles2:/usr/share/nginx/html
+
  db-is-ready:
    # This should match the version of postgres used in the CI workflow
    image: postgis/postgis:14-3.3-alpine
--- a/docs/src/config-file.md
+++ b/docs/src/config-file.md
@ -153,16 +153,20 @@ postgres:
      # Values may be integers or floating point numbers.
      bounds: [-180.0, -90.0, 180.0, 90.0]

-# Publish PMTiles files
+# Publish PMTiles files from local disk or proxy to a web server
 pmtiles:
  paths:
    # scan this whole dir, matching all *.pmtiles files
    - /dir-path
    # specific pmtiles file will be published as a pmt source (filename without extension)
    - /path/to/pmt.pmtiles
+    # A web server with a PMTiles file that supports range requests
+    - https://example.org/path/tiles.pmtiles
  sources:
    # named source matching source name to a single file
    pm-src1: /path/to/pmt.pmtiles
+    # A named source to a web server with a PMTiles file that supports range requests
+    pm-web2: https://example.org/path/tiles.pmtiles
    
 # Publish MBTiles files
 mbtiles:
--- a/docs/src/installation.md
+++ b/docs/src/installation.md
@ -36,7 +36,7 @@ brew install martin

 ### Docker

-Martin is also available as a [Docker image](https://ghcr.io/maplibre/martin). You could either share a configuration file from the host with the container via the `-v` param, or you can let Martin auto-discover all sources e.g. by passing `DATABASE_URL` or specifying the .mbtiles/.pmtiles files.
+Martin is also available as a [Docker image](https://ghcr.io/maplibre/martin). You could either share a configuration file from the host with the container via the `-v` param, or you can let Martin auto-discover all sources e.g. by passing `DATABASE_URL` or specifying the .mbtiles/.pmtiles files or URLs to .pmtiles.

 ```shell
 export PGPASSWORD=postgres  # secret!
--- a/docs/src/introduction.md
+++ b/docs/src/introduction.md
@ -1,6 +1,6 @@
 ![Martin](https://raw.githubusercontent.com/maplibre/martin/main/logo.png)

-Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTiles](https://protomaps.com/blog/pmtiles-v3-whats-new), and [MBTiles](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust).
+Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTiles](https://protomaps.com/blog/pmtiles-v3-whats-new) (local or remote), and [MBTiles](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust).

 See also [Martin demo site](https://martin.maplibre.org/)

--- a/docs/src/run-with-docker.md
+++ b/docs/src/run-with-docker.md
@ -7,7 +7,7 @@ You can use official Docker image [`ghcr.io/maplibre/martin`](https://ghcr.io/ma
 ```shell
 docker run \
  -p 3000:3000 \
-  -e DATABASE_URL=postgresql://postgres@postgres.example.com/db \
+  -e DATABASE_URL=postgresql://postgres@postgres.example.org/db \
  ghcr.io/maplibre/martin
 ```

--- a/docs/src/sources-files.md
+++ b/docs/src/sources-files.md
@ -1,9 +1,9 @@
 ## MBTiles and PMTiles File Sources

-Martin can serve any type of tiles from [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) and [MBTile](https://github.com/mapbox/mbtiles-spec) files.  To serve a file from CLI, simply put the path to the file or the directory with `*.mbtiles` or `*.pmtiles` files. For example:
+Martin can serve any type of tiles from [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) and [MBTile](https://github.com/mapbox/mbtiles-spec) files.  To serve a file from CLI, simply put the path to the file or the directory with `*.mbtiles` or `*.pmtiles` files. A path to PMTiles file may be a URL. For example:

 ```shell
-martin  /path/to/mbtiles/file.mbtiles  /path/to/directory
+martin  /path/to/mbtiles/file.mbtiles  /path/to/directory   https://example.org/path/tiles.pmtiles
 ```

 You may also want to generate a [config file](config-file.md) using the `--save-config my-config.yaml`, and later edit it and use it with `--config my-config.yaml` option.
--- a/6
+++ b/6
@ -67,7 +67,7 @@ start-legacy: (docker-up "db-legacy") docker-is-ready

 # Start a specific test database, e.g. db or db-legacy
 [private]
-docker-up name:
+docker-up name: start-pmtiles-server
    docker-compose up -d {{ name }}

 # Wait for the test database to be ready
@ -88,6 +88,10 @@ restart:
 stop:
    docker-compose down --remove-orphans

+# Start test server for testing HTTP pmtiles
+start-pmtiles-server:
+    docker-compose up -d fileserver
+
 # Run benchmark tests
 bench: start
    cargo bench
--- a/martin/Cargo.toml
+++ b/martin/Cargo.toml
@ -80,6 +80,7 @@ json-patch.workspace = true
 log.workspace = true
 martin-tile-utils.workspace = true
 mbtiles.workspace = true
+moka.workspace = true
 num_cpus.workspace = true
 pbf_font_tools.workspace = true
 pmtiles.workspace = true
@ -87,6 +88,7 @@ postgis.workspace = true
 postgres-protocol.workspace = true
 postgres.workspace = true
 regex.workspace = true
+reqwest.workspace = true
 rustls-native-certs.workspace = true
 rustls-pemfile.workspace = true
 rustls.workspace = true
@ -101,6 +103,7 @@ thiserror.workspace = true
 tilejson.workspace = true
 tokio = { workspace = true, features = ["io-std"] }
 tokio-postgres-rustls.workspace = true
+url.workspace = true

 [dev-dependencies]
 cargo-husky.workspace = true
--- a/martin/benches/bench.rs
+++ b/martin/benches/bench.rs
@ -16,7 +16,7 @@ struct NullSource {
 impl NullSource {
    fn new() -> Self {
        Self {
-            tilejson: tilejson! { "https://example.com/".to_string() },
+            tilejson: tilejson! { "https://example.org/".to_string() },
        }
    }
 }
--- a/martin/src/args/root.rs
+++ b/martin/src/args/root.rs
@ -2,6 +2,7 @@ use std::path::PathBuf;

 use clap::Parser;
 use log::warn;
+use url::Url;

 use crate::args::connections::Arguments;
 use crate::args::environment::Env;
@ -89,11 +90,11 @@ impl Args {
        }

        if !cli_strings.is_empty() {
-            config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles");
+            config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles", true);
        }

        if !cli_strings.is_empty() {
-            config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles");
+            config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles", false);
        }

        if !self.extras.sprite.is_empty() {
@ -108,10 +109,29 @@ impl Args {
    }
 }

-pub fn parse_file_args(cli_strings: &mut Arguments, extension: &str) -> FileConfigEnum {
-    let paths = cli_strings.process(|v| match PathBuf::try_from(v) {
+fn is_url(s: &str, extension: &str) -> bool {
+    if s.starts_with("http") {
+        if let Ok(url) = Url::parse(s) {
+            if url.scheme() == "http" || url.scheme() == "https" {
+                if let Some(ext) = url.path().rsplit('.').next() {
+                    return ext == extension;
+                }
+            }
+        }
+    }
+    false
+}
+
+pub fn parse_file_args(
+    cli_strings: &mut Arguments,
+    extension: &str,
+    allow_url: bool,
+) -> FileConfigEnum {
+    let paths = cli_strings.process(|s| match PathBuf::try_from(s) {
        Ok(v) => {
-            if v.is_dir() {
+            if allow_url && is_url(s, extension) {
+                Take(v)
+            } else if v.is_dir() {
                Share(v)
            } else if v.is_file() && v.extension().map_or(false, |e| e == extension) {
                Take(v)
--- a/martin/src/bin/martin-cp.rs
+++ b/martin/src/bin/martin-cp.rs
@ -71,7 +71,7 @@ pub struct CopyArgs {
    pub url_query: Option<String>,
    /// Optional accepted encoding parameter as if the browser sent it in the HTTP request.
    /// If set to multiple values like `gzip,br`, martin-cp will use the first encoding,
-    /// or re-encode if the tile is already encoded and that encoding is not listed.  
+    /// or re-encode if the tile is already encoded and that encoding is not listed.
    /// Use `identity` to disable compression. Ignored for non-encodable tiles like PNG and JPEG.
    #[arg(long, alias = "encodings", default_value = "gzip")]
    pub encoding: String,
--- a/martin/src/config.rs
+++ b/martin/src/config.rs
@ -11,11 +11,11 @@ use log::info;
 use serde::{Deserialize, Serialize};
 use subst::VariableMap;

-use crate::file_config::{resolve_files, FileConfigEnum};
+use crate::file_config::{resolve_files, resolve_files_urls, FileConfigEnum};
 use crate::fonts::FontSources;
 use crate::mbtiles::MbtSource;
 use crate::pg::PgConfig;
-use crate::pmtiles::PmtSource;
+use crate::pmtiles::{PmtFileSource, PmtHttpSource};
 use crate::source::{TileInfoSources, TileSources};
 use crate::sprites::SpriteSources;
 use crate::srv::SrvConfig;
@ -92,7 +92,8 @@ impl Config {
    }

    async fn resolve_tile_sources(&mut self, idr: IdResolver) -> MartinResult<TileSources> {
-        let new_pmt_src = &mut PmtSource::new_box;
+        let new_pmt_src = &mut PmtFileSource::new_box;
+        let new_pmt_url_src = &mut PmtHttpSource::new_url_box;
        let new_mbt_src = &mut MbtSource::new_box;
        let mut sources: Vec<Pin<Box<dyn Future<Output = MartinResult<TileInfoSources>>>>> =
            Vec::new();
@ -102,12 +103,14 @@ impl Config {
        }

        if !self.pmtiles.is_empty() {
-            let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", new_pmt_src);
+            let cfg = &mut self.pmtiles;
+            let val = resolve_files_urls(cfg, idr.clone(), "pmtiles", new_pmt_src, new_pmt_url_src);
            sources.push(Box::pin(val));
        }

        if !self.mbtiles.is_empty() {
-            let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", new_mbt_src);
+            let cfg = &mut self.mbtiles;
+            let val = resolve_files(cfg, idr.clone(), "mbtiles", new_mbt_src);
            sources.push(Box::pin(val));
        }

--- a/martin/src/file_config.rs
+++ b/martin/src/file_config.rs
@ -1,14 +1,17 @@
 use std::collections::{BTreeMap, HashSet};
 use std::future::Future;
 use std::mem;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};

 use futures::TryFutureExt;
 use log::{info, warn};
 use serde::{Deserialize, Serialize};
+use url::Url;

 use crate::config::{copy_unrecognized_config, UnrecognizedValues};
-use crate::file_config::FileError::{InvalidFilePath, InvalidSourceFilePath, IoError};
+use crate::file_config::FileError::{
+    InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError,
+};
 use crate::source::{Source, TileInfoSources};
 use crate::utils::{IdResolver, OptOneMany};
 use crate::MartinResult;
@ -24,14 +27,23 @@ pub enum FileError {
    #[error("Source path is not a file: {}", .0.display())]
    InvalidFilePath(PathBuf),

+    #[error("Error {0} while parsing URL {1}")]
+    InvalidSourceUrl(url::ParseError, String),
+
    #[error("Source {0} uses bad file {}", .1.display())]
    InvalidSourceFilePath(String, PathBuf),

    #[error(r"Unable to parse metadata in file {}: {0}", .1.display())]
    InvalidMetadata(String, PathBuf),

+    #[error(r"Unable to parse metadata in file {1}: {0}")]
+    InvalidUrlMetadata(String, Url),
+
    #[error(r#"Unable to aquire connection to file: {0}"#)]
    AquireConnError(String),
+
+    #[error(r#"PMTiles error {0} processing {1}"#)]
+    PmtError(pmtiles::PmtError, String),
 }

 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@ -169,6 +181,10 @@ pub struct FileConfigSource {
    pub path: PathBuf,
 }

+async fn dummy_resolver(_id: String, _url: Url) -> FileResult<Box<dyn Source>> {
+    unreachable!()
+}
+
 pub async fn resolve_files<Fut>(
    config: &mut FileConfigEnum,
    idr: IdResolver,
@ -178,19 +194,39 @@ pub async fn resolve_files<Fut>(
 where
    Fut: Future<Output = Result<Box<dyn Source>, FileError>>,
 {
-    resolve_int(config, idr, extension, new_source)
+    let dummy = &mut dummy_resolver;
+    resolve_int(config, idr, extension, false, new_source, dummy)
        .map_err(crate::MartinError::from)
        .await
 }

-async fn resolve_int<Fut>(
+pub async fn resolve_files_urls<Fut1, Fut2>(
    config: &mut FileConfigEnum,
    idr: IdResolver,
    extension: &str,
-    new_source: &mut impl FnMut(String, PathBuf) -> Fut,
+    new_source: &mut impl FnMut(String, PathBuf) -> Fut1,
+    new_url_source: &mut impl FnMut(String, Url) -> Fut2,
+) -> MartinResult<TileInfoSources>
+where
+    Fut1: Future<Output = Result<Box<dyn Source>, FileError>>,
+    Fut2: Future<Output = Result<Box<dyn Source>, FileError>>,
+{
+    resolve_int(config, idr, extension, true, new_source, new_url_source)
+        .map_err(crate::MartinError::from)
+        .await
+}
+
+async fn resolve_int<Fut1, Fut2>(
+    config: &mut FileConfigEnum,
+    idr: IdResolver,
+    extension: &str,
+    parse_urls: bool,
+    new_source: &mut impl FnMut(String, PathBuf) -> Fut1,
+    new_url_source: &mut impl FnMut(String, Url) -> Fut2,
 ) -> FileResult<TileInfoSources>
 where
-    Fut: Future<Output = Result<Box<dyn Source>, FileError>>,
+    Fut1: Future<Output = Result<Box<dyn Source>, FileError>>,
+    Fut2: Future<Output = Result<Box<dyn Source>, FileError>>,
 {
    let Some(cfg) = config.extract_file_config() else {
        return Ok(TileInfoSources::default());
@ -203,67 +239,76 @@ where

    if let Some(sources) = cfg.sources {
        for (id, source) in sources {
-            let can = source.abs_path()?;
-            if !can.is_file() {
-                // todo: maybe warn instead?
-                return Err(InvalidSourceFilePath(id.to_string(), can));
+            if let Some(url) = parse_url(parse_urls, source.get_path())? {
+                let dup = !files.insert(source.get_path().clone());
+                let dup = if dup { "duplicate " } else { "" };
+                let id = idr.resolve(&id, url.to_string());
+                configs.insert(id.clone(), source);
+                results.push(new_url_source(id.clone(), url.clone()).await?);
+                info!("Configured {dup}source {id} from {}", sanitize_url(&url));
+            } else {
+                let can = source.abs_path()?;
+                if !can.is_file() {
+                    // todo: maybe warn instead?
+                    return Err(InvalidSourceFilePath(id.to_string(), can));
+                }
+
+                let dup = !files.insert(can.clone());
+                let dup = if dup { "duplicate " } else { "" };
+                let id = idr.resolve(&id, can.to_string_lossy().to_string());
+                info!("Configured {dup}source {id} from {}", can.display());
+                configs.insert(id.clone(), source.clone());
+                results.push(new_source(id, source.into_path()).await?);
            }
-
-            let dup = !files.insert(can.clone());
-            let dup = if dup { "duplicate " } else { "" };
-            let id = idr.resolve(&id, can.to_string_lossy().to_string());
-            info!("Configured {dup}source {id} from {}", can.display());
-            configs.insert(id.clone(), source.clone());
-
-            let path = match source {
-                FileConfigSrc::Obj(pmt) => pmt.path,
-                FileConfigSrc::Path(path) => path,
-            };
-            results.push(new_source(id, path).await?);
        }
    }

    for path in cfg.paths {
-        let is_dir = path.is_dir();
-        let dir_files = if is_dir {
-            // directories will be kept in the config just in case there are new files
-            directories.push(path.clone());
-            path.read_dir()
-                .map_err(|e| IoError(e, path.clone()))?
-                .filter_map(Result::ok)
-                .filter(|f| {
-                    f.path().extension().filter(|e| *e == extension).is_some() && f.path().is_file()
+        if let Some(url) = parse_url(parse_urls, &path)? {
+            let id = url
+                .path_segments()
+                .and_then(Iterator::last)
+                .and_then(|s| {
+                    // Strip extension and trailing dot, or keep the original string
+                    s.strip_suffix(extension)
+                        .and_then(|s| s.strip_suffix('.'))
+                        .or(Some(s))
                })
-                .map(|f| f.path())
-                .collect()
-        } else if path.is_file() {
-            vec![path]
-        } else {
-            return Err(InvalidFilePath(path.canonicalize().unwrap_or(path)));
-        };
-        for path in dir_files {
-            let can = path.canonicalize().map_err(|e| IoError(e, path.clone()))?;
-            if files.contains(&can) {
-                if !is_dir {
-                    warn!("Ignoring duplicate MBTiles path: {}", can.display());
-                }
-                continue;
-            }
-            let id = path.file_stem().map_or_else(
-                || "_unknown".to_string(),
-                |s| s.to_string_lossy().to_string(),
-            );
-            let source = FileConfigSrc::Path(path);
-            let id = idr.resolve(&id, can.to_string_lossy().to_string());
-            info!("Configured source {id} from {}", can.display());
-            files.insert(can);
-            configs.insert(id.clone(), source.clone());
+                .unwrap_or("pmt_web_source");

-            let path = match source {
-                FileConfigSrc::Obj(pmt) => pmt.path,
-                FileConfigSrc::Path(path) => path,
+            let id = idr.resolve(id, url.to_string());
+            configs.insert(id.clone(), FileConfigSrc::Path(path));
+            results.push(new_url_source(id.clone(), url.clone()).await?);
+            info!("Configured source {id} from URL {}", sanitize_url(&url));
+        } else {
+            let is_dir = path.is_dir();
+            let dir_files = if is_dir {
+                // directories will be kept in the config just in case there are new files
+                directories.push(path.clone());
+                dir_to_paths(&path, extension)?
+            } else if path.is_file() {
+                vec![path]
+            } else {
+                return Err(InvalidFilePath(path.canonicalize().unwrap_or(path)));
            };
-            results.push(new_source(id, path).await?);
+            for path in dir_files {
+                let can = path.canonicalize().map_err(|e| IoError(e, path.clone()))?;
+                if files.contains(&can) {
+                    if !is_dir {
+                        warn!("Ignoring duplicate MBTiles path: {}", can.display());
+                    }
+                    continue;
+                }
+                let id = path.file_stem().map_or_else(
+                    || "_unknown".to_string(),
+                    |s| s.to_string_lossy().to_string(),
+                );
+                let id = idr.resolve(&id, can.to_string_lossy().to_string());
+                info!("Configured source {id} from {}", can.display());
+                files.insert(can);
+                configs.insert(id.clone(), FileConfigSrc::Path(path.clone()));
+                results.push(new_source(id, path).await?);
+            }
        }
    }

@ -272,6 +317,41 @@ where
    Ok(results)
 }

+fn dir_to_paths(path: &Path, extension: &str) -> Result<Vec<PathBuf>, FileError> {
+    Ok(path
+        .read_dir()
+        .map_err(|e| IoError(e, path.to_path_buf()))?
+        .filter_map(Result::ok)
+        .filter(|f| {
+            f.path().extension().filter(|e| *e == extension).is_some() && f.path().is_file()
+        })
+        .map(|f| f.path())
+        .collect())
+}
+
+fn sanitize_url(url: &Url) -> String {
+    let mut result = format!("{}://", url.scheme());
+    if let Some(host) = url.host_str() {
+        result.push_str(host);
+    }
+    if let Some(port) = url.port() {
+        result.push(':');
+        result.push_str(&port.to_string());
+    }
+    result.push_str(url.path());
+    result
+}
+
+fn parse_url(is_enabled: bool, path: &Path) -> Result<Option<Url>, FileError> {
+    if !is_enabled {
+        return Ok(None);
+    }
+    path.to_str()
+        .filter(|v| v.starts_with("http://") || v.starts_with("https://"))
+        .map(|v| Url::parse(v).map_err(|e| InvalidSourceUrl(e, v.to_string())))
+        .transpose()
+}
+
 #[cfg(test)]
 mod tests {
    use std::collections::BTreeMap;
@ -287,10 +367,14 @@ mod tests {
            paths:
              - /dir-path
              - /path/to/file2.ext
+              - http://example.org/file.ext
            sources:
                pm-src1: /tmp/file.ext
                pm-src2:
                  path: /tmp/file.ext
+                pm-src3: https://example.org/file3.ext
+                pm-src4:
+                  path: https://example.org/file4.ext
        "})
        .unwrap();
        let res = cfg.finalize("").unwrap();
@ -304,6 +388,7 @@ mod tests {
            vec![
                PathBuf::from("/dir-path"),
                PathBuf::from("/path/to/file2.ext"),
+                PathBuf::from("http://example.org/file.ext"),
            ]
        );
        assert_eq!(
@ -318,7 +403,17 @@ mod tests {
                    FileConfigSrc::Obj(FileConfigSource {
                        path: PathBuf::from("/tmp/file.ext"),
                    })
-                )
+                ),
+                (
+                    "pm-src3".to_string(),
+                    FileConfigSrc::Path(PathBuf::from("https://example.org/file3.ext"))
+                ),
+                (
+                    "pm-src4".to_string(),
+                    FileConfigSrc::Obj(FileConfigSource {
+                        path: PathBuf::from("https://example.org/file4.ext"),
+                    })
+                ),
            ]))
        );
    }
--- a/martin/src/pmtiles/file_pmtiles.rs
+++ b/martin/src/pmtiles/file_pmtiles.rs
@ -0,0 +1,59 @@
+use std::fmt::{Debug, Display, Formatter};
+use std::io;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use log::{trace, warn};
+use martin_tile_utils::{Encoding, Format, TileInfo};
+use pmtiles::async_reader::AsyncPmTilesReader;
+use pmtiles::cache::NoCache;
+use pmtiles::mmap::MmapBackend;
+use pmtiles::{Compression, TileType};
+use tilejson::TileJSON;
+
+use crate::file_config::FileError::{InvalidMetadata, IoError};
+use crate::file_config::{FileError, FileResult};
+use crate::pmtiles::impl_pmtiles_source;
+use crate::source::{Source, UrlQuery};
+use crate::{MartinResult, TileCoord, TileData};
+
+impl_pmtiles_source!(PmtFileSource, MmapBackend, NoCache, PathBuf);
+
+impl PmtFileSource {
+    pub async fn new_box(id: String, path: PathBuf) -> FileResult<Box<dyn Source>> {
+        Ok(Box::new(PmtFileSource::new(id, path).await?))
+    }
+
+    async fn new(id: String, path: PathBuf) -> FileResult<Self> {
+        let backend = MmapBackend::try_from(path.as_path())
+            .await
+            .map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("{e:?}: Cannot open file {}", path.display()),
+                )
+            })
+            .map_err(|e| IoError(e, path.clone()))?;
+
+        let reader = AsyncPmTilesReader::try_from_source(backend).await;
+        let reader = reader
+            .map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("{e:?}: Cannot open file {}", path.display()),
+                )
+            })
+            .map_err(|e| IoError(e, path.clone()))?;
+
+        Self::new_int(id, path, reader).await
+    }
+
+    fn display_path(path: &Path) -> impl Display + '_ {
+        path.display()
+    }
+
+    fn metadata_err(message: String, path: PathBuf) -> FileError {
+        InvalidMetadata(message, path)
+    }
+}
--- a/martin/src/pmtiles/http_pmtiles.rs
+++ b/martin/src/pmtiles/http_pmtiles.rs
@ -0,0 +1,69 @@
+use std::fmt::{Debug, Display, Formatter};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use log::{trace, warn};
+use martin_tile_utils::{Encoding, Format, TileInfo};
+use moka::future::Cache;
+use pmtiles::async_reader::AsyncPmTilesReader;
+use pmtiles::cache::{DirCacheResult, DirectoryCache};
+use pmtiles::http::HttpBackend;
+use pmtiles::{Compression, Directory, TileType};
+use reqwest::Client;
+use tilejson::TileJSON;
+use url::Url;
+
+use crate::file_config::FileError::InvalidUrlMetadata;
+use crate::file_config::{FileError, FileResult};
+use crate::pmtiles::impl_pmtiles_source;
+use crate::source::{Source, UrlQuery};
+use crate::{MartinResult, TileCoord, TileData};
+
+struct PmtCache(Cache<usize, Directory>);
+
+impl PmtCache {
+    fn new() -> Self {
+        Self(Cache::new(10_000))
+    }
+}
+
+#[async_trait]
+impl DirectoryCache for PmtCache {
+    async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult {
+        match self.0.get(&offset).await {
+            Some(dir) => dir.find_tile_id(tile_id).into(),
+            None => DirCacheResult::NotCached,
+        }
+    }
+
+    async fn insert_dir(&self, offset: usize, directory: Directory) {
+        self.0.insert(offset, directory).await;
+    }
+}
+
+impl_pmtiles_source!(PmtHttpSource, HttpBackend, PmtCache, Url);
+
+impl PmtHttpSource {
+    pub async fn new_url_box(id: String, url: Url) -> FileResult<Box<dyn Source>> {
+        let client = Client::new();
+        let cache = PmtCache::new();
+        Ok(Box::new(
+            PmtHttpSource::new_url(client, cache, id, url).await?,
+        ))
+    }
+
+    async fn new_url(client: Client, cache: PmtCache, id: String, url: Url) -> FileResult<Self> {
+        let reader = AsyncPmTilesReader::new_with_cached_url(cache, client, url.clone()).await;
+        let reader = reader.map_err(|e| FileError::PmtError(e, url.to_string()))?;
+
+        Self::new_int(id, url, reader).await
+    }
+
+    fn display_path(path: &Url) -> impl Display + '_ {
+        path
+    }
+
+    fn metadata_err(message: String, path: Url) -> FileError {
+        InvalidUrlMetadata(message, path)
+    }
+}
--- a/martin/src/pmtiles/mod.rs
+++ b/martin/src/pmtiles/mod.rs
@ -1,155 +1,137 @@
-use std::fmt::{Debug, Formatter};
-use std::io;
-use std::path::PathBuf;
-use std::sync::Arc;
+mod file_pmtiles;
+mod http_pmtiles;

-use async_trait::async_trait;
-use log::{trace, warn};
-use martin_tile_utils::{Encoding, Format, TileInfo};
-use pmtiles::async_reader::AsyncPmTilesReader;
-use pmtiles::mmap::MmapBackend;
-use pmtiles::{Compression, TileType};
-use tilejson::TileJSON;
+pub use file_pmtiles::PmtFileSource;
+pub use http_pmtiles::PmtHttpSource;

-use crate::file_config::FileError::{InvalidMetadata, IoError};
-use crate::file_config::FileResult;
-use crate::source::{Source, TileData, UrlQuery};
-use crate::{MartinResult, TileCoord};
-
-#[derive(Clone)]
-pub struct PmtSource {
-    id: String,
-    path: PathBuf,
-    pmtiles: Arc<AsyncPmTilesReader<MmapBackend>>,
-    tilejson: TileJSON,
-    tile_info: TileInfo,
-}
-
-impl Debug for PmtSource {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "PmtSource {{ id: {}, path: {:?} }}", self.id, self.path)
-    }
-}
-
-impl PmtSource {
-    pub async fn new_box(id: String, path: PathBuf) -> FileResult<Box<dyn Source>> {
-        Ok(Box::new(PmtSource::new(id, path).await?))
-    }
-
-    async fn new(id: String, path: PathBuf) -> FileResult<Self> {
-        let backend = MmapBackend::try_from(path.as_path())
-            .await
-            .map_err(|e| {
-                io::Error::new(
-                    io::ErrorKind::Other,
-                    format!("{e:?}: Cannot open file {}", path.display()),
-                )
-            })
-            .map_err(|e| IoError(e, path.clone()))?;
-
-        let reader = AsyncPmTilesReader::try_from_source(backend).await;
-        let reader = reader
-            .map_err(|e| {
-                io::Error::new(
-                    io::ErrorKind::Other,
-                    format!("{e:?}: Cannot open file {}", path.display()),
-                )
-            })
-            .map_err(|e| IoError(e, path.clone()))?;
-        let hdr = &reader.get_header();
-
-        if hdr.tile_type != TileType::Mvt && hdr.tile_compression != Compression::None {
-            return Err(InvalidMetadata(
-                format!(
-                    "Format {:?} and compression {:?} are not yet supported",
-                    hdr.tile_type, hdr.tile_compression
-                ),
-                path,
-            ));
+macro_rules! impl_pmtiles_source {
+    ($name: ident, $backend: ty, $cache: ty, $path: ty) => {
+        #[derive(Clone)]
+        pub struct $name {
+            id: String,
+            path: $path,
+            pmtiles: Arc<AsyncPmTilesReader<$backend, $cache>>,
+            tilejson: TileJSON,
+            tile_info: TileInfo,
        }

-        let format = match hdr.tile_type {
-            TileType::Mvt => TileInfo::new(
-                Format::Mvt,
-                match hdr.tile_compression {
-                    Compression::None => Encoding::Uncompressed,
-                    Compression::Unknown => {
-                        warn!(
-                            "MVT tiles have unknown compression in file {}",
-                            path.display()
-                        );
-                        Encoding::Uncompressed
-                    }
-                    Compression::Gzip => Encoding::Gzip,
-                    Compression::Brotli => Encoding::Brotli,
-                    Compression::Zstd => Encoding::Zstd,
-                },
-            ),
-            TileType::Png => Format::Png.into(),
-            TileType::Jpeg => Format::Jpeg.into(),
-            TileType::Webp => Format::Webp.into(),
-            TileType::Unknown => {
-                return Err(InvalidMetadata(
-                    "Unknown tile type".to_string(),
-                    path.clone(),
-                ))
+        impl Debug for $name {
+            fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+                write!(
+                    f,
+                    "{} {{ id: {}, path: {:?} }}",
+                    stringify!($name),
+                    self.id,
+                    self.path
+                )
            }
-        };
-
-        let tilejson = reader.parse_tilejson(Vec::new()).await.unwrap_or_else(|e| {
-            warn!("{e:?}: Unable to parse metadata for {}", path.display());
-            hdr.get_tilejson(Vec::new())
-        });
-
-        Ok(Self {
-            id,
-            path,
-            pmtiles: Arc::new(reader),
-            tilejson,
-            tile_info: format,
-        })
-    }
-}
-
-#[async_trait]
-impl Source for PmtSource {
-    fn get_id(&self) -> &str {
-        &self.id
-    }
-
-    fn get_tilejson(&self) -> &TileJSON {
-        &self.tilejson
-    }
-
-    fn get_tile_info(&self) -> TileInfo {
-        self.tile_info
-    }
-
-    fn clone_source(&self) -> Box<dyn Source> {
-        Box::new(self.clone())
-    }
-
-    async fn get_tile(
-        &self,
-        xyz: &TileCoord,
-        _url_query: &Option<UrlQuery>,
-    ) -> MartinResult<TileData> {
-        // TODO: optimize to return Bytes
-        if let Some(t) = self
-            .pmtiles
-            .get_tile(xyz.z, u64::from(xyz.x), u64::from(xyz.y))
-            .await
-        {
-            Ok(t.to_vec())
-        } else {
-            trace!(
-                "Couldn't find tile data in {}/{}/{} of {}",
-                xyz.z,
-                xyz.x,
-                xyz.y,
-                &self.id
-            );
-            Ok(Vec::new())
        }
-    }
+
+        impl $name {
+            async fn new_int(
+                id: String,
+                path: $path,
+                reader: AsyncPmTilesReader<$backend, $cache>,
+            ) -> FileResult<Self> {
+                let hdr = &reader.get_header();
+
+                if hdr.tile_type != TileType::Mvt && hdr.tile_compression != Compression::None {
+                    return Err(Self::metadata_err(
+                        format!(
+                            "Format {:?} and compression {:?} are not yet supported",
+                            hdr.tile_type, hdr.tile_compression
+                        ),
+                        path,
+                    ));
+                }
+
+                let format = match hdr.tile_type {
+                    TileType::Mvt => TileInfo::new(
+                        Format::Mvt,
+                        match hdr.tile_compression {
+                            Compression::None => Encoding::Uncompressed,
+                            Compression::Unknown => {
+                                warn!(
+                                    "MVT tiles have unknown compression in file {}",
+                                    Self::display_path(&path)
+                                );
+                                Encoding::Uncompressed
+                            }
+                            Compression::Gzip => Encoding::Gzip,
+                            Compression::Brotli => Encoding::Brotli,
+                            Compression::Zstd => Encoding::Zstd,
+                        },
+                    ),
+                    // All these assume uncompressed data (validated above)
+                    TileType::Png => Format::Png.into(),
+                    TileType::Jpeg => Format::Jpeg.into(),
+                    TileType::Webp => Format::Webp.into(),
+                    TileType::Unknown => {
+                        return Err(Self::metadata_err("Unknown tile type".to_string(), path))
+                    }
+                };
+
+                let tilejson = reader.parse_tilejson(Vec::new()).await.unwrap_or_else(|e| {
+                    warn!(
+                        "{e:?}: Unable to parse metadata for {}",
+                        Self::display_path(&path)
+                    );
+                    hdr.get_tilejson(Vec::new())
+                });
+
+                Ok(Self {
+                    id,
+                    path,
+                    pmtiles: Arc::new(reader),
+                    tilejson,
+                    tile_info: format,
+                })
+            }
+        }
+
+        #[async_trait]
+        impl Source for $name {
+            fn get_id(&self) -> &str {
+                &self.id
+            }
+
+            fn get_tilejson(&self) -> &TileJSON {
+                &self.tilejson
+            }
+
+            fn get_tile_info(&self) -> TileInfo {
+                self.tile_info
+            }
+
+            fn clone_source(&self) -> Box<dyn Source> {
+                Box::new(self.clone())
+            }
+
+            async fn get_tile(
+                &self,
+                xyz: &TileCoord,
+                _url_query: &Option<UrlQuery>,
+            ) -> MartinResult<TileData> {
+                // TODO: optimize to return Bytes
+                if let Some(t) = self
+                    .pmtiles
+                    .get_tile(xyz.z, u64::from(xyz.x), u64::from(xyz.y))
+                    .await
+                {
+                    Ok(t.to_vec())
+                } else {
+                    trace!(
+                        "Couldn't find tile data in {}/{}/{} of {}",
+                        xyz.z,
+                        xyz.x,
+                        xyz.y,
+                        &self.id
+                    );
+                    Ok(Vec::new())
+                }
+            }
+        }
+    };
 }
+
+pub(crate) use impl_pmtiles_source;
--- a/mbtiles/tests/copy.rs
+++ b/mbtiles/tests/copy.rs
@ -52,7 +52,7 @@ const TILES_V2: &str = "
   -- , (6, 2, 6, cast('1-keep-1-rm' as blob)) -- this row is removed
      , (5, 3, 7, cast('new' as blob))         -- this row is added, dup value
      , (5, 3, 8, cast('new' as blob))         -- this row is added, dup value
-      
+
      -- Expected delta:
      --   5/1/1 edit
      --   5/1/2 edit
--- a/tests/config.yaml
+++ b/tests/config.yaml
@ -166,9 +166,11 @@ postgres:


 pmtiles:
+  paths:
+    - http://localhost:5412/webp2.pmtiles
  sources:
    pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
-    pmt2: tests/fixtures/pmtiles2/webp2.pmtiles
+    pmt2: http://localhost:5412/webp2.pmtiles

 sprites:
  paths: tests/fixtures/sprites/src1
--- a/tests/expected/auto/save_config.yaml
+++ b/tests/expected/auto/save_config.yaml
@ -191,16 +191,14 @@ pmtiles:
  paths:
  - tests/fixtures/mbtiles
  - tests/fixtures/pmtiles
-  - tests/fixtures/pmtiles2
  sources:
    png: tests/fixtures/pmtiles/png.pmtiles
    stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
-    webp2: tests/fixtures/pmtiles2/webp2.pmtiles
+    webp2: http://localhost:5412/webp2.pmtiles
 mbtiles:
  paths:
  - tests/fixtures/mbtiles
  - tests/fixtures/pmtiles
-  - tests/fixtures/pmtiles2
  sources:
    geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles
    geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles
--- a/tests/expected/configured/catalog_cfg.json
+++ b/tests/expected/configured/catalog_cfg.json
@ -44,6 +44,10 @@
    },
    "table_source": {
      "content_type": "application/x-protobuf"
+    },
+    "webp2": {
+      "content_type": "image/webp",
+      "name": "ne2sr"
    }
  },
  "sprites": {
--- a/tests/expected/configured/save_config.yaml
+++ b/tests/expected/configured/save_config.yaml
@ -163,7 +163,8 @@ postgres:
 pmtiles:
  sources:
    pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
-    pmt2: tests/fixtures/pmtiles2/webp2.pmtiles
+    pmt2: http://localhost:5412/webp2.pmtiles
+    webp2: http://localhost:5412/webp2.pmtiles
 sprites:
  paths: tests/fixtures/sprites/src1
  sources:
--- a/tests/test.sh
+++ b/tests/test.sh
@ -9,6 +9,7 @@ CURL=${CURL:-curl --silent --show-error --fail --compressed}

 MARTIN_BUILD_ALL="${MARTIN_BUILD_ALL:-cargo build}"

+STATICS_URL="${STATICS_URL:-http://localhost:5412}"
 MARTIN_PORT="${MARTIN_PORT:-3111}"
 MARTIN_URL="http://localhost:${MARTIN_PORT}"
 MARTIN_ARGS="${MARTIN_ARGS:---listen-addresses localhost:${MARTIN_PORT}}"
@ -199,6 +200,10 @@ if [[ "$MARTIN_BUILD_ALL" != "-" ]]; then
 fi


+echo "------------------------------------------------------------------------------------------------------------------------"
+echo "Check HTTP server is running"
+$CURL --head "$STATICS_URL/webp2.pmtiles"
+
 echo "------------------------------------------------------------------------------------------------------------------------"
 echo "Test auto configured Martin"

@ -207,7 +212,7 @@ LOG_FILE="${LOG_DIR}/${TEST_NAME}.txt"
 TEST_OUT_DIR="${TEST_OUT_BASE_DIR}/${TEST_NAME}"
 mkdir -p "$TEST_OUT_DIR"

-ARG=(--default-srid 900913 --auto-bounds calc --save-config "${TEST_OUT_DIR}/save_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2 --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts)
+ARG=(--default-srid 900913 --auto-bounds calc --save-config "${TEST_OUT_DIR}/save_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles "$STATICS_URL/webp2.pmtiles" --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts)
 export DATABASE_URL="$MARTIN_DATABASE_URL"

 set -x