maplibre/martin
1
1
Fork 0
mirror of https://github.com/maplibre/martin.git synced 2024-12-19 12:51:37 +03:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Improve SSL mode testing (#913)

Browse Source
This commit is contained in:
Yuri Astrakhan 2023-09-30 22:49:56 -04:00 committed by GitHub
parent ae8e0709d7
commit 64d652f5af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 191 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
.github/workflows/ci.yml vendored
View File

@ -24,7 +24,68 @@ defaults:
shell: bash
jobs:
docker:
lint-debug-test:
name: Lint and Unit test
runs-on: ubuntu-latest
env:
PGDATABASE: test
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
services:
postgres:
image: postgis/postgis:16-3.4
ports:
# will assign a random free host port
- 5432/tcp
# Sadly there is currently no way to pass arguments to the service image other than this hack
# See also https://stackoverflow.com/a/62720566/177275
options: >-
-e POSTGRES_DB=test
-e POSTGRES_USER=postgres
-e POSTGRES_PASSWORD=postgres
-e PGDATABASE=test
-e PGUSER=postgres
-e PGPASSWORD=postgres
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
--entrypoint sh
postgis/postgis:16-3.4
-c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Rust Versions
run: rustc --version && cargo --version
- uses: Swatinem/rust-cache@v2
if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
- run: cargo fmt --all -- --check
- run: cargo clippy --package martin-tile-utils -- -D warnings
- run: cargo clippy --package martin-mbtiles --no-default-features -- -D warnings
- run: cargo clippy --package martin-mbtiles -- -D warnings
- run: cargo clippy --package martin -- -D warnings
- run: cargo clippy --package martin --features bless-tests -- -D warnings
- run: cargo doc --no-deps --workspace
env:
RUSTDOCFLAGS: "-D warnings"
- name: Init database
run: tests/fixtures/initdb.sh
env:
PGPORT: ${{ job.services.postgres.ports[5432] }}
- name: Run cargo test
run: |
set -x
cargo test --package martin-tile-utils
cargo test --package martin-mbtiles --no-default-features
cargo test --package martin-mbtiles
cargo test --package martin
cargo test --doc
env:
DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require
docker-build-test:
name: Build and test docker images
runs-on: ubuntu-latest
env:
@ -39,10 +100,7 @@ jobs:
postgres:
image: postgis/postgis:15-3.3
ports:
# will assign a random free host port
- 5432/tcp
# Sadly there is currently no way to pass arguments to the service image other than this hack
# See also https://stackoverflow.com/a/62720566/177275
options: >-
-e POSTGRES_DB=test
-e POSTGRES_USER=postgres
@ -68,7 +126,7 @@ jobs:
# Install latest cross version from git (disabled as it is probably less stable)
# cargo install cross --git https://github.com/cross-rs/cross
cross --version
- name: Setup database
- name: Init database
run: tests/fixtures/initdb.sh
env:
PGPORT: ${{ job.services.postgres.ports[5432] }}
@ -207,19 +265,10 @@ jobs:
run: rustc --version && cargo --version
- uses: Swatinem/rust-cache@v2
if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
- name: Lint (Linux)
if: matrix.target == 'x86_64-unknown-linux-gnu'
run: |
set -x
cargo fmt --all -- --check
cargo clippy --package martin-tile-utils -- -D warnings
cargo clippy --package martin-mbtiles --no-default-features -- -D warnings
cargo clippy --package martin-mbtiles -- -D warnings
cargo clippy --package martin -- -D warnings
cargo clippy --package martin --features bless-tests -- -D warnings
- name: Build (.deb)
if: matrix.target == 'debian-x86_64'
run: |
set -x
sudo apt-get install -y dpkg dpkg-dev liblzma-dev
cargo install cargo-deb
cargo deb -v -p martin --output target/debian/debian-x86_64.deb
@ -233,7 +282,6 @@ jobs:
export RUSTFLAGS='-C strip=debuginfo'
cargo build --release --target ${{ matrix.target }} --package martin-mbtiles
cargo build --release --target ${{ matrix.target }} --package martin
mkdir -p target_releases
mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/
mv target/${{ matrix.target }}/release/martin${{ matrix.ext }} target_releases/
@ -243,8 +291,8 @@ jobs:
name: build-${{ matrix.target }}
path: target_releases/*
test:
name: Test ${{ matrix.target }}
test-multi-os:
name: Test on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: [ build ]
strategy:
@ -279,19 +327,6 @@ jobs:
tests/fixtures/initdb.sh
env:
DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
- name: Unit Tests (Linux)
if: matrix.target == 'x86_64-unknown-linux-gnu'
run: |
set -x
cargo test --package martin-tile-utils
cargo test --package martin-mbtiles --no-default-features
cargo test --package martin-mbtiles
cargo test --package martin
cargo test --doc
RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace
cargo clean
env:
DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
- name: Download build artifact build-${{ matrix.target }}
uses: actions/download-artifact@v3
with:
@ -337,8 +372,8 @@ jobs:
path: tests/output/*
retention-days: 5
test-legacy:
name: Test Legacy DB
test-with-svc:
name: Test postgis:${{ matrix.img_ver }} sslmode=${{ matrix.sslmode }}
runs-on: ubuntu-latest
needs: [ build ]
strategy:
@ -346,30 +381,35 @@ jobs:
matrix:
include:
# These must match the versions of postgres used in the docker-compose.yml
- image: postgis/postgis:11-3.0-alpine
- img_ver: 11-3.0-alpine
args: postgres
sslmode: disable
- image: postgis/postgis:14-3.3-alpine
- img_ver: 14-3.3-alpine
args: postgres
sslmode: disable
# alpine images don't support SSL, so for this we use the debian images
- image: postgis/postgis:15-3.3
- img_ver: 15-3.3
args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
sslmode: require
#
# FIXME!
# DISABLED because Rustls fails to validate name (CN?) with the NotValidForName error
#- img_ver: 15-3.3
# args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
# sslmode: verify-ca
#- img_ver: 15-3.3
# args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
# sslmode: verify-full
env:
# PG_* variables are used by psql
PGDATABASE: test
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
services:
postgres:
image: ${{ matrix.image }}
image: postgis/postgis:${{ matrix.img_ver }}
ports:
# will assign a random free host port
- 5432/tcp
# Sadly there is currently no way to pass arguments to the service image other than this hack
# See also https://stackoverflow.com/a/62720566/177275
options: >-
-e POSTGRES_DB=test
-e POSTGRES_USER=postgres
@ -382,17 +422,24 @@ jobs:
--health-timeout 5s
--health-retries 5
--entrypoint sh
${{ matrix.image }}
postgis/postgis:${{ matrix.img_ver }}
-c "exec docker-entrypoint.sh ${{ matrix.args }}"
steps:
- name: Checkout sources
uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
- name: Setup database
- name: Init database
run: tests/fixtures/initdb.sh
env:
PGPORT: ${{ job.services.postgres.ports[5432] }}
- name: Get DB SSL cert (sslmode=verify-*)
if: matrix.sslmode == 'verify-ca' || matrix.sslmode == 'verify-full'
run: |
set -x
mkdir -p target/certs
docker cp ${{ job.services.postgres.id }}:/etc/ssl/certs/ssl-cert-snakeoil.pem target/certs/server.crt
docker cp ${{ job.services.postgres.id }}:/etc/ssl/private/ssl-cert-snakeoil.key target/certs/server.key
- name: Download build artifact build-x86_64-unknown-linux-gnu
uses: actions/download-artifact@v3
with:
@ -400,6 +447,9 @@ jobs:
path: target_releases/
- name: Integration Tests
run: |
if [[ "${{ matrix.sslmode }}" == "verify-ca" || "${{ matrix.sslmode }}" == "verify-full" ]]; then
export PGSSLROOTCERT=target/certs/server.crt
fi
export MARTIN_BUILD=-
export MARTIN_BIN=target_releases/martin
export MBTILES_BUILD=-
@ -417,6 +467,9 @@ jobs:
- name: Tests Debian package
run: |
sudo dpkg -i target_releases/debian-x86_64.deb
if [[ "${{ matrix.sslmode }}" == "verify-ca" || "${{ matrix.sslmode }}" == "verify-full" ]]; then
export PGSSLROOTCERT=target/certs/server.crt
fi
export MARTIN_BUILD=-
export MARTIN_BIN=/usr/bin/martin
export MBTILES_BUILD=-
@ -427,6 +480,7 @@ jobs:
env:
DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
- name: Unit Tests
if: matrix.sslmode != 'verify-ca' && matrix.sslmode != 'verify-full'
run: |
echo "Running unit tests, connecting to DATABASE_URL=$DATABASE_URL"
echo "Same but as base64 to prevent GitHub obfuscation (this is not a secret):"
@ -447,7 +501,7 @@ jobs:
package:
name: Package ${{ matrix.target }}
runs-on: ${{ matrix.os }}
needs: [ docker, test, test-legacy ]
needs: [ lint-debug-test, docker-build-test, test-multi-os, test-with-svc ]
strategy:
fail-fast: true
matrix:

28
Cargo.lock generated
View File

@ -495,9 +495,9 @@ dependencies = [
[[package]]
name = "brotli"
version = "3.3.4"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@ -506,9 +506,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
version = "2.3.4"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@ -1696,9 +1696,9 @@ dependencies = [
[[package]]
name = "linux-raw-sys"
version = "0.4.7"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db"
[[package]]
name = "local-channel"
@ -2394,9 +2394,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.9.5"
version = "1.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff"
dependencies = [
"aho-corasick",
"memchr",
@ -2406,9 +2406,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.8"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
dependencies = [
"aho-corasick",
"memchr",
@ -2464,9 +2464,9 @@ dependencies = [
[[package]]
name = "roxmltree"
version = "0.18.0"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8f595a457b6b8c6cda66a48503e92ee8d19342f905948f29c383200ec9eb1d8"
checksum = "862340e351ce1b271a378ec53f304a5558f7db87f3769dc655a8f6ecbb68b302"
dependencies = [
"xmlparser",
]
@ -3916,9 +3916,9 @@ dependencies = [
[[package]]
name = "xmlparser"
version = "0.13.5"
version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xmlwriter"

28
docker-compose.yml
View File

@ -75,6 +75,34 @@ services:
- ./tests/fixtures/initdb-dc-ssl.sh:/docker-entrypoint-initdb.d/10_martin.sh
- ./tests/fixtures/initdb-dc.sh:/docker-entrypoint-initdb.d/20_martin.sh
db-ssl-cert:
# This should match the version of postgres used in the CI workflow
image: postgis/postgis:15-3.3
command:
- "postgres"
- "-c"
- "ssl=on"
- "-c"
- "ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem"
- "-c"
- "ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"
restart: unless-stopped
ports:
- "${PGPORT:-5411}:5432"
environment:
# POSTGRES_* variables are used by the postgis/postgres image
# PG_* variables are used by psql
- POSTGRES_DB=db
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
- PGDATABASE=db
- PGUSER=postgres
- PGPASSWORD=postgres
volumes:
- ./tests/fixtures:/fixtures
- ./tests/fixtures/initdb-dc-ssl-cert.sh:/docker-entrypoint-initdb.d/10_martin.sh
- ./tests/fixtures/initdb-dc.sh:/docker-entrypoint-initdb.d/20_martin.sh
db-legacy:
# This should match the version of postgres used in the CI workflow
image: postgis/postgis:11-3.0-alpine

29
justfile
View File

@ -43,18 +43,25 @@ clean-test:
rm -rf tests/output
# Start a test database
start: (docker-up "db")
start: (docker-up "db") docker-is-ready
# Start an ssl-enabled test database
start-ssl: (docker-up "db-ssl")
start-ssl: (docker-up "db-ssl") docker-is-ready
# Start an ssl-enabled test database that requires a client certificate
start-ssl-cert: (docker-up "db-ssl-cert") docker-is-ready
# Start a legacy test database
start-legacy: (docker-up "db-legacy")
start-legacy: (docker-up "db-legacy") docker-is-ready
# Start a specific test database, e.g. db or db-legacy
[private]
docker-up name:
docker-compose up -d {{ name }}
# Wait for the test database to be ready
[private]
docker-is-ready:
docker-compose run -T --rm db-is-ready
alias _down := stop
@ -87,6 +94,22 @@ test: start test-unit test-int
test-ssl: start-ssl test-unit clean-test
tests/test.sh
# Run all tests using an SSL connection with client cert to a test database. Expected output won't match.
test-ssl-cert: start-ssl-cert
#!/usr/bin/env bash
set -euxo pipefail
# copy client cert to the tests folder from the docker container
KEY_DIR=target/certs
mkdir -p $KEY_DIR
docker cp martin-db-ssl-cert-1:/etc/ssl/certs/ssl-cert-snakeoil.pem $KEY_DIR/ssl-cert-snakeoil.pem
docker cp martin-db-ssl-cert-1:/etc/ssl/private/ssl-cert-snakeoil.key $KEY_DIR/ssl-cert-snakeoil.key
# export DATABASE_URL="$DATABASE_URL?sslmode=verify-full&sslrootcert=$KEY_DIR/ssl-cert-snakeoil.pem&sslcert=$KEY_DIR/ssl-cert-snakeoil.pem&sslkey=$KEY_DIR/ssl-cert-snakeoil.key"
export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key"
{{just_executable()}} test-unit clean-test
tests/test.sh
# Run all tests using the oldest supported version of the database
test-legacy: start-legacy test-unit test-int

18
tests/fixtures/initdb-dc-ssl-cert.sh vendored Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env sh
set -e
mv /var/lib/postgresql/data/pg_hba.conf /var/lib/postgresql/data/pg_hba.conf.bak
cat > /var/lib/postgresql/data/pg_hba.conf <<EOF
# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only
#local all all trust
# localhost connections
#host all all 127.0.0.1/32 trust
# external connections
hostssl all all all cert
# clientcert=verify-full
EOF

11
tests/test.sh
View File

@ -13,6 +13,9 @@ MARTIN_BIN="${MARTIN_BIN:-cargo run --} ${MARTIN_ARGS}"
MBTILES_BUILD="${MBTILES_BUILD:-cargo build -p martin-mbtiles}"
MBTILES_BIN="${MBTILES_BIN:-target/debug/mbtiles}"
TMP_DIR="${TMP_DIR:-target/tmp}"
mkdir -p "$TMP_DIR"
function wait_for_martin {
# Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this:
# timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS "$MARTIN_URL/health"
@ -152,7 +155,7 @@ mkdir -p "$TEST_OUT_DIR"
ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles)
set -x
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_1.txt &
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${TMP_DIR}/test_log_1.txt" &
PROCESS_ID=`jobs -p`
{ set +x; } 2> /dev/null
@ -227,7 +230,7 @@ test_pbf mb_mvt_2_3_1 world_cities/2/3/1
test_pbf points_empty_srid_0_0_0 points_empty_srid/0/0/0
kill_process $PROCESS_ID
validate_log test_log_1.txt
validate_log "${TMP_DIR}/test_log_1.txt"
echo "------------------------------------------------------------------------------------------------------------------------"
@ -237,7 +240,7 @@ mkdir -p "$TEST_OUT_DIR"
ARG=(--config tests/config.yaml --max-feature-count 1000 --save-config "$(dirname "$0")/output/given_config.yaml" -W 1)
set -x
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_2.txt &
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${TMP_DIR}/test_log_2.txt" &
PROCESS_ID=`jobs -p`
{ set +x; } 2> /dev/null
trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
@ -266,7 +269,7 @@ test_jsn spr_cmp_2x sprite/src1,mysrc@2x.json
test_png spr_cmp_2x sprite/src1,mysrc@2x.png
kill_process $PROCESS_ID
validate_log test_log_2.txt
validate_log "${TMP_DIR}/test_log_2.txt"
remove_line "$(dirname "$0")/output/given_config.yaml" " connection_string: "
remove_line "$(dirname "$0")/output/generated_config.yaml" " connection_string: "