Docker improvements and CI tests ()

* Change docker image to use `entrypoint` -- so that Martin can be used
as a command:

```bash
docker run maplibre/martin <parameters>
```

* The docker image is now tested the same way as in the CI tests
* Added a few changes to the justfile

Fixes 
This commit is contained in:
Yuri Astrakhan 2022-11-02 14:00:05 -04:00 committed by GitHub
parent b8ac719be8
commit 6b114cc7f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 145 additions and 52 deletions

View File

@ -2,6 +2,10 @@
.git/
# Having Dockerfile in .dockerignore lets us develop docker images without full rebuild on each change
Dockerfile
# Justfile is not needed in the image
justfile
# Remove self from the image too
.dockerignore
#### COPIED FROM .gitignore ####
.DS_Store
@ -12,3 +16,4 @@ Dockerfile
pg_data/
config.yml
tests/output/
tests/expected/

View File

@ -150,36 +150,3 @@ jobs:
body_path: CHANGELOG.md
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
docker:
needs: [test]
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v3
# https://github.com/docker/metadata-action
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v4
with:
images: maplibre/martin
# https://github.com/docker/setup-qemu-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
install: true
- name: Login to DockerHub
uses: docker/login-action@v2
if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }}
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Build and push the Docker image
uses: docker/build-push-action@v3
with:
push: ${{ github.actor != 'dependabot[bot]' && github.event_name != 'pull_request' }}
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}

82
.github/workflows/docker.yml vendored Normal file
View File

@ -0,0 +1,82 @@
name: Build and test docker
on:
push:
branches: [main]
pull_request:
branches: [main]
release:
branches: [main]
workflow_dispatch:
jobs:
docker:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v3
# https://github.com/docker/metadata-action
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v4
with:
images: maplibre/martin
# https://github.com/docker/setup-qemu-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
install: true
- name: Build the Docker image
id: docker_build
uses: docker/build-push-action@v3
with:
push: false
load: true
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
- name: Start postgres
uses: nyurik/action-setup-postgis@v1
id: pg
with:
username: test
password: test
database: test
rights: --superuser
- name: Init database
shell: bash
run: tests/fixtures/initdb.sh
env:
DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
- name: Test Docker image
run: |
export MARTIN_BUILD=-
export MARTIN_BIN="docker run --rm --net host -e DATABASE_URL -v $PWD/tests:/tests ${{ steps.docker_meta.outputs.tags }}"
tests/test.sh
env:
DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
- name: Login to DockerHub
uses: docker/login-action@v2
if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }}
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Push the Docker image
if: ${{ github.actor != 'dependabot[bot]' && github.event_name != 'pull_request' }}
uses: docker/build-push-action@v3
with:
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}

View File

@ -1,3 +1,5 @@
name: Code coverage
on:
push:
branches: [main]
@ -5,8 +7,6 @@ on:
branches: [main]
workflow_dispatch:
name: Code coverage
jobs:
grcov:
runs-on: ubuntu-latest

View File

@ -17,4 +17,4 @@ COPY --from=builder \
/usr/local/bin/
EXPOSE 3000
CMD /usr/local/bin/martin
ENTRYPOINT ["/usr/local/bin/martin"]

View File

@ -9,8 +9,8 @@ export CARGO_TERM_COLOR := "always"
just --list --unsorted
# Start Martin server and a test database
run: start-db
cargo run
run *ARGS: start-db
cargo run -- {{ARGS}}
# Perform cargo clean to delete all build files
clean: clean-test
@ -59,6 +59,14 @@ bless: start-db clean-test
rm -rf tests/expected
mv tests/output tests/expected
# Build martin docker image
docker-build:
docker build -t martin .
# Build and run martin docker image
docker-run *ARGS:
docker run -it --rm --net host -e DATABASE_URL -v $PWD/tests:/tests martin {{ARGS}}
# Do any git command, ensuring that the testing environment is set up. Accepts the same arguments as git.
git *ARGS: start-db
git {{ARGS}}

View File

@ -10,23 +10,48 @@ MARTIN_BIN="${MARTIN_BIN:-cargo run --}"
function wait_for_martin {
# Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this:
# timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS http://localhost:3000/healthz
echo "Waiting for Martin to start..."
for i in {1..300}; do
PROCESS_ID=$1
echo "Waiting for Martin ($PROCESS_ID) to start..."
for i in {1..30}; do
if curl -sSf http://localhost:3000/healthz 2>/dev/null >/dev/null; then
echo "Martin is up!"
curl -s http://localhost:3000/healthz
return
fi
sleep 0.2
if ps -p $PROCESS_ID > /dev/null ; then
echo "Martin is not up yet, waiting..."
sleep 1
else
echo "Martin died!"
ps au
lsof -i
exit 1
fi
done
echo "Martin did not start in time"
ps au
lsof -i
exit 1
}
function kill_process {
PROCESS_ID=$1
echo "Waiting for Martin ($PROCESS_ID) to stop..."
kill $PROCESS_ID
for i in {1..50}; do
if ps -p $PROCESS_ID > /dev/null ; then
sleep 0.1
else
echo "Martin ($PROCESS_ID) has stopped"
return
fi
done
echo "Martin did not stop in time, killing it"
kill -9 $PROCESS_ID
# wait for it to die using timeout and wait
timeout -k 1s 1s wait $PROCESS_ID || true
}
test_pbf()
{
FILENAME="$TEST_OUT_DIR/$1.pbf"
@ -49,11 +74,15 @@ if [[ "$MARTIN_BUILD" != "-" ]]; then
$MARTIN_BUILD
fi
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test auto configured Martin"
set -x
$MARTIN_BIN --default-srid 900913 &
PROCESS_ID=$!
trap "kill $PROCESS_ID || true" EXIT
wait_for_martin
echo "Test auto configured Martin"
{ set +x; } 2> /dev/null
trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
wait_for_martin $PROCESS_ID
TEST_OUT_DIR="$(dirname "$0")/output/auto"
mkdir -p "$TEST_OUT_DIR"
@ -97,15 +126,17 @@ test_pbf points3857_srid_0_0_0 http://localhost:3000/public.points3857/0/0/0.pb
echo "IGNORING: This test is currently failing, and has been failing for a while"
echo "IGNORING: " test_pbf points_empty_srid_0_0_0 http://localhost:3000/public.points_empty_srid/0/0/0.pbf
kill $PROCESS_ID
kill_process $PROCESS_ID
# ------------------------------------------------------------------------------------------------------------------------
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test pre-configured Martin"
set -x
$MARTIN_BIN --config tests/config.yaml "$DATABASE_URL" &
PROCESS_ID=$!
trap "kill $PROCESS_ID || true" EXIT
wait_for_martin
echo "Test pre-configured Martin"
{ set +x; } 2> /dev/null
trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
wait_for_martin $PROCESS_ID
TEST_OUT_DIR="$(dirname "$0")/output/configured"
mkdir -p "$TEST_OUT_DIR"
@ -119,4 +150,4 @@ test_pbf cmp_0_0_0 http://localhost:3000/public.points1,public.points2/0/0/0.pb
test_pbf fnc_0_0_0 http://localhost:3000/rpc/public.function_source/0/0/0.pbf
test_pbf fnc2_0_0_0 http://localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin
kill $PROCESS_ID
kill_process $PROCESS_ID