From 6b114cc7f158b75f4ead3073bda6c524380202ac Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 2 Nov 2022 14:00:05 -0400 Subject: [PATCH] Docker improvements and CI tests (#472) * Change docker image to use `entrypoint` -- so that Martin can be used as a command: ```bash docker run maplibre/martin ``` * The docker image is now tested the same way as in the CI tests * Added a few changes to the justfile Fixes #436 --- .dockerignore | 5 +++ .github/workflows/ci.yml | 33 --------------- .github/workflows/docker.yml | 82 ++++++++++++++++++++++++++++++++++++ .github/workflows/grcov.yml | 4 +- Dockerfile | 2 +- justfile | 12 +++++- tests/test.sh | 59 ++++++++++++++++++++------ 7 files changed, 145 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.dockerignore b/.dockerignore index ee5cf9a4..c7346b75 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,10 @@ .git/ # Having Dockerfile in .dockerignore lets us develop docker images without full rebuild on each change Dockerfile +# Justfile is not needed in the image +justfile +# Remove self from the image too +.dockerignore #### COPIED FROM .gitignore #### .DS_Store @@ -12,3 +16,4 @@ Dockerfile pg_data/ config.yml tests/output/ +tests/expected/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddc2ffe0..e5990a33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,36 +150,3 @@ jobs: body_path: CHANGELOG.md env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - docker: - needs: [test] - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v3 - # https://github.com/docker/metadata-action - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v4 - with: - images: maplibre/martin - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Login to DockerHub - uses: docker/login-action@v2 - if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }} - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - name: Build and push the Docker image - uses: docker/build-push-action@v3 - with: - push: ${{ github.actor != 'dependabot[bot]' && github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..e08a98a5 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,82 @@ +name: Build and test docker + +on: + push: + branches: [main] + pull_request: + branches: [main] + release: + branches: [main] + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + + - name: Checkout sources + uses: actions/checkout@v3 + + # https://github.com/docker/metadata-action + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: maplibre/martin + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + + - name: Build the Docker image + id: docker_build + uses: docker/build-push-action@v3 + with: + push: false + load: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Start postgres + uses: nyurik/action-setup-postgis@v1 + id: pg + with: + username: test + password: test + database: test + rights: --superuser + + - name: Init database + shell: bash + run: tests/fixtures/initdb.sh + env: + DATABASE_URL: ${{ steps.pg.outputs.connection-uri }} + + - name: Test Docker image + run: | + export MARTIN_BUILD=- + export MARTIN_BIN="docker run --rm --net host -e DATABASE_URL -v $PWD/tests:/tests ${{ steps.docker_meta.outputs.tags }}" + tests/test.sh + env: + DATABASE_URL: ${{ steps.pg.outputs.connection-uri }} + + - name: Login to DockerHub + uses: docker/login-action@v2 + if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }} + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Push the Docker image + if: ${{ github.actor != 'dependabot[bot]' && github.event_name != 'pull_request' }} + uses: docker/build-push-action@v3 + with: + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/grcov.yml b/.github/workflows/grcov.yml index efb655bc..41d7b034 100644 --- a/.github/workflows/grcov.yml +++ b/.github/workflows/grcov.yml @@ -1,3 +1,5 @@ +name: Code coverage + on: push: branches: [main] @@ -5,8 +7,6 @@ on: branches: [main] workflow_dispatch: -name: Code coverage - jobs: grcov: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index d2721d4d..e515d8b1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -17,4 +17,4 @@ COPY --from=builder \ /usr/local/bin/ EXPOSE 3000 -CMD /usr/local/bin/martin +ENTRYPOINT ["/usr/local/bin/martin"] diff --git a/justfile b/justfile index f88e8374..83babaee 100644 --- a/justfile +++ b/justfile @@ -9,8 +9,8 @@ export CARGO_TERM_COLOR := "always" just --list --unsorted # Start Martin server and a test database -run: start-db - cargo run +run *ARGS: start-db + cargo run -- {{ARGS}} # Perform cargo clean to delete all build files clean: clean-test @@ -59,6 +59,14 @@ bless: start-db clean-test rm -rf tests/expected mv tests/output tests/expected +# Build martin docker image +docker-build: + docker build -t martin . + +# Build and run martin docker image +docker-run *ARGS: + docker run -it --rm --net host -e DATABASE_URL -v $PWD/tests:/tests martin {{ARGS}} + # Do any git command, ensuring that the testing environment is set up. Accepts the same arguments as git. git *ARGS: start-db git {{ARGS}} diff --git a/tests/test.sh b/tests/test.sh index 93bb0436..dcd22121 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -10,23 +10,48 @@ MARTIN_BIN="${MARTIN_BIN:-cargo run --}" function wait_for_martin { # Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this: # timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS http://localhost:3000/healthz - - echo "Waiting for Martin to start..." - for i in {1..300}; do + PROCESS_ID=$1 + echo "Waiting for Martin ($PROCESS_ID) to start..." + for i in {1..30}; do if curl -sSf http://localhost:3000/healthz 2>/dev/null >/dev/null; then echo "Martin is up!" curl -s http://localhost:3000/healthz return fi - sleep 0.2 + if ps -p $PROCESS_ID > /dev/null ; then + echo "Martin is not up yet, waiting..." + sleep 1 + else + echo "Martin died!" + ps au + lsof -i + exit 1 + fi done - echo "Martin did not start in time" ps au lsof -i exit 1 } +function kill_process { + PROCESS_ID=$1 + echo "Waiting for Martin ($PROCESS_ID) to stop..." + kill $PROCESS_ID + for i in {1..50}; do + if ps -p $PROCESS_ID > /dev/null ; then + sleep 0.1 + else + echo "Martin ($PROCESS_ID) has stopped" + return + fi + done + echo "Martin did not stop in time, killing it" + kill -9 $PROCESS_ID + # wait for it to die using timeout and wait + timeout -k 1s 1s wait $PROCESS_ID || true +} + test_pbf() { FILENAME="$TEST_OUT_DIR/$1.pbf" @@ -49,11 +74,15 @@ if [[ "$MARTIN_BUILD" != "-" ]]; then $MARTIN_BUILD fi + +echo "------------------------------------------------------------------------------------------------------------------------" +echo "Test auto configured Martin" +set -x $MARTIN_BIN --default-srid 900913 & PROCESS_ID=$! -trap "kill $PROCESS_ID || true" EXIT -wait_for_martin -echo "Test auto configured Martin" +{ set +x; } 2> /dev/null +trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT +wait_for_martin $PROCESS_ID TEST_OUT_DIR="$(dirname "$0")/output/auto" mkdir -p "$TEST_OUT_DIR" @@ -97,15 +126,17 @@ test_pbf points3857_srid_0_0_0 http://localhost:3000/public.points3857/0/0/0.pb echo "IGNORING: This test is currently failing, and has been failing for a while" echo "IGNORING: " test_pbf points_empty_srid_0_0_0 http://localhost:3000/public.points_empty_srid/0/0/0.pbf -kill $PROCESS_ID +kill_process $PROCESS_ID -# ------------------------------------------------------------------------------------------------------------------------ +echo "------------------------------------------------------------------------------------------------------------------------" +echo "Test pre-configured Martin" +set -x $MARTIN_BIN --config tests/config.yaml "$DATABASE_URL" & PROCESS_ID=$! -trap "kill $PROCESS_ID || true" EXIT -wait_for_martin -echo "Test pre-configured Martin" +{ set +x; } 2> /dev/null +trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT +wait_for_martin $PROCESS_ID TEST_OUT_DIR="$(dirname "$0")/output/configured" mkdir -p "$TEST_OUT_DIR" @@ -119,4 +150,4 @@ test_pbf cmp_0_0_0 http://localhost:3000/public.points1,public.points2/0/0/0.pb test_pbf fnc_0_0_0 http://localhost:3000/rpc/public.function_source/0/0/0.pbf test_pbf fnc2_0_0_0 http://localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin -kill $PROCESS_ID +kill_process $PROCESS_ID