Docker improvements and CI tests (#472)

* Change docker image to use `entrypoint` -- so that Martin can be used
as a command:

```bash
docker run maplibre/martin <parameters>
```

* The docker image is now tested the same way as in the CI tests
* Added a few changes to the justfile

Fixes #436

.dockerignore

@@ -2,6 +2,10 @@
 .git/
 # Having Dockerfile in .dockerignore lets us develop docker images without full rebuild on each change
 Dockerfile
+# Justfile is not needed in the image
+justfile
+# Remove self from the image too
+.dockerignore
 
 #### COPIED FROM .gitignore ####
 .DS_Store
@@ -12,3 +16,4 @@ Dockerfile
 pg_data/
 config.yml
 tests/output/
+tests/expected/

.github/workflows/ci.yml

@@ -150,36 +150,3 @@ jobs:
           body_path: CHANGELOG.md
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  docker:
-    needs: [test]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      # https://github.com/docker/metadata-action
-      - name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v4
-        with:
-          images: maplibre/martin
-      # https://github.com/docker/setup-qemu-action
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      # https://github.com/docker/setup-buildx-action
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }}
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-      - name: Build and push the Docker image
-        uses: docker/build-push-action@v3
-        with:
-          push: ${{ github.actor != 'dependabot[bot]' && github.event_name != 'pull_request' }}
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/docker.yml

@@ -0,0 +1,82 @@
+name: Build and test docker
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  release:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      # https://github.com/docker/metadata-action
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v4
+        with:
+          images: maplibre/martin
+
+      # https://github.com/docker/setup-qemu-action
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Build the Docker image
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          push: false
+          load: true
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+
+      - name: Start postgres
+        uses: nyurik/action-setup-postgis@v1
+        id: pg
+        with:
+          username: test
+          password: test
+          database: test
+          rights: --superuser
+
+      - name: Init database
+        shell: bash
+        run: tests/fixtures/initdb.sh
+        env:
+          DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
+
+      - name: Test Docker image
+        run: |
+          export MARTIN_BUILD=-
+          export MARTIN_BIN="docker run --rm --net host -e DATABASE_URL -v $PWD/tests:/tests ${{ steps.docker_meta.outputs.tags }}"
+          tests/test.sh
+        env:
+          DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }}
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Push the Docker image
+        if: ${{ github.actor != 'dependabot[bot]' && github.event_name != 'pull_request' }}
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/grcov.yml

@@ -1,3 +1,5 @@
+name: Code coverage
+
 on:
   push:
     branches: [main]
@@ -5,8 +7,6 @@ on:
     branches: [main]
   workflow_dispatch:
 
-name: Code coverage
-
 jobs:
   grcov:
     runs-on: ubuntu-latest

Dockerfile

@@ -17,4 +17,4 @@ COPY --from=builder \
   /usr/local/bin/
 
 EXPOSE 3000
-CMD /usr/local/bin/martin
+ENTRYPOINT ["/usr/local/bin/martin"]

justfile

@@ -9,8 +9,8 @@ export CARGO_TERM_COLOR := "always"
   just --list --unsorted
 
 # Start Martin server and a test database
-run: start-db
+run *ARGS: start-db
-    cargo run
+    cargo run -- {{ARGS}}
 
 # Perform  cargo clean  to delete all build files
 clean: clean-test
@@ -59,6 +59,14 @@ bless: start-db clean-test
     rm -rf tests/expected
     mv tests/output tests/expected
 
+# Build martin docker image
+docker-build:
+    docker build -t martin .
+
+# Build and run martin docker image
+docker-run *ARGS:
+    docker run -it --rm --net host -e DATABASE_URL -v $PWD/tests:/tests martin {{ARGS}}
+
 # Do any git command, ensuring that the testing environment is set up. Accepts the same arguments as git.
 git *ARGS: start-db
     git {{ARGS}}

tests/test.sh

@@ -10,23 +10,48 @@ MARTIN_BIN="${MARTIN_BIN:-cargo run --}"
 function wait_for_martin {
     # Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this:
     # timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS http://localhost:3000/healthz
-
+    PROCESS_ID=$1
-    echo "Waiting for Martin to start..."
+    echo "Waiting for Martin ($PROCESS_ID) to start..."
-    for i in {1..300}; do
+    for i in {1..30}; do
         if curl -sSf http://localhost:3000/healthz 2>/dev/null >/dev/null; then
             echo "Martin is up!"
             curl -s http://localhost:3000/healthz
             return
         fi
-        sleep 0.2
+        if ps -p $PROCESS_ID > /dev/null ; then
+            echo "Martin is not up yet, waiting..."
+            sleep 1
+        else
+            echo "Martin died!"
+            ps au
+            lsof -i
+            exit 1
+        fi
     done
-
     echo "Martin did not start in time"
     ps au
     lsof -i
     exit 1
 }
 
+function kill_process {
+    PROCESS_ID=$1
+    echo "Waiting for Martin ($PROCESS_ID) to stop..."
+    kill $PROCESS_ID
+    for i in {1..50}; do
+        if ps -p $PROCESS_ID > /dev/null ; then
+            sleep 0.1
+        else
+            echo "Martin ($PROCESS_ID) has stopped"
+            return
+        fi
+    done
+    echo "Martin did not stop in time, killing it"
+    kill -9 $PROCESS_ID
+    # wait for it to die using timeout and wait
+    timeout -k 1s 1s wait $PROCESS_ID || true
+}
+
 test_pbf()
 {
   FILENAME="$TEST_OUT_DIR/$1.pbf"
@@ -49,11 +74,15 @@ if [[ "$MARTIN_BUILD" != "-" ]]; then
   $MARTIN_BUILD
 fi
 
+
+echo "------------------------------------------------------------------------------------------------------------------------"
+echo "Test auto configured Martin"
+set -x
 $MARTIN_BIN --default-srid 900913 &
 PROCESS_ID=$!
-trap "kill $PROCESS_ID || true" EXIT
+{ set +x; } 2> /dev/null
-wait_for_martin
+trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
-echo "Test auto configured Martin"
+wait_for_martin $PROCESS_ID
 
 TEST_OUT_DIR="$(dirname "$0")/output/auto"
 mkdir -p "$TEST_OUT_DIR"
@@ -97,15 +126,17 @@ test_pbf points3857_srid_0_0_0  http://localhost:3000/public.points3857/0/0/0.pb
 echo "IGNORING: This test is currently failing, and has been failing for a while"
 echo "IGNORING:   " test_pbf points_empty_srid_0_0_0  http://localhost:3000/public.points_empty_srid/0/0/0.pbf
 
-kill $PROCESS_ID
+kill_process $PROCESS_ID
 
-# ------------------------------------------------------------------------------------------------------------------------
 
+echo "------------------------------------------------------------------------------------------------------------------------"
+echo "Test pre-configured Martin"
+set -x
 $MARTIN_BIN --config tests/config.yaml "$DATABASE_URL" &
 PROCESS_ID=$!
-trap "kill $PROCESS_ID || true" EXIT
+{ set +x; } 2> /dev/null
-wait_for_martin
+trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
-echo "Test pre-configured Martin"
+wait_for_martin $PROCESS_ID
 
 TEST_OUT_DIR="$(dirname "$0")/output/configured"
 mkdir -p "$TEST_OUT_DIR"
@@ -119,4 +150,4 @@ test_pbf cmp_0_0_0  http://localhost:3000/public.points1,public.points2/0/0/0.pb
 test_pbf fnc_0_0_0  http://localhost:3000/rpc/public.function_source/0/0/0.pbf
 test_pbf fnc2_0_0_0 http://localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin
 
-kill $PROCESS_ID
+kill_process $PROCESS_ID