martin/.github/workflows/ci.yml

name: CI

on:
  push:
    branches: [ main ]
    paths-ignore:
      - '**.md'
      - 'demo/**'
      - 'docs/**'
      - 'HomebrewFormula/**'
  pull_request:
    branches: [ main ]
    paths-ignore:
      - '**.md'
      - 'demo/**'
      - 'docs/**'
      - 'HomebrewFormula/**'
  release:
    types: [ published ]
  workflow_dispatch:

defaults:
  run:
    shell: bash

jobs:
  lint-debug-test:
    name: Lint and Unit test
    runs-on: ubuntu-latest
    env:
      PGDATABASE: test
      PGHOST: localhost
      PGUSER: postgres
      PGPASSWORD: postgres
    services:
      postgres:
        image: postgis/postgis:16-3.4
        ports:
          # will assign a random free host port
          - 5432/tcp
        # Sadly there is currently no way to pass arguments to the service image other than this hack
        # See also https://stackoverflow.com/a/62720566/177275
        options: >-
          -e POSTGRES_DB=test
          -e POSTGRES_USER=postgres
          -e POSTGRES_PASSWORD=postgres
          -e PGDATABASE=test
          -e PGUSER=postgres
          -e PGPASSWORD=postgres
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --entrypoint sh
          postgis/postgis:16-3.4
          -c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"          
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Rust Versions
        run: rustc --version && cargo --version
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
      - run: cargo fmt --all -- --check
      - run: cargo clippy --package martin-tile-utils -- -D warnings
      - run: cargo clippy --package martin-mbtiles --no-default-features -- -D warnings
      - run: cargo clippy --package martin-mbtiles -- -D warnings
      - run: cargo clippy --package martin -- -D warnings
      - run: cargo clippy --package martin --features bless-tests -- -D warnings
      - run: cargo doc --no-deps --workspace
        env:
          RUSTDOCFLAGS: "-D warnings"
      - name: Init database
        run: tests/fixtures/initdb.sh
        env:
          PGPORT: ${{ job.services.postgres.ports[5432] }}
      - name: Run cargo test
        run: |
          set -x
          cargo test --package martin-tile-utils
          cargo test --package martin-mbtiles --no-default-features
          cargo test --package martin-mbtiles
          cargo test --package martin
          cargo test --doc          
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

  docker-build-test:
    name: Build and test docker images
    runs-on: ubuntu-latest
    env:
      # PG_* variables are used by psql
      PGDATABASE: test
      PGHOST: localhost
      PGUSER: postgres
      PGPASSWORD: postgres
      TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
      # TODO:  aarch64-unknown-linux-gnu
    services:
      postgres:
        image: postgis/postgis:15-3.3
        ports:
          - 5432/tcp
        options: >-
          -e POSTGRES_DB=test
          -e POSTGRES_USER=postgres
          -e POSTGRES_PASSWORD=postgres
          -e PGDATABASE=test
          -e PGUSER=postgres
          -e PGPASSWORD=postgres
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --entrypoint sh
          postgis/postgis:15-3.3
          -c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"          
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
      - name: Install cross
        run: |
          cargo install cross
          # Install latest cross version from git (disabled as it is probably less stable)
          # cargo install cross --git https://github.com/cross-rs/cross
          cross --version          
      - name: Init database
        run: tests/fixtures/initdb.sh
        env:
          PGPORT: ${{ job.services.postgres.ports[5432] }}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        # https://github.com/docker/setup-qemu-action
        with:
          platforms: linux/amd64,linux/arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        # https://github.com/docker/setup-buildx-action
        with:
          install: true
          platforms: linux/amd64,linux/arm64

      - name: Build targets
        run: |
          for target in $TARGETS; do
            echo -e "\n----------------------------------------------"
            echo "Building $target"

            export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo'
            cross build --release --target $target --package martin-mbtiles
            cross build --release --target $target --package martin

            mkdir -p target_releases/$target
            mv target/$target/release/mbtiles target_releases/$target
            mv target/$target/release/martin target_releases/$target
          done          

      - name: Save build artifacts to build-${{ matrix.target }}
        uses: actions/upload-artifact@v3
        with:
          name: cross-build
          path: target_releases/*
      - name: Reorganize artifacts for docker build
        run: |
          mkdir -p target_releases/linux/arm64
          mv target_releases/aarch64-unknown-linux-musl/* target_releases/linux/arm64/
          mkdir -p target_releases/linux/amd64
          mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/          

      - name: Build linux/arm64 Docker image
        uses: docker/build-push-action@v5
        # https://github.com/docker/build-push-action
        with:
          context: .
          file: multi-platform.Dockerfile
          load: true
          tags: ${{ github.repository }}:linux-arm64
          platforms: linux/arm64
      - name: Test linux/arm64 Docker image
        run: |
          PLATFORM=linux/arm64
          TAG=${{ github.repository }}:linux-arm64
          export MBTILES_BUILD=-
          export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
          export MARTIN_BUILD=-
          export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
          tests/test.sh          
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

      - name: Build linux/amd64 Docker image
        uses: docker/build-push-action@v5
        # https://github.com/docker/build-push-action
        with:
          context: .
          file: multi-platform.Dockerfile
          load: true
          tags: ${{ github.repository }}:linux-amd64
          platforms: linux/amd64
      - name: Test linux/amd64 Docker image
        run: |
          PLATFORM=linux/amd64
          TAG=${{ github.repository }}:linux-amd64
          export MBTILES_BUILD=-
          export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
          export MARTIN_BUILD=-
          export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
          tests/test.sh          
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

      - name: Login to GitHub Docker registry
        if: github.event_name != 'pull_request'
        uses: docker/login-action@v3
        # https://github.com/docker/login-action
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Docker meta
        id: docker_meta
        uses: docker/metadata-action@v5
        # https://github.com/docker/metadata-action
        with:
          images: ghcr.io/${{ github.repository }}
      - name: Push the Docker image
        if: github.event_name != 'pull_request'
        uses: docker/build-push-action@v5
        with:
          context: .
          file: multi-platform.Dockerfile
          push: true
          tags: ${{ steps.docker_meta.outputs.tags }}
          labels: ${{ steps.docker_meta.outputs.labels }}
          platforms: linux/amd64,linux/arm64

  build:
    name: Build ${{ matrix.target }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: true
      matrix:
        include:
          - target: aarch64-apple-darwin
            os: macOS-latest
            cross: 'true'
          - target: debian-x86_64
            os: ubuntu-latest
            cross: 'true'
          - target: x86_64-apple-darwin
            os: macOS-latest
          - target: x86_64-pc-windows-msvc
            os: windows-latest
            ext: '.exe'
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Rust Versions
        run: rustc --version && cargo --version
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
      - name: Build (.deb)
        if: matrix.target == 'debian-x86_64'
        run: |
          set -x
          sudo apt-get install -y dpkg dpkg-dev liblzma-dev
          cargo install cargo-deb
          cargo deb -v -p martin --output target/debian/debian-x86_64.deb
          mkdir -p target_releases
          mv target/debian/debian-x86_64.deb target_releases/          
      - name: Build
        if: matrix.target != 'debian-x86_64'
        run: |
          set -x
          rustup target add "${{ matrix.target }}"
          export RUSTFLAGS='-C strip=debuginfo'
          cargo build --release --target ${{ matrix.target }} --package martin-mbtiles
          cargo build --release --target ${{ matrix.target }} --package martin
          mkdir -p target_releases
          mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/
          mv target/${{ matrix.target }}/release/martin${{ matrix.ext }} target_releases/          
      - name: Save build artifacts to build-${{ matrix.target }}
        uses: actions/upload-artifact@v3
        with:
          name: build-${{ matrix.target }}
          path: target_releases/*

  test-multi-os:
    name: Test on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    needs: [ build ]
    strategy:
      fail-fast: true
      matrix:
        include:
          - target: x86_64-apple-darwin
            os: macOS-latest
          - target: x86_64-pc-windows-msvc
            os: windows-latest
            ext: '.exe'
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-latest
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
      - name: Start postgres
        uses: nyurik/action-setup-postgis@v1
        id: pg
        with:
          username: test
          password: test
          database: test
          rights: --superuser
      - name: Init database
        run: |
          echo "DATABASE_URL=$DATABASE_URL"
          echo "Print the same in base64 to bypass Github's obfuscation (uses hardcoded password):"
          echo "$DATABASE_URL" | base64
          tests/fixtures/initdb.sh          
        env:
          DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
      - name: Download build artifact build-${{ matrix.target }}
        uses: actions/download-artifact@v3
        with:
          name: build-${{ matrix.target }}
          path: target/
      - name: Integration Tests
        run: |
          export MARTIN_BUILD=-
          export MARTIN_BIN=target/martin${{ matrix.ext }}
          export MBTILES_BUILD=-
          export MBTILES_BIN=target/mbtiles${{ matrix.ext }}
          if [[ "${{ runner.os }}" != "Windows" ]]; then
            chmod +x "$MARTIN_BIN" "$MBTILES_BIN"
          fi
          tests/test.sh          
        env:
          DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
      - name: Compare test output results (Linux)
        if: matrix.target == 'x86_64-unknown-linux-gnu'
        run: diff --brief --recursive --new-file tests/output tests/expected
      - name: Download Debian package (Linux)
        if: matrix.target == 'x86_64-unknown-linux-gnu'
        uses: actions/download-artifact@v3
        with:
          name: build-debian-x86_64
          path: target/
      - name: Tests Debian package (Linux)
        if: matrix.target == 'x86_64-unknown-linux-gnu'
        run: |
          sudo dpkg -i target/debian-x86_64.deb
          export MARTIN_BUILD=-
          export MARTIN_BIN=/usr/bin/martin${{ matrix.ext }}
          export MBTILES_BUILD=-
          export MBTILES_BIN=/usr/bin/mbtiles${{ matrix.ext }}
          tests/test.sh          
        env:
          DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
      - name: Save test output on failure (Linux)
        if: failure() && matrix.target == 'x86_64-unknown-linux-gnu'
        uses: actions/upload-artifact@v3
        with:
          name: failed-test-output
          path: tests/output/*
          retention-days: 5

  test-with-svc:
    name: Test postgis:${{ matrix.img_ver }} sslmode=${{ matrix.sslmode }}
    runs-on: ubuntu-latest
    needs: [ build ]
    strategy:
      fail-fast: true
      matrix:
        include:
          # These must match the versions of postgres used in the docker-compose.yml
          - img_ver: 11-3.0-alpine
            args: postgres
            sslmode: disable
          - img_ver: 14-3.3-alpine
            args: postgres
            sslmode: disable
          # alpine images don't support SSL, so for this we use the debian images
          - img_ver: 15-3.3
            args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
            sslmode: require
          #
          # FIXME!
          # DISABLED because Rustls fails to validate name (CN?) with the NotValidForName error
          #- img_ver: 15-3.3
          #  args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
          #  sslmode: verify-ca
          #- img_ver: 15-3.3
          #  args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
          #  sslmode: verify-full
    env:
      PGDATABASE: test
      PGHOST: localhost
      PGUSER: postgres
      PGPASSWORD: postgres
    services:
      postgres:
        image: postgis/postgis:${{ matrix.img_ver }}
        ports:
          - 5432/tcp
        options: >-
          -e POSTGRES_DB=test
          -e POSTGRES_USER=postgres
          -e POSTGRES_PASSWORD=postgres
          -e PGDATABASE=test
          -e PGUSER=postgres
          -e PGPASSWORD=postgres
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --entrypoint sh
          postgis/postgis:${{ matrix.img_ver }}
          -c "exec docker-entrypoint.sh ${{ matrix.args }}"          
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
      - name: Init database
        run: tests/fixtures/initdb.sh
        env:
          PGPORT: ${{ job.services.postgres.ports[5432] }}
      - name: Get DB SSL cert (sslmode=verify-*)
        if: matrix.sslmode == 'verify-ca' || matrix.sslmode == 'verify-full'
        run: |
          set -x
          mkdir -p target/certs
          docker cp ${{ job.services.postgres.id }}:/etc/ssl/certs/ssl-cert-snakeoil.pem target/certs/server.crt
          docker cp ${{ job.services.postgres.id }}:/etc/ssl/private/ssl-cert-snakeoil.key target/certs/server.key          
      - name: Download build artifact build-x86_64-unknown-linux-gnu
        uses: actions/download-artifact@v3
        with:
          name: build-x86_64-unknown-linux-gnu
          path: target_releases/
      - name: Integration Tests
        run: |
          if [[ "${{ matrix.sslmode }}" == "verify-ca" || "${{ matrix.sslmode }}" == "verify-full" ]]; then
            export PGSSLROOTCERT=target/certs/server.crt
          fi
          export MARTIN_BUILD=-
          export MARTIN_BIN=target_releases/martin
          export MBTILES_BUILD=-
          export MBTILES_BIN=target_releases/mbtiles
          chmod +x "$MARTIN_BIN" "$MBTILES_BIN"
          tests/test.sh
          rm -rf target_releases          
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
      - name: Download Debian package
        uses: actions/download-artifact@v3
        with:
          name: build-debian-x86_64
          path: target_releases/
      - name: Tests Debian package
        run: |
          sudo dpkg -i target_releases/debian-x86_64.deb
          if [[ "${{ matrix.sslmode }}" == "verify-ca" || "${{ matrix.sslmode }}" == "verify-full" ]]; then
            export PGSSLROOTCERT=target/certs/server.crt
          fi
          export MARTIN_BUILD=-
          export MARTIN_BIN=/usr/bin/martin
          export MBTILES_BUILD=-
          export MBTILES_BIN=/usr/bin/mbtiles
          tests/test.sh
          sudo dpkg -P martin
          rm -rf target_releases          
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
      - name: Unit Tests
        if: matrix.sslmode != 'verify-ca' && matrix.sslmode != 'verify-full'
        run: |
          echo "Running unit tests, connecting to DATABASE_URL=$DATABASE_URL"
          echo "Same but as base64 to prevent GitHub obfuscation (this is not a secret):"
          echo "$DATABASE_URL" | base64
          set -x
          cargo test --package martin
          cargo clean          
        env:
          DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
      - name: On error, save test output
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: test-output
          path: tests/output/*
          retention-days: 5

  package:
    name: Package ${{ matrix.target }}
    runs-on: ${{ matrix.os }}
    needs: [ lint-debug-test, docker-build-test, test-multi-os, test-with-svc ]
    strategy:
      fail-fast: true
      matrix:
        include:
          - target: aarch64-apple-darwin
            os: ubuntu-latest
            name: martin-Darwin-aarch64.tar.gz
            sha: 'true'
          - target: debian-x86_64
            os: ubuntu-latest
            name: martin-Debian-x86_64.deb
          - target: x86_64-apple-darwin
            os: macOS-latest
            name: martin-Darwin-x86_64.tar.gz
            sha: 'true'
          - target: x86_64-pc-windows-msvc
            os: windows-latest
            name: martin-Windows-x86_64.zip
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-latest
            name: martin-Linux-x86_64.tar.gz
          #
          # From the cross-build
          #
          - target: aarch64-unknown-linux-musl
            os: ubuntu-latest
            cross: 'true'
            name: martin-Linux-aarch64-musl.tar.gz
          - target: x86_64-unknown-linux-musl
            os: ubuntu-latest
            cross: 'true'
            name: martin-Linux-x86_64-musl.tar.gz
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
      - name: Download build artifact build-${{ matrix.target }}
        if: matrix.cross != 'true'
        uses: actions/download-artifact@v3
        with:
          name: build-${{ matrix.target }}
          path: target/
      - name: Download cross-build artifact build-${{ matrix.target }}
        if: matrix.cross == 'true'
        uses: actions/download-artifact@v3
        with:
          name: cross-build
          path: target/
      - name: Package
        run: |
          cd target/
          if [[ "${{ runner.os }}" == "Windows" ]]; then
            7z a ../${{ matrix.name }} martin.exe mbtiles.exe
          elif [[ "${{ matrix.target }}" == "debian-x86_64" ]]; then
            mv debian-x86_64.deb ../${{ matrix.name }}
          else
            if [[ "${{ matrix.cross }}" == "true" ]]; then
              mv ${{ matrix.target }}/* .
            fi
            chmod +x martin mbtiles
            tar czvf ../${{ matrix.name }} martin mbtiles
          fi          
      # TODO: why is this needed and where should the result go?
      # - name: Generate SHA-256 (MacOS)
      #   if: matrix.sha == 'true'
      #   run: shasum -a 256 ${{ matrix.name }}
      - name: Publish
        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@v1
        with:
          draft: true
          files: 'martin*'
          body_path: CHANGELOG.md
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  # This final step is needed to mark the whole workflow as successful
  # Don't change its name - it is used by the merge protection rules
  done:
    name: CI Finished
    runs-on: ubuntu-latest
    needs: [ package ]
    steps:
      - name: Finished
        run: echo "CI finished successfully"