catala/.github/workflows/harness.yml

name: CI

on:
  push:
    branches: [master]
    tags: ['*.*.*']
  workflow_dispatch:
  pull_request_target:
  # It is important to use `pull_request_target` and not `pull_request` here: it
  # means the version of this file from master is used rather than the one from
  # the PR. This allows writing to the docker caches, etc. from PRs ; but mind
  # that if you attempt modification in a PR, of course.

# don't use the default fetch source, as with 'pull_request_target' that will
# test the target branch only!
env:
  GIT_FETCH_REF: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:

  build:
    name: Build Catala and generate image
    runs-on: self-hosted
    permissions:
      packages: write
    outputs:
      image: ghcr.io/catalalang/catala@${{ steps.image.outputs.digest }}
      version: ${{ steps.describe.outputs.version }}
    steps:
      - name: Checkout
        # This is *only* needed to extract the git version...
        # Approaches like proudust/gh-describe@v1 don't work
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Need full commit history for 'git describe'
          ref: ${{ env.GIT_FETCH_REF }}
      - name: Get git-describe version
        id: describe
        run: echo "version=$(git describe --tags)" >> "$GITHUB_OUTPUT"
      - name: Get an image tag that Docker accepts
        id: branch
        run: sed 's/[^a-zA-Z0-9-]/-/g; s/^/tag=/' <<<"${{ github.head_ref || github.ref_name }}" >> "$GITHUB_OUTPUT"
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Make build context image
        uses: docker/build-push-action@v5
        with:
          # The default context would fetch the default source (ie master on
          # 'pull_request_target' to master >:( ) ; we have already have fetched
          # anyway so use the current dir
          context: .
          target: dev-build-context
          # Caching using GH cache doesn't work, use registry caching directly
          # instead
          cache-from: |
            type=registry,ref=ghcr.io/catalalang/catala:dev-cache-master
            type=registry,ref=ghcr.io/catalalang/catala:dev-cache-${{ steps.branch.outputs.tag }}
          cache-to: |
            type=registry,ref=ghcr.io/catalalang/catala:dev-cache-${{ steps.branch.outputs.tag }},mode=max
      - name: Build and push
        id: image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ghcr.io/catalalang/catala:${{ steps.describe.outputs.version }}
          build-args: "CATALA_VERSION=${{ steps.describe.outputs.version }}"
          labels: org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
          cache-from: |
            type=registry,ref=ghcr.io/catalalang/catala:build-cache-master
            type=registry,ref=ghcr.io/catalalang/catala:build-cache-${{ steps.branch.outputs.tag }}
          cache-to: |
            type=registry,ref=ghcr.io/catalalang/catala:build-cache-${{ steps.branch.outputs.tag }},mode=max

  tests:
    name: Run integrated tests
    needs: build
    runs-on: self-hosted
    container:
      image: ${{ needs.build.outputs.image }}
      options: --user root
    steps:
      - name: Fix home
        # We need to run as root as a Workaround Github actions issues
        # (https://github.com/actions/checkout/issues/1014) but need ~ocaml as
        # home.
        run: sudo sh -c "echo HOME=/home/ocaml >> ${GITHUB_ENV}"
      - name: Check promoted files
        run: |
          cd /home/ocaml/catala && opam exec -- make check-promoted
      - name: Run tests
        if: ${{ always() }}
        run: cd /home/ocaml/catala && opam exec -- make testsuite
      - name: Generate test report
        if: ${{ always() }}
        run: |
          cd /home/ocaml/catala
          opam exec -- clerk report --xml _build/*@test _build/test-*/*@test >report.junit.xml
      - name: Test Summary
        uses: test-summary/action@v2
        with:
          paths: /home/ocaml/catala/report.junit.xml
        if: ${{ always() }}


  examples:
    name: Build examples and generate artifacts
    needs: build
    runs-on: self-hosted
    container:
      image: ${{ needs.build.outputs.image }}
      options: --user root
      env:
        DUNE_PROFILE: release
        OPAMSWITCH: /home/ocaml/catala
    steps:
      - name: Fix home
        # Workaround Github actions issue, see
        # https://github.com/actions/runner/issues/863
        run: sudo sh -c "echo HOME=/home/ocaml >> ${GITHUB_ENV}"
      - name: Install LaTeX deps
        # This is done late because caching would not benefit compared to
        # installation through apk (1,5G upload is slow)
        run: sudo apk add texlive-xetex texlive-dvi texmf-dist-latexextra texmf-dist-binextra texmf-dist-pictures texmf-dist-fontsrecommended font-dejavu groff libpng
        # Fewer texmf deps should be required once
        # https://gitlab.alpinelinux.org/alpine/aports/-/issues/16190 is fixed
      - name: Build Catala extra docs
        run: |
          cd ~/catala
          opam --cli=2.1 exec -- make syntax
          opam --cli=2.1 exec -- make doc
      - name: Checkout examples repo
        # Github fetch action is expected to work for containers, but doesn't
        # (permission issues)
        run: |
          git clone https://github.com/CatalaLang/catala-examples --depth 1 ~/catala-examples -b "${{ github.head_ref || github.ref_name }}" ||
          git clone https://github.com/CatalaLang/catala-examples --depth 1 ~/catala-examples
      - name: Build examples
        run: |
          cd ~/catala-examples
          opam --cli=2.1 exec -- make all testsuite install
      - name: Generate examples test report
        if: ${{ always() }}
        run: |
          cd ~/catala-examples
          opam exec -- clerk report --xml _build/clerk_tests/*@test _build/clerk_tests/test-*/*@test >report.junit.xml
      - name: Test Summary
        uses: test-summary/action@v2
        with:
          paths: "/home/ocaml/catala-examples/report.junit.xml"
        if: ${{ always() }}
      - name: Checkout french-law repo
        run: |
          git clone https://github.com/CatalaLang/french-law --depth 1 ~/french-law -b "${{ github.head_ref || github.ref_name }}" ||
          git clone https://github.com/CatalaLang/french-law --depth 1 ~/french-law
      - name: Build french-law
        run: |
          cd ~/french-law
          opam --cli=2.1 exec -- make dependencies-ocaml dependencies-js dependencies-python
          opam --cli=2.1 exec -- make bench_ocaml bench_js bench_python
      - name: Pack website assets
        run: |
          mkdir -p ~/website-assets
          cd ~/website-assets
          ln -s \
            ~/catala-examples/_build/tutorial_en/tutorial_en.html \
            ~/catala-examples/_build/tutoriel_fr/tutoriel_fr.html \
            ~/catala-examples/_build/us_tax_code/us_tax_code.html \
            ~/catala-examples/_build/allocations_familiales/Allocations_familiales.html \
            ~/catala-examples/_build/allocations_familiales/Allocations_familiales_schema.json \
            ~/catala-examples/_build/aides_logement/Aides_logement.html\
            ~/catala-examples/_build/aides_logement/Aides_logement_schema.json \
            .
          tar cz -hf ~/catala/website-assets.tar.gz *
      - name: Gather all artifacts
        run: |
          cd
          mkdir -p artifacts
          mv catala/_build/default/_doc/_html artifacts/api-doc
          mv catala/doc/syntax/syntax.pdf artifacts/
          mv catala/_build/default/*.html artifacts/
          mv catala/_opam/doc/catala-examples/tuto*/*.html artifacts/
          tar cz -hf "artifacts/french-law_ocaml.tar.gz" -C french-law/_build/install/default/lib french-law
          cp catala-examples/_build/french-law_npm.tar.gz artifacts/
          cp catala-examples/_build/french_law_python.tar.gz artifacts/
          mv catala/website-assets.tar.gz artifacts/
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Catala examples
          path: /home/ocaml/artifacts/*

  binaries:
    name: Build static binaries
    runs-on: self-hosted
    if: ${{ github.event_name != 'pull_request_target' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Need full commit history for 'git describe'
          ref: ${{ env.GIT_FETCH_REF }}
      - name: Get git-describe version
        id: describe
        run: echo "version=$(git describe --tags)" >> "$GITHUB_OUTPUT"
      - name: Build release binaries
        run: |
          mkdir -p artifacts
          export CATALA_VERSION="${{ steps.describe.outputs.version }}"
          ./build_release.sh -C artifacts
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Catala binaries
          path: artifacts/*

  pages:
    name: Publish static content to github-pages
    needs: [ examples, binaries, tests ]
    # Doesn't really depend on tests, but we don't want to publish if they fail
    if: ${{ github.event_name != 'pull_request_target' && github.ref == 'refs/heads/master' }}
    # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
    permissions:
      contents: read
      pages: write
      id-token: write
    # Allow one concurrent deployment
    concurrency:
      group: "pages"
      cancel-in-progress: true
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Setup Pages
        uses: actions/configure-pages@v3
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          merge-multiple: true
          path: artifacts/
      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: tree
          version: 1.0
      - name: Generate HTML index
        run: |
          cd artifacts
          tree -H . -L 1 --noreport --dirsfirst -T 'Catala latest development artifacts' --charset utf-8 -o index.html
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: 'artifacts/'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v1