catala/.github/workflows/harness.yml

name: CI

on:
  push:
    branches: [master]
    tags: ['*.*.*']
  workflow_dispatch:
  pull_request_target:
  # It is important to use `pull_request_target` and not `pull_request` here: it
  # means the version of this file from master is used rather than the one from
  # the PR. This allows writing to the docker caches, etc. from PRs ; but mind
  # that if you attempt modification in a PR, of course.

# don't use the default fetch source, as with 'pull_request_target' that will
# test the target branch only!
env:
  GIT_FETCH_REF: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:

  build:
    name: Build Catala and generate image
    runs-on: self-hosted
    permissions:
      packages: write
    outputs:
      image: ghcr.io/catalalang/catala@${{ steps.image.outputs.digest }}
      version: ${{ steps.describe.outputs.version }}
    steps:
      - name: Checkout
        # This is *only* needed to extract the git version...
        # Approaches like proudust/gh-describe@v1 don't work
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Need full commit history for 'git describe'
          ref: ${{ env.GIT_FETCH_REF }}
      - name: Get git-describe version
        id: describe
        run: echo "version=$(git describe --tags)" >> "$GITHUB_OUTPUT"
      - name: Get an image tag that Docker accepts
        id: branch
        run: sed 's/[^a-zA-Z0-9-]/-/g; s/^/tag=/' <<<"${{ github.head_ref || github.ref_name }}" >> "$GITHUB_OUTPUT"
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Make build context image
        uses: docker/build-push-action@v5
        with:
          # The default context would fetch the default source (ie master on
          # 'pull_request_target' to master >:( ) ; we have already have fetched
          # anyway so use the current dir
          context: .
          target: dev-build-context
          # Caching using GH cache doesn't work, use registry caching directly
          # instead
          cache-from: |
            type=registry,ref=ghcr.io/catalalang/catala:dev-cache-master
            type=registry,ref=ghcr.io/catalalang/catala:dev-cache-${{ steps.branch.outputs.tag }}            
          cache-to: |
            type=registry,ref=ghcr.io/catalalang/catala:dev-cache-${{ steps.branch.outputs.tag }},mode=max            
      - name: Build and push
        id: image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ghcr.io/catalalang/catala:${{ steps.describe.outputs.version }}
          build-args: "CATALA_VERSION=${{ steps.describe.outputs.version }}"
          labels: org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
          cache-from: |
            type=registry,ref=ghcr.io/catalalang/catala:build-cache-master
            type=registry,ref=ghcr.io/catalalang/catala:build-cache-${{ steps.branch.outputs.tag }}            
          cache-to: |
            type=registry,ref=ghcr.io/catalalang/catala:build-cache-${{ steps.branch.outputs.tag }},mode=max            

  tests:
    name: Run integrated tests
    needs: build
    runs-on: self-hosted
    container:
      image: ${{ needs.build.outputs.image }}
      options: --user ocaml
    steps:
      - name: Check promoted files
        run: |
          cd /home/ocaml/catala && opam exec -- make check-promoted
          git diff --exit-code          
      - name: Run tests
        if: ${{ always() }}
        run: cd /home/ocaml/catala && opam exec -- make tests

  examples:
    name: Build examples and generate artifacts
    needs: build
    runs-on: self-hosted
    container:
      image: ${{ needs.build.outputs.image }}
      options: --user ocaml
      env:
        DUNE_PROFILE: release
    steps:
      - name: Fix home
        # Workaround Github actions issue, see
        # https://github.com/actions/runner/issues/863
        run: sudo sh -c "echo HOME=/home/ocaml >> ${GITHUB_ENV}"
      - name: Install LaTeX deps
        # This is done late because caching would not benefit compared to
        # installation through apk (1,5G upload is slow)
        run: sudo apk add texlive-xetex texmf-dist-latexextra texmf-dist-pictures font-dejavu groff
      - name: Build Catala extra docs
        run: |
          cd ~/catala
          opam --cli=2.1 exec -- make syntax
          opam --cli=2.1 exec -- make doc          
      - name: Checkout examples repo
        # Github fetch action is expected to work for containers, but doesn't
        # (permission issues)
        run: |
          git clone https://github.com/CatalaLang/catala-examples --depth 1 ~/catala-examples -b "${{ github.head_ref || github.ref_name }}" ||
          git clone https://github.com/CatalaLang/catala-examples --depth 1 ~/catala-examples          
      - name: Build examples
        run: |
          cd ~/catala-examples
          opam --cli=2.1 exec -- make all testsuite install          
      - name: Checkout french-law repo
        run: |
          git clone https://github.com/CatalaLang/french-law --depth 1 ~/french-law -b "${{ github.head_ref || github.ref_name }}" ||
          git clone https://github.com/CatalaLang/french-law --depth 1 ~/french-law          
      - name: Build french-law
        run: |
          cd ~/french-law
          opam --cli=2.1 exec -- make dependencies-ocaml dependencies-js dependencies-python
          opam --cli=2.1 exec -- make bench_ocaml bench_js bench_python          
      - name: Pack website assets
        run: >-
          tar cz -hf website-assets.tar.gz
          -C ~/catala-examples/_build/default/tutorial_en/ tutorial_en.html
          -C ~/catala-examples/_build/default/tutoriel_fr/ tutoriel_fr.html
          -C ~/catala-examples/_build/default/us_tax_code/ us_tax_code.html
          -C ~/catala-examples/_build/default/allocations_familiales/ Allocations_familiales.html Allocations_familiales_schema.json
          -C ~/catala-examples/_build/default/aides_logement/ Aides_logement.html Aides_logement_schema.json          
      - name: Gather all artifacts
        run: |
          cd
          mkdir -p artifacts
          mv catala/_build/default/_doc/_html artifacts/api-doc
          mv catala/doc/syntax/syntax.pdf artifacts/
          mv catala/_build/default/*.html artifacts/
          mv ~/.opam/catala/doc/catala-examples/tuto*/*.html artifacts/
          tar cz -hf "artifacts/french-law_ocaml.tar.gz" -C french-law/_build/install/default/lib french-law
          cp catala-examples/_build/french-law_npm.tar.gz artifacts/
          cp catala-examples/_build/french_law_python.tar.gz artifacts/
          mv website-assets.tar.gz artifacts/          
      - name: Upload artifacts
        continue-on-error: true
        # Uploading artifacts works but then return failure with:
        # EACCES: permission denied, open '/__w/_temp/_runner_file_commands/set_output_xxx'
        # a chmod doesn't work around it so we resort to just ignoring the error...
        uses: actions/upload-artifact@v4
        with:
          name: Catala examples
          path: /home/ocaml/artifacts/*

  binaries:
    name: Build static binaries
    runs-on: self-hosted
    if: ${{ github.event_name != 'pull_request_target' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Need full commit history for 'git describe'
          ref: ${{ env.GIT_FETCH_REF }}
      - name: Get git-describe version
        id: describe
        run: echo "version=$(git describe --tags)" >> "$GITHUB_OUTPUT"
      - name: Build release binaries
        run: |
          mkdir -p artifacts
          export CATALA_VERSION="${{ steps.describe.outputs.version }}"
          ./build_release.sh -C artifacts          
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Catala binaries
          path: artifacts/*

  pages:
    name: Publish static content to github-pages
    needs: [ examples, binaries, tests ]
    # Doesn't really depend on tests, but we don't want to publish if they fail
    if: ${{ github.event_name != 'pull_request_target' && github.ref == 'refs/heads/master' }}
    # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
    permissions:
      contents: read
      pages: write
      id-token: write
    # Allow one concurrent deployment
    concurrency:
      group: "pages"
      cancel-in-progress: true
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Setup Pages
        uses: actions/configure-pages@v3
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          merge-multiple: true
          path: artifacts/
      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: tree
          version: 1.0
      - name: Generate HTML index
        run: |
          cd artifacts
          tree -H . -L 1 --noreport --dirsfirst -T 'Catala latest development artifacts' --charset utf-8 -o index.html          
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: 'artifacts/'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v1