streamly/.github/workflows/regression-check.yml

name: Regression checking

on:
  workflow_dispatch:
  pull_request:

# References:
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
# https://docs.github.com/en/actions/using-workflows/reusing-workflows#reusable-workflows-and-starter-workflows
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions

# You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along
# with the benchmark target.
# Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the
# seperator, and "12" is the new cutoff percent
jobs:
  check-regressions:
    env:
      CI_BENCHMARKS_WITH_CUTOFF: >-
        Data.Array
        Data.Array.Unboxed
        Data.Fold
        Data.Parser
        Data.Parser.ParserD
        Data.Parser.ParserK
        Data.SmallArray
        Data.Stream.StreamD
        Data.Stream.StreamDK
        Data.Stream.StreamK:6
        Data.Unfold
        Data.Stream
        FileSystem.Handle
        Prelude.Ahead
        Prelude.Async:12
        Prelude.Parallel
        Prelude.WAsync:6
        Prelude.WSerial
        Prelude.ZipAsync
        Prelude.ZipSerial
        Unicode.Char
        Unicode.Stream
        Unicode.Utf8        
      CI_FIELDS: allocated
      CI_DIFF_CUTOFF_PERCENT: 3

    runs-on: ubuntu-latest

    steps:

    - name: Update environment
      run: |
        CI_BENCHMARKS=""
        for i in $CI_BENCHMARKS_WITH_CUTOFF
        do
          bname=$(echo "$i" | cut -d: -f1)
          CI_BENCHMARKS="$CI_BENCHMARKS $bname"
        done
        echo "CI_BENCHMARKS=$CI_BENCHMARKS" >> $GITHUB_ENV        

    - name: Download ghc
      run: |
        curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/0.1.17.4/x86_64-linux-ghcup-0.1.17.4
        chmod +x ./ghcup
        GHCVER=8.10.7
        ./ghcup install ghc $GHCVER
        ./ghcup set ghc $GHCVER
        cabal update        

    - uses: actions/cache@v2
      name: Cache ~/.cabal
      with:
        path: |
            ~/.cabal            
        # Bump the key version to clear the cache
        key: cache-v1

    - name: Cache charts from master
      id: cache-charts-master
      uses: actions/cache@v2
      with:
        path: charts-master
        # Bump the key version to clear the cache
        key: charts-master-v1

    # -----------------------------------------------------------------
    # -- Generate reports for the base branch and upload
    # -----------------------------------------------------------------

    - name: Checkout the base branch
      if: steps.cache-charts-master.outputs.cache-hit != 'true'
      uses: actions/checkout@v2
      with:
        ref: master

    - name: Run benchmarks
      if: steps.cache-charts-master.outputs.cache-hit != 'true'
      run: |
        cabal run bench-runner --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --raw        

    - name: Move charts to charts-master
      if: steps.cache-charts-master.outputs.cache-hit != 'true'
      run: mv charts charts-master

    # -----------------------------------------------------------------
    # -- Download, generate reports for the current branch and append
    # -----------------------------------------------------------------

    - name: Checkout the current branch
      uses: actions/checkout@v2
      with:
        clean: false

    - name: Copy charts-master to charts
      run: cp -r charts-master charts

    - name: Run benchmarks and append
      run: |
        cabal run bench-runner --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --raw --append        

    # -----------------------------------------------------------------
    # -- Compare
    # -----------------------------------------------------------------

    - name: List all benchmarks
      run: |
        cabal run bench-runner --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --no-measure        

    - name: Compare benchmarks
      run: |

        EXIT_STATUS=0

        for i in $CI_BENCHMARKS_WITH_CUTOFF
        do
          arrI=(${i//:/ })

          bname=${arrI[0]}
          cutoff=${arrI[1]}

          test -z "$cutoff" && cutoff=$CI_DIFF_CUTOFF_PERCENT

          echo
          echo "Checking $bname for regressions greater than $cutoff percent"
          ! cabal run bench-runner --project-file=cabal.project.report -- \
                      --targets "$bname" \
                      --fields "$CI_FIELDS" \
                      --no-measure --silent \
                      --diff-cutoff-percent $cutoff \
                | grep -v "^$"
          test $? -eq 1 && EXIT_STATUS=1
        done

        exit $EXIT_STATUS