streamly/.github/workflows/regression-check.yml

name: Regression checking

on:
  workflow_dispatch:
  pull_request:

# References:
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
# https://docs.github.com/en/actions/using-workflows/reusing-workflows#reusable-workflows-and-starter-workflows
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions

# You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along
# with the benchmark target.
# Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the
# seperator, and "12" is the new cutoff percent
jobs:
  check-regressions:
    env:
      CI_BENCHMARKS_WITH_CUTOFF: >-
        Data.Array
        Data.MutArray
        Data.Array.Generic
        Data.Ring.Unboxed
        Data.Fold
        Data.Fold.Window
        Data.Parser
        Data.ParserK
        Data.StreamD
        Data.StreamK:6
        Data.Unfold
        Data.Stream
        Data.Stream.Concurrent
        Data.Stream.ConcurrentEager
        Data.Stream.ConcurrentInterleaved
        Data.Stream.ConcurrentOrdered
        FileSystem.Handle
        Unicode.Char
        Unicode.Stream
        Unicode.Utf8        
      CI_FIELDS: allocated
      CI_DIFF_CUTOFF_PERCENT: 3

    runs-on: ubuntu-latest

    steps:

    - name: Update environment
      run: |
        CI_BENCHMARKS=""
        for i in $CI_BENCHMARKS_WITH_CUTOFF
        do
          bname=$(echo "$i" | cut -d: -f1)
          CI_BENCHMARKS="$CI_BENCHMARKS $bname"
        done
        echo "CI_BENCHMARKS=$CI_BENCHMARKS" >> $GITHUB_ENV        

    - name: Download ghc
      run: |
        GHCUP_VER=0.1.18.0
        curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/x86_64-linux-ghcup-$GHCUP_VER
        chmod +x ./ghcup
        GHCVER=9.2.7
        ./ghcup install ghc $GHCVER
        ./ghcup set ghc $GHCVER
        cabal update        

    - uses: actions/cache@v2
      name: Cache ~/.cabal
      with:
        path: |
            ~/.cabal            
        # Bump the key version to clear the cache
        key: cache-v2

    - name: Cache bench-runner from pr
      id: cache-bench-runner-pr
      uses: actions/cache@v2
      with:
        path: bench-runner
        # Bump the key version to clear the cache
        key: bench-runner-v1

    # -----------------------------------------------------------------
    # -- Install bench-report in the current directory
    # -----------------------------------------------------------------

    - name: Checkout the current branch
      uses: actions/checkout@v2
      with:
        clean: false

    - name: Install bench-runner
      run: |
        cabal install bench-runner --project-file=cabal.project.report --installdir=./        

    # -----------------------------------------------------------------
    # -- Generate reports for the base branch and upload
    # -----------------------------------------------------------------

    - name: Checkout the base branch
      uses: actions/checkout@v2
      with:
        ref: master
        clean: false

    - name: Run benchmarks
      run: |
        ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw        

    - name: Move charts to charts-master
      run: mv charts charts-master

    # -----------------------------------------------------------------
    # -- Download, generate reports for the current branch and append
    # -----------------------------------------------------------------

    - name: Checkout the current branch
      uses: actions/checkout@v2
      with:
        clean: false

    - name: Copy charts-master to charts
      run: cp -r charts-master charts

    - name: Run benchmarks and append
      run: |
        ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append        

    # -----------------------------------------------------------------
    # -- Compare
    # -----------------------------------------------------------------

    - name: List all benchmarks
      run: |
        ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure        

    - name: Compare benchmarks
      run: |

        EXIT_STATUS=0

        for i in $CI_BENCHMARKS_WITH_CUTOFF
        do
          arrI=(${i//:/ })

          bname=${arrI[0]}
          cutoff=${arrI[1]}

          test -z "$cutoff" && cutoff=$CI_DIFF_CUTOFF_PERCENT

          echo
          echo "Checking $bname for regressions greater than $cutoff percent"
          ! ./bench-runner  \
                    --package-name streamly-benchmarks \
                    --package-version 0.0.0 \
                    --targets "$bname" \
                    --fields "$CI_FIELDS" \
                    --no-measure --silent \
                    --diff-cutoff-percent $cutoff \
                | grep -v "^$"
          test $? -eq 1 && EXIT_STATUS=1
        done

        exit $EXIT_STATUS