streamly/.github/workflows/regression-check.yml
2023-08-19 15:38:58 +05:30

166 lines
5.0 KiB
YAML

name: Regression checking
on:
workflow_dispatch:
pull_request:
# References:
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
# https://docs.github.com/en/actions/using-workflows/reusing-workflows#reusable-workflows-and-starter-workflows
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions
# You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along
# with the benchmark target.
# Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the
# seperator, and "12" is the new cutoff percent
jobs:
check-regressions:
env:
CI_BENCHMARKS_WITH_CUTOFF: >-
Data.Array
Data.MutArray
Data.Array.Generic
Data.Ring.Unboxed
Data.Fold
Data.Fold.Window
Data.Parser
Data.ParserK
Data.StreamD
Data.StreamK:6
Data.Unfold
Data.Stream
Data.Stream.Concurrent
Data.Stream.ConcurrentEager
Data.Stream.ConcurrentInterleaved
Data.Stream.ConcurrentOrdered
FileSystem.Handle
Unicode.Char
Unicode.Stream
Unicode.Utf8
CI_FIELDS: allocated
CI_DIFF_CUTOFF_PERCENT: 3
runs-on: ubuntu-latest
steps:
- name: Update environment
run: |
CI_BENCHMARKS=""
for i in $CI_BENCHMARKS_WITH_CUTOFF
do
bname=$(echo "$i" | cut -d: -f1)
CI_BENCHMARKS="$CI_BENCHMARKS $bname"
done
echo "CI_BENCHMARKS=$CI_BENCHMARKS" >> $GITHUB_ENV
- name: Download ghc
run: |
GHCUP_VER=0.1.18.0
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/x86_64-linux-ghcup-$GHCUP_VER
chmod +x ./ghcup
GHCVER=9.2.7
./ghcup install ghc $GHCVER
./ghcup set ghc $GHCVER
cabal update
- uses: actions/cache@v2
name: Cache ~/.cabal
with:
path: |
~/.cabal
# Bump the key version to clear the cache
key: cache-v2
- name: Cache bench-runner from pr
id: cache-bench-runner-pr
uses: actions/cache@v2
with:
path: bench-runner
# Bump the key version to clear the cache
key: bench-runner-v1
# -----------------------------------------------------------------
# -- Install bench-report in the current directory
# -----------------------------------------------------------------
- name: Checkout the current branch
uses: actions/checkout@v2
with:
clean: false
- name: Install bench-runner
run: |
cabal install bench-runner --project-file=cabal.project.report --installdir=./
# -----------------------------------------------------------------
# -- Generate reports for the base branch and upload
# -----------------------------------------------------------------
- name: Checkout the base branch
uses: actions/checkout@v2
with:
ref: master
clean: false
- name: Run benchmarks
run: |
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw
- name: Move charts to charts-master
run: mv charts charts-master
# -----------------------------------------------------------------
# -- Download, generate reports for the current branch and append
# -----------------------------------------------------------------
- name: Checkout the current branch
uses: actions/checkout@v2
with:
clean: false
- name: Copy charts-master to charts
run: cp -r charts-master charts
- name: Run benchmarks and append
run: |
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append
# -----------------------------------------------------------------
# -- Compare
# -----------------------------------------------------------------
- name: List all benchmarks
run: |
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure
- name: Compare benchmarks
run: |
EXIT_STATUS=0
for i in $CI_BENCHMARKS_WITH_CUTOFF
do
arrI=(${i//:/ })
bname=${arrI[0]}
cutoff=${arrI[1]}
test -z "$cutoff" && cutoff=$CI_DIFF_CUTOFF_PERCENT
echo
echo "Checking $bname for regressions greater than $cutoff percent"
! ./bench-runner \
--package-name streamly-benchmarks \
--package-version 0.0.0 \
--targets "$bname" \
--fields "$CI_FIELDS" \
--no-measure --silent \
--diff-cutoff-percent $cutoff \
| grep -v "^$"
test $? -eq 1 && EXIT_STATUS=1
done
exit $EXIT_STATUS