git.iko.soy/streamly
on master help sign in
streamly/.github/workflows/regression-check.yml
165 lines 5.1 kB history raw
1name: Regression checking
3on:
4 workflow_dispatch:
5 pull_request:
7# References:
8# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
9# https://docs.github.com/en/actions/using-workflows/reusing-workflows#reusable-workflows-and-starter-workflows
10# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
11# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions
13# You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along
14# with the benchmark target.
15# Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the
16# seperator, and "12" is the new cutoff percent
17jobs:
18 check-regressions:
19 env:
20 CI_BENCHMARKS_WITH_CUTOFF: >-
21 Data.Array
22 Data.MutArray
23 Data.Array.Generic
24 Data.Ring.Unboxed
25 Data.Fold
26 Data.Fold.Window
27 Data.Parser
28 Data.ParserK
29 Data.StreamD
30 Data.StreamK:6
31 Data.Unfold
32 Data.Stream
33 Data.Stream.Concurrent
34 Data.Stream.ConcurrentEager
35 Data.Stream.ConcurrentInterleaved
36 Data.Stream.ConcurrentOrdered
37 FileSystem.Handle
38 Unicode.Char
39 Unicode.Stream
40 Unicode.Utf8
41 CI_FIELDS: allocated
42 CI_DIFF_CUTOFF_PERCENT: 3
44 runs-on: ubuntu-latest
46 steps:
48 - name: Update environment
49 run: |
50 CI_BENCHMARKS=""
51 for i in $CI_BENCHMARKS_WITH_CUTOFF
52 do
53 bname=$(echo "$i" | cut -d: -f1)
54 CI_BENCHMARKS="$CI_BENCHMARKS $bname"
55 done
56 echo "CI_BENCHMARKS=$CI_BENCHMARKS" >> $GITHUB_ENV
58 - name: Download ghc
59 run: |
60 GHCUP_VER=0.1.18.0
61 curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/x86_64-linux-ghcup-$GHCUP_VER
62 chmod +x ./ghcup
63 GHCVER=9.2.7
64 ./ghcup install ghc $GHCVER
65 ./ghcup set ghc $GHCVER
66 cabal update
68 - uses: actions/cache@v2
69 name: Cache ~/.cabal
70 with:
71 path: |
72 ~/.cabal
73 # Bump the key version to clear the cache
74 key: cache-v2
76 - name: Cache bench-runner from pr
77 id: cache-bench-runner-pr
78 uses: actions/cache@v2
79 with:
80 path: bench-runner
81 # Bump the key version to clear the cache
82 key: bench-runner-v1
84 # -----------------------------------------------------------------
85 # -- Install bench-report in the current directory
86 # -----------------------------------------------------------------
88 - name: Checkout the current branch
89 uses: actions/checkout@v2
90 with:
91 clean: false
93 - name: Install bench-runner
94 run: |
95 cabal install bench-runner --project-file=cabal.project.report --installdir=./
97 # -----------------------------------------------------------------
98 # -- Generate reports for the base branch and upload
99 # -----------------------------------------------------------------
101 - name: Checkout the base branch
102 uses: actions/checkout@v2
103 with:
104 ref: master
105 clean: false
107 - name: Run benchmarks
108 run: |
109 ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw
111 - name: Move charts to charts-master
112 run: mv charts charts-master
114 # -----------------------------------------------------------------
115 # -- Download, generate reports for the current branch and append
116 # -----------------------------------------------------------------
118 - name: Checkout the current branch
119 uses: actions/checkout@v2
120 with:
121 clean: false
123 - name: Copy charts-master to charts
124 run: cp -r charts-master charts
126 - name: Run benchmarks and append
127 run: |
128 ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append
130 # -----------------------------------------------------------------
131 # -- Compare
132 # -----------------------------------------------------------------
134 - name: List all benchmarks
135 run: |
136 ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure
138 - name: Compare benchmarks
139 run: |
141 EXIT_STATUS=0
143 for i in $CI_BENCHMARKS_WITH_CUTOFF
144 do
145 arrI=(${i//:/ })
147 bname=${arrI[0]}
148 cutoff=${arrI[1]}
150 test -z "$cutoff" && cutoff=$CI_DIFF_CUTOFF_PERCENT
152 echo
153 echo "Checking $bname for regressions greater than $cutoff percent"
154 ! ./bench-runner \
155 --package-name streamly-benchmarks \
156 --package-version 0.0.0 \
157 --targets "$bname" \
158 --fields "$CI_FIELDS" \
159 --no-measure --silent \
160 --diff-cutoff-percent $cutoff \
161 | grep -v "^$"
162 test $? -eq 1 && EXIT_STATUS=1
163 done
165 exit $EXIT_STATUS