git.iko.soy/streamly
on master help sign in
Add benchmark specific cutoffs
6e8d1d1Adithya Kumar
.github/workflows/regression-check.yml+42−10
compare with master
unifiedsplit
@@ -9,11 +9,15 @@ on: pull_request
9# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
9# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
10# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions
10# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions
11
11
12+# You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along
13+# with the benchmark target.
14+# Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the
15+# seperator, and "12" is the new cutoff percent
12jobs:
16jobs:
13 check-regressions:
17 check-regressions:
14 env:
18 env:
15 BENCH_SH: bin/bench.sh
19 BENCH_SH: bin/bench.sh
16− BENCHMARKS: >-
20+ BENCHMARKS_WITH_CUTOFF: >-
17 Data.Array
21 Data.Array
18 Data.Array.Foreign
22 Data.Array.Foreign
19 Data.Array.Prim
23 Data.Array.Prim
@@ -25,14 +29,14 @@ jobs:
25 Data.SmallArray
29 Data.SmallArray
26 Data.Stream.StreamD
30 Data.Stream.StreamD
27 Data.Stream.StreamDK
31 Data.Stream.StreamDK
28− Data.Stream.StreamK
32+ Data.Stream.StreamK:6
29 Data.Unfold
33 Data.Unfold
30 FileSystem.Handle
34 FileSystem.Handle
31 Prelude.Ahead
35 Prelude.Ahead
32− Prelude.Async
36+ Prelude.Async:12
33 Prelude.Parallel
37 Prelude.Parallel
34 Prelude.Serial
38 Prelude.Serial
35− Prelude.WAsync
39+ Prelude.WAsync:6
36 Prelude.WSerial
40 Prelude.WSerial
37 Prelude.ZipAsync
41 Prelude.ZipAsync
38 Prelude.ZipSerial
42 Prelude.ZipSerial
@@ -46,6 +50,16 @@ jobs:
46
50
47 steps:
51 steps:
48
52
53+ - name: Update environment
54+ run: |
55+ BENCHMARKS=""
56+ for i in $BENCHMARKS_WITH_CUTOFF
57+ do
58+ bname=$(echo "$i" | cut -d: -f1)
59+ BENCHMARKS="$BENCHMARKS $bname"
60+ done
61+ echo "BENCHMARKS=$BENCHMARKS" >> $GITHUB_ENV
62+
49 - name: Setup haskell
63 - name: Setup haskell
50 uses: actions/setup-haskell@v1
64 uses: actions/setup-haskell@v1
51 with:
65 with:
@@ -113,9 +127,27 @@ jobs:
113 - name: Compare benchmarks
127 - name: Compare benchmarks
114 run: |
128 run: |
115 chmod +x $BENCH_SH
129 chmod +x $BENCH_SH
130+
131+ EXIT_STATUS=0
132+
133+ for i in $BENCHMARKS_WITH_CUTOFF
134+ do
135+ arrI=(${i//:/ })
136+
137+ bname=${arrI[0]}
138+ cutoff=${arrI[1]}
139+
140+ test -z "$cutoff" && cutoff=$DIFF_CUTOFF_PERCENT
141+
142+ echo
143+ echo "Checking $bname for regressions greater than $cutoff percent"
116− ! $BENCH_SH --benchmarks "$BENCHMARKS" \
144+ ! $BENCH_SH --benchmarks "$bname" \
117− --fields "$FIELDS" \
145+ --fields "$FIELDS" \
118− --no-measure \
146+ --no-measure \
119− --silent --append \
147+ --silent --append \
120− --diff-cutoff-percent $DIFF_CUTOFF_PERCENT \
148+ --diff-cutoff-percent $cutoff \
121− | grep -v "^$"
149+ | grep -v "^$"
150+ test $? -eq 1 && EXIT_STATUS=1
151+ done
152+
153+ exit $EXIT_STATUS