git.iko.soy/streamly
on 5cbcd5b help sign in
streamly/bench.sh
287 lines 7.6 kB history raw
1#!/bin/bash
3print_help () {
4 echo "Usage: $0 "
5 echo " [--compare] [--base commit] [--candidate commit]"
6 echo " [--benchmarks <all|linear|linear-async|linear-rate|nested|base>]"
7 echo " [--graphs]"
8 echo " [--slow]"
9 echo " [--no-measure]"
10 echo " [--append] "
11 echo " -- <gauge options>"
12 echo
13 echo "Multiple benchmarks can be specified as a space separate list"
14 echo " e.g. --benchmarks \"linear nested\""
15 echo
16 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
17 echo "commit is generated, in the 'charts' directory."
18 echo "Use --base and --candidate to select the commits to compare."
19 echo
20 echo "Any arguments after a '--' are passed directly to guage"
21 echo "You can omit '--' if the gauge args used do not start with a '-'."
22 exit
25# $1: message
26die () {
27 >&2 echo -e "Error: $1"
28 exit 1
31set_benchmarks() {
32 if test -z "$BENCHMARKS"
33 then
34 BENCHMARKS=$DEFAULT_BENCHMARKS
35 elif test "$BENCHMARKS" = "all"
36 then
37 BENCHMARKS=$ALL_BENCHMARKS
38 fi
39 echo "Using benchmark suites [$BENCHMARKS]"
42# $1: benchmark name (linear, nested, base)
43find_report_prog() {
44 local prog_name="chart"
45 hash -r
46 local prog_path=$($STACK exec which $prog_name)
47 if test -x "$prog_path"
48 then
49 echo $prog_path
50 else
51 return 1
52 fi
55# $1: benchmark name (linear, nested, base)
56build_report_prog() {
57 local prog_name="chart"
58 local prog_path=$($STACK exec which $prog_name)
60 hash -r
61 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
62 then
63 echo "Building bench-graph executables"
64 BUILD_ONCE=1
65 $STACK build --flag "streamly:dev" || die "build failed"
66 elif test ! -x "$prog_path"
67 then
68 return 1
69 fi
70 return 0
73build_report_progs() {
74 if test "$RAW" = "0"
75 then
76 build_report_prog || exit 1
77 local prog
78 prog=$(find_report_prog) || \
79 die "Cannot find bench-graph executable"
80 echo "Using bench-graph executable [$prog]"
81 fi
84# We run the benchmarks in isolation in a separate process so that different
85# benchmarks do not interfere with other. To enable that we need to pass the
86# benchmark exe path to guage as an argument. Unfortunately it cannot find its
87# own path currently.
89# The path is dependent on the architecture and cabal version.
90# Use this command to find the exe if this script fails with an error:
91# find .stack-work/ -type f -name "benchmarks"
93find_bench_prog () {
94 local bench_name=$1
95 local bench_prog=`$STACK path --dist-dir`/build/$bench_name/$bench_name
96 if test -x "$bench_prog"
97 then
98 echo $bench_prog
99 else
100 return 1
101 fi
104bench_output_file() {
105 local bench_name=$1
106 echo "charts/$bench_name/results.csv"
109# --min-duration 0 means exactly one iteration per sample. We use a million
110# iterations in the benchmarking code explicitly and do not use the iterations
111# done by the benchmarking tool.
113# Benchmarking tool by default discards the first iteration to remove
114# aberrations due to initial evaluations etc. We do not discard it because we
115# are anyway doing iterations in the benchmarking code and many of them so that
116# any constant factor gets amortized and anyway it is a cost that we pay in
117# real life.
119# We can pass --min-samples value from the command line as second argument
120# after the benchmark name in case we want to use more than one sample.
122run_bench () {
123 local bench_name=$1
124 local output_file=$(bench_output_file $bench_name)
125 local bench_prog
126 bench_prog=$(find_bench_prog $bench_name) || \
127 die "Cannot find benchmark executable for benchmark $bench_name"
129 mkdir -p `dirname $output_file`
131 echo "Running benchmark $bench_name ..."
133 $bench_prog $SPEED_OPTIONS \
134 --csvraw=$output_file \
135 -v 2 \
136 --measure-with $bench_prog $GAUGE_ARGS || die "Benchmarking failed"
139run_benches() {
140 for i in $1
141 do
142 run_bench $i
143 done
146run_benches_comparing() {
147 local bench_list=$1
149 if test -z "$CANDIDATE"
150 then
151 CANDIDATE=$(git rev-parse HEAD)
152 fi
153 if test -z "$BASE"
154 then
155 # XXX Should be where the current branch is forked from master
156 BASE="$CANDIDATE^"
157 fi
158 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
159 echo "Checking out base commit [$BASE] for benchmarking"
160 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
162 $STACK build --bench --no-run-benchmarks || die "build failed"
163 run_benches "$bench_list"
165 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
166 git checkout "$CANDIDATE" || \
167 die "Checkout of candidate [$CANDIDATE] commit failed"
169 $STACK build --bench --no-run-benchmarks || die "build failed"
170 run_benches "$bench_list"
171 # XXX reset back to the original commit
174backup_output_file() {
175 local bench_name=$1
176 local output_file=$(bench_output_file $bench_name)
178 if test -e $output_file -a "$APPEND" != 1
179 then
180 mv -f -v $output_file ${output_file}.prev
181 fi
184run_measurements() {
185 local bench_list=$1
187 for i in $bench_list
188 do
189 backup_output_file $i
190 done
192 if test "$COMPARE" = "0"
193 then
194 run_benches "$bench_list"
195 else
196 run_benches_comparing "$bench_list"
197 fi
200run_reports() {
201 local prog
202 prog=$(find_report_prog) || \
203 die "Cannot find bench-graph executable"
204 echo
206 for i in $1
207 do
208 echo "Generating reports for ${i}..."
209 $prog --benchmark $i
210 done
213#-----------------------------------------------------------------------------
214# Execution starts here
215#-----------------------------------------------------------------------------
217DEFAULT_BENCHMARKS="linear"
218ALL_BENCHMARKS="linear linear-async linear-rate nested base"
220COMPARE=0
221BASE=
222CANDIDATE=
224APPEND=0
225RAW=0
226GRAPH=0
227MEASURE=1
228SPEED_OPTIONS="--quick --min-samples 10 --time-limit 1 --min-duration 0"
230STACK=stack
231GAUGE_ARGS=
233BUILD_ONCE=0
235#-----------------------------------------------------------------------------
236# Read command line
237#-----------------------------------------------------------------------------
239while test -n "$1"
240do
241 case $1 in
242 -h|--help|help) print_help ;;
243 --slow) SPEED_OPTIONS="--min-duration 0"; shift ;;
244 --append) APPEND=1; shift ;;
245 --benchmarks) shift; BENCHMARKS=$1; shift ;;
246 --base) shift; BASE=$1; shift ;;
247 --candidate) shift; CANDIDATE=$1; shift ;;
248 --compare) COMPARE=1; shift ;;
249 --raw) RAW=1; shift ;;
250 --graphs) GRAPH=1; shift ;;
251 --no-measure) MEASURE=0; shift ;;
252 --) shift; break ;;
253 -*|--*) print_help ;;
254 *) break ;;
255 esac
256done
257GAUGE_ARGS=$*
259echo "Using stack command [$STACK]"
260set_benchmarks
262#-----------------------------------------------------------------------------
263# Build stuff
264#-----------------------------------------------------------------------------
266# We need to build the report progs first at the current (latest) commit before
267# checking out any other commit for benchmarking.
268build_report_progs "$BENCHMARKS"
270#-----------------------------------------------------------------------------
271# Run benchmarks
272#-----------------------------------------------------------------------------
274if test "$MEASURE" = "1"
275then
276 $STACK build --bench --no-run-benchmarks || die "build failed"
277 run_measurements "$BENCHMARKS"
278fi
280#-----------------------------------------------------------------------------
281# Run reports
282#-----------------------------------------------------------------------------
284if test "$RAW" = "0"
285then
286 run_reports "$BENCHMARKS"
287fi