git.iko.soy/streamly
on 57fad21 help sign in
streamly/bench.sh
301 lines 8.0 kB history raw
1#!/bin/bash
3print_help () {
4 echo "Usage: $0 "
5 echo " [--benchmarks <all|linear|linear-async|linear-rate|nested|base>]"
6 echo " [--group-diff]"
7 echo " [--graphs]"
8 echo " [--no-measure]"
9 echo " [--append] "
10 echo " [--compare] [--base commit] [--candidate commit]"
11 echo " [--slow]"
12 echo " -- <gauge options>"
13 echo
14 echo "Multiple benchmarks can be specified as a space separate list"
15 echo " e.g. --benchmarks \"linear nested\""
16 echo
17 echo "--group-diff is used to compare groups within a single benchmark"
18 echo " e.g. StreamD vs StreamK in base benchmark."
19 echo
20 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
21 echo "commit is generated, in the 'charts' directory."
22 echo "Use --base and --candidate to select the commits to compare."
23 echo
24 echo "Any arguments after a '--' are passed directly to guage"
25 exit
28# $1: message
29die () {
30 >&2 echo -e "Error: $1"
31 exit 1
34set_benchmarks() {
35 if test -z "$BENCHMARKS"
36 then
37 BENCHMARKS=$DEFAULT_BENCHMARKS
38 elif test "$BENCHMARKS" = "all"
39 then
40 BENCHMARKS=$ALL_BENCHMARKS
41 fi
42 echo "Using benchmark suites [$BENCHMARKS]"
45# $1: benchmark name (linear, nested, base)
46find_report_prog() {
47 local prog_name="chart"
48 hash -r
49 local prog_path=$($STACK exec which $prog_name)
50 if test -x "$prog_path"
51 then
52 echo $prog_path
53 else
54 return 1
55 fi
58# $1: benchmark name (linear, nested, base)
59build_report_prog() {
60 local prog_name="chart"
61 local prog_path=$($STACK exec which $prog_name)
63 hash -r
64 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
65 then
66 echo "Building bench-graph executables"
67 BUILD_ONCE=1
68 $STACK build --flag "streamly:dev" || die "build failed"
69 elif test ! -x "$prog_path"
70 then
71 return 1
72 fi
73 return 0
76build_report_progs() {
77 if test "$RAW" = "0"
78 then
79 build_report_prog || exit 1
80 local prog
81 prog=$(find_report_prog) || \
82 die "Cannot find bench-graph executable"
83 echo "Using bench-graph executable [$prog]"
84 fi
87# We run the benchmarks in isolation in a separate process so that different
88# benchmarks do not interfere with other. To enable that we need to pass the
89# benchmark exe path to guage as an argument. Unfortunately it cannot find its
90# own path currently.
92# The path is dependent on the architecture and cabal version.
93# Use this command to find the exe if this script fails with an error:
94# find .stack-work/ -type f -name "benchmarks"
96find_bench_prog () {
97 local bench_name=$1
98 local bench_prog=`$STACK path --dist-dir`/build/$bench_name/$bench_name
99 if test -x "$bench_prog"
100 then
101 echo $bench_prog
102 else
103 return 1
104 fi
107bench_output_file() {
108 local bench_name=$1
109 echo "charts/$bench_name/results.csv"
112# --min-duration 0 means exactly one iteration per sample. We use a million
113# iterations in the benchmarking code explicitly and do not use the iterations
114# done by the benchmarking tool.
116# Benchmarking tool by default discards the first iteration to remove
117# aberrations due to initial evaluations etc. We do not discard it because we
118# are anyway doing iterations in the benchmarking code and many of them so that
119# any constant factor gets amortized and anyway it is a cost that we pay in
120# real life.
122# We can pass --min-samples value from the command line as second argument
123# after the benchmark name in case we want to use more than one sample.
125run_bench () {
126 local bench_name=$1
127 local output_file=$(bench_output_file $bench_name)
128 local bench_prog
129 bench_prog=$(find_bench_prog $bench_name) || \
130 die "Cannot find benchmark executable for benchmark $bench_name"
132 mkdir -p `dirname $output_file`
134 echo "Running benchmark $bench_name ..."
136 $bench_prog $SPEED_OPTIONS \
137 --csvraw=$output_file \
138 -v 2 \
139 --measure-with $bench_prog $GAUGE_ARGS || die "Benchmarking failed"
142run_benches() {
143 for i in $1
144 do
145 run_bench $i
146 done
149run_benches_comparing() {
150 local bench_list=$1
152 if test -z "$CANDIDATE"
153 then
154 CANDIDATE=$(git rev-parse HEAD)
155 fi
156 if test -z "$BASE"
157 then
158 # XXX Should be where the current branch is forked from master
159 BASE="$CANDIDATE^"
160 fi
161 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
162 echo "Checking out base commit [$BASE] for benchmarking"
163 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
165 $STACK build $STACK_BUILD_FLAGS --bench --no-run-benchmarks || die "build failed"
166 run_benches "$bench_list"
168 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
169 git checkout "$CANDIDATE" || \
170 die "Checkout of candidate [$CANDIDATE] commit failed"
172 $STACK build $STACK_BUILD_FLAGS --bench --no-run-benchmarks || die "build failed"
173 run_benches "$bench_list"
174 # XXX reset back to the original commit
177backup_output_file() {
178 local bench_name=$1
179 local output_file=$(bench_output_file $bench_name)
181 if test -e $output_file -a "$APPEND" != 1
182 then
183 mv -f -v $output_file ${output_file}.prev
184 fi
187run_measurements() {
188 local bench_list=$1
190 for i in $bench_list
191 do
192 backup_output_file $i
193 done
195 if test "$COMPARE" = "0"
196 then
197 run_benches "$bench_list"
198 else
199 run_benches_comparing "$bench_list"
200 fi
203run_reports() {
204 local prog
205 prog=$(find_report_prog) || \
206 die "Cannot find bench-graph executable"
207 echo
209 for i in $1
210 do
211 echo "Generating reports for ${i}..."
212 $prog $(test "$GRAPH" = 1 && echo "--graphs") \
213 $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \
214 --benchmark $i
215 done
218#-----------------------------------------------------------------------------
219# Execution starts here
220#-----------------------------------------------------------------------------
222DEFAULT_BENCHMARKS="linear"
223ALL_BENCHMARKS="linear linear-async linear-rate nested base"
224GROUP_DIFF=0
226COMPARE=0
227BASE=
228CANDIDATE=
230APPEND=0
231RAW=0
232GRAPH=0
233MEASURE=1
234SPEED_OPTIONS="--quick --min-samples 10 --time-limit 1 --min-duration 0"
236STACK=stack
237GAUGE_ARGS=
239BUILD_ONCE=0
241#-----------------------------------------------------------------------------
242# Read command line
243#-----------------------------------------------------------------------------
245while test -n "$1"
246do
247 case $1 in
248 -h|--help|help) print_help ;;
249 # options with arguments
250 --slow) SPEED_OPTIONS="--min-duration 0"; shift ;;
251 --benchmarks) shift; BENCHMARKS=$1; shift ;;
252 --base) shift; BASE=$1; shift ;;
253 --candidate) shift; CANDIDATE=$1; shift ;;
254 # flags
255 --compare) COMPARE=1; shift ;;
256 --raw) RAW=1; shift ;;
257 --append) APPEND=1; shift ;;
258 --group-diff) GROUP_DIFF=1; shift ;;
259 --graphs) GRAPH=1; shift ;;
260 --no-measure) MEASURE=0; shift ;;
261 --) shift; break ;;
262 -*|--*) print_help ;;
263 *) break ;;
264 esac
265done
266GAUGE_ARGS=$*
268echo "Using stack command [$STACK]"
269set_benchmarks
271if echo "$BENCHMARKS" | grep -q base
272then
273 STACK_BUILD_FLAGS="--flag streamly:dev"
274fi
276#-----------------------------------------------------------------------------
277# Build stuff
278#-----------------------------------------------------------------------------
280# We need to build the report progs first at the current (latest) commit before
281# checking out any other commit for benchmarking.
282build_report_progs "$BENCHMARKS"
284#-----------------------------------------------------------------------------
285# Run benchmarks
286#-----------------------------------------------------------------------------
288if test "$MEASURE" = "1"
289then
290 $STACK build $STACK_BUILD_FLAGS --bench --no-run-benchmarks || die "build failed"
291 run_measurements "$BENCHMARKS"
292fi
294#-----------------------------------------------------------------------------
295# Run reports
296#-----------------------------------------------------------------------------
298if test "$RAW" = "0"
299then
300 run_reports "$BENCHMARKS"
301fi