git.iko.soy/streamly
on ec08681 help sign in
streamly/bench.sh
178 lines 5.0 kB history raw
1#!/bin/bash
3print_help () {
4 echo "Usage: $0 "
5 echo " [--quick] [--append] "
6 echo " [--no-graphs] [--no-measure]"
7 echo " [--benchmark <linear|nested>]"
8 echo " [--compare] [--base commit] [--candidate commit]"
9 echo " -- <gauge options>"
10 echo
11 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
12 echo "commit is generated, in the 'charts' directory."
13 echo "Use --base and --candidate to select the commits to compare."
14 echo
15 echo "Any arguments after a '--' are passed directly to guage"
16 echo "You can omit '--' if the gauge args used do not start with a '-'."
17 exit
20# $1: message
21die () {
22 >&2 echo -e "Error: $1"
23 exit 1
26DEFAULT_BENCHMARK=linear
27COMPARE=0
29while test -n "$1"
30do
31 case $1 in
32 -h|--help|help) print_help ;;
33 --quick) QUICK=1; shift ;;
34 --append) APPEND=1; shift ;;
35 --benchmark) shift; BENCHMARK=$1; shift ;;
36 --base) shift; BASE=$1; shift ;;
37 --candidate) shift; CANDIDATE=$1; shift ;;
38 --compare) COMPARE=1; shift ;;
39 --no-graphs) GRAPH=0; shift ;;
40 --no-measure) MEASURE=0; shift ;;
41 --) shift; break ;;
42 -*|--*) print_help ;;
43 *) break ;;
44 esac
45done
47GAUGE_ARGS=$*
49if test -z "$BENCHMARK"
50then
51 BENCHMARK=$DEFAULT_BENCHMARK
52 echo "Using default benchmark suite [$BENCHMARK], use --benchmark to specify another"
53else
54 echo "Using benchmark suite [$BENCHMARK]"
55fi
57STACK=stack
58echo "Using stack command [$STACK]"
60# We build it first at the current commit before checking out any other commit
61# for benchmarking.
62if test "$GRAPH" != "0"
63then
64 CHART_PROG="chart-$BENCHMARK"
65 prog=$($STACK exec which $CHART_PROG)
66 hash -r
67 if test ! -x "$prog"
68 then
69 echo "Building charting executable"
70 $STACK build --flag "streamly:dev" || die "build failed"
71 fi
73 prog=$($STACK exec which $CHART_PROG)
74 if test ! -x "$prog"
75 then
76 die "Could not find [$CHART_PROG] executable"
77 fi
78 CHART_PROG=$prog
79 echo "Using chart executable [$CHART_PROG]"
80fi
82# We run the benchmarks in isolation in a separate process so that different
83# benchmarks do not interfere with other. To enable that we need to pass the
84# benchmark exe path to guage as an argument. Unfortunately it cannot find its
85# own path currently.
87# The path is dependent on the architecture and cabal version.
88# Use this command to find the exe if this script fails with an error:
89# find .stack-work/ -type f -name "benchmarks"
91find_bench_prog () {
92 BENCH_PROG=`$STACK path --dist-dir`/build/$BENCHMARK/$BENCHMARK
93 if test ! -x "$BENCH_PROG"
94 then
95 echo
96 echo "WARNING! benchmark binary [$BENCH_PROG] not found or not executable"
97 echo "WARNING! not using isolated measurement."
98 echo
99 fi
102# --min-duration 0 means exactly one iteration per sample. We use a million
103# iterations in the benchmarking code explicitly and do not use the iterations
104# done by the benchmarking tool.
106# Benchmarking tool by default discards the first iteration to remove
107# aberrations due to initial evaluations etc. We do not discard it because we
108# are anyway doing iterations in the benchmarking code and many of them so that
109# any constant factor gets amortized and anyway it is a cost that we pay in
110# real life.
112# We can pass --min-samples value from the command line as second argument
113# after the benchmark name in case we want to use more than one sample.
115if test "$QUICK" = "1"
116then
117 ENABLE_QUICK="--quick"
118fi
120OUTPUT_FILE="charts/results.csv"
122run_bench () {
123 $STACK build --bench --no-run-benchmarks || die "build failed"
124 find_bench_prog
125 mkdir -p charts
127 # We set min-samples to 3 if we use less than three samples, statistical
128 # analysis crashes. Note that the benchmark runs for a minimum of 5 seconds.
129 # We use min-duration=0 to run just one iteration for each sample. Anyway the
130 # default is to run iterations worth minimum 30 ms and most of our benchmarks
131 # are close to that or more.
132 $BENCH_PROG $ENABLE_QUICK \
133 --include-first-iter \
134 --min-samples 3 \
135 --min-duration 0 \
136 --match exact \
137 --csvraw=$OUTPUT_FILE \
138 -v 2 \
139 --measure-with $BENCH_PROG $GAUGE_ARGS || die "Benchmarking failed"
142if test "$MEASURE" != "0"
143 then
144 if test -e $OUTPUT_FILE -a "$APPEND" != 1
145 then
146 mv -f -v $OUTPUT_FILE ${OUTPUT_FILE}.prev
147 fi
149 if test "$COMPARE" = "0"
150 then
151 run_bench
152 else
153 if test -z "$CANDIDATE"
154 then
155 CANDIDATE=$(git rev-parse HEAD)
156 fi
157 if test -z "$BASE"
158 then
159 # XXX Should be where the current branch is forked from master
160 BASE="$CANDIDATE^"
161 fi
162 echo "Checking out base commit for benchmarking"
163 git checkout "$BASE" || die "Checkout of base commit failed"
164 run_bench
165 echo "Checking out candidate commit for benchmarking"
166 git checkout "$CANDIDATE" || die "Checkout of candidate commit failed"
167 run_bench
168 fi
169fi
171if test "$GRAPH" != "0"
172then
173 echo
174 echo "Generating charts from ${OUTPUT_FILE}..."
175 $CHART_PROG
176fi
178# XXX reset back to the original commit