@@ -2,16 +2,20 @@
22
33print_help () {
44 echo "Usage: $0 "
5− echo " [--quick] [--append] "
6− echo " [--no-graphs] [--no-measure]"
7− echo " [--benchmark <linear|nested>]"
85 echo " [--compare] [--base commit] [--candidate commit]"
6+ echo " [--benchmark <linear|nested|base>]"
7+ echo " [--graphs]"
8+ echo " [--slow]"
9+ echo " [--no-measure]"
10+ echo " [--append] "
911 echo " -- <gauge options>"
1012 echo
11− echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
12− echo "commit is generated, in the 'charts' directory."
13− echo "Use --base and --candidate to select the commits to compare."
14− echo
13+ # XXX in the report the base and candidate are shown in reverse order as we
14+ # run the benchmarks for candidate first and then base.
15+ #echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
16+ #echo "commit is generated, in the 'charts' directory."
17+ #echo "Use --base and --candidate to select the commits to compare."
18+ #echo
1519 echo "Any arguments after a '--' are passed directly to guage"
1620 echo "You can omit '--' if the gauge args used do not start with a '-'."
1721 exit
@@ -23,61 +27,64 @@ die () {
2327 exit 1
2428}
2529
26−DEFAULT_BENCHMARK=linear
27−COMPARE=0
28−
29−while test -n "$1"
30−do
31− case $1 in
32− -h|--help|help) print_help ;;
33− --quick) QUICK=1; shift ;;
34− --append) APPEND=1; shift ;;
35− --benchmark) shift; BENCHMARK=$1; shift ;;
36− --base) shift; BASE=$1; shift ;;
37− --candidate) shift; CANDIDATE=$1; shift ;;
38− --compare) COMPARE=1; shift ;;
39− --no-graphs) GRAPH=0; shift ;;
40− --no-measure) MEASURE=0; shift ;;
41− --) shift; break ;;
42− -*|--*) print_help ;;
43− *) break ;;
44− esac
45−done
30+set_benchmarks() {
31+ if test -z "$BENCHMARKS"
32+ then
33+ BENCHMARKS=$DEFAULT_BENCHMARKS
34+ echo "Using default benchmark suite [$BENCHMARKS], use --benchmark to specify another"
35+ else
36+ echo "Using benchmark suite [$BENCHMARKS]"
37+ fi
38+}
4639
47−GAUGE_ARGS=$*
40+# $1: benchmark name (linear, nested, base)
41+find_report_prog() {
42+ local bench_name=$1
43+ local prog_name="chart-$bench_name"
44+ hash -r
45+ local prog_path=$($STACK exec which $prog_name)
46+ if test -x "$prog_path"
47+ then
48+ echo $prog_path
49+ else
50+ return 1
51+ fi
52+}
4853
49−if test -z "$BENCHMARK"
50−then
51− BENCHMARK=$DEFAULT_BENCHMARK
52− echo "Using default benchmark suite [$BENCHMARK], use --benchmark to specify another"
53−else
54− echo "Using benchmark suite [$BENCHMARK]"
55−fi
54+# $1: benchmark name (linear, nested, base)
55+build_report_prog() {
56+ local bench_name=$1
57+ local prog_name="chart-$bench_name"
58+ local prog_path=$($STACK exec which $prog_name)
5659
57−STACK=stack
58−echo "Using stack command [$STACK]"
60+ hash -r
61+ if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
62+ then
63+ echo "Building bench-graph executables"
64+ BUILD_ONCE=1
65+ $STACK build --flag "streamly:dev" || die "build failed"
66+ elif test ! -x "$prog_path"
67+ then
68+ return 1
69+ fi
70+ return 0
71+}
5972
60−# We build it first at the current commit before checking out any other commit
61−# for benchmarking.
62−if test "$GRAPH" != "0"
63−then
64− CHART_PROG="chart-$BENCHMARK"
65− prog=$($STACK exec which $CHART_PROG)
66− hash -r
67− if test ! -x "$prog"
68− then
69− echo "Building charting executable"
70− $STACK build --flag "streamly:dev" || die "build failed"
71− fi
73+build_report_progs() {
74+ local bench_list=$1
7275
73− prog=$($STACK exec which $CHART_PROG)
74− if test ! -x "$prog"
76+ if test "$RAW" = "0"
7577 then
76− die "Could not find [$CHART_PROG] executable"
78+ for i in $bench_list
79+ do
80+ build_report_prog $i || exit 1
81+ local prog
82+ prog=$(find_report_prog $i) || \
83+ die "Cannot find bench-graph executable for benchmark $i"
84+ echo "Using bench-graph executable [$prog]"
85+ done
7786 fi
78− CHART_PROG=$prog
79− echo "Using chart executable [$CHART_PROG]"
80−fi
87+}
8188
8289# We run the benchmarks in isolation in a separate process so that different
8390# benchmarks do not interfere with other. To enable that we need to pass the
@@ -89,16 +96,21 @@ fi
8996# find .stack-work/ -type f -name "benchmarks"
9097
9198find_bench_prog () {
92− BENCH_PROG=`$STACK path --dist-dir`/build/$BENCHMARK/$BENCHMARK
93− if test ! -x "$BENCH_PROG"
99+ local bench_name=$1
100+ local bench_prog=`$STACK path --dist-dir`/build/$bench_name/$bench_name
101+ if test -x "$bench_prog"
94102 then
95− echo
96− echo "WARNING! benchmark binary [$BENCH_PROG] not found or not executable"
97− echo "WARNING! not using isolated measurement."
98− echo
103+ echo $bench_prog
104+ else
105+ return 1
99106 fi
100107}
101108
109+bench_output_file() {
110+ local bench_name=$1
111+ echo "charts/$bench_name/results.csv"
112+}
113+
102114# --min-duration 0 means exactly one iteration per sample. We use a million
103115# iterations in the benchmarking code explicitly and do not use the iterations
104116# done by the benchmarking tool.
@@ -112,44 +124,33 @@ find_bench_prog () {
112124# We can pass --min-samples value from the command line as second argument
113125# after the benchmark name in case we want to use more than one sample.
114126
115−if test "$QUICK" = "1"
116−then
117− ENABLE_QUICK="--quick"
118−fi
127+run_bench () {
128+ local bench_name=$1
129+ local output_file=$(bench_output_file $bench_name)
130+ local bench_prog
131+ bench_prog=$(find_bench_prog $bench_name) || \
132+ die "Cannot find benchmark executable for benchmark $bench_name"
119133
120−OUTPUT_FILE="charts/results.csv"
134+ mkdir -p `dirname $output_file`
121135
122−run_bench () {
123− $STACK build --bench --no-run-benchmarks || die "build failed"
124− find_bench_prog
125− mkdir -p charts
126−
127− # We set min-samples to 3 if we use less than three samples, statistical
128− # analysis crashes. Note that the benchmark runs for a minimum of 5 seconds.
129− # We use min-duration=0 to run just one iteration for each sample. Anyway the
130− # default is to run iterations worth minimum 30 ms and most of our benchmarks
131− # are close to that or more.
132− $BENCH_PROG $ENABLE_QUICK \
133− --include-first-iter \
134− --min-samples 3 \
135− --min-duration 0 \
136− --match exact \
137− --csvraw=$OUTPUT_FILE \
136+ echo "Running benchmark $bench_name ..."
137+
138+ $bench_prog $SPEED_OPTIONS \
139+ --csvraw=$output_file \
138140 -v 2 \
139− --measure-with $BENCH_PROG $GAUGE_ARGS || die "Benchmarking failed"
141+ --measure-with $bench_prog $GAUGE_ARGS || die "Benchmarking failed"
140142}
141143
142−if test "$MEASURE" != "0"
143− then
144− if test -e $OUTPUT_FILE -a "$APPEND" != 1
145− then
146− mv -f -v $OUTPUT_FILE ${OUTPUT_FILE}.prev
147− fi
144+run_benches() {
145+ for i in $1
146+ do
147+ run_bench $i
148+ done
149+}
150+
151+run_benches_comparing() {
152+ local bench_list=$1
148153
149− if test "$COMPARE" = "0"
150− then
151− run_bench
152− else
153154 if test -z "$CANDIDATE"
154155 then
155156 CANDIDATE=$(git rev-parse HEAD)
@@ -161,18 +162,132 @@ if test "$MEASURE" != "0"
161162 fi
162163 echo "Checking out base commit for benchmarking"
163164 git checkout "$BASE" || die "Checkout of base commit failed"
164− run_bench
165+
166+ run_benches "$bench_list"
167+
165168 echo "Checking out candidate commit for benchmarking"
166169 git checkout "$CANDIDATE" || die "Checkout of candidate commit failed"
167− run_bench
170+
171+ run_benches "$bench_list"
172+ # XXX reset back to the original commit
173+}
174+
175+backup_output_file() {
176+ local bench_name=$1
177+ local output_file=$(bench_output_file $bench_name)
178+
179+ if test -e $output_file -a "$APPEND" != 1
180+ then
181+ mv -f -v $output_file ${output_file}.prev
168182 fi
169−fi
183+}
184+
185+run_measurements() {
186+ local bench_list=$1
187+
188+ for i in $bench_list
189+ do
190+ backup_output_file $i
191+ done
192+
193+ if test "$COMPARE" = "0"
194+ then
195+ run_benches "$bench_list"
196+ else
197+ run_benches_comparing "$bench_list"
198+ fi
199+}
200+
201+run_report() {
202+ local bench_name=$1
203+ local input_file=$2
204+
205+ if test "$RAW" = "0"
206+ then
207+ local prog
208+ prog=$(find_report_prog $bench_name) || \
209+ die "Cannot find bench-graph executable for benchmark $bench_name"
210+ echo
211+ echo "Generating reports for ${bench_name}..."
212+ $prog
213+ fi
214+}
215+
216+run_reports() {
217+ for i in $1
218+ do
219+ run_report $i
220+ done
221+}
222+
223+#-----------------------------------------------------------------------------
224+# Execution starts here
225+#-----------------------------------------------------------------------------
226+
227+DEFAULT_BENCHMARKS="linear"
228+
229+COMPARE=0
230+BASE=
231+CANDIDATE=
232+
233+APPEND=0
234+RAW=0
235+GRAPH=0
236+MEASURE=1
237+SPEED_OPTIONS="--quick --min-samples 10 --time-limit 1 --min-duration 0"
170238
171−if test "$GRAPH" != "0"
239+STACK=stack
240+GAUGE_ARGS=
241+
242+BUILD_ONCE=0
243+
244+#-----------------------------------------------------------------------------
245+# Read command line
246+#-----------------------------------------------------------------------------
247+
248+while test -n "$1"
249+do
250+ case $1 in
251+ -h|--help|help) print_help ;;
252+ --slow) SPEED_OPTIONS="--min-duration 0"; shift ;;
253+ --append) APPEND=1; shift ;;
254+ --benchmark) shift; BENCHMARKS=$1; shift ;;
255+ --base) shift; BASE=$1; shift ;;
256+ --candidate) shift; CANDIDATE=$1; shift ;;
257+ --compare) COMPARE=1; shift ;;
258+ --raw) RAW=1; shift ;;
259+ --graphs) GRAPH=1; shift ;;
260+ --no-measure) MEASURE=0; shift ;;
261+ --) shift; break ;;
262+ -*|--*) print_help ;;
263+ *) break ;;
264+ esac
265+done
266+GAUGE_ARGS=$*
267+
268+echo "Using stack command [$STACK]"
269+set_benchmarks
270+
271+#-----------------------------------------------------------------------------
272+# Build stuff
273+#-----------------------------------------------------------------------------
274+
275+# We need to build the report progs first at the current (latest) commit before
276+# checking out any other commit for benchmarking.
277+build_report_progs "$BENCHMARKS"
278+
279+#-----------------------------------------------------------------------------
280+# Run benchmarks
281+#-----------------------------------------------------------------------------
282+
283+if test "$MEASURE" = "1"
172284then
173− echo
174− echo "Generating charts from ${OUTPUT_FILE}..."
175− $CHART_PROG
285+ $STACK build --bench --no-run-benchmarks || die "build failed"
286+ run_measurements "$BENCHMARKS"
176287fi
177288
178−# XXX reset back to the original commit
289+#-----------------------------------------------------------------------------
290+# Run reports
291+#-----------------------------------------------------------------------------
292+
293+run_reports "$BENCHMARKS"