git.iko.soy/streamly
on 83d00da help sign in
streamly/bin/bench.sh
428 lines 12 kB history raw
1#!/usr/bin/env bash
3SCRIPT_DIR=$(cd `dirname $0`; pwd)
5RUNNING_BENCHMARKS=y
6source $SCRIPT_DIR/build-lib.sh
8print_help () {
9 echo "Usage: $0 "
10 echo " [--benchmarks <"bench1 bench2 ..." | help>]"
11 echo " [--fields <"field1 field2 ..." | help>]"
12 echo " [--graphs]"
13 echo " [--no-measure]"
14 echo " [--append]"
15 echo " [--long]"
16 echo " [--slow]"
17 echo " [--quick]"
18 echo " [--raw]"
19 echo " [--dev-build]"
20 echo " [--with-compiler <compiler exe name>]"
21 echo " [--cabal-build-options <options>]"
22 echo " [--rtsopts <opts>]"
23 echo " [--compare] [--base <commit>] [--candidate <commit>]"
24 echo " -- <gauge options or benchmarks>"
25 echo
26 echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
27 echo "--fields: measurement fields to report, use 'help' for a list"
28 echo "--graphs: Generate graphical reports"
29 echo "--no-measure: Don't run benchmarks, run reports from previous results"
30 echo "--append: Don't overwrite previous results, append for comparison"
31 echo "--long: Use much longer stream size for infinite stream benchmarks"
32 echo "--slow: Slightly more accurate results at the expense of speed"
33 echo "--quick: Faster results, useful for longer benchmarks"
34 echo "--raw: Run the benchmarks but don't report them. This is useful when"
35 echo " you only want to work with the csv files generated."
36 echo "--cabal-build-options: Pass any cabal build options to be used for build"
37 echo " e.g. --cabal-build-options \"--flag dev\""
38 echo
39 echo "When specific space complexity group is chosen then (and only then) "
40 echo "RTS memory restrictions are used accordingly. For example, "
41 echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
42 echo "restricts Heap/Stack space for O(1) characterstics"
43 echo
44 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
45 echo "commit is generated, in the 'charts' directory."
46 echo "Use --base and --candidate to select the commits to compare."
47 echo
48 echo "Any arguments after a '--' are passed directly to gauge"
49 exit
52#-----------------------------------------------------------------------------
53# Reporting utility functions
54#-----------------------------------------------------------------------------
56list_comparisons () {
57 echo "Comparison groups:"
58 for i in $COMPARISONS
59 do
60 echo -n "$i ["
61 eval "echo -n \$$i"
62 echo "]"
63 done
64 echo
67# chart is expensive to build and usually not required to be rebuilt,
68# use master branch as fallback
69cabal_which_report() {
70 local path=$(cabal_which streamly-benchmarks x $1)
71 if test -z "$path"
72 then
73 cabal_which_builddir dist-newstyle streamly-benchmarks x $1
74 else
75 echo $path
76 fi
79find_report_prog() {
80 local prog_name="chart"
81 hash -r
82 local prog_path=$(cabal_which_report $prog_name)
83 if test -x "$prog_path"
84 then
85 echo $prog_path
86 else
87 return 1
88 fi
91build_report_prog() {
92 local prog_name="chart"
93 local prog_path=$(cabal_which_report $prog_name)
95 hash -r
96 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
97 then
98 echo "Building bench-show executables"
99 BUILD_ONCE=1
100 $CABAL_EXECUTABLE v2-build --flags dev chart \
101 || die "bench-show build failed"
103 elif test ! -x "$prog_path"
104 then
105 return 1
106 fi
107 return 0
110build_report_progs() {
111 if test "$RAW" = "0"
112 then
113 build_report_prog || exit 1
114 local prog
115 prog=$(find_report_prog) || \
116 die "Cannot find bench-show executable"
117 echo "Using bench-show executable [$prog]"
118 fi
121# We run the benchmarks in isolation in a separate process so that different
122# benchmarks do not interfere with other. To enable that we need to pass the
123# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
124# own path currently.
126# The path is dependent on the architecture and cabal version.
128bench_output_file() {
129 local bench_name=$1
130 echo "charts/$bench_name/results.csv"
133# --min-duration 0 means exactly one iteration per sample. We use a million
134# iterations in the benchmarking code explicitly and do not use the iterations
135# done by the benchmarking tool.
137# Benchmarking tool by default discards the first iteration to remove
138# aberrations due to initial evaluations etc. We do not discard it because we
139# are anyway doing iterations in the benchmarking code and many of them so that
140# any constant factor gets amortized and anyway it is a cost that we pay in
141# real life.
143# We can pass --min-samples value from the command line as second argument
144# after the benchmark name in case we want to use more than one sample.
146# $1: bench name
147# $2: bench executable
148target_exe_extra_args () {
149 local bench_name=$1
150 local bench_prog=$2
152 local output_file=$(bench_output_file $bench_name)
153 mkdir -p `dirname $output_file`
155 local QUICK_OPTS="--quick --min-duration 0"
156 local SPEED_OPTIONS
157 if test "$LONG" -eq 0
158 then
159 if test "$SLOW" -eq 0
160 then
161 if test "$QUICK_MODE" -eq 0
162 then
163 # default mode, not super quick, not slow
164 SPEED_OPTIONS="$QUICK_OPTS --min-samples 10 --time-limit 1"
165 else
166 # super quick but less accurate
167 # When the time-limit is too low and the benchmark is tiny,
168 # then if the number of iterations is very small the GC stats
169 # may remain 0. So keep the time-limit at a minimum of 10 ms
170 # to collect significant stats. The problem was observed in
171 # the Prelude.Serial/reverse' benchmark.
172 SPEED_OPTIONS="$QUICK_OPTS --time-limit 0.01 --include-first-iter"
173 fi
174 else
175 # Slow but more accurate mode
176 SPEED_OPTIONS="--min-duration 0"
177 fi
178 else
179 # large stream size, always super quick
180 GAUGE_ARGS="$GAUGE_ARGS $bench_name/o-1-space"
181 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"
182 fi
184 echo "$SPEED_OPTIONS \
185 --csvraw=$output_file \
186 -v 2 \
187 --measure-with "$SCRIPT_DIR/bench-exec-one.sh" \
188 $GAUGE_ARGS"
191run_benches_comparing() {
192 local bench_list=$1
194 if test -z "$CANDIDATE"
195 then
196 CANDIDATE=$(git rev-parse HEAD)
197 fi
198 if test -z "$BASE"
199 then
200 # XXX Should be where the current branch is forked from master
201 BASE="$CANDIDATE^"
202 fi
203 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
204 echo "Checking out base commit [$BASE] for benchmarking"
205 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
207 $BUILD_BENCH || die "build failed"
208 run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
210 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
211 git checkout "$CANDIDATE" || \
212 die "Checkout of candidate [$CANDIDATE] commit failed"
214 $BUILD_BENCH || die "build failed"
215 run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
216 # XXX reset back to the original commit
219backup_output_file() {
220 local bench_name=$1
221 local output_file=$(bench_output_file $bench_name)
223 if test -e $output_file -a "$APPEND" != 1
224 then
225 mv -f -v $output_file ${output_file}.prev
226 fi
229run_measurements() {
230 local bench_list=$1
232 for i in $bench_list
233 do
234 backup_output_file $i
235 done
237 if test "$COMPARE" = "0"
238 then
239 run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
240 else
241 run_benches_comparing "$bench_list"
242 fi
245run_reports() {
246 local prog
247 prog=$(find_report_prog) || \
248 die "Cannot find bench-graph executable"
249 echo
251 for i in $1
252 do
253 echo "Generating reports for ${i}..."
254 $prog $(test "$GRAPH" = 1 && echo "--graphs") \
255 --benchmark $i --fields "$FIELDS"
256 done
259#-----------------------------------------------------------------------------
260# Execution starts here
261#-----------------------------------------------------------------------------
263cd $SCRIPT_DIR/..
265USE_GIT_CABAL=1
266set_common_vars
268DEFAULT_FIELDS="allocated bytescopied cputime"
269ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
270FIELDS=$DEFAULT_FIELDS
272COMPARE=0
273BASE=
274CANDIDATE=
276APPEND=0
277LONG=0
278RAW=0
279GRAPH=0
280MEASURE=1
282GAUGE_ARGS=
283BUILD_ONCE=0
285CABAL_BUILD_OPTIONS="--flag fusion-plugin "
287#-----------------------------------------------------------------------------
288# Read command line
289#-----------------------------------------------------------------------------
291while test -n "$1"
292do
293 case $1 in
294 -h|--help|help) print_help ;;
295 # options with arguments
296 --benchmarks) shift; TARGETS=$1; shift ;;
297 --targets) shift; TARGETS=$1; shift ;;
298 --fields) shift; FIELDS=$1; shift ;;
299 --base) shift; BASE=$1; shift ;;
300 --candidate) shift; CANDIDATE=$1; shift ;;
301 --with-compiler) shift; CABAL_WITH_COMPILER=$1; shift ;;
302 --cabal-build-flags) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
303 --cabal-build-options) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
304 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
305 # flags
306 --slow) SLOW=1; shift ;;
307 --quick) QUICK_MODE=1; shift ;;
308 --compare) COMPARE=1; shift ;;
309 --raw) RAW=1; shift ;;
310 --append) APPEND=1; shift ;;
311 --long) LONG=1; shift ;;
312 --graphs) GRAPH=1; shift ;;
313 --no-measure) MEASURE=0; shift ;;
314 --dev-build) RUNNING_DEVBUILD=1; shift ;;
315 --) shift; break ;;
316 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
317 *) break ;;
318 esac
319done
320GAUGE_ARGS=$*
322set_derived_vars
324#-----------------------------------------------------------------------------
325# Determine targets
326#-----------------------------------------------------------------------------
328only_real_benchmarks () {
329 for i in $TARGETS
330 do
331 local SKIP=0
332 for j in $COMPARISONS
333 do
334 if test $i == $j
335 then
336 SKIP=1
337 fi
338 done
339 if test "$SKIP" -eq 0
340 then
341 echo -n "$i "
342 fi
343 done
346# Requires RUNNING_DEVBUILD var
347source $SCRIPT_DIR/targets.sh
349if test "$(has_item "$TARGETS" help)" = "help"
350then
351 list_target_groups
352 list_comparisons
353 list_targets
354 exit
355fi
357if test "$(has_item "$FIELDS" help)" = "help"
358then
359 echo "Supported fields: $ALL_FIELDS"
360 echo "Default fields: $DEFAULT_FIELDS"
361 exit
362fi
364if test "$LONG" -ne 0
365then
366 if test -n "$TARGETS"
367 then
368 echo "Cannot specify benchmarks [$TARGETS] with --long"
369 exit
370 fi
371 TARGETS=$infinite_grp
372fi
374DEFAULT_TARGETS="$(all_grp)"
375TARGETS=$(set_targets)
377TARGETS_ORIG=$TARGETS
378TARGETS=$(only_real_benchmarks)
380echo "Using benchmark suites [$TARGETS]"
382#-----------------------------------------------------------------------------
383# Build reporting utility
384#-----------------------------------------------------------------------------
386# We need to build the report progs first at the current (latest) commit before
387# checking out any other commit for benchmarking.
388build_report_progs
390#-----------------------------------------------------------------------------
391# Build and run targets
392#-----------------------------------------------------------------------------
394BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-benchmarks"
395if test "$MEASURE" = "1"
396then
397 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
398 export QUICK_MODE
399 export RTS_OPTIONS
400 run_measurements "$TARGETS"
401fi
403#-----------------------------------------------------------------------------
404# Run reports
405#-----------------------------------------------------------------------------
407COMPARISON_REPORTS=""
408for i in $COMPARISONS
409do
410 if test "$(has_item "$TARGETS_ORIG" $i)" = $i
411 then
412 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
413 mkdir -p "charts/$i"
414 constituents=$(eval "echo -n \$${i}")
415 dest_file="charts/$i/results.csv"
416 : > $dest_file
417 for j in $constituents
418 do
419 cat "charts/$j/results.csv" >> $dest_file
420 done
421 fi
422done
424if test "$RAW" = "0"
425then
426 run_reports "$TARGETS"
427 run_reports "$COMPARISON_REPORTS"
428fi