git.iko.soy/streamly
on ec19422 help sign in
streamly/bin/bench.sh
417 lines 11 kB history raw
1#!/usr/bin/env bash
3SCRIPT_DIR=$(cd `dirname $0`; pwd)
5RUNNING_BENCHMARKS=y
6source $SCRIPT_DIR/build-lib.sh
8print_help () {
9 echo "Usage: $0 "
10 echo " [--benchmarks <"bench1 bench2 ..." | help>]"
11 echo " [--fields <"field1 field2 ..." | help>]"
12 echo " [--graphs]"
13 echo " [--no-measure]"
14 echo " [--append]"
15 echo " [--long]"
16 echo " [--slow]"
17 echo " [--quick]"
18 echo " [--raw]"
19 echo " [--dev-build]"
20 echo " [--cabal-build-flags <flag>]"
21 echo " [--rtsopts <opts>]"
22 echo " [--compare] [--base <commit>] [--candidate <commit>]"
23 echo " -- <gauge options or benchmarks>"
24 echo
25 echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
26 echo "--fields: measurement fields to report, use 'help' for a list"
27 echo "--graphs: Generate graphical reports"
28 echo "--no-measure: Don't run benchmarks, run reports from previous results"
29 echo "--append: Don't overwrite previous results, append for comparison"
30 echo "--long: Use much longer stream size for infinite stream benchmarks"
31 echo "--slow: Slightly more accurate results at the expense of speed"
32 echo "--quick: Faster results, useful for longer benchmarks"
33 echo "--raw: Run the benchmarks but don't report them. This is useful when"
34 echo " you only want to work with the csv files generated."
35 echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"
36 echo " e.g. --cabal-build-flags \"--flag fusion-plugin\""
37 echo
38 echo "When specific space complexity group is chosen then (and only then) "
39 echo "RTS memory restrictions are used accordingly. For example, "
40 echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
41 echo "restricts Heap/Stack space for O(1) characterstics"
42 echo
43 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
44 echo "commit is generated, in the 'charts' directory."
45 echo "Use --base and --candidate to select the commits to compare."
46 echo
47 echo "Any arguments after a '--' are passed directly to gauge"
48 exit
51#-----------------------------------------------------------------------------
52# Reporting utility functions
53#-----------------------------------------------------------------------------
55list_comparisons () {
56 echo "Comparison groups:"
57 for i in $COMPARISONS
58 do
59 echo -n "$i ["
60 eval "echo -n \$$i"
61 echo "]"
62 done
63 echo
66# chart is expensive to build and usually not required to be rebuilt,
67# use master branch as fallback
68cabal_which_report() {
69 local path=$(cabal_which streamly-benchmarks x $1)
70 if test -z "$path"
71 then
72 cabal_which_builddir dist-newstyle streamly-benchmarks x $1
73 else
74 echo $path
75 fi
78find_report_prog() {
79 local prog_name="chart"
80 hash -r
81 local prog_path=$(cabal_which_report $prog_name)
82 if test -x "$prog_path"
83 then
84 echo $prog_path
85 else
86 return 1
87 fi
90build_report_prog() {
91 local prog_name="chart"
92 local prog_path=$(cabal_which_report $prog_name)
94 hash -r
95 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
96 then
97 echo "Building bench-show executables"
98 BUILD_ONCE=1
99 $CABAL_EXECUTABLE v2-build --flags dev chart \
100 || die "bench-show build failed"
102 elif test ! -x "$prog_path"
103 then
104 return 1
105 fi
106 return 0
109build_report_progs() {
110 if test "$RAW" = "0"
111 then
112 build_report_prog || exit 1
113 local prog
114 prog=$(find_report_prog) || \
115 die "Cannot find bench-show executable"
116 echo "Using bench-show executable [$prog]"
117 fi
120# We run the benchmarks in isolation in a separate process so that different
121# benchmarks do not interfere with other. To enable that we need to pass the
122# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
123# own path currently.
125# The path is dependent on the architecture and cabal version.
127bench_output_file() {
128 local bench_name=$1
129 echo "charts/$bench_name/results.csv"
132# --min-duration 0 means exactly one iteration per sample. We use a million
133# iterations in the benchmarking code explicitly and do not use the iterations
134# done by the benchmarking tool.
136# Benchmarking tool by default discards the first iteration to remove
137# aberrations due to initial evaluations etc. We do not discard it because we
138# are anyway doing iterations in the benchmarking code and many of them so that
139# any constant factor gets amortized and anyway it is a cost that we pay in
140# real life.
142# We can pass --min-samples value from the command line as second argument
143# after the benchmark name in case we want to use more than one sample.
145# $1: bench name
146# $2: bench executable
147target_exe_extra_args () {
148 local bench_name=$1
149 local bench_prog=$2
151 local output_file=$(bench_output_file $bench_name)
152 mkdir -p `dirname $output_file`
154 local QUICK_OPTS="--quick --min-duration 0"
155 local SPEED_OPTIONS
156 if test "$LONG" -eq 0
157 then
158 if test "$SLOW" -eq 0
159 then
160 if test "$QUICK_MODE" -eq 0
161 then
162 # default mode, not super quick, not slow
163 SPEED_OPTIONS="$QUICK_OPTS --min-samples 10 --time-limit 1"
164 else
165 # super quick but less accurate
166 SPEED_OPTIONS="$QUICK_OPTS --time-limit 0 --include-first-iter"
167 fi
168 else
169 # Slow but more accurate mode
170 SPEED_OPTIONS="--min-duration 0"
171 fi
172 else
173 # large stream size, always super quick
174 GAUGE_ARGS="$GAUGE_ARGS $bench_name/o-1-space"
175 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"
176 fi
178 echo "$SPEED_OPTIONS \
179 --csvraw=$output_file \
180 -v 2 \
181 --measure-with "$SCRIPT_DIR/bench-exec-one.sh" \
182 $GAUGE_ARGS"
185run_benches_comparing() {
186 local bench_list=$1
188 if test -z "$CANDIDATE"
189 then
190 CANDIDATE=$(git rev-parse HEAD)
191 fi
192 if test -z "$BASE"
193 then
194 # XXX Should be where the current branch is forked from master
195 BASE="$CANDIDATE^"
196 fi
197 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
198 echo "Checking out base commit [$BASE] for benchmarking"
199 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
201 $BUILD_BENCH || die "build failed"
202 run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
204 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
205 git checkout "$CANDIDATE" || \
206 die "Checkout of candidate [$CANDIDATE] commit failed"
208 $BUILD_BENCH || die "build failed"
209 run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
210 # XXX reset back to the original commit
213backup_output_file() {
214 local bench_name=$1
215 local output_file=$(bench_output_file $bench_name)
217 if test -e $output_file -a "$APPEND" != 1
218 then
219 mv -f -v $output_file ${output_file}.prev
220 fi
223run_measurements() {
224 local bench_list=$1
226 for i in $bench_list
227 do
228 backup_output_file $i
229 done
231 if test "$COMPARE" = "0"
232 then
233 run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
234 else
235 run_benches_comparing "$bench_list"
236 fi
239run_reports() {
240 local prog
241 prog=$(find_report_prog) || \
242 die "Cannot find bench-graph executable"
243 echo
245 for i in $1
246 do
247 echo "Generating reports for ${i}..."
248 $prog $(test "$GRAPH" = 1 && echo "--graphs") \
249 --benchmark $i --fields "$FIELDS"
250 done
253#-----------------------------------------------------------------------------
254# Execution starts here
255#-----------------------------------------------------------------------------
257cd $SCRIPT_DIR/..
259USE_GIT_CABAL=1
260set_common_vars
262DEFAULT_FIELDS="allocated bytescopied cputime"
263ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
264FIELDS=$DEFAULT_FIELDS
266COMPARE=0
267BASE=
268CANDIDATE=
270APPEND=0
271LONG=0
272RAW=0
273GRAPH=0
274MEASURE=1
276GAUGE_ARGS=
277BUILD_ONCE=0
279#-----------------------------------------------------------------------------
280# Read command line
281#-----------------------------------------------------------------------------
283while test -n "$1"
284do
285 case $1 in
286 -h|--help|help) print_help ;;
287 # options with arguments
288 --benchmarks) shift; TARGETS=$1; shift ;;
289 --targets) shift; TARGETS=$1; shift ;;
290 --fields) shift; FIELDS=$1; shift ;;
291 --base) shift; BASE=$1; shift ;;
292 --candidate) shift; CANDIDATE=$1; shift ;;
293 --cabal-build-flags) shift; CABAL_BUILD_OPTIONS=$1; shift ;;
294 --cabal-build-options) shift; CABAL_BUILD_OPTIONS=$1; shift ;;
295 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
296 # flags
297 --slow) SLOW=1; shift ;;
298 --quick) QUICK_MODE=1; shift ;;
299 --compare) COMPARE=1; shift ;;
300 --raw) RAW=1; shift ;;
301 --append) APPEND=1; shift ;;
302 --long) LONG=1; shift ;;
303 --graphs) GRAPH=1; shift ;;
304 --no-measure) MEASURE=0; shift ;;
305 --dev-build) RUNNING_DEVBUILD=1; shift ;;
306 --) shift; break ;;
307 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
308 *) break ;;
309 esac
310done
311GAUGE_ARGS=$*
313#-----------------------------------------------------------------------------
314# Determine targets
315#-----------------------------------------------------------------------------
317only_real_benchmarks () {
318 for i in $TARGETS
319 do
320 local SKIP=0
321 for j in $COMPARISONS
322 do
323 if test $i == $j
324 then
325 SKIP=1
326 fi
327 done
328 if test "$SKIP" -eq 0
329 then
330 echo -n "$i "
331 fi
332 done
335# Requires RUNNING_DEVBUILD var
336source $SCRIPT_DIR/targets.sh
338if test "$(has_item "$TARGETS" help)" = "help"
339then
340 list_target_groups
341 list_comparisons
342 list_targets
343 exit
344fi
346if test "$(has_item "$FIELDS" help)" = "help"
347then
348 echo "Supported fields: $ALL_FIELDS"
349 echo "Default fields: $DEFAULT_FIELDS"
350 exit
351fi
353if test "$LONG" -ne 0
354then
355 if test -n "$TARGETS"
356 then
357 echo "Cannot specify benchmarks [$TARGETS] with --long"
358 exit
359 fi
360 TARGETS=$infinite_grp
361fi
363DEFAULT_TARGETS="$(all_grp)"
364TARGETS=$(set_targets)
366TARGETS_ORIG=$TARGETS
367TARGETS=$(only_real_benchmarks)
369echo "Using benchmark suites [$TARGETS]"
371#-----------------------------------------------------------------------------
372# Build reporting utility
373#-----------------------------------------------------------------------------
375# We need to build the report progs first at the current (latest) commit before
376# checking out any other commit for benchmarking.
377build_report_progs
379#-----------------------------------------------------------------------------
380# Build and run targets
381#-----------------------------------------------------------------------------
383BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-benchmarks"
384if test "$MEASURE" = "1"
385then
386 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
387 export QUICK_MODE
388 export RTS_OPTIONS
389 run_measurements "$TARGETS"
390fi
392#-----------------------------------------------------------------------------
393# Run reports
394#-----------------------------------------------------------------------------
396COMPARISON_REPORTS=""
397for i in $COMPARISONS
398do
399 if test "$(has_item "$TARGETS_ORIG" $i)" = $i
400 then
401 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
402 mkdir -p "charts/$i"
403 constituents=$(eval "echo -n \$${i}")
404 dest_file="charts/$i/results.csv"
405 : > $dest_file
406 for j in $constituents
407 do
408 cat "charts/$j/results.csv" >> $dest_file
409 done
410 fi
411done
413if test "$RAW" = "0"
414then
415 run_reports "$TARGETS"
416 run_reports "$COMPARISON_REPORTS"
417fi