git.iko.soy/streamly
on e9828b3 help sign in
streamly/bench.sh
598 lines 15 kB history raw
1#!/bin/bash
3#------------------------------------------------------------------------------
4# Benchmark groups
5#------------------------------------------------------------------------------
7# IMPORTANT NOTE: the names "_grp" and "_cmp" suffixes are special, do
8# not rename them to something else.
10base_stream_grp="\
11 Data.Stream.StreamD \
12 Data.Stream.StreamK \
13 Data.Stream.StreamDK"
15prelude_serial_grp="\
16 Prelude.Serial \
17 Prelude.WSerial \
18 Prelude.ZipSerial"
20prelude_concurrent_grp="\
21 Prelude.Async \
22 Prelude.WAsync \
23 Prelude.Ahead \
24 Prelude.Parallel \
25 Prelude.ZipAsync"
27prelude_other_grp="\
28 Prelude.Rate \
29 Prelude.Concurrent \
30 Prelude.Adaptive"
32array_grp="\
33 Memory.Array \
34 Data.Array \
35 Data.Array.Prim \
36 Data.SmallArray \
37 Data.Array.Prim.Pinned"
39base_parser_grp="Data.Parser.ParserD Data.Parser.ParserK"
40parser_grp="Data.Fold Data.Parser"
42#------------------------------------------------------------------------------
43# Streaming vs non-streaming
44#------------------------------------------------------------------------------
45# The "o-1-space" groups of these benchmarks are run with long stream
46# sizes when --long option is used.
48infinite_grp="\
49 $prelude_serial_grp \
50 $prelude_concurrent_grp \
51 Prelude.Rate"
53#------------------------------------------------------------------------------
54# Comparison groups
55#------------------------------------------------------------------------------
56# *_cmp denotes a comparison benchmarks, the benchmarks provided in *_cmp
57# variables are compared with each other
58base_stream_cmp="Data.Stream.StreamD Data.Stream.StreamK"
59serial_wserial_cmp="Prelude.Serial Prelude.WSerial"
60serial_async_cmp="Prelude.Serial Prelude.Async"
61concurrent_cmp="Prelude.Async Prelude.WAsync Prelude.Ahead Prelude.Parallel"
62array_cmp="Memory.Array Data.Array.Prim Data.Array Data.Array.Prim.Pinned"
63pinned_array_cmp="Memory.Array Data.Array.Prim.Pinned"
64base_parser_cmp=$base_parser_grp
65COMPARISONS="\
66 base_stream_cmp \
67 serial_wserial_cmp \
68 serial_async_cmp \
69 concurrent_cmp \
70 array_cmp \
71 pinned_array_cmp \
72 base_parser_cmp"
74#------------------------------------------------------------------------------
75# All
76#------------------------------------------------------------------------------
77# All high level benchmarks
78all_grp="\
79 $prelude_serial_grp \
80 $prelude_concurrent_grp \
81 $array_grp \
82 $parser_grp \
83 Data.Unfold \
84 FileSystem.Handle"
86ALL_BENCH_GROUPS="\
87 all_grp \
88 prelude_serial_grp \
89 prelude_concurrent_grp \
90 array_grp \
91 infinite_grp \
92 base_stream_grp \
93 base_parser_grp"
95#------------------------------------------------------------------------------
96# Script
97#------------------------------------------------------------------------------
99BENCH_SH_DIR=$(dirname $0)
101list_benches () {
102 echo "Individual benchmarks:"
103 for i in $all_grp
104 do
105 echo "$i"
106 done
107 echo
110list_bench_groups () {
111 echo "Benchmark groups:"
112 for i in $ALL_BENCH_GROUPS
113 do
114 echo -n "$i ["
115 eval "echo -n \$$i"
116 echo "]"
117 done
118 echo
121list_comparisons () {
122 echo "Comparison groups:"
123 for i in $COMPARISONS
124 do
125 echo -n "$i ["
126 eval "echo -n \$$i"
127 echo "]"
128 done
129 echo
132print_help () {
133 echo "Usage: $0 "
134 echo " [--benchmarks <"bench1 bench2 ..." | help>]"
135 echo " [--fields <"field1 field2 ..." | help>]"
136 echo " [--graphs]"
137 echo " [--no-measure]"
138 echo " [--append]"
139 echo " [--long]"
140 echo " [--slow]"
141 echo " [--quick]"
142 echo " [--cabal-build-flags <flag>]"
143 echo " [--compare] [--base <commit>] [--candidate <commit>]"
144 echo " -- <gauge options or benchmarks>"
145 echo
146 echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
147 echo "--fields: measurement fields to report, use 'help' for a list"
148 echo "--graphs: Generate graphical reports"
149 echo "--no-measure: Don't run benchmarks, run reports from previous results"
150 echo "--append: Don't overwrite previous results, append for comparison"
151 echo "--long: Use much longer stream size for infinite stream benchmarks"
152 echo "--slow: Slightly more accurate results at the expense of speed"
153 echo "--quick: Faster results, useful for longer benchmarks"
154 echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"
155 echo
156 echo "When specific space complexity group is chosen then (and only then) "
157 echo "RTS memory restrictions are used accordingly. For example, "
158 echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
159 echo "restricts Heap/Stack space for O(1) characterstics"
160 echo
161 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
162 echo "commit is generated, in the 'charts' directory."
163 echo "Use --base and --candidate to select the commits to compare."
164 echo
165 echo "Any arguments after a '--' are passed directly to gauge"
166 exit
169# $1: message
170die () {
171 >&2 echo -e "Error: $1"
172 exit 1
175set_benchmarks() {
176 if test -z "$BENCHMARKS"
177 then
178 echo $DEFAULT_BENCHMARKS
179 else
180 for i in $(echo $BENCHMARKS)
181 do
182 case $i in
183 *_grp) eval "echo -n \$${i}" ;;
184 *_cmp) eval "echo -n \$${i} $i" ;;
185 *) echo -n $i ;;
186 esac
187 echo -n " "
188 done
189 fi
192find_report_prog() {
193 local prog_name="chart"
194 hash -r
195 local prog_path=$($WHICH_COMMAND $prog_name)
196 if test -x "$prog_path"
197 then
198 echo $prog_path
199 else
200 return 1
201 fi
204build_report_prog() {
205 local prog_name="chart"
206 local prog_path=$($WHICH_COMMAND $prog_name)
208 hash -r
209 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
210 then
211 echo "Building bench-graph executables"
212 BUILD_ONCE=1
213 $BUILD_CHART_EXE || die "build failed"
214 elif test ! -x "$prog_path"
215 then
216 return 1
217 fi
218 return 0
221build_report_progs() {
222 if test "$RAW" = "0"
223 then
224 build_report_prog || exit 1
225 local prog
226 prog=$(find_report_prog) || \
227 die "Cannot find bench-graph executable"
228 echo "Using bench-graph executable [$prog]"
229 fi
232# We run the benchmarks in isolation in a separate process so that different
233# benchmarks do not interfere with other. To enable that we need to pass the
234# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
235# own path currently.
237# The path is dependent on the architecture and cabal version.
238# Use this command to find the exe if this script fails with an error:
239# find .stack-work/ -type f -name "benchmarks"
241stack_bench_prog () {
242 local bench_name=$1
243 local bench_prog=`stack path --dist-dir`/build/$bench_name/$bench_name
244 if test -x "$bench_prog"
245 then
246 echo $bench_prog
247 else
248 return 1
249 fi
252cabal_bench_prog () {
253 local bench_name=$1
254 local bench_prog=`$WHICH_COMMAND $1`
255 if test -x "$bench_prog"
256 then
257 echo $bench_prog
258 else
259 return 1
260 fi
263bench_output_file() {
264 local bench_name=$1
265 echo "charts/$bench_name/results.csv"
268# $1: command
269function run_verbose() {
270 echo "$*"
271 bash -c "$*"
274# --min-duration 0 means exactly one iteration per sample. We use a million
275# iterations in the benchmarking code explicitly and do not use the iterations
276# done by the benchmarking tool.
278# Benchmarking tool by default discards the first iteration to remove
279# aberrations due to initial evaluations etc. We do not discard it because we
280# are anyway doing iterations in the benchmarking code and many of them so that
281# any constant factor gets amortized and anyway it is a cost that we pay in
282# real life.
284# We can pass --min-samples value from the command line as second argument
285# after the benchmark name in case we want to use more than one sample.
287run_bench () {
288 local bench_name=$1
289 local bench_exe=$bench_name
290 local output_file=$(bench_output_file $bench_name)
291 local bench_prog
292 bench_prog=$($GET_BENCH_PROG $bench_exe) || \
293 die "Cannot find benchmark executable for benchmark $bench_name"
295 mkdir -p `dirname $output_file`
297 echo "Running benchmark $bench_name ..."
299 local QUICK_OPTS="--quick --min-duration 0"
300 local SPEED_OPTIONS
301 if test "$LONG" -eq 0
302 then
303 if test "$SLOW" -eq 0
304 then
305 export QUICK_MODE
306 if test "$QUICK_MODE" -eq 0
307 then
308 # default mode, not super quick, not slow
309 SPEED_OPTIONS="$QUICK_OPTS --min-samples 10 --time-limit 1"
310 else
311 # super quick but less accurate
312 SPEED_OPTIONS="$QUICK_OPTS --time-limit 0 --include-first-iter"
313 fi
314 else
315 # Slow but more accurate mode
316 SPEED_OPTIONS="--min-duration 0"
317 fi
318 else
319 # large stream size, always super quick
320 GAUGE_ARGS="$GAUGE_ARGS $bench_name/o-1-space"
321 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"
322 fi
324 export BENCH_EXEC_PATH=$bench_prog
325 export RTS_OPTIONS
326 run_verbose $bench_prog $SPEED_OPTIONS \
327 --csvraw=$output_file \
328 -v 2 \
329 --measure-with "$BENCH_SH_DIR/bin/bench-exec-one.sh" \
330 $GAUGE_ARGS || die "Benchmarking failed"
333run_benches() {
334 for i in $1
335 do
336 run_bench $i
337 done
340run_benches_comparing() {
341 local bench_list=$1
343 if test -z "$CANDIDATE"
344 then
345 CANDIDATE=$(git rev-parse HEAD)
346 fi
347 if test -z "$BASE"
348 then
349 # XXX Should be where the current branch is forked from master
350 BASE="$CANDIDATE^"
351 fi
352 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
353 echo "Checking out base commit [$BASE] for benchmarking"
354 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
356 $BUILD_BENCH || die "build failed"
357 run_benches "$bench_list"
359 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
360 git checkout "$CANDIDATE" || \
361 die "Checkout of candidate [$CANDIDATE] commit failed"
363 $BUILD_BENCH || die "build failed"
364 run_benches "$bench_list"
365 # XXX reset back to the original commit
368backup_output_file() {
369 local bench_name=$1
370 local output_file=$(bench_output_file $bench_name)
372 if test -e $output_file -a "$APPEND" != 1
373 then
374 mv -f -v $output_file ${output_file}.prev
375 fi
378run_measurements() {
379 local bench_list=$1
381 for i in $bench_list
382 do
383 backup_output_file $i
384 done
386 if test "$COMPARE" = "0"
387 then
388 run_benches "$bench_list"
389 else
390 run_benches_comparing "$bench_list"
391 fi
394run_reports() {
395 local prog
396 prog=$(find_report_prog) || \
397 die "Cannot find bench-graph executable"
398 echo
400 for i in $1
401 do
402 echo "Generating reports for ${i}..."
403 $prog $(test "$GRAPH" = 1 && echo "--graphs") \
404 --benchmark $i --fields "$FIELDS"
405 done
408#-----------------------------------------------------------------------------
409# Execution starts here
410#-----------------------------------------------------------------------------
412DEFAULT_BENCHMARKS="$all_grp"
413DEFAULT_FIELDS="allocated bytescopied cputime"
414ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
415FIELDS=$DEFAULT_FIELDS
417COMPARE=0
418BASE=
419CANDIDATE=
421APPEND=0
422SLOW=0
423QUICK_MODE=0
424LONG=0
425RAW=0
426GRAPH=0
427MEASURE=1
429GAUGE_ARGS=
430RTS_OPTIONS=
431BUILD_ONCE=0
432USE_STACK=0
433CABAL_BUILD_FLAGS=""
435GHC_VERSION=$(ghc --numeric-version)
437cabal_which() {
438 find dist-newstyle -type f -path "*${GHC_VERSION}/streamly-benchmarks*/$1"
441#-----------------------------------------------------------------------------
442# Read command line
443#-----------------------------------------------------------------------------
445while test -n "$1"
446do
447 case $1 in
448 -h|--help|help) print_help ;;
449 # options with arguments
450 --benchmarks) shift; BENCHMARKS=$1; shift ;;
451 --fields) shift; FIELDS=$1; shift ;;
452 --base) shift; BASE=$1; shift ;;
453 --candidate) shift; CANDIDATE=$1; shift ;;
454 --cabal-build-flags) shift; CABAL_BUILD_FLAGS=$1; shift ;;
455 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
456 # flags
457 --slow) SLOW=1; shift ;;
458 --quick) QUICK_MODE=1; shift ;;
459 --compare) COMPARE=1; shift ;;
460 --raw) RAW=1; shift ;;
461 --append) APPEND=1; shift ;;
462 --long) LONG=1; shift ;;
463 --graphs) GRAPH=1; shift ;;
464 --no-measure) MEASURE=0; shift ;;
465 --) shift; break ;;
466 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
467 *) break ;;
468 esac
469done
470GAUGE_ARGS=$*
472only_real_benchmarks () {
473 for i in $BENCHMARKS
474 do
475 local SKIP=0
476 for j in $COMPARISONS
477 do
478 if test $i == $j
479 then
480 SKIP=1
481 fi
482 done
483 if test "$SKIP" -eq 0
484 then
485 echo -n "$i "
486 fi
487 done
490has_item () {
491 for i in $1
492 do
493 if test "$i" = "$2"
494 then
495 echo "$i"
496 break
497 fi
498 done
501BENCHMARKS_ORIG=$BENCHMARKS
502if test "$(has_item "$BENCHMARKS_ORIG" help)" = "help"
503then
504 list_bench_groups
505 list_comparisons
506 list_benches
507 exit
508fi
510if test "$(has_item "$FIELDS" help)" = "help"
511then
512 echo "Supported fields: $ALL_FIELDS"
513 echo "Default fields: $DEFAULT_FIELDS"
514 exit
515fi
517if test "$LONG" -ne 0
518then
519 if test -n "$BENCHMARKS"
520 then
521 echo "Cannot specify benchmarks [$BENCHMARKS] with --long"
522 exit
523 fi
524 BENCHMARKS=$infinite_grp
525fi
527BENCHMARKS=$(set_benchmarks)
528BENCHMARKS_ORIG=$BENCHMARKS
529BENCHMARKS=$(only_real_benchmarks)
530EXECUTABLES=$BENCHMARKS
532echo "Using benchmark suites [$BENCHMARKS]"
534if test "$USE_STACK" = "1"
535then
536 WHICH_COMMAND="stack exec which"
537 BUILD_CHART_EXE="stack build --flag streamly:dev"
538 GET_BENCH_PROG=stack_bench_prog
539 BUILD_BENCH="stack build $STACK_BUILD_FLAGS --bench --no-run-benchmarks"
540else
541 # XXX cabal issue "cabal v2-exec which" cannot find benchmark/test executables
542 #WHICH_COMMAND="cabal v2-exec which"
543 WHICH_COMMAND=cabal_which
544 BUILD_CHART_EXE="cabal v2-build --flags dev chart"
545 GET_BENCH_PROG=cabal_bench_prog
546 BUILD_BENCH="cabal v2-build $CABAL_BUILD_FLAGS --enable-benchmarks"
547fi
549#-----------------------------------------------------------------------------
550# Build stuff
551#-----------------------------------------------------------------------------
553# We need to build the report progs first at the current (latest) commit before
554# checking out any other commit for benchmarking.
555build_report_progs "$BENCHMARKS"
557#-----------------------------------------------------------------------------
558# Run benchmarks
559#-----------------------------------------------------------------------------
561if test "$MEASURE" = "1"
562then
563 echo $BUILD_BENCH
564 if test "$USE_STACK" = "1"
565 then
566 $BUILD_BENCH || die "build failed"
567 else
568 $BUILD_BENCH $EXECUTABLES || die "build failed"
569 fi
570 run_measurements "$BENCHMARKS"
571fi
573#-----------------------------------------------------------------------------
574# Run reports
575#-----------------------------------------------------------------------------
577COMPARISON_REPORTS=""
578for i in $COMPARISONS
579do
580 if test "$(has_item "$BENCHMARKS_ORIG" $i)" = $i
581 then
582 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
583 mkdir -p "charts/$i"
584 constituents=$(eval "echo -n \$${i}")
585 dest_file="charts/$i/results.csv"
586 : > $dest_file
587 for j in $constituents
588 do
589 cat "charts/$j/results.csv" >> $dest_file
590 done
591 fi
592done
594if test "$RAW" = "0"
595then
596 run_reports "$BENCHMARKS"
597 run_reports "$COMPARISON_REPORTS"
598fi