@@ -1,134 +1,9 @@
11#!/usr/bin/env bash
22
3−#------------------------------------------------------------------------------
4−# Benchmark groups
5−#------------------------------------------------------------------------------
6−
7−# IMPORTANT NOTE: the names "_grp" and "_cmp" suffixes are special, do
8−# not rename them to something else.
9−
10−base_stream_grp="\
11− Data.Stream.StreamD \
12− Data.Stream.StreamK \
13− Data.Stream.StreamDK"
14−
15−prelude_serial_grp="\
16− Prelude.Serial \
17− Prelude.WSerial \
18− Prelude.ZipSerial"
19−
20−prelude_concurrent_grp="\
21− Prelude.Async \
22− Prelude.WAsync \
23− Prelude.Ahead \
24− Prelude.Parallel \
25− Prelude.ZipAsync"
26−
27−prelude_other_grp="\
28− Prelude.Rate \
29− Prelude.Concurrent \
30− Prelude.Adaptive"
31−
32−array_grp="\
33− Data.Array \
34− Data.Array.Storable.Foreign \
35− Data.Array.Prim \
36− Data.SmallArray \
37− Data.Array.Prim.Pinned"
38−
39−base_parser_grp="Data.Parser.ParserD Data.Parser.ParserK"
40−parser_grp="Data.Fold Data.Parser"
41−
42−#------------------------------------------------------------------------------
43−# Streaming vs non-streaming
44−#------------------------------------------------------------------------------
45−# The "o-1-space" groups of these benchmarks are run with long stream
46−# sizes when --long option is used.
47−
48−infinite_grp="\
49− $prelude_serial_grp \
50− $prelude_concurrent_grp \
51− Prelude.Rate"
52−
53−#------------------------------------------------------------------------------
54−# Comparison groups
55−#------------------------------------------------------------------------------
56−# *_cmp denotes a comparison benchmarks, the benchmarks provided in *_cmp
57−# variables are compared with each other
58−base_stream_cmp="Data.Stream.StreamD Data.Stream.StreamK"
59−serial_wserial_cmp="Prelude.Serial Prelude.WSerial"
60−serial_async_cmp="Prelude.Serial Prelude.Async"
61−concurrent_cmp="Prelude.Async Prelude.WAsync Prelude.Ahead Prelude.Parallel"
62−array_cmp="Memory.Array Data.Array.Prim Data.Array Data.Array.Prim.Pinned"
63−pinned_array_cmp="Memory.Array Data.Array.Prim.Pinned"
64−base_parser_cmp=$base_parser_grp
65−COMPARISONS="\
66− base_stream_cmp \
67− serial_wserial_cmp \
68− serial_async_cmp \
69− concurrent_cmp \
70− array_cmp \
71− pinned_array_cmp \
72− base_parser_cmp"
73−
74−#------------------------------------------------------------------------------
75−# All
76−#------------------------------------------------------------------------------
77−# All high level benchmarks
78−all_grp="\
79− $prelude_serial_grp \
80− $prelude_concurrent_grp \
81− $array_grp \
82− $parser_grp \
83− Unicode.Stream \
84− Data.Unfold \
85− FileSystem.Handle"
86−
87−ALL_BENCH_GROUPS="\
88− all_grp \
89− prelude_serial_grp \
90− prelude_concurrent_grp \
91− array_grp \
92− infinite_grp \
93− base_stream_grp \
94− base_parser_grp"
95−
96−#------------------------------------------------------------------------------
97−# Script
98−#------------------------------------------------------------------------------
99−
100−BENCH_SH_DIR=$(cd `dirname $0`; pwd)
101−
102−list_benches () {
103− echo "Individual benchmarks:"
104− for i in $all_grp
105− do
106− echo "$i"
107− done
108− echo
109−}
3+SCRIPT_DIR=$(cd `dirname $0`; pwd)
1104
111−list_bench_groups () {
112− echo "Benchmark groups:"
113− for i in $ALL_BENCH_GROUPS
114− do
115− echo -n "$i ["
116− eval "echo -n \$$i"
117− echo "]"
118− done
119− echo
120−}
121−
122−list_comparisons () {
123− echo "Comparison groups:"
124− for i in $COMPARISONS
125− do
126− echo -n "$i ["
127− eval "echo -n \$$i"
128− echo "]"
129− done
130− echo
131−}
5+RUNNING_BENCHMARKS=y
6+source $SCRIPT_DIR/build-lib.sh
1327
1338print_help () {
1349 echo "Usage: $0 "
@@ -141,7 +16,9 @@ print_help () {
14116 echo " [--slow]"
14217 echo " [--quick]"
14318 echo " [--raw]"
19+ echo " [--dev-build]"
14420 echo " [--cabal-build-flags <flag>]"
21+ echo " [--rtsopts <opts>]"
14522 echo " [--compare] [--base <commit>] [--candidate <commit>]"
14623 echo " -- <gauge options or benchmarks>"
14724 echo
@@ -156,6 +33,7 @@ print_help () {
15633 echo "--raw: Run the benchmarks but don't report them. This is useful when"
15734 echo " you only want to work with the csv files generated."
15835 echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"
36+ echo " e.g. --cabal-build-flags \"--flag fusion-plugin\""
15937 echo
16038 echo "When specific space complexity group is chosen then (and only then) "
16139 echo "RTS memory restrictions are used accordingly. For example, "
@@ -170,33 +48,37 @@ print_help () {
17048 exit
17149}
17250
173−# $1: message
174−die () {
175− >&2 echo -e "Error: $1"
176− exit 1
51+#-----------------------------------------------------------------------------
52+# Reporting utility functions
53+#-----------------------------------------------------------------------------
54+
55+list_comparisons () {
56+ echo "Comparison groups:"
57+ for i in $COMPARISONS
58+ do
59+ echo -n "$i ["
60+ eval "echo -n \$$i"
61+ echo "]"
62+ done
63+ echo
17764}
17865
179−set_benchmarks() {
180− if test -z "$BENCHMARKS"
66+# chart is expensive to build and usually not required to be rebuilt,
67+# use master branch as fallback
68+cabal_which_report() {
69+ local path=$(cabal_which streamly-benchmarks $1)
70+ if test -z "$path"
18171 then
182− echo $DEFAULT_BENCHMARKS
72+ cabal_which_builddir dist-newstyle streamly-benchmarks $1
18373 else
184− for i in $(echo $BENCHMARKS)
185− do
186− case $i in
187− *_grp) eval "echo -n \$${i}" ;;
188− *_cmp) eval "echo -n \$${i} $i" ;;
189− *) echo -n $i ;;
190− esac
191− echo -n " "
192− done
74+ echo $path
19375 fi
19476}
19577
19678find_report_prog() {
19779 local prog_name="chart"
19880 hash -r
199− local prog_path=$($WHICH_COMMAND_REPORT $prog_name)
81+ local prog_path=$(cabal_which_report $prog_name)
20082 if test -x "$prog_path"
20183 then
20284 echo $prog_path
@@ -207,14 +89,16 @@ find_report_prog() {
20789
20890build_report_prog() {
20991 local prog_name="chart"
210− local prog_path=$($WHICH_COMMAND_REPORT $prog_name)
92+ local prog_path=$(cabal_which_report $prog_name)
21193
21294 hash -r
21395 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
21496 then
215− echo "Building bench-graph executables"
97+ echo "Building bench-show executables"
21698 BUILD_ONCE=1
217− $BUILD_CHART_EXE || die "build failed"
99+ $CABAL_EXECUTABLE v2-build --flags dev chart \
100+ || die "bench-show build failed"
101+
218102 elif test ! -x "$prog_path"
219103 then
220104 return 1
@@ -228,8 +112,8 @@ build_report_progs() {
228112 build_report_prog || exit 1
229113 local prog
230114 prog=$(find_report_prog) || \
231− die "Cannot find bench-graph executable"
232− echo "Using bench-graph executable [$prog]"
115+ die "Cannot find bench-show executable"
116+ echo "Using bench-show executable [$prog]"
233117 fi
234118}
235119
@@ -239,42 +123,12 @@ build_report_progs() {
239123# own path currently.
240124
241125# The path is dependent on the architecture and cabal version.
242−# Use this command to find the exe if this script fails with an error:
243−# find .stack-work/ -type f -name "benchmarks"
244−
245−stack_bench_prog () {
246− local bench_name=$1
247− local bench_prog=`stack path --dist-dir`/build/$bench_name/$bench_name
248− if test -x "$bench_prog"
249− then
250− echo $bench_prog
251− else
252− return 1
253− fi
254−}
255−
256−cabal_bench_prog () {
257− local bench_name=$1
258− local bench_prog=`$WHICH_COMMAND $1`
259− if test -x "$bench_prog"
260− then
261− echo $bench_prog
262− else
263− return 1
264− fi
265−}
266126
267127bench_output_file() {
268128 local bench_name=$1
269129 echo "charts/$bench_name/results.csv"
270130}
271131
272−# $1: command
273−function run_verbose() {
274− echo "$*"
275− bash -c "$*"
276−}
277−
278132# --min-duration 0 means exactly one iteration per sample. We use a million
279133# iterations in the benchmarking code explicitly and do not use the iterations
280134# done by the benchmarking tool.
@@ -288,25 +142,21 @@ function run_verbose() {
288142# We can pass --min-samples value from the command line as second argument
289143# after the benchmark name in case we want to use more than one sample.
290144
291−run_bench () {
145+# $1: bench name
146+# $2: bench executable
147+target_exe_extra_args () {
292148 local bench_name=$1
293− local bench_exe=$bench_name
294− local output_file=$(bench_output_file $bench_name)
295− local bench_prog
296− bench_prog=$($GET_BENCH_PROG $bench_exe) || \
297− die "Cannot find benchmark executable for benchmark $bench_name"
149+ local bench_prog=$2
298150
151+ local output_file=$(bench_output_file $bench_name)
299152 mkdir -p `dirname $output_file`
300153
301− echo "Running benchmark $bench_name ..."
302−
303154 local QUICK_OPTS="--quick --min-duration 0"
304155 local SPEED_OPTIONS
305156 if test "$LONG" -eq 0
306157 then
307158 if test "$SLOW" -eq 0
308159 then
309− export QUICK_MODE
310160 if test "$QUICK_MODE" -eq 0
311161 then
312162 # default mode, not super quick, not slow
@@ -325,20 +175,11 @@ run_bench () {
325175 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"
326176 fi
327177
328− export BENCH_EXEC_PATH=$bench_prog
329− export RTS_OPTIONS
330− run_verbose $bench_prog $SPEED_OPTIONS \
178+ echo "$SPEED_OPTIONS \
331179 --csvraw=$output_file \
332180 -v 2 \
333− --measure-with "$BENCH_SH_DIR/bench-exec-one.sh" \
334− $GAUGE_ARGS || die "Benchmarking failed"
335−}
336−
337−run_benches() {
338− for i in $1
339− do
340− run_bench $i
341− done
181+ --measure-with "$SCRIPT_DIR/bench-exec-one.sh" \
182+ $GAUGE_ARGS"
342183}
343184
344185run_benches_comparing() {
@@ -358,14 +199,14 @@ run_benches_comparing() {
358199 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
359200
360201 $BUILD_BENCH || die "build failed"
361− run_benches "$bench_list"
202+ run_targets streamly-benchmarks "$bench_list" target_exe_extra_args
362203
363204 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
364205 git checkout "$CANDIDATE" || \
365206 die "Checkout of candidate [$CANDIDATE] commit failed"
366207
367208 $BUILD_BENCH || die "build failed"
368− run_benches "$bench_list"
209+ run_targets streamly-benchmarks "$bench_list" target_exe_extra_args
369210 # XXX reset back to the original commit
370211}
371212
@@ -389,7 +230,7 @@ run_measurements() {
389230
390231 if test "$COMPARE" = "0"
391232 then
392− run_benches "$bench_list"
233+ run_targets streamly-benchmarks "$bench_list" target_exe_extra_args
393234 else
394235 run_benches_comparing "$bench_list"
395236 fi
@@ -413,9 +254,10 @@ run_reports() {
413254# Execution starts here
414255#-----------------------------------------------------------------------------
415256
416−cd $BENCH_SH_DIR/..
257+cd $SCRIPT_DIR/..
258+
259+set_common_vars
417260
418−DEFAULT_BENCHMARKS="$all_grp"
419261DEFAULT_FIELDS="allocated bytescopied cputime"
420262ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
421263FIELDS=$DEFAULT_FIELDS
@@ -425,53 +267,13 @@ BASE=
425267CANDIDATE=
426268
427269APPEND=0
428−SLOW=0
429−QUICK_MODE=0
430270LONG=0
431271RAW=0
432272GRAPH=0
433273MEASURE=1
434274
435275GAUGE_ARGS=
436−RTS_OPTIONS=
437276BUILD_ONCE=0
438−USE_STACK=0
439−CABAL_BUILD_FLAGS=""
440−
441−GHC_VERSION=$(ghc --numeric-version)
442−
443−CABAL_EXECUTABLE=cabal
444−
445−# Use branch specific builds if git-cabal is present in PATH
446−BUILD_DIR=dist-newstyle
447−if which git-cabal 2>/dev/null
448−then
449− echo "Using git-cabal for branch specific builds"
450− CABAL_EXECUTABLE=git-cabal
451− BUILD_DIR=$(git-cabal show-builddir)
452−fi
453−
454−# $1: builddir
455−# $2: command to find
456−cabal_which_builddir() {
457− find $1 -type f -path "*${GHC_VERSION}/streamly-benchmarks*/$2" 2>/dev/null
458−}
459−
460−cabal_which() {
461− cabal_which_builddir $BUILD_DIR $1
462−}
463−
464−# chart is expensive to build and usually not required to be rebuilt,
465−# use master branch as fallback
466−cabal_which_report() {
467− local path=$(cabal_which $1)
468− if test -z "$path"
469− then
470− cabal_which_builddir dist-newstyle $1
471− else
472− echo $path
473− fi
474−}
475277
476278#-----------------------------------------------------------------------------
477279# Read command line
@@ -482,11 +284,13 @@ do
482284 case $1 in
483285 -h|--help|help) print_help ;;
484286 # options with arguments
485− --benchmarks) shift; BENCHMARKS=$1; shift ;;
287+ --benchmarks) shift; TARGETS=$1; shift ;;
288+ --targets) shift; TARGETS=$1; shift ;;
486289 --fields) shift; FIELDS=$1; shift ;;
487290 --base) shift; BASE=$1; shift ;;
488291 --candidate) shift; CANDIDATE=$1; shift ;;
489− --cabal-build-flags) shift; CABAL_BUILD_FLAGS=$1; shift ;;
292+ --cabal-build-flags) shift; CABAL_BUILD_OPTIONS=$1; shift ;;
293+ --cabal-build-options) shift; CABAL_BUILD_OPTIONS=$1; shift ;;
490294 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
491295 # flags
492296 --slow) SLOW=1; shift ;;
@@ -497,6 +301,7 @@ do
497301 --long) LONG=1; shift ;;
498302 --graphs) GRAPH=1; shift ;;
499303 --no-measure) MEASURE=0; shift ;;
304+ --dev-build) RUNNING_DEVBUILD=1; shift ;;
500305 --) shift; break ;;
501306 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
502307 *) break ;;
@@ -504,8 +309,12 @@ do
504309done
505310GAUGE_ARGS=$*
506311
312+#-----------------------------------------------------------------------------
313+# Determine targets
314+#-----------------------------------------------------------------------------
315+
507316only_real_benchmarks () {
508− for i in $BENCHMARKS
317+ for i in $TARGETS
509318 do
510319 local SKIP=0
511320 for j in $COMPARISONS
@@ -522,23 +331,14 @@ only_real_benchmarks () {
522331 done
523332}
524333
525−has_item () {
526− for i in $1
527− do
528− if test "$i" = "$2"
529− then
530− echo "$i"
531− break
532− fi
533− done
534−}
334+# Requires RUNNING_DEVBUILD var
335+source $SCRIPT_DIR/targets.sh
535336
536−BENCHMARKS_ORIG=$BENCHMARKS
537−if test "$(has_item "$BENCHMARKS_ORIG" help)" = "help"
337+if test "$(has_item "$TARGETS" help)" = "help"
538338then
539− list_bench_groups
339+ list_target_groups
540340 list_comparisons
541− list_benches
341+ list_targets
542342 exit
543343fi
544344
@@ -551,40 +351,24 @@ fi
551351
552352if test "$LONG" -ne 0
553353then
554− if test -n "$BENCHMARKS"
354+ if test -n "$TARGETS"
555355 then
556− echo "Cannot specify benchmarks [$BENCHMARKS] with --long"
356+ echo "Cannot specify benchmarks [$TARGETS] with --long"
557357 exit
558358 fi
559− BENCHMARKS=$infinite_grp
359+ TARGETS=$infinite_grp
560360fi
561361
562−BENCHMARKS=$(set_benchmarks)
563−BENCHMARKS_ORIG=$BENCHMARKS
564−BENCHMARKS=$(only_real_benchmarks)
565−EXECUTABLES=$BENCHMARKS
362+DEFAULT_TARGETS="$(all_grp)"
363+TARGETS=$(set_targets)
566364
567−echo "Using benchmark suites [$BENCHMARKS]"
365+TARGETS_ORIG=$TARGETS
366+TARGETS=$(only_real_benchmarks)
568367
569−if test "$USE_STACK" = "1"
570−then
571− WHICH_COMMAND="stack exec which"
572− WHICH_COMMAND_REPORT="stack exec which"
573− BUILD_CHART_EXE="stack build --flag streamly:dev"
574− GET_BENCH_PROG=stack_bench_prog
575− BUILD_BENCH="stack build $STACK_BUILD_FLAGS --bench --no-run-benchmarks"
576−else
577− # XXX cabal issue "cabal v2-exec which" cannot find benchmark/test executables
578− #WHICH_COMMAND="cabal v2-exec which"
579− WHICH_COMMAND=cabal_which
580− WHICH_COMMAND_REPORT=cabal_which_report
581− BUILD_CHART_EXE="$CABAL_EXECUTABLE v2-build --flags dev chart"
582− GET_BENCH_PROG=cabal_bench_prog
583− BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_FLAGS --enable-benchmarks"
584−fi
368+echo "Using benchmark suites [$TARGETS]"
585369
586370#-----------------------------------------------------------------------------
587−# Build stuff
371+# Build reporting utility
588372#-----------------------------------------------------------------------------
589373
590374# We need to build the report progs first at the current (latest) commit before
@@ -592,23 +376,16 @@ fi
592376build_report_progs
593377
594378#-----------------------------------------------------------------------------
595−# Run benchmarks
379+# Build and run targets
596380#-----------------------------------------------------------------------------
597381
382+BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-benchmarks"
598383if test "$MEASURE" = "1"
599384then
600− echo $BUILD_BENCH
601− if test "$USE_STACK" = "1"
602− then
603− $BUILD_BENCH || die "build failed"
604− else
605− for c in $EXECUTABLES
606− do
607− COMPONENTS+="streamly-benchmarks:bench:$c "
608− done
609− $BUILD_BENCH $COMPONENTS || die "build failed"
610− fi
611− run_measurements "$BENCHMARKS"
385+ run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
386+ export QUICK_MODE
387+ export RTS_OPTIONS
388+ run_measurements "$TARGETS"
612389fi
613390
614391#-----------------------------------------------------------------------------
@@ -618,7 +395,7 @@ fi
618395COMPARISON_REPORTS=""
619396for i in $COMPARISONS
620397do
621− if test "$(has_item "$BENCHMARKS_ORIG" $i)" = $i
398+ if test "$(has_item "$TARGETS_ORIG" $i)" = $i
622399 then
623400 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
624401 mkdir -p "charts/$i"
@@ -634,6 +411,6 @@ done
634411
635412if test "$RAW" = "0"
636413then
637− run_reports "$BENCHMARKS"
414+ run_reports "$TARGETS"
638415 run_reports "$COMPARISON_REPORTS"
639416fi