git.iko.soy/streamly
on master help sign in
Create a bench-runner wrapper that uses bench-config

Consolidate the specific customizations to bench-config.sh. Integrate bench-exec-one.sh into the main bench.sh script. After the gauge/tasty unification changes, we do not have a requirement for a separate executable now. Now we can push the generic part of the script to the benchmark/bench-report package. And that package can be reused by streaming-benchmarks as well. We can make that as a submodule. Ultimately we should be converting these shell scripts to Haskell.

commit1a17e2db4b92e1faad8df8ecf66c8033bf493c9d authorHarendra Kumar <harendra@composewell.com> dateSun 4 Jul 2021, 02:36 (5 years ago) parent2ab247b
7 files changed+796−781
compare with master
unifiedsplit
bin/bench-exec-options.sh → bin/bench-config.sh+124 bin/bench-exec-one.sh−150 bin/bench-runner.sh+665 bin/bench.sh+1−517 bin/build-lib.sh+4 bin/targets.sh−112 streamly.cabal+2−2
bin/bench-exec-options.sh → bin/bench-config.shrenamed+124
@@ -1,5 +1,129 @@
11#!/usr/bin/env bash
22
3+# Customization options
4+bench_config () {
5+ BENCHMARK_DIR=benchmark
6+ BENCHMARK_PACKAGE_NAME=streamly-benchmarks
7+ BENCHMARK_PACKAGE_VERSION=0.0.0
8+
9+ USE_GAUGE=0
10+ DEFAULT_FIELDS="allocated cputime bytescopied"
11+}
12+
13+#------------------------------------------------------------------------------
14+# test and benchmark groups
15+#------------------------------------------------------------------------------
16+
17+# IMPORTANT NOTE: the names "_grp" and "_cmp" suffixes are special, do
18+# not rename them to something else.
19+
20+bench_targets () {
21+ base_stream_grp="\
22+ `bench_only Data.Stream.StreamD` \
23+ `bench_only Data.Stream.StreamK` \
24+ `bench_only Data.Stream.StreamDK`"
25+
26+ prelude_serial_grp="\
27+ Prelude.Serial \
28+ Prelude.WSerial \
29+ Prelude.ZipSerial"
30+
31+ prelude_concurrent_grp="\
32+ Prelude.Async \
33+ Prelude.WAsync \
34+ Prelude.Ahead \
35+ Prelude.Parallel \
36+ Prelude.ZipAsync"
37+
38+ prelude_other_grp="\
39+ `test_only Prelude` \
40+ $(test_only $(dev_build Prelude.Rate)) \
41+ `bench_only Prelude.Rate` \
42+ `test_only Prelude.Fold` \
43+ `test_only Prelude.Concurrent` \
44+ $(bench_only $(dev_build Prelude.Concurrent)) \
45+ `bench_only Prelude.Adaptive`"
46+
47+ array_grp="\
48+ Data.Array \
49+ Data.Array.Foreign \
50+ Data.Array.Prim \
51+ Data.SmallArray \
52+ Data.Array.Prim.Pinned"
53+
54+ base_parser_grp="\
55+ Data.Parser.ParserD \
56+ `bench_only Data.Parser.ParserK`"
57+
58+ parser_grp="\
59+ Data.Fold \
60+ Data.Parser"
61+
62+ list_grp="\
63+ `test_only Data.List.Base` \
64+ `test_only Data.List`"
65+
66+ #------------------------------------------------------------------------------
67+ # Streaming vs non-streaming
68+ #------------------------------------------------------------------------------
69+ # The "o-1-space" groups of these benchmarks are run with long stream
70+ # sizes when --long option is used.
71+
72+ infinite_grp="\
73+ $prelude_serial_grp \
74+ $prelude_concurrent_grp \
75+ `bench_only Prelude.Rate`"
76+
77+ #------------------------------------------------------------------------------
78+ # Benchmark comparison groups
79+ #------------------------------------------------------------------------------
80+
81+ # *_cmp denotes a comparison benchmark, the benchmarks provided in *_cmp
82+ # variables are compared with each other
83+ base_stream_cmp="Data.Stream.StreamD Data.Stream.StreamK"
84+ serial_wserial_cmp="Prelude.Serial Prelude.WSerial"
85+ serial_async_cmp="Prelude.Serial Prelude.Async"
86+ concurrent_cmp="Prelude.Async Prelude.WAsync Prelude.Ahead Prelude.Parallel"
87+ array_cmp="Memory.Array Data.Array.Prim Data.Array Data.Array.Prim.Pinned"
88+ pinned_array_cmp="Memory.Array Data.Array.Prim.Pinned"
89+ base_parser_cmp=$base_parser_grp
90+
91+ COMPARISONS="\
92+ base_stream_cmp \
93+ serial_wserial_cmp \
94+ serial_async_cmp \
95+ concurrent_cmp \
96+ array_cmp \
97+ pinned_array_cmp \
98+ base_parser_cmp"
99+
100+ #------------------------------------------------------------------------------
101+ # All test/benchmark modules must be in at least one of these
102+ #------------------------------------------------------------------------------
103+
104+ # All groups
105+ GROUP_TARGETS="\
106+ base_stream_grp \
107+ prelude_serial_grp \
108+ prelude_concurrent_grp \
109+ prelude_other_grp \
110+ array_grp \
111+ base_parser_grp \
112+ parser_grp \
113+ list_grp \
114+ infinite_grp"
115+
116+ # Not in any groups
117+ INDIVIDUAL_TARGETS="\
118+ Data.Unfold \
119+ Unicode.Stream \
120+ FileSystem.Handle \
121+ `test_only FileSystem.Event` \
122+ `test_only Network.Socket` \
123+ `test_only Network.Inet.TCP` \
124+ `test_only version-bounds`"
125+}
126+
3127#------------------------------------------------------------------------------
4128# RTS options based on the benchmark executable name and benchmark name
5129#------------------------------------------------------------------------------
bin/bench-exec-one.shdeleted−150
@@ -1,150 +0,0 @@
1−#!/usr/bin/env bash
2−
3−# Environment passed:
4−# BENCH_EXEC_PATH: the benchmark executable
5−# RTS_OPTIONS: additional RTS options
6−# QUICK_MODE: whether we are in quick mode
7−# USE_GAUGE: whether to use gauge or tasty-bench
8−# LONG: whether to use a large stream size
9−
10−set -e
11−set -o pipefail
12−
13−SCRIPT_DIR=$(cd `dirname $0`; pwd)
14−source $SCRIPT_DIR/bench-exec-options.sh
15−
16−# $1: message
17−die () {
18− >&2 echo -e "Error: $1"
19− exit 1
20−}
21−
22−warn () {
23− >&2 echo -e "Warning: $1"
24−}
25−
26−test -n "$BENCH_EXEC_PATH" || die "BENCH_EXEC_PATH env var must be set"
27−test -n "$QUICK_MODE" || warn "QUICK_MODE env var not set (to 0 or 1)"
28−
29−#------------------------------------------------------------------------------
30−# Speed options
31−#------------------------------------------------------------------------------
32−
33−if test "$USE_GAUGE" -eq 0
34−then
35− SUPER_QUICK_OPTIONS="--stdev 1000000"
36− QUICKER_OPTIONS="--stdev 100"
37−else
38− # Do not keep time limit as 0 otherwise GC stats may remain 0 in some cases.
39− SUPER_QUICK_OPTIONS="--quick --min-duration 0 --time-limit 0.01 --include-first-iter"
40− QUICKER_OPTIONS="--min-samples 3 --time-limit 1"
41−fi
42−
43−# tasty-bench does not like an option set twice
44−set_super_quick_mode () {
45− echo -n super_quick
46−}
47−
48−# For certain long benchmarks if the user has not requested super quick
49−# mode we anyway use a slightly quicker mode.
50−use_quicker_mode () {
51− if test "$QUICK_MODE" -eq 0
52− then
53− echo quicker
54− fi
55−}
56−
57−bench_output_file() {
58− local bench_name=$1
59− echo "charts/$bench_name/results.csv"
60−}
61−
62−#------------------------------------------------------------------------------
63−# Determine options from benchmark name
64−#------------------------------------------------------------------------------
65−
66−BENCH_NAME_ORIG="$1"
67−shift
68−
69−if test "$USE_GAUGE" -eq 0
70−then
71− # XXX this is a hack to make the "/" separated names used in the functions
72− # determining options based on benchmark name. For tasty-bench the benchmark
73− # names are separated by "." instead of "/".
74− BENCH_NAME0=$(echo $BENCH_NAME_ORIG | sed -e s/^All\.//)
75− BENCH_NAME1=$(echo $BENCH_NAME0 | cut -f1 -d '/')
76− BENCH_NAME2=$(echo $BENCH_NAME0 | cut -f2- -d '/' | sed -e 's/\./\//g')
77− BENCH_NAME="$BENCH_NAME1/$BENCH_NAME2"
78−else
79− BENCH_NAME=$BENCH_NAME_ORIG
80−fi
81−
82−RTS_OPTIONS=\
83−"+RTS -T \
84−$(bench_rts_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME) \
85−$RTS_OPTIONS \
86−-RTS"
87−
88−QUICK_MODE_TYPE="\
89−$(if test "$QUICK_MODE" -ne 0; then set_super_quick_mode; fi) \
90−$(bench_speed_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME)"
91−
92−for i in $QUICK_MODE_TYPE
93−do
94− case "$i" in
95− super_quick) QUICK_BENCH_OPTIONS="$SUPER_QUICK_OPTIONS"; break ;;
96− quicker) QUICK_BENCH_OPTIONS="$QUICKER_OPTIONS"; break ;;
97− esac
98−done
99−
100−if test "$LONG" -ne 0
101−then
102− STREAM_SIZE=10000000
103− STREAM_LEN=$(env LC_ALL=en_US.UTF-8 printf "--stream-size %'.f\n" $STREAM_SIZE)
104− STREAM_SIZE_OPT="--stream-size $STREAM_SIZE"
105−fi
106−
107−echo "$BENCH_NAME_ORIG: \
108−$RTS_OPTIONS \
109−$STREAM_LEN \
110−$QUICK_BENCH_OPTIONS" \
111−"$@"
112−
113−#------------------------------------------------------------------------------
114−# Run benchmark with options and collect results
115−#------------------------------------------------------------------------------
116−
117−output_file=$(bench_output_file $(basename $BENCH_EXEC_PATH))
118−mkdir -p `dirname $output_file`
119−rm -f ${output_file}.tmp
120−
121−if test $USE_GAUGE -eq 0
122−then
123− # Escape "\" and double quotes in benchmark names
124− BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')
125− $BENCH_EXEC_PATH \
126− -j 1 \
127− $RTS_OPTIONS \
128− $STREAM_SIZE_OPT \
129− $QUICK_BENCH_OPTIONS \
130− "$@" \
131− --csv=${output_file}.tmp \
132− -p '$0 == "'"$BENCH_NAME_ESC"'"'
133−
134− # Convert cpuTime field from picoseconds to seconds
135− awk --version 2>&1 | grep -q "GNU Awk" \
136− || die "Need GNU awk. [$(which awk)] is not GNU awk."
137− tail -n +2 ${output_file}.tmp | \
138− awk 'BEGIN {FPAT = "([^,]+)|(\"[^\"]+\")";OFS=","} {$2=$2/1000000000000;print}' \
139− >> $output_file
140−else
141− $BENCH_EXEC_PATH \
142− $RTS_OPTIONS \
143− $STREAM_SIZE_OPT \
144− $QUICK_BENCH_OPTIONS \
145− "$@" \
146− --csvraw=${output_file}.tmp \
147− -m exact "$BENCH_NAME"
148− tail -n +2 ${output_file}.tmp \
149− >> $output_file
150−fi
bin/bench-runner.shnew file+665
@@ -0,0 +1,665 @@
1+#!/usr/bin/env bash
2+
3+# Note that this script is used in the "streamly" package as well as
4+# in "streaming-benchmarks" package. Any changes to the script should be
5+# generic enough so that it works in both the cases.
6+
7+#------------------------------------------------------------------------------
8+
9+set -o pipefail
10+
11+SCRIPT_DIR=$(cd `dirname $0`; pwd)
12+
13+RUNNING_BENCHMARKS=y
14+source $SCRIPT_DIR/build-lib.sh
15+
16+print_help () {
17+ echo "Usage: $0 "
18+ echo " [--benchmarks <"bench1 bench2 ..." | help>]"
19+ echo " [--prefix <benchmark name prefix to match>"
20+ echo " [--fields <"field1 field2 ..." | help>]"
21+ echo " [--sort-by-name]"
22+ echo " [--compare]"
23+ echo " [--graphs]"
24+ echo " [--no-measure]"
25+ echo " [--append]"
26+ echo " [--long]"
27+ echo " [--slow]"
28+ echo " [--quick]"
29+ echo " [--raw]"
30+ echo " [--dev-build]"
31+ echo " [--use-nix]"
32+ echo " [--with-compiler <compiler exe name>]"
33+ echo " [--cabal-build-options <options>]"
34+ echo " [--rtsopts <opts>]"
35+ echo " [--commit-compare] [--base <commit>] [--candidate <commit>]"
36+ #echo " -- <gauge options or benchmarks>"
37+ echo
38+ echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
39+ echo "--compare: compare the specified benchmarks with each other"
40+ echo "--fields: measurement fields to report, use 'help' for a list"
41+ echo "--graphs: Generate graphical reports"
42+ echo "--no-measure: Don't run benchmarks, run reports from previous results"
43+ echo "--append: Don't overwrite previous results, append for comparison"
44+ echo "--long: Use much longer stream size for infinite stream benchmarks"
45+ echo "--slow: Slightly more accurate results at the expense of speed"
46+ echo "--quick: Faster results, useful for longer benchmarks"
47+ echo "--raw: Run the benchmarks but don't report them. This is useful when"
48+ echo " you only want to work with the csv files generated."
49+ echo "--cabal-build-options: Pass any cabal build options to be used for build"
50+ echo " e.g. --cabal-build-options \"--flag dev\""
51+ echo
52+ echo "When specific space complexity group is chosen then (and only then) "
53+ echo "RTS memory restrictions are used accordingly. For example, "
54+ echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
55+ echo "restricts Heap/Stack space for O(1) characterstics"
56+ echo
57+ echo "When using --commit-compare, by default comparative chart of HEAD^ vs HEAD"
58+ echo "commit is generated, in the 'charts' directory."
59+ echo "Use --base and --candidate to select the commits to compare."
60+ echo
61+ #echo "Any arguments after a '--' are passed directly to gauge"
62+ exit
63+}
64+
65+#-----------------------------------------------------------------------------
66+# Reporting utility functions
67+#-----------------------------------------------------------------------------
68+
69+list_comparisons () {
70+ echo "Comparison groups:"
71+ for i in $COMPARISONS
72+ do
73+ echo -n "$i ["
74+ eval "echo -n \$$i"
75+ echo "]"
76+ done
77+ echo
78+}
79+
80+# chart is expensive to build and usually not required to be rebuilt
81+cabal_which_report() {
82+ local path="./bin/$1"
83+ if test -e "$path"
84+ then
85+ echo $path
86+ fi
87+}
88+
89+find_report_prog() {
90+ local prog_name="bench-report"
91+ hash -r
92+ local prog_path=$(cabal_which_report $prog_name)
93+ if test -x "$prog_path"
94+ then
95+ echo $prog_path
96+ else
97+ return 1
98+ fi
99+}
100+
101+build_report_prog() {
102+ local prog_name="bench-report"
103+ local prog_path=$(cabal_which_report $prog_name)
104+
105+ hash -r
106+ if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
107+ then
108+ echo "Building bench-report executables"
109+ BUILD_ONCE=1
110+ pushd $BENCHMARK_DIR/bench-report
111+ local cmd
112+ cmd="$CABAL_EXECUTABLE install --installdir $SCRIPT_DIR bench-report"
113+ if test "$USE_NIX" -eq 0
114+ then
115+ $cmd || die "bench-report build failed"
116+ else
117+ nix-shell --run "$cmd" || die "bench-report build failed"
118+ fi
119+ popd
120+
121+ elif test ! -x "$prog_path"
122+ then
123+ return 1
124+ fi
125+ return 0
126+}
127+
128+build_report_progs() {
129+ if test "$RAW" = "0"
130+ then
131+ build_report_prog || exit 1
132+ local prog
133+ prog=$(find_report_prog) || \
134+ die "Cannot find bench-report executable"
135+ echo "Using bench-report executable [$prog]"
136+ fi
137+}
138+
139+# We run the benchmarks in isolation in a separate process so that different
140+# benchmarks do not interfere with other. To enable that we need to pass the
141+# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
142+# own path currently.
143+
144+# The path is dependent on the architecture and cabal version.
145+
146+bench_output_file() {
147+ local bench_name=$1
148+ echo "charts/$bench_name/results.csv"
149+}
150+
151+#------------------------------------------------------------------------------
152+# Speed options
153+#------------------------------------------------------------------------------
154+
155+# tasty-bench does not like an option set twice
156+set_super_quick_mode () {
157+ echo -n super_quick
158+}
159+
160+# For certain long benchmarks if the user has not requested super quick
161+# mode we anyway use a slightly quicker mode.
162+use_quicker_mode () {
163+ if test "$QUICK_MODE" -eq 0
164+ then
165+ echo quicker
166+ fi
167+}
168+
169+#------------------------------------------------------------------------------
170+# Determine options from benchmark name
171+#------------------------------------------------------------------------------
172+
173+# Global environment passed:
174+# BENCH_EXEC_PATH: the benchmark executable
175+# RTS_OPTIONS: additional RTS options
176+# QUICK_MODE: whether we are in quick mode
177+# USE_GAUGE: whether to use gauge or tasty-bench
178+# LONG: whether to use a large stream size
179+
180+# $1: bench name
181+bench_exec_one() {
182+ local BENCH_NAME_ORIG
183+ BENCH_NAME_ORIG="$1"
184+ shift
185+
186+ local SUPER_QUICK_OPTIONS
187+ local QUICKER_OPTIONS
188+ if test "$USE_GAUGE" -eq 0
189+ then
190+ SUPER_QUICK_OPTIONS="--stdev 1000000"
191+ QUICKER_OPTIONS="--stdev 100"
192+ else
193+ # Do not keep time limit as 0 otherwise GC stats may remain 0 in some cases.
194+ SUPER_QUICK_OPTIONS="--quick --min-duration 0 --time-limit 0.01 --include-first-iter"
195+ QUICKER_OPTIONS="--min-samples 3 --time-limit 1"
196+ fi
197+
198+ local BENCH_NAME0
199+ local BENCH_NAME1
200+ local BENCH_NAME2
201+ local BENCH_NAME
202+ if test "$USE_GAUGE" -eq 0
203+ then
204+ # XXX this is a hack to make the "/" separated names used in the functions
205+ # determining options based on benchmark name. For tasty-bench the benchmark
206+ # names are separated by "." instead of "/".
207+ BENCH_NAME0=$(echo $BENCH_NAME_ORIG | sed -e s/^All\.//)
208+ BENCH_NAME1=$(echo $BENCH_NAME0 | cut -f1 -d '/')
209+ BENCH_NAME2=$(echo $BENCH_NAME0 | cut -f2- -d '/' | sed -e 's/\./\//g')
210+ BENCH_NAME="$BENCH_NAME1/$BENCH_NAME2"
211+ else
212+ BENCH_NAME=$BENCH_NAME_ORIG
213+ fi
214+
215+ local RTS_OPTIONS1
216+ RTS_OPTIONS1="\
217++RTS -T \
218+$(bench_rts_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME) \
219+$RTS_OPTIONS \
220+-RTS"
221+
222+ local QUICK_MODE_TYPE
223+ QUICK_MODE_TYPE="\
224+ $(if test "$QUICK_MODE" -ne 0; then set_super_quick_mode; fi) \
225+ $(bench_speed_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME)"
226+
227+ local QUICK_BENCH_OPTIONS
228+ for i in $QUICK_MODE_TYPE
229+ do
230+ case "$i" in
231+ super_quick) QUICK_BENCH_OPTIONS="$SUPER_QUICK_OPTIONS"; break ;;
232+ quicker) QUICK_BENCH_OPTIONS="$QUICKER_OPTIONS"; break ;;
233+ esac
234+ done
235+
236+ local STREAM_SIZE
237+ local STREAM_LEN
238+ local STREAM_SIZE_OPT
239+ if test "$LONG" -ne 0
240+ then
241+ STREAM_SIZE=10000000
242+ STREAM_LEN=$(env LC_ALL=en_US.UTF-8 printf "--stream-size %'.f\n" $STREAM_SIZE)
243+ STREAM_SIZE_OPT="--stream-size $STREAM_SIZE"
244+ fi
245+
246+ echo "$BENCH_NAME_ORIG: \
247+ $RTS_OPTIONS1 \
248+ $STREAM_LEN \
249+ $QUICK_BENCH_OPTIONS" \
250+ "$@"
251+
252+ #------------------------------------------------------------------------------
253+ # Run benchmark with options and collect results
254+ #------------------------------------------------------------------------------
255+
256+ local output_file
257+ output_file=$(bench_output_file $(basename $BENCH_EXEC_PATH))
258+ mkdir -p `dirname $output_file`
259+ rm -f ${output_file}.tmp
260+
261+ local BENCH_NAME_ESC
262+ if test $USE_GAUGE -eq 0
263+ then
264+ # Escape "\" and double quotes in benchmark names
265+ BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')
266+ $BENCH_EXEC_PATH \
267+ -j 1 \
268+ $RTS_OPTIONS1 \
269+ $STREAM_SIZE_OPT \
270+ $QUICK_BENCH_OPTIONS \
271+ "$@" \
272+ --csv=${output_file}.tmp \
273+ -p '$0 == "'"$BENCH_NAME_ESC"'"'
274+
275+ # Convert cpuTime field from picoseconds to seconds
276+ awk --version 2>&1 | grep -q "GNU Awk" \
277+ || die "Need GNU awk. [$(which awk)] is not GNU awk."
278+ tail -n +2 ${output_file}.tmp | \
279+ awk 'BEGIN {FPAT = "([^,]+)|(\"[^\"]+\")";OFS=","} {$2=$2/1000000000000;print}' \
280+ >> $output_file
281+ else
282+ $BENCH_EXEC_PATH \
283+ $RTS_OPTIONS1 \
284+ $STREAM_SIZE_OPT \
285+ $QUICK_BENCH_OPTIONS \
286+ "$@" \
287+ --csvraw=${output_file}.tmp \
288+ -m exact "$BENCH_NAME"
289+ tail -n +2 ${output_file}.tmp \
290+ >> $output_file
291+ fi
292+}
293+
294+invoke_gauge () {
295+ local target_prog="$1"
296+ local target_name="$2"
297+ local output_file="$3"
298+
299+ local MATCH=""
300+ if test "$LONG" -ne 0
301+ then
302+ MATCH="$target_name/o-1-space"
303+ else
304+ MATCH="$BENCH_PREFIX"
305+ fi
306+ echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file
307+ # keep only benchmark names with shortest prefix e.g. "a/b/c" and "a/b", we
308+ # should only keep "a/b" otherwise benchmarks will run multiple times. why?
309+ $target_prog -l \
310+ | grep "^$MATCH" \
311+ | while read -r name; \
312+ do bench_exec_one "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
313+ || die "Benchmark execution failed."
314+}
315+
316+invoke_tasty_bench () {
317+ local target_prog="$1"
318+ local target_name="$2"
319+ local output_file="$3"
320+
321+ local MATCH=""
322+ if test "$LONG" -ne 0
323+ then
324+ MATCH="-p /$target_name\/o-1-space/"
325+ else
326+ if test -n "$BENCH_PREFIX"
327+ then
328+ # escape "/"
329+ local escaped_name=$(echo "$BENCH_PREFIX" | sed -e 's/\//\\\//g')
330+ MATCH="-p /$escaped_name/"
331+ fi
332+ fi
333+ echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file
334+ $target_prog -l $MATCH \
335+ | grep "^All" \
336+ | while read -r name; \
337+ do bench_exec_one "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
338+ || die "Benchmark execution failed."
339+}
340+
341+run_bench_target () {
342+ local package_name=$1
343+ local component=$2
344+ local target_name=$3
345+
346+ local target_prog
347+ target_prog=$(cabal_target_prog $package_name $component $target_name) || \
348+ die "Cannot find executable for target $target_name"
349+
350+ echo "Running executable $target_name ..."
351+
352+ # Needed by bench_exec_one
353+ BENCH_EXEC_PATH=$target_prog
354+
355+ local output_file=$(bench_output_file $target_name)
356+ mkdir -p `dirname $output_file`
357+ if test "$USE_GAUGE" -eq 0
358+ then invoke_tasty_bench "$target_prog" "$target_name" "$output_file"
359+ else invoke_gauge "$target_prog" "$target_name" "$output_file"
360+ fi
361+}
362+
363+# $1: package name
364+# $2: component
365+# $3: targets
366+run_bench_targets() {
367+ for i in $3
368+ do
369+ run_bench_target $1 $2 $i
370+ done
371+}
372+
373+run_benches_comparing() {
374+ local bench_list=$1
375+
376+ if test -z "$CANDIDATE"
377+ then
378+ CANDIDATE=$(git rev-parse HEAD)
379+ fi
380+ if test -z "$BASE"
381+ then
382+ # XXX Should be where the current branch is forked from master
383+ BASE="$CANDIDATE^"
384+ fi
385+ echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
386+ echo "Checking out base commit [$BASE] for benchmarking"
387+ git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
388+
389+ $BUILD_BENCH || die "build failed"
390+ run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
391+
392+ echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
393+ git checkout "$CANDIDATE" || \
394+ die "Checkout of candidate [$CANDIDATE] commit failed"
395+
396+ $BUILD_BENCH || die "build failed"
397+ run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
398+ # XXX reset back to the original commit
399+}
400+
401+backup_output_file() {
402+ local bench_name=$1
403+ local output_file=$(bench_output_file $bench_name)
404+
405+ if test -e $output_file -a "$APPEND" != 1
406+ then
407+ mv -f -v $output_file ${output_file}.prev
408+ fi
409+}
410+
411+run_measurements() {
412+ local bench_list=$1
413+
414+ for i in $bench_list
415+ do
416+ backup_output_file $i
417+ done
418+
419+ if test "$COMMIT_COMPARE" = "0"
420+ then
421+ run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
422+ else
423+ run_benches_comparing "$bench_list"
424+ fi
425+}
426+
427+run_reports() {
428+ local prog
429+ prog=$(find_report_prog) || \
430+ die "Cannot find bench-graph executable"
431+ echo
432+
433+ for i in $1
434+ do
435+ echo "Generating reports for ${i}..."
436+ $prog \
437+ --benchmark $i \
438+ $(test "$GRAPH" = 1 && echo "--graphs") \
439+ $(test "$SORT_BY_NAME" = 1 && echo "--sort-by-name") \
440+ --fields "$FIELDS"
441+ done
442+}
443+
444+#-----------------------------------------------------------------------------
445+# Execution starts here
446+#-----------------------------------------------------------------------------
447+
448+cd $SCRIPT_DIR/..
449+
450+USE_GIT_CABAL=1 # This is used by set_common_vars
451+set_common_vars
452+
453+USE_NIX=0
454+COMPARE=0
455+COMMIT_COMPARE=0
456+BASE=
457+CANDIDATE=
458+
459+APPEND=0
460+LONG=0
461+RAW=0
462+SORT_BY_NAME=0
463+GRAPH=0
464+MEASURE=1
465+
466+GAUGE_ARGS=
467+BUILD_ONCE=0
468+
469+CABAL_BUILD_OPTIONS="--flag fusion-plugin "
470+
471+#-----------------------------------------------------------------------------
472+# Read command line
473+#-----------------------------------------------------------------------------
474+
475+while test -n "$1"
476+do
477+ case $1 in
478+ -h|--help|help) print_help ;;
479+ # options with arguments
480+ --benchmarks) shift; TARGETS=$1; shift ;;
481+ --targets) shift; TARGETS=$1; shift ;;
482+ --prefix) shift; BENCH_PREFIX="$1"; shift ;;
483+ --fields) shift; FIELDS=$1; shift ;;
484+ --base) shift; BASE=$1; shift ;;
485+ --candidate) shift; CANDIDATE=$1; shift ;;
486+ --with-compiler) shift; CABAL_WITH_COMPILER=$1; shift ;;
487+ --cabal-build-flags) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
488+ --cabal-build-options) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
489+ --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
490+ --config) shift; BENCH_CONFIG_FILE=$1; shift ;;
491+ # flags
492+ --slow) SLOW=1; shift ;;
493+ --quick) QUICK_MODE=1; shift ;;
494+ --compare) COMPARE=1; shift ;;
495+ --commit-compare) COMMIT_COMPARE=1; shift ;;
496+ --raw) RAW=1; shift ;;
497+ --append) APPEND=1; shift ;;
498+ --long) LONG=1; shift ;;
499+ --sort-by-name) SORT_BY_NAME=1; shift ;;
500+ --graphs) GRAPH=1; shift ;;
501+ --no-measure) MEASURE=0; shift ;;
502+ --dev-build) RUNNING_DEVBUILD=1; shift ;;
503+ --use-nix) USE_NIX=1; shift ;;
504+ --use-gauge) USE_GAUGE=1; shift ;;
505+ --) shift; break ;;
506+ *) echo "Unknown flags: $*"; echo; print_help ;;
507+ esac
508+done
509+GAUGE_ARGS=("$@")
510+
511+if test -z "$BENCH_CONFIG_FILE"
512+then
513+ die "Please use --config to specify config file"
514+fi
515+
516+source "$BENCH_CONFIG_FILE" || \
517+ die "Failed to source config file $BENCH_CONFIG_FILE"
518+
519+# Defined in $BENCH_CONFIG_FILE
520+bench_config
521+
522+if test -z "$FIELDS"
523+then
524+ FIELDS=$DEFAULT_FIELDS
525+fi
526+
527+set_derived_vars
528+
529+#-----------------------------------------------------------------------------
530+# Determine targets
531+#-----------------------------------------------------------------------------
532+
533+only_real_benchmarks () {
534+ for i in $TARGETS
535+ do
536+ local SKIP=0
537+ for j in $COMPARISONS
538+ do
539+ if test $i == $j
540+ then
541+ SKIP=1
542+ fi
543+ done
544+ if test "$SKIP" -eq 0
545+ then
546+ echo -n "$i "
547+ fi
548+ done
549+}
550+
551+# defined in $BENCH_CONFIG_FILE
552+bench_targets
553+
554+if test "$(has_item "$TARGETS" help)" = "help"
555+then
556+ list_target_groups
557+ list_comparisons
558+ list_targets
559+ exit
560+fi
561+
562+COMMON_FIELDS="allocated bytescopied cputime maxrss"
563+if test "$USE_GAUGE" -eq 1
564+then
565+ ALL_FIELDS="$COMMON_FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
566+else
567+ ALL_FIELDS="$COMMON_FIELDS"
568+fi
569+if test "$(has_item "$FIELDS" help)" = "help"
570+then
571+ echo "Supported fields: $ALL_FIELDS"
572+ echo "Default fields: $DEFAULT_FIELDS"
573+ exit
574+fi
575+
576+if test "$LONG" -ne 0
577+then
578+ if test -n "$TARGETS"
579+ then
580+ echo "Cannot specify benchmarks [$TARGETS] with --long"
581+ exit
582+ fi
583+ TARGETS=$infinite_grp
584+fi
585+
586+DEFAULT_TARGETS="$(all_grp)"
587+TARGETS=$(set_targets)
588+
589+TARGETS_ORIG=$TARGETS
590+TARGETS=$(only_real_benchmarks)
591+
592+echo "Using benchmark suites [$TARGETS]"
593+
594+#-----------------------------------------------------------------------------
595+# Build reporting utility
596+#-----------------------------------------------------------------------------
597+
598+# We need to build the report progs first at the current (latest) commit before
599+# checking out any other commit for benchmarking.
600+build_report_progs
601+
602+#-----------------------------------------------------------------------------
603+# Build and run targets
604+#-----------------------------------------------------------------------------
605+
606+if test "$USE_GAUGE" -eq 1
607+then
608+ BUILD_FLAGS="--flag use-gauge"
609+fi
610+
611+BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks"
612+if test "$MEASURE" = "1"
613+then
614+ run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS"
615+ run_measurements "$TARGETS"
616+fi
617+
618+#-----------------------------------------------------------------------------
619+# Run reports
620+#-----------------------------------------------------------------------------
621+
622+# $1: var name
623+build_comparison_results () {
624+ local name
625+ local constituents
626+
627+ name=$1
628+ constituents=$(eval "echo -n \$${name}")
629+ mkdir -p "charts/$name"
630+ dest_file="charts/$name/results.csv"
631+ : > $dest_file
632+ for j in $constituents
633+ do
634+ cat "charts/$j/results.csv" >> $dest_file
635+ done
636+}
637+
638+if test "$COMPARE" -eq 1
639+then
640+ DYN_CMP_GRP="$(echo "$TARGETS" | sed -e 's/ /_/g')_cmp"
641+ eval "$DYN_CMP_GRP=\"$TARGETS\""
642+ COMPARISON_REPORTS=$DYN_CMP_GRP
643+ build_comparison_results $DYN_CMP_GRP
644+else
645+ COMPARISON_REPORTS=""
646+fi
647+
648+for i in $COMPARISONS
649+do
650+ if test "$(has_item "$TARGETS_ORIG" $i)" = $i
651+ then
652+ COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
653+ build_comparison_results $i
654+ fi
655+done
656+
657+if test "$RAW" = "0"
658+then
659+ run_reports "$TARGETS"
660+ run_reports "$COMPARISON_REPORTS"
661+ if test -n "$DYN_CMP_GRP"
662+ then
663+ rm -rf "charts/$DYN_CMP_GRP"
664+ fi
665+fi
bin/bench.sh+1−517
@@ -1,519 +1,3 @@
11#!/usr/bin/env bash
22
3−# Note that this script is used in the "streamly" package as well as
4−# in "streaming-benchmarks" package. Any changes to the script should be
5−# generic enough so that it works in both the cases.
6−
7−# Customization options
8−BENCHMARK_DIR=benchmark
9−BENCHMARK_PACKAGE_NAME=streamly-benchmarks
10−BENCHMARK_PACKAGE_VERSION=0.0.0
11−
12−USE_GAUGE=0
13−DEFAULT_FIELDS="allocated cputime bytescopied"
14−
15−#------------------------------------------------------------------------------
16−
17−set -o pipefail
18−
19−SCRIPT_DIR=$(cd `dirname $0`; pwd)
20−
21−RUNNING_BENCHMARKS=y
22−source $SCRIPT_DIR/build-lib.sh
23−
24−print_help () {
25− echo "Usage: $0 "
26− echo " [--benchmarks <"bench1 bench2 ..." | help>]"
27− echo " [--prefix <benchmark name prefix to match>"
28− echo " [--fields <"field1 field2 ..." | help>]"
29− echo " [--sort-by-name]"
30− echo " [--compare]"
31− echo " [--graphs]"
32− echo " [--no-measure]"
33− echo " [--append]"
34− echo " [--long]"
35− echo " [--slow]"
36− echo " [--quick]"
37− echo " [--raw]"
38− echo " [--dev-build]"
39− echo " [--use-nix]"
40− echo " [--with-compiler <compiler exe name>]"
41− echo " [--cabal-build-options <options>]"
42− echo " [--rtsopts <opts>]"
43− echo " [--commit-compare] [--base <commit>] [--candidate <commit>]"
44− echo " -- <gauge options or benchmarks>"
45− echo
46− echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
47− echo "--compare: compare the specified benchmarks with each other"
48− echo "--fields: measurement fields to report, use 'help' for a list"
49− echo "--graphs: Generate graphical reports"
50− echo "--no-measure: Don't run benchmarks, run reports from previous results"
51− echo "--append: Don't overwrite previous results, append for comparison"
52− echo "--long: Use much longer stream size for infinite stream benchmarks"
53− echo "--slow: Slightly more accurate results at the expense of speed"
54− echo "--quick: Faster results, useful for longer benchmarks"
55− echo "--raw: Run the benchmarks but don't report them. This is useful when"
56− echo " you only want to work with the csv files generated."
57− echo "--cabal-build-options: Pass any cabal build options to be used for build"
58− echo " e.g. --cabal-build-options \"--flag dev\""
59− echo
60− echo "When specific space complexity group is chosen then (and only then) "
61− echo "RTS memory restrictions are used accordingly. For example, "
62− echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
63− echo "restricts Heap/Stack space for O(1) characterstics"
64− echo
65− echo "When using --commit-compare, by default comparative chart of HEAD^ vs HEAD"
66− echo "commit is generated, in the 'charts' directory."
67− echo "Use --base and --candidate to select the commits to compare."
68− echo
69− echo "Any arguments after a '--' are passed directly to gauge"
70− exit
71−}
72−
73−#-----------------------------------------------------------------------------
74−# Reporting utility functions
75−#-----------------------------------------------------------------------------
76−
77−list_comparisons () {
78− echo "Comparison groups:"
79− for i in $COMPARISONS
80− do
81− echo -n "$i ["
82− eval "echo -n \$$i"
83− echo "]"
84− done
85− echo
86−}
87−
88−# chart is expensive to build and usually not required to be rebuilt
89−cabal_which_report() {
90− local path="./bin/$1"
91− if test -e "$path"
92− then
93− echo $path
94− fi
95−}
96−
97−find_report_prog() {
98− local prog_name="bench-report"
99− hash -r
100− local prog_path=$(cabal_which_report $prog_name)
101− if test -x "$prog_path"
102− then
103− echo $prog_path
104− else
105− return 1
106− fi
107−}
108−
109−build_report_prog() {
110− local prog_name="bench-report"
111− local prog_path=$(cabal_which_report $prog_name)
112−
113− hash -r
114− if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
115− then
116− echo "Building bench-report executables"
117− BUILD_ONCE=1
118− pushd $BENCHMARK_DIR/bench-report
119− local cmd
120− cmd="$CABAL_EXECUTABLE install --installdir $SCRIPT_DIR bench-report"
121− if test "$USE_NIX" -eq 0
122− then
123− $cmd || die "bench-report build failed"
124− else
125− nix-shell --run "$cmd" || die "bench-report build failed"
126− fi
127− popd
128−
129− elif test ! -x "$prog_path"
130− then
131− return 1
132− fi
133− return 0
134−}
135−
136−build_report_progs() {
137− if test "$RAW" = "0"
138− then
139− build_report_prog || exit 1
140− local prog
141− prog=$(find_report_prog) || \
142− die "Cannot find bench-report executable"
143− echo "Using bench-report executable [$prog]"
144− fi
145−}
146−
147−# We run the benchmarks in isolation in a separate process so that different
148−# benchmarks do not interfere with other. To enable that we need to pass the
149−# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
150−# own path currently.
151−
152−# The path is dependent on the architecture and cabal version.
153−
154−bench_output_file() {
155− local bench_name=$1
156− echo "charts/$bench_name/results.csv"
157−}
158−
159−invoke_gauge () {
160− local target_prog="$1"
161− local target_name="$2"
162− local output_file="$3"
163−
164− local MATCH=""
165− if test "$LONG" -ne 0
166− then
167− MATCH="$target_name/o-1-space"
168− else
169− MATCH="$BENCH_PREFIX"
170− fi
171− echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file
172− # keep only benchmark names with shortest prefix e.g. "a/b/c" and "a/b", we
173− # should only keep "a/b" otherwise benchmarks will run multiple times. why?
174− $target_prog -l \
175− | grep "^$MATCH" \
176− | while read -r name; \
177− do bin/bench-exec-one.sh "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
178− || die "Benchmark execution failed."
179−}
180−
181−invoke_tasty_bench () {
182− local target_prog="$1"
183− local target_name="$2"
184− local output_file="$3"
185−
186− local MATCH=""
187− if test "$LONG" -ne 0
188− then
189− MATCH="-p /$target_name\/o-1-space/"
190− else
191− if test -n "$BENCH_PREFIX"
192− then
193− # escape "/"
194− local escaped_name=$(echo "$BENCH_PREFIX" | sed -e 's/\//\\\//g')
195− MATCH="-p /$escaped_name/"
196− fi
197− fi
198− echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file
199− $target_prog -l $MATCH \
200− | grep "^All" \
201− | while read -r name; \
202− do bin/bench-exec-one.sh "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
203− || die "Benchmark execution failed."
204−}
205−
206−run_bench_target () {
207− local package_name=$1
208− local component=$2
209− local target_name=$3
210−
211− local target_prog
212− target_prog=$(cabal_target_prog $package_name $component $target_name) || \
213− die "Cannot find executable for target $target_name"
214−
215− echo "Running executable $target_name ..."
216−
217− # Needed by bench-exec-one.sh
218− export BENCH_EXEC_PATH=$target_prog
219− export RTS_OPTIONS
220− export QUICK_MODE
221− export USE_GAUGE
222− export LONG
223−
224− local output_file=$(bench_output_file $target_name)
225− mkdir -p `dirname $output_file`
226− if test "$USE_GAUGE" -eq 0
227− then invoke_tasty_bench "$target_prog" "$target_name" "$output_file"
228− else invoke_gauge "$target_prog" "$target_name" "$output_file"
229− fi
230−}
231−
232−# $1: package name
233−# $2: component
234−# $3: targets
235−run_bench_targets() {
236− for i in $3
237− do
238− run_bench_target $1 $2 $i
239− done
240−}
241−
242−run_benches_comparing() {
243− local bench_list=$1
244−
245− if test -z "$CANDIDATE"
246− then
247− CANDIDATE=$(git rev-parse HEAD)
248− fi
249− if test -z "$BASE"
250− then
251− # XXX Should be where the current branch is forked from master
252− BASE="$CANDIDATE^"
253− fi
254− echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
255− echo "Checking out base commit [$BASE] for benchmarking"
256− git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
257−
258− $BUILD_BENCH || die "build failed"
259− run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
260−
261− echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
262− git checkout "$CANDIDATE" || \
263− die "Checkout of candidate [$CANDIDATE] commit failed"
264−
265− $BUILD_BENCH || die "build failed"
266− run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
267− # XXX reset back to the original commit
268−}
269−
270−backup_output_file() {
271− local bench_name=$1
272− local output_file=$(bench_output_file $bench_name)
273−
274− if test -e $output_file -a "$APPEND" != 1
275− then
276− mv -f -v $output_file ${output_file}.prev
277− fi
278−}
279−
280−run_measurements() {
281− local bench_list=$1
282−
283− for i in $bench_list
284− do
285− backup_output_file $i
286− done
287−
288− if test "$COMMIT_COMPARE" = "0"
289− then
290− run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
291− else
292− run_benches_comparing "$bench_list"
293− fi
294−}
295−
296−run_reports() {
297− local prog
298− prog=$(find_report_prog) || \
299− die "Cannot find bench-graph executable"
300− echo
301−
302− for i in $1
303− do
304− echo "Generating reports for ${i}..."
305− $prog \
306− --benchmark $i \
307− $(test "$GRAPH" = 1 && echo "--graphs") \
308− $(test "$SORT_BY_NAME" = 1 && echo "--sort-by-name") \
309− --fields "$FIELDS"
310− done
311−}
312−
313−#-----------------------------------------------------------------------------
314−# Execution starts here
315−#-----------------------------------------------------------------------------
316−
317−cd $SCRIPT_DIR/..
318−
319−USE_GIT_CABAL=1
320−USE_NIX=0
321−set_common_vars
322−
323−COMMON_FIELDS="allocated bytescopied cputime maxrss"
324−if test "$USE_GAUGE" -eq 1
325−then
326− ALL_FIELDS="$COMMON_FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
327−else
328− ALL_FIELDS="$COMMON_FIELDS"
329−fi
330−FIELDS=$DEFAULT_FIELDS
331−
332−COMPARE=0
333−COMMIT_COMPARE=0
334−BASE=
335−CANDIDATE=
336−
337−APPEND=0
338−LONG=0
339−RAW=0
340−SORT_BY_NAME=0
341−GRAPH=0
342−MEASURE=1
343−
344−GAUGE_ARGS=
345−BUILD_ONCE=0
346−
347−CABAL_BUILD_OPTIONS="--flag fusion-plugin "
348−
349−#-----------------------------------------------------------------------------
350−# Read command line
351−#-----------------------------------------------------------------------------
352−
353−while test -n "$1"
354−do
355− case $1 in
356− -h|--help|help) print_help ;;
357− # options with arguments
358− --benchmarks) shift; TARGETS=$1; shift ;;
359− --targets) shift; TARGETS=$1; shift ;;
360− --prefix) shift; BENCH_PREFIX="$1"; shift ;;
361− --fields) shift; FIELDS=$1; shift ;;
362− --base) shift; BASE=$1; shift ;;
363− --candidate) shift; CANDIDATE=$1; shift ;;
364− --with-compiler) shift; CABAL_WITH_COMPILER=$1; shift ;;
365− --cabal-build-flags) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
366− --cabal-build-options) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
367− --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
368− # flags
369− --slow) SLOW=1; shift ;;
370− --quick) QUICK_MODE=1; shift ;;
371− --compare) COMPARE=1; shift ;;
372− --commit-compare) COMMIT_COMPARE=1; shift ;;
373− --raw) RAW=1; shift ;;
374− --append) APPEND=1; shift ;;
375− --long) LONG=1; shift ;;
376− --sort-by-name) SORT_BY_NAME=1; shift ;;
377− --graphs) GRAPH=1; shift ;;
378− --no-measure) MEASURE=0; shift ;;
379− --dev-build) RUNNING_DEVBUILD=1; shift ;;
380− --use-nix) USE_NIX=1; shift ;;
381− --use-gauge) USE_GAUGE=1; shift ;;
382− --) shift; break ;;
383− *) echo "Unknown flags: $*"; echo; print_help ;;
384− esac
385−done
386−GAUGE_ARGS=("$@")
387−
388−set_derived_vars
389−
390−#-----------------------------------------------------------------------------
391−# Determine targets
392−#-----------------------------------------------------------------------------
393−
394−only_real_benchmarks () {
395− for i in $TARGETS
396− do
397− local SKIP=0
398− for j in $COMPARISONS
399− do
400− if test $i == $j
401− then
402− SKIP=1
403− fi
404− done
405− if test "$SKIP" -eq 0
406− then
407− echo -n "$i "
408− fi
409− done
410−}
411−
412−# Requires RUNNING_DEVBUILD var
413−source $SCRIPT_DIR/targets.sh
414−
415−if test "$(has_item "$TARGETS" help)" = "help"
416−then
417− list_target_groups
418− list_comparisons
419− list_targets
420− exit
421−fi
422−
423−if test "$(has_item "$FIELDS" help)" = "help"
424−then
425− echo "Supported fields: $ALL_FIELDS"
426− echo "Default fields: $DEFAULT_FIELDS"
427− exit
428−fi
429−
430−if test "$LONG" -ne 0
431−then
432− if test -n "$TARGETS"
433− then
434− echo "Cannot specify benchmarks [$TARGETS] with --long"
435− exit
436− fi
437− TARGETS=$infinite_grp
438−fi
439−
440−DEFAULT_TARGETS="$(all_grp)"
441−TARGETS=$(set_targets)
442−
443−TARGETS_ORIG=$TARGETS
444−TARGETS=$(only_real_benchmarks)
445−
446−echo "Using benchmark suites [$TARGETS]"
447−
448−#-----------------------------------------------------------------------------
449−# Build reporting utility
450−#-----------------------------------------------------------------------------
451−
452−# We need to build the report progs first at the current (latest) commit before
453−# checking out any other commit for benchmarking.
454−build_report_progs
455−
456−#-----------------------------------------------------------------------------
457−# Build and run targets
458−#-----------------------------------------------------------------------------
459−
460−if test "$USE_GAUGE" -eq 1
461−then
462− BUILD_FLAGS="--flag use-gauge"
463−fi
464−
465−BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks"
466−if test "$MEASURE" = "1"
467−then
468− run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS"
469− run_measurements "$TARGETS"
470−fi
471−
472−#-----------------------------------------------------------------------------
473−# Run reports
474−#-----------------------------------------------------------------------------
475−
476−# $1: var name
477−build_comparison_results () {
478− local name
479− local constituents
480−
481− name=$1
482− constituents=$(eval "echo -n \$${name}")
483− mkdir -p "charts/$name"
484− dest_file="charts/$name/results.csv"
485− : > $dest_file
486− for j in $constituents
487− do
488− cat "charts/$j/results.csv" >> $dest_file
489− done
490−}
491−
492−if test "$COMPARE" -eq 1
493−then
494− DYN_CMP_GRP="$(echo "$TARGETS" | sed -e 's/ /_/g')_cmp"
495− eval "$DYN_CMP_GRP=\"$TARGETS\""
496− COMPARISON_REPORTS=$DYN_CMP_GRP
497− build_comparison_results $DYN_CMP_GRP
498−else
499− COMPARISON_REPORTS=""
500−fi
501−
502−for i in $COMPARISONS
503−do
504− if test "$(has_item "$TARGETS_ORIG" $i)" = $i
505− then
506− COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
507− build_comparison_results $i
508− fi
509−done
510−
511−if test "$RAW" = "0"
512−then
513− run_reports "$TARGETS"
514− run_reports "$COMPARISON_REPORTS"
515− if test -n "$DYN_CMP_GRP"
516− then
517− rm -rf "charts/$DYN_CMP_GRP"
518− fi
519−fi
3+bin/bench-runner.sh --config bin/bench-config.sh "$@"
bin/build-lib.sh+4
@@ -4,6 +4,10 @@ die () {
44 exit 1
55}
66
7+warn () {
8+ >&2 echo -e "Warning: $1"
9+}
10+
711# $1: command
812function run_verbose() {
913 echo "$*"
bin/targets.shdeleted−112
@@ -1,112 +0,0 @@
1−#------------------------------------------------------------------------------
2−# test and benchmark groups
3−#------------------------------------------------------------------------------
4−
5−# IMPORTANT NOTE: the names "_grp" and "_cmp" suffixes are special, do
6−# not rename them to something else.
7−
8−base_stream_grp="\
9− `bench_only Data.Stream.StreamD` \
10− `bench_only Data.Stream.StreamK` \
11− `bench_only Data.Stream.StreamDK`"
12−
13−prelude_serial_grp="\
14− Prelude.Serial \
15− Prelude.WSerial \
16− Prelude.ZipSerial"
17−
18−prelude_concurrent_grp="\
19− Prelude.Async \
20− Prelude.WAsync \
21− Prelude.Ahead \
22− Prelude.Parallel \
23− Prelude.ZipAsync"
24−
25−
26−prelude_other_grp="\
27− `test_only Prelude` \
28− $(test_only $(dev_build Prelude.Rate)) \
29− `bench_only Prelude.Rate` \
30− `test_only Prelude.Fold` \
31− `test_only Prelude.Concurrent` \
32− $(bench_only $(dev_build Prelude.Concurrent)) \
33− `bench_only Prelude.Adaptive`"
34−
35−array_grp="\
36− Data.Array \
37− Data.Array.Foreign \
38− Data.Array.Prim \
39− Data.SmallArray \
40− Data.Array.Prim.Pinned"
41−
42−base_parser_grp="\
43− Data.Parser.ParserD \
44− `bench_only Data.Parser.ParserK`"
45−
46−parser_grp="\
47− Data.Fold \
48− Data.Parser"
49−
50−list_grp="\
51− `test_only Data.List.Base` \
52− `test_only Data.List`"
53−
54−#------------------------------------------------------------------------------
55−# Streaming vs non-streaming
56−#------------------------------------------------------------------------------
57−# The "o-1-space" groups of these benchmarks are run with long stream
58−# sizes when --long option is used.
59−
60−infinite_grp="\
61− $prelude_serial_grp \
62− $prelude_concurrent_grp \
63− `bench_only Prelude.Rate`"
64−
65−#------------------------------------------------------------------------------
66−# Benchmark comparison groups
67−#------------------------------------------------------------------------------
68−
69−# *_cmp denotes a comparison benchmark, the benchmarks provided in *_cmp
70−# variables are compared with each other
71−base_stream_cmp="Data.Stream.StreamD Data.Stream.StreamK"
72−serial_wserial_cmp="Prelude.Serial Prelude.WSerial"
73−serial_async_cmp="Prelude.Serial Prelude.Async"
74−concurrent_cmp="Prelude.Async Prelude.WAsync Prelude.Ahead Prelude.Parallel"
75−array_cmp="Memory.Array Data.Array.Prim Data.Array Data.Array.Prim.Pinned"
76−pinned_array_cmp="Memory.Array Data.Array.Prim.Pinned"
77−base_parser_cmp=$base_parser_grp
78−
79−COMPARISONS="\
80− base_stream_cmp \
81− serial_wserial_cmp \
82− serial_async_cmp \
83− concurrent_cmp \
84− array_cmp \
85− pinned_array_cmp \
86− base_parser_cmp"
87−
88−#------------------------------------------------------------------------------
89−# All test/benchmark modules must be in at least one of these
90−#------------------------------------------------------------------------------
91−
92−# All groups
93−GROUP_TARGETS="\
94− base_stream_grp \
95− prelude_serial_grp \
96− prelude_concurrent_grp \
97− prelude_other_grp \
98− array_grp \
99− base_parser_grp \
100− parser_grp \
101− list_grp \
102− infinite_grp"
103−
104−# Not in any groups
105−INDIVIDUAL_TARGETS="\
106− Data.Unfold \
107− Unicode.Stream \
108− FileSystem.Handle \
109− `test_only FileSystem.Event` \
110− `test_only Network.Socket` \
111− `test_only Network.Inet.TCP` \
112− `test_only version-bounds`"
streamly.cabal+2−2
@@ -77,13 +77,13 @@ extra-source-files:
7777 benchmark/lib/Streamly/Benchmark/Common/*.hs
7878 benchmark/streamly-benchmarks.cabal
7979 bin/bench.sh
80− bin/bench-exec-one.sh
80+ bin/bench-config.sh
81+ bin/bench-runner.sh
8182 bin/build-lib.sh
8283 bin/ghc.sh
8384 bin/run-ci.sh
8485 bin/mk-hscope.sh
8586 bin/mk-tags.sh
86− bin/targets.sh
8787 bin/test.sh
8888 configure
8989 configure.ac