git.iko.soy/streamly
on 3feeec2 help sign in
streamly/bin/bench.sh
479 lines 13 kB history raw
1#!/usr/bin/env bash
3set -o pipefail
5SCRIPT_DIR=$(cd `dirname $0`; pwd)
7RUNNING_BENCHMARKS=y
8source $SCRIPT_DIR/build-lib.sh
10print_help () {
11 echo "Usage: $0 "
12 echo " [--benchmarks <"bench1 bench2 ..." | help>]"
13 echo " [--prefix <benchmark name prefix to match>"
14 echo " [--fields <"field1 field2 ..." | help>]"
15 echo " [--sort-by-name]"
16 echo " [--graphs]"
17 echo " [--no-measure]"
18 echo " [--append]"
19 echo " [--long]"
20 echo " [--slow]"
21 echo " [--quick]"
22 echo " [--raw]"
23 echo " [--dev-build]"
24 echo " [--with-compiler <compiler exe name>]"
25 echo " [--cabal-build-options <options>]"
26 echo " [--rtsopts <opts>]"
27 echo " [--compare] [--base <commit>] [--candidate <commit>]"
28 echo " -- <gauge options or benchmarks>"
29 echo
30 echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
31 echo "--fields: measurement fields to report, use 'help' for a list"
32 echo "--graphs: Generate graphical reports"
33 echo "--no-measure: Don't run benchmarks, run reports from previous results"
34 echo "--append: Don't overwrite previous results, append for comparison"
35 echo "--long: Use much longer stream size for infinite stream benchmarks"
36 echo "--slow: Slightly more accurate results at the expense of speed"
37 echo "--quick: Faster results, useful for longer benchmarks"
38 echo "--raw: Run the benchmarks but don't report them. This is useful when"
39 echo " you only want to work with the csv files generated."
40 echo "--cabal-build-options: Pass any cabal build options to be used for build"
41 echo " e.g. --cabal-build-options \"--flag dev\""
42 echo
43 echo "When specific space complexity group is chosen then (and only then) "
44 echo "RTS memory restrictions are used accordingly. For example, "
45 echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
46 echo "restricts Heap/Stack space for O(1) characterstics"
47 echo
48 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
49 echo "commit is generated, in the 'charts' directory."
50 echo "Use --base and --candidate to select the commits to compare."
51 echo
52 echo "Any arguments after a '--' are passed directly to gauge"
53 exit
56#-----------------------------------------------------------------------------
57# Reporting utility functions
58#-----------------------------------------------------------------------------
60list_comparisons () {
61 echo "Comparison groups:"
62 for i in $COMPARISONS
63 do
64 echo -n "$i ["
65 eval "echo -n \$$i"
66 echo "]"
67 done
68 echo
71# chart is expensive to build and usually not required to be rebuilt,
72# use master branch as fallback
73cabal_which_report() {
74 local path=$(cabal_which streamly-benchmarks x $1)
75 if test -z "$path"
76 then
77 echo "Cannot find $1 executable, trying in dist-newstyle" 1>&2
78 local path1=$(cabal_which_builddir dist-newstyle streamly-benchmarks x $1)
79 if test -z "$path1"
80 then
81 local path2="./bin/$1"
82 echo "Cannot find $1 executable, trying $path2" 1>&2
83 if test -e "$path2"
84 then
85 echo $path2
86 fi
87 else
88 echo $path1
89 fi
90 else
91 echo $path
92 fi
95find_report_prog() {
96 local prog_name="chart"
97 hash -r
98 local prog_path=$(cabal_which_report $prog_name)
99 if test -x "$prog_path"
100 then
101 echo $prog_path
102 else
103 return 1
104 fi
107build_report_prog() {
108 local prog_name="chart"
109 local prog_path=$(cabal_which_report $prog_name)
111 hash -r
112 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
113 then
114 echo "Building bench-show executables"
115 BUILD_ONCE=1
116 $CABAL_EXECUTABLE v2-build --flags dev chart \
117 || die "bench-show build failed"
119 elif test ! -x "$prog_path"
120 then
121 return 1
122 fi
123 return 0
126build_report_progs() {
127 if test "$RAW" = "0"
128 then
129 build_report_prog || exit 1
130 local prog
131 prog=$(find_report_prog) || \
132 die "Cannot find bench-show executable"
133 echo "Using bench-show executable [$prog]"
134 fi
137# We run the benchmarks in isolation in a separate process so that different
138# benchmarks do not interfere with other. To enable that we need to pass the
139# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
140# own path currently.
142# The path is dependent on the architecture and cabal version.
144bench_output_file() {
145 local bench_name=$1
146 echo "charts/$bench_name/results.csv"
149invoke_gauge () {
150 local target_prog="$1"
151 local target_name="$2"
152 local output_file="$3"
154 local MATCH=""
155 if test "$LONG" -ne 0
156 then
157 MATCH="$target_name/o-1-space"
158 else
159 MATCH="$BENCH_PREFIX"
160 fi
161 echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file
162 # keep only benchmark names with shortest prefix e.g. "a/b/c" and "a/b", we
163 # should only keep "a/b" otherwise benchmarks will run multiple times. why?
164 $target_prog -l \
165 | grep "^$MATCH" \
166 | while read -r name; \
167 do bin/bench-exec-one.sh "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
168 || die "Benchmark execution failed."
171invoke_tasty_bench () {
172 local target_prog="$1"
173 local target_name="$2"
174 local output_file="$3"
176 local MATCH=""
177 if test "$LONG" -ne 0
178 then
179 MATCH="-p /$target_name\/o-1-space/"
180 else
181 if test -n "$BENCH_PREFIX"
182 then
183 # escape "/"
184 local escaped_name=$(echo "$BENCH_PREFIX" | sed -e 's/\//\\\//g')
185 MATCH="-p /$escaped_name/"
186 fi
187 fi
188 echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file
189 $target_prog -l $MATCH \
190 | grep "^All" \
191 | while read -r name; \
192 do bin/bench-exec-one.sh "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
193 || die "Benchmark execution failed."
196run_bench_target () {
197 local package_name=$1
198 local component=$2
199 local target_name=$3
201 local target_prog
202 target_prog=$(cabal_target_prog $package_name $component $target_name) || \
203 die "Cannot find executable for target $target_name"
205 echo "Running executable $target_name ..."
207 # Needed by bench-exec-one.sh
208 export BENCH_EXEC_PATH=$target_prog
209 export RTS_OPTIONS
210 export QUICK_MODE
211 export USE_GAUGE
212 export LONG
214 local output_file=$(bench_output_file $target_name)
215 mkdir -p `dirname $output_file`
216 if test "$USE_GAUGE" -eq 0
217 then invoke_tasty_bench "$target_prog" "$target_name" "$output_file"
218 else invoke_gauge "$target_prog" "$target_name" "$output_file"
219 fi
222# $1: package name
223# $2: component
224# $3: targets
225run_bench_targets() {
226 for i in $3
227 do
228 run_bench_target $1 $2 $i
229 done
232run_benches_comparing() {
233 local bench_list=$1
235 if test -z "$CANDIDATE"
236 then
237 CANDIDATE=$(git rev-parse HEAD)
238 fi
239 if test -z "$BASE"
240 then
241 # XXX Should be where the current branch is forked from master
242 BASE="$CANDIDATE^"
243 fi
244 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
245 echo "Checking out base commit [$BASE] for benchmarking"
246 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
248 $BUILD_BENCH || die "build failed"
249 run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
251 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
252 git checkout "$CANDIDATE" || \
253 die "Checkout of candidate [$CANDIDATE] commit failed"
255 $BUILD_BENCH || die "build failed"
256 run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
257 # XXX reset back to the original commit
260backup_output_file() {
261 local bench_name=$1
262 local output_file=$(bench_output_file $bench_name)
264 if test -e $output_file -a "$APPEND" != 1
265 then
266 mv -f -v $output_file ${output_file}.prev
267 fi
270run_measurements() {
271 local bench_list=$1
273 for i in $bench_list
274 do
275 backup_output_file $i
276 done
278 if test "$COMPARE" = "0"
279 then
280 run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
281 else
282 run_benches_comparing "$bench_list"
283 fi
286run_reports() {
287 local prog
288 prog=$(find_report_prog) || \
289 die "Cannot find bench-graph executable"
290 echo
292 for i in $1
293 do
294 echo "Generating reports for ${i}..."
295 $prog \
296 --benchmark $i \
297 $(test "$GRAPH" = 1 && echo "--graphs") \
298 $(test "$SORT_BY_NAME" = 1 && echo "--sort-by-name") \
299 --fields "$FIELDS"
300 done
303#-----------------------------------------------------------------------------
304# Execution starts here
305#-----------------------------------------------------------------------------
307cd $SCRIPT_DIR/..
309USE_GAUGE=0
310USE_GIT_CABAL=1
311set_common_vars
313DEFAULT_FIELDS="allocated bytescopied cputime"
314ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
315FIELDS=$DEFAULT_FIELDS
317COMPARE=0
318BASE=
319CANDIDATE=
321APPEND=0
322LONG=0
323RAW=0
324SORT_BY_NAME=0
325GRAPH=0
326MEASURE=1
328GAUGE_ARGS=
329BUILD_ONCE=0
331CABAL_BUILD_OPTIONS="--flag fusion-plugin "
333#-----------------------------------------------------------------------------
334# Read command line
335#-----------------------------------------------------------------------------
337while test -n "$1"
338do
339 case $1 in
340 -h|--help|help) print_help ;;
341 # options with arguments
342 --benchmarks) shift; TARGETS=$1; shift ;;
343 --targets) shift; TARGETS=$1; shift ;;
344 --prefix) shift; BENCH_PREFIX="$1"; shift ;;
345 --fields) shift; FIELDS=$1; shift ;;
346 --base) shift; BASE=$1; shift ;;
347 --candidate) shift; CANDIDATE=$1; shift ;;
348 --with-compiler) shift; CABAL_WITH_COMPILER=$1; shift ;;
349 --cabal-build-flags) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
350 --cabal-build-options) shift; CABAL_BUILD_OPTIONS+=$1; shift ;;
351 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
352 # flags
353 --slow) SLOW=1; shift ;;
354 --quick) QUICK_MODE=1; shift ;;
355 --compare) COMPARE=1; shift ;;
356 --raw) RAW=1; shift ;;
357 --append) APPEND=1; shift ;;
358 --long) LONG=1; shift ;;
359 --sort-by-name) SORT_BY_NAME=1; shift ;;
360 --graphs) GRAPH=1; shift ;;
361 --no-measure) MEASURE=0; shift ;;
362 --dev-build) RUNNING_DEVBUILD=1; shift ;;
363 --use-gauge) USE_GAUGE=1; shift ;;
364 --) shift; break ;;
365 *) echo "Unknown flags: $*"; echo; print_help ;;
366 esac
367done
368GAUGE_ARGS=("$@")
370set_derived_vars
372#-----------------------------------------------------------------------------
373# Determine targets
374#-----------------------------------------------------------------------------
376only_real_benchmarks () {
377 for i in $TARGETS
378 do
379 local SKIP=0
380 for j in $COMPARISONS
381 do
382 if test $i == $j
383 then
384 SKIP=1
385 fi
386 done
387 if test "$SKIP" -eq 0
388 then
389 echo -n "$i "
390 fi
391 done
394# Requires RUNNING_DEVBUILD var
395source $SCRIPT_DIR/targets.sh
397if test "$(has_item "$TARGETS" help)" = "help"
398then
399 list_target_groups
400 list_comparisons
401 list_targets
402 exit
403fi
405if test "$(has_item "$FIELDS" help)" = "help"
406then
407 echo "Supported fields: $ALL_FIELDS"
408 echo "Default fields: $DEFAULT_FIELDS"
409 exit
410fi
412if test "$LONG" -ne 0
413then
414 if test -n "$TARGETS"
415 then
416 echo "Cannot specify benchmarks [$TARGETS] with --long"
417 exit
418 fi
419 TARGETS=$infinite_grp
420fi
422DEFAULT_TARGETS="$(all_grp)"
423TARGETS=$(set_targets)
425TARGETS_ORIG=$TARGETS
426TARGETS=$(only_real_benchmarks)
428echo "Using benchmark suites [$TARGETS]"
430#-----------------------------------------------------------------------------
431# Build reporting utility
432#-----------------------------------------------------------------------------
434# We need to build the report progs first at the current (latest) commit before
435# checking out any other commit for benchmarking.
436build_report_progs
438#-----------------------------------------------------------------------------
439# Build and run targets
440#-----------------------------------------------------------------------------
442if test "$USE_GAUGE" -eq 1
443then
444 BUILD_FLAGS="--flag use-gauge"
445fi
447BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks"
448if test "$MEASURE" = "1"
449then
450 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
451 run_measurements "$TARGETS"
452fi
454#-----------------------------------------------------------------------------
455# Run reports
456#-----------------------------------------------------------------------------
458COMPARISON_REPORTS=""
459for i in $COMPARISONS
460do
461 if test "$(has_item "$TARGETS_ORIG" $i)" = $i
462 then
463 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
464 mkdir -p "charts/$i"
465 constituents=$(eval "echo -n \$${i}")
466 dest_file="charts/$i/results.csv"
467 : > $dest_file
468 for j in $constituents
469 do
470 cat "charts/$j/results.csv" >> $dest_file
471 done
472 fi
473done
475if test "$RAW" = "0"
476then
477 run_reports "$TARGETS"
478 run_reports "$COMPARISON_REPORTS"
479fi