@@ -1,5 +1,19 @@
11#!/usr/bin/env bash
22
3+# Note that this script is used in the "streamly" package as well as
4+# in "streaming-benchmarks" package. Any changes to the script should be
5+# generic enough so that it works in both the cases.
6+
7+# Customization options
8+BENCHMARK_DIR=benchmark
9+BENCHMARK_PACKAGE_NAME=streamly-benchmarks
10+BENCHMARK_PACKAGE_VERSION=0.0.0
11+
12+USE_GAUGE=0
13+DEFAULT_FIELDS="allocated cputime bytescopied"
14+
15+#------------------------------------------------------------------------------
16+
317set -o pipefail
418
519SCRIPT_DIR=$(cd `dirname $0`; pwd)
@@ -13,6 +27,7 @@ print_help () {
1327 echo " [--prefix <benchmark name prefix to match>"
1428 echo " [--fields <"field1 field2 ..." | help>]"
1529 echo " [--sort-by-name]"
30+ echo " [--compare]"
1631 echo " [--graphs]"
1732 echo " [--no-measure]"
1833 echo " [--append]"
@@ -25,10 +40,11 @@ print_help () {
2540 echo " [--with-compiler <compiler exe name>]"
2641 echo " [--cabal-build-options <options>]"
2742 echo " [--rtsopts <opts>]"
28− echo " [--compare] [--base <commit>] [--candidate <commit>]"
43+ echo " [--commit-compare] [--base <commit>] [--candidate <commit>]"
2944 echo " -- <gauge options or benchmarks>"
3045 echo
3146 echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
47+ echo "--compare: compare the specified benchmarks with each other"
3248 echo "--fields: measurement fields to report, use 'help' for a list"
3349 echo "--graphs: Generate graphical reports"
3450 echo "--no-measure: Don't run benchmarks, run reports from previous results"
@@ -46,7 +62,7 @@ print_help () {
4662 echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
4763 echo "restricts Heap/Stack space for O(1) characterstics"
4864 echo
49− echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
65+ echo "When using --commit-compare, by default comparative chart of HEAD^ vs HEAD"
5066 echo "commit is generated, in the 'charts' directory."
5167 echo "Use --base and --candidate to select the commits to compare."
5268 echo
@@ -99,9 +115,9 @@ build_report_prog() {
99115 then
100116 echo "Building bench-report executables"
101117 BUILD_ONCE=1
102− pushd benchmark/bench-report
118+ pushd $BENCHMARK_DIR/bench-report
103119 local cmd
104− cmd="$CABAL_EXECUTABLE install --installdir ../../bin bench-report"
120+ cmd="$CABAL_EXECUTABLE install --installdir $SCRIPT_DIR bench-report"
105121 if test "$USE_NIX" -eq 0
106122 then
107123 $cmd || die "bench-report build failed"
@@ -240,14 +256,14 @@ run_benches_comparing() {
240256 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
241257
242258 $BUILD_BENCH || die "build failed"
243− run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
259+ run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
244260
245261 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
246262 git checkout "$CANDIDATE" || \
247263 die "Checkout of candidate [$CANDIDATE] commit failed"
248264
249265 $BUILD_BENCH || die "build failed"
250− run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
266+ run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
251267 # XXX reset back to the original commit
252268}
253269
@@ -269,9 +285,9 @@ run_measurements() {
269285 backup_output_file $i
270286 done
271287
272− if test "$COMPARE" = "0"
288+ if test "$COMMIT_COMPARE" = "0"
273289 then
274− run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
290+ run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
275291 else
276292 run_benches_comparing "$bench_list"
277293 fi
@@ -300,16 +316,21 @@ run_reports() {
300316
301317cd $SCRIPT_DIR/..
302318
303−USE_GAUGE=0
304319USE_GIT_CABAL=1
305320USE_NIX=0
306321set_common_vars
307322
308−DEFAULT_FIELDS="allocated bytescopied cputime"
309−ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
323+COMMON_FIELDS="allocated bytescopied cputime maxrss"
324+if test "$USE_GAUGE" -eq 1
325+then
326+ ALL_FIELDS="$COMMON_FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
327+else
328+ ALL_FIELDS="$COMMON_FIELDS"
329+fi
310330FIELDS=$DEFAULT_FIELDS
311331
312332COMPARE=0
333+COMMIT_COMPARE=0
313334BASE=
314335CANDIDATE=
315336
@@ -348,6 +369,7 @@ do
348369 --slow) SLOW=1; shift ;;
349370 --quick) QUICK_MODE=1; shift ;;
350371 --compare) COMPARE=1; shift ;;
372+ --commit-compare) COMMIT_COMPARE=1; shift ;;
351373 --raw) RAW=1; shift ;;
352374 --append) APPEND=1; shift ;;
353375 --long) LONG=1; shift ;;
@@ -443,7 +465,7 @@ fi
443465BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks"
444466if test "$MEASURE" = "1"
445467then
446− run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
468+ run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS"
447469 run_measurements "$TARGETS"
448470fi
449471
@@ -451,20 +473,38 @@ fi
451473# Run reports
452474#-----------------------------------------------------------------------------
453475
454−COMPARISON_REPORTS=""
455−for i in $COMPARISONS
456−do
457− if test "$(has_item "$TARGETS_ORIG" $i)" = $i
458− then
459− COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
460− mkdir -p "charts/$i"
461− constituents=$(eval "echo -n \$${i}")
462− dest_file="charts/$i/results.csv"
476+# $1: var name
477+build_comparison_results () {
478+ local name
479+ local constituents
480+
481+ name=$1
482+ constituents=$(eval "echo -n \$${name}")
483+ mkdir -p "charts/$name"
484+ dest_file="charts/$name/results.csv"
463485 : > $dest_file
464486 for j in $constituents
465487 do
466488 cat "charts/$j/results.csv" >> $dest_file
467489 done
490+}
491+
492+if test "$COMPARE" -eq 1
493+then
494+ DYN_CMP_GRP="$(echo "$TARGETS" | sed -e 's/ /_/g')_cmp"
495+ eval "$DYN_CMP_GRP=\"$TARGETS\""
496+ COMPARISON_REPORTS=$DYN_CMP_GRP
497+ build_comparison_results $DYN_CMP_GRP
498+else
499+ COMPARISON_REPORTS=""
500+fi
501+
502+for i in $COMPARISONS
503+do
504+ if test "$(has_item "$TARGETS_ORIG" $i)" = $i
505+ then
506+ COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
507+ build_comparison_results $i
468508 fi
469509done
470510
@@ -472,4 +512,8 @@ if test "$RAW" = "0"
472512then
473513 run_reports "$TARGETS"
474514 run_reports "$COMPARISON_REPORTS"
515+ if test -n "$DYN_CMP_GRP"
516+ then
517+ rm -rf "charts/$DYN_CMP_GRP"
518+ fi
475519fi