@@ -0,0 +1,154 @@
1+#!/bin/bash
2+
3+print_help () {
4+ echo "Usage: $0 [--quick] [--append] [--no-graphs] [--no-measure] \
5+[--no-compare] [--base commit] [--candidate commit] -- <gauge options>"
6+ echo "Any arguments after a '--' are passed directly to guage"
7+ echo "You can omit '--' if the gauge args used do not start with a '-'."
8+ exit
9+}
10+
11+# $1: message
12+die () {
13+ >&2 echo -e "Error: $1"
14+ exit 1
15+}
16+
17+COMPARE=1
18+
19+while test -n "$1"
20+do
21+ case $1 in
22+ -h|--help|help) print_help ;;
23+ --quick) QUICK=1; shift ;;
24+ --append) APPEND=1; shift ;;
25+ --base) shift; BASE=$1; shift ;;
26+ --candidate) shift; CANDIDATE=$1; shift ;;
27+ --no-compare) COMPARE=0; shift ;;
28+ --no-graphs) GRAPH=0; shift ;;
29+ --no-measure) MEASURE=0; shift ;;
30+ --) shift; break ;;
31+ -*|--*) print_help ;;
32+ *) break ;;
33+ esac
34+done
35+
36+GAUGE_ARGS=$*
37+
38+STACK=stack
39+echo "Using stack command [$STACK]"
40+
41+# We build it first at the current commit before checking out any other commit
42+# for benchmarking.
43+if test "$GRAPH" != "0"
44+then
45+ CHARTS=$($STACK exec which charts)
46+ if test ! -x $CHARTS
47+ then
48+ echo "Building charts executable"
49+ $STACK build --flag "streamly:dev" || die "build failed"
50+ fi
51+
52+ CHARTS=$($STACK exec which charts)
53+ if test ! -x $CHARTS
54+ then
55+ die "Could not find 'charts' executable"
56+ fi
57+fi
58+
59+# We run the benchmarks in isolation in a separate process so that different
60+# benchmarks do not interfere with other. To enable that we need to pass the
61+# benchmark exe path to guage as an argument. Unfortunately it cannot find its
62+# own path currently.
63+
64+# The path is dependent on the architecture and cabal version.
65+# Use this command to find the exe if this script fails with an error:
66+# find .stack-work/ -type f -name "benchmarks"
67+
68+find_bench_prog () {
69+ BENCH_PROG=`$STACK path --dist-dir`/build/benchOps/benchOps
70+ if test ! -x "$BENCH_PROG"
71+ then
72+ echo
73+ echo "WARNING! benchmark binary [$BENCH_PROG] not found or not executable"
74+ echo "WARNING! not using isolated measurement."
75+ echo
76+ fi
77+}
78+
79+# --min-duration 0 means exactly one iteration per sample. We use a million
80+# iterations in the benchmarking code explicitly and do not use the iterations
81+# done by the benchmarking tool.
82+#
83+# Benchmarking tool by default discards the first iteration to remove
84+# aberrations due to initial evaluations etc. We do not discard it because we
85+# are anyway doing iterations in the benchmarking code and many of them so that
86+# any constant factor gets amortized and anyway it is a cost that we pay in
87+# real life.
88+#
89+# We can pass --min-samples value from the command line as second argument
90+# after the benchmark name in case we want to use more than one sample.
91+
92+if test "$QUICK" = "1"
93+then
94+ ENABLE_QUICK="--quick"
95+fi
96+
97+OUTPUT_FILE="charts/results.csv"
98+
99+run_bench () {
100+ $STACK build --bench --no-run-benchmarks || die "build failed"
101+ find_bench_prog
102+ mkdir -p charts
103+
104+ # We set min-samples to 1 so that we run with default benchmark duration of 5
105+ # seconds, whatever number of samples are possible in that.
106+ # We run just one iteration for each sample. Anyway the default is to run
107+ # for 30 ms and most our benchmarks are close to that or more.
108+ # If we use less than three samples, statistical analysis crashes
109+ $BENCH_PROG $ENABLE_QUICK \
110+ --include-first-iter \
111+ --min-samples 3 \
112+ --min-duration 0 \
113+ --csvraw=$OUTPUT_FILE \
114+ -v 2 \
115+ --measure-with $BENCH_PROG $GAUGE_ARGS || die "Benchmarking failed"
116+}
117+
118+if test "$MEASURE" != "0"
119+ then
120+ if test -e $OUTPUT_FILE -a "$APPEND" != 1
121+ then
122+ mv -f -v $OUTPUT_FILE ${OUTPUT_FILE}.prev
123+ fi
124+
125+ if test "$COMPARE" = "0"
126+ then
127+ run_bench
128+ else
129+ if test -z "$CANDIDATE"
130+ then
131+ CANDIDATE=HEAD
132+ fi
133+ if test -z "$BASE"
134+ then
135+ # XXX Should be where the current branch is forked from master
136+ BASE="$CANDIDATE^"
137+ fi
138+ echo "Checking out base commit for benchmarking"
139+ git checkout "$BASE" || die "Checkout of base commit failed"
140+ run_bench
141+ echo "Checking out candidate commit for benchmarking"
142+ git checkout "$CANDIDATE" || die "Checkout of candidate commit failed"
143+ run_bench
144+ fi
145+fi
146+
147+if test "$GRAPH" != "0"
148+then
149+ echo
150+ echo "Generating charts from ${OUTPUT_FILE}..."
151+ $CHARTS
152+fi
153+
154+# XXX reset back to the original commit