@@ -0,0 +1,154 @@
4+ echo "Usage: $0 [--quick] [--append] [--no-graphs] [--no-measure] \
5+[--no-compare] [--base commit] [--candidate commit] -- <gauge options>"
6+ echo "Any arguments after a '--' are passed directly to guage"
7+ echo "You can omit '--' if the gauge args used do not start with a '-'."
13+ >&2 echo -e "Error: $1"
22+ -h|--help|help) print_help ;;
23+ --quick) QUICK=1; shift ;;
24+ --append) APPEND=1; shift ;;
25+ --base) shift; BASE=$1; shift ;;
26+ --candidate) shift; CANDIDATE=$1; shift ;;
27+ --no-compare) COMPARE=0; shift ;;
28+ --no-graphs) GRAPH=0; shift ;;
29+ --no-measure) MEASURE=0; shift ;;
31+ -*|--*) print_help ;;
39+echo "Using stack command [$STACK]"
41+# We build it first at the current commit before checking out any other commit
43+if test "$GRAPH" != "0"
45+ CHARTS=$($STACK exec which charts)
48+ echo "Building charts executable"
49+ $STACK build --flag "streamly:dev" || die "build failed"
52+ CHARTS=$($STACK exec which charts)
55+ die "Could not find 'charts' executable"
59+# We run the benchmarks in isolation in a separate process so that different
60+# benchmarks do not interfere with other. To enable that we need to pass the
61+# benchmark exe path to guage as an argument. Unfortunately it cannot find its
64+# The path is dependent on the architecture and cabal version.
65+# Use this command to find the exe if this script fails with an error:
66+# find .stack-work/ -type f -name "benchmarks"
69+ BENCH_PROG=`$STACK path --dist-dir`/build/benchOps/benchOps
70+ if test ! -x "$BENCH_PROG"
73+ echo "WARNING! benchmark binary [$BENCH_PROG] not found or not executable"
74+ echo "WARNING! not using isolated measurement."
79+# --min-duration 0 means exactly one iteration per sample. We use a million
80+# iterations in the benchmarking code explicitly and do not use the iterations
81+# done by the benchmarking tool.
83+# Benchmarking tool by default discards the first iteration to remove
84+# aberrations due to initial evaluations etc. We do not discard it because we
85+# are anyway doing iterations in the benchmarking code and many of them so that
86+# any constant factor gets amortized and anyway it is a cost that we pay in
89+# We can pass --min-samples value from the command line as second argument
90+# after the benchmark name in case we want to use more than one sample.
92+if test "$QUICK" = "1"
94+ ENABLE_QUICK="--quick"
97+OUTPUT_FILE="charts/results.csv"
100+ $STACK build --bench --no-run-benchmarks || die "build failed"
104+ # We set min-samples to 1 so that we run with default benchmark duration of 5
105+ # seconds, whatever number of samples are possible in that.
106+ # We run just one iteration for each sample. Anyway the default is to run
107+ # for 30 ms and most our benchmarks are close to that or more.
108+ # If we use less than three samples, statistical analysis crashes
109+ $BENCH_PROG $ENABLE_QUICK \
110+ --include-first-iter \
113+ --csvraw=$OUTPUT_FILE \
115+ --measure-with $BENCH_PROG $GAUGE_ARGS || die "Benchmarking failed"
118+if test "$MEASURE" != "0"
120+ if test -e $OUTPUT_FILE -a "$APPEND" != 1
122+ mv -f -v $OUTPUT_FILE ${OUTPUT_FILE}.prev
125+ if test "$COMPARE" = "0"
129+ if test -z "$CANDIDATE"
135+ # XXX Should be where the current branch is forked from master
138+ echo "Checking out base commit for benchmarking"
139+ git checkout "$BASE" || die "Checkout of base commit failed"
141+ echo "Checking out candidate commit for benchmarking"
142+ git checkout "$CANDIDATE" || die "Checkout of candidate commit failed"
147+if test "$GRAPH" != "0"
150+ echo "Generating charts from ${OUTPUT_FILE}..."
154+# XXX reset back to the original commit