@@ -142,62 +142,71 @@ bench_output_file() {
142 echo "charts/$bench_name/results.csv"
142 echo "charts/$bench_name/results.csv"
145−# --min-duration 0 means exactly one iteration per sample. We use a million
145+run_bench_target () {
146−# iterations in the benchmarking code explicitly and do not use the iterations
146+ local package_name=$1
147−# done by the benchmarking tool.
147+ local component=$2
148−#
148+ local target_name=$3
149−# Benchmarking tool by default discards the first iteration to remove
149+ local output_file=$(bench_output_file $target_name)
150−# aberrations due to initial evaluations etc. We do not discard it because we
151−# are anyway doing iterations in the benchmarking code and many of them so that
152−# any constant factor gets amortized and anyway it is a cost that we pay in
155−# We can pass --min-samples value from the command line as second argument
156−# after the benchmark name in case we want to use more than one sample.
159−# $2: bench executable
160−target_exe_extra_args () {
164− local output_file=$(bench_output_file $bench_name)
151+ local target_prog
165− mkdir -p `dirname $output_file`
152+ target_prog=$(cabal_target_prog $package_name $component $target_name) || \
153+ die "Cannot find executable for target $target_name"
167− local QUICK_OPTS="--quick --min-duration 0"
155+ echo "Running executable $target_name ..."
168− local SPEED_OPTIONS
156+
157+ # Needed by bench-exec-one.sh
158+ export BENCH_EXEC_PATH=$target_prog
169− if test "$LONG" -eq 0
159+ if test "$LONG" -ne 0
161+ BENCH_ARGS="-p /$target_name\/o-1-space/"
162+ STREAM_SIZE=10000000
167+ if test "$USE_GAUGE" -eq 0
171− if test "$SLOW" -eq 0
169+ if test "$LONG" -ne 0
173− if test "$QUICK_MODE" -eq 0
171+ MATCH="-p /$target_name\/o-1-space/"
173+ if test -n "$GAUGE_ARGS"
175− # default mode, not super quick, not slow
175+ local GAUGE_ARGS1=$(echo "$GAUGE_ARGS" | sed -e 's/\//\\\//g')
176− SPEED_OPTIONS="$QUICK_OPTS --min-samples 10 --time-limit 1"
176+ MATCH="-p /$GAUGE_ARGS1/"
178− # super quick but less accurate
179− # When the time-limit is too low and the benchmark is tiny,
180− # then if the number of iterations is very small the GC stats
181− # may remain 0. So keep the time-limit at a minimum of 10 ms
182− # to collect significant stats. The problem was observed in
183− # the Prelude.Serial/reverse' benchmark.
184− SPEED_OPTIONS="$QUICK_OPTS --time-limit 0.01 --include-first-iter"
187− # Slow but more accurate mode
188− SPEED_OPTIONS="--min-duration 0"
179+ echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file
180+ $target_prog -l $MATCH \
182+ | while read -r name; do bin/bench-exec-one.sh "$name"; done
191− # large stream size, always super quick
184+ if test "$LONG" -ne 0
192− GAUGE_ARGS="$GAUGE_ARGS $bench_name/o-1-space"
185+ then
193− SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"
186+ MATCH="$target_name/o-1-space"
190+ echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file
191+ # XXX We may have to use "sort | awk" to keep only benchmark names with
192+ # shortest prefix e.g. "a/b/c" and "a/b", we should only keep "a/b"
193+ # otherwise benchmarks will run multiple times.
195+ | grep "^$target_name" \
197+ | sort | paste -sd "," - | awk 'BEGIN {FS=","} {t="XU987"; for(i=1;i<=NF;i++) if (substr($i,1,length(t)) != t) {print $i; t=$i}}' \
198+ | while read -r name; do bin/bench-exec-one.sh "$name"; done
196− echo "$SPEED_OPTIONS \
202+# $1: package name
197− --csvraw=$output_file \
203+# $2: component
198− -v 2 \
204+# $3: targets
199− --measure-with "$SCRIPT_DIR/bench-exec-one.sh" \
205+run_bench_targets() {
200− $GAUGE_ARGS"
206+ for i in $3
208+ run_bench_target $1 $2 $i
203run_benches_comparing() {
212run_benches_comparing() {
@@ -217,14 +226,14 @@ run_benches_comparing() {
217 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
226 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
219 $BUILD_BENCH || die "build failed"
228 $BUILD_BENCH || die "build failed"
220− run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
229+ run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
222 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
231 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
223 git checkout "$CANDIDATE" || \
232 git checkout "$CANDIDATE" || \
224 die "Checkout of candidate [$CANDIDATE] commit failed"
233 die "Checkout of candidate [$CANDIDATE] commit failed"
226 $BUILD_BENCH || die "build failed"
235 $BUILD_BENCH || die "build failed"
227− run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
236+ run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
228 # XXX reset back to the original commit
237 # XXX reset back to the original commit
@@ -248,7 +257,7 @@ run_measurements() {
249 if test "$COMPARE" = "0"
258 if test "$COMPARE" = "0"
251− run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
260+ run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
253 run_benches_comparing "$bench_list"
262 run_benches_comparing "$bench_list"
@@ -274,6 +283,8 @@ run_reports() {
275cd $SCRIPT_DIR/..
284cd $SCRIPT_DIR/..
277USE_GIT_CABAL=1
288USE_GIT_CABAL=1
278set_common_vars
289set_common_vars
@@ -323,7 +334,7 @@ do
323 --long) LONG=1; shift ;;
334 --long) LONG=1; shift ;;
324 --graphs) GRAPH=1; shift ;;
335 --graphs) GRAPH=1; shift ;;
325 --no-measure) MEASURE=0; shift ;;
336 --no-measure) MEASURE=0; shift ;;
326− --dev-build) RUNNING_DEVBUILD=1; shift ;;
337+ --dev-build) RUNNING_DEVBUILD=1; shift ;;
327 --) shift; break ;;
338 --) shift; break ;;
328 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
339 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
329 *) break ;;
340 *) break ;;
@@ -409,6 +420,7 @@ then
409 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
420 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
410 export QUICK_MODE
421 export QUICK_MODE
411 export RTS_OPTIONS
422 export RTS_OPTIONS
412 run_measurements "$TARGETS"
424 run_measurements "$TARGETS"