@@ -142,62 +142,71 @@ bench_output_file() {
142142 echo "charts/$bench_name/results.csv"
143143}
144144
145−# --min-duration 0 means exactly one iteration per sample. We use a million
146−# iterations in the benchmarking code explicitly and do not use the iterations
147−# done by the benchmarking tool.
148−#
149−# Benchmarking tool by default discards the first iteration to remove
150−# aberrations due to initial evaluations etc. We do not discard it because we
151−# are anyway doing iterations in the benchmarking code and many of them so that
152−# any constant factor gets amortized and anyway it is a cost that we pay in
153−# real life.
154−#
155−# We can pass --min-samples value from the command line as second argument
156−# after the benchmark name in case we want to use more than one sample.
157−
158−# $1: bench name
159−# $2: bench executable
160−target_exe_extra_args () {
161− local bench_name=$1
162− local bench_prog=$2
145+run_bench_target () {
146+ local package_name=$1
147+ local component=$2
148+ local target_name=$3
149+ local output_file=$(bench_output_file $target_name)
163150
164− local output_file=$(bench_output_file $bench_name)
165− mkdir -p `dirname $output_file`
151+ local target_prog
152+ target_prog=$(cabal_target_prog $package_name $component $target_name) || \
153+ die "Cannot find executable for target $target_name"
166154
167− local QUICK_OPTS="--quick --min-duration 0"
168− local SPEED_OPTIONS
169− if test "$LONG" -eq 0
155+ echo "Running executable $target_name ..."
156+
157+ # Needed by bench-exec-one.sh
158+ export BENCH_EXEC_PATH=$target_prog
159+ if test "$LONG" -ne 0
170160 then
171− if test "$SLOW" -eq 0
161+ BENCH_ARGS="-p /$target_name\/o-1-space/"
162+ STREAM_SIZE=10000000
163+ export STREAM_SIZE
164+ fi
165+
166+ local MATCH=""
167+ if test "$USE_GAUGE" -eq 0
168+ then
169+ if test "$LONG" -ne 0
172170 then
173− if test "$QUICK_MODE" -eq 0
171+ MATCH="-p /$target_name\/o-1-space/"
172+ else
173+ if test -n "$GAUGE_ARGS"
174174 then
175− # default mode, not super quick, not slow
176− SPEED_OPTIONS="$QUICK_OPTS --min-samples 10 --time-limit 1"
177− else
178− # super quick but less accurate
179− # When the time-limit is too low and the benchmark is tiny,
180− # then if the number of iterations is very small the GC stats
181− # may remain 0. So keep the time-limit at a minimum of 10 ms
182− # to collect significant stats. The problem was observed in
183− # the Prelude.Serial/reverse' benchmark.
184− SPEED_OPTIONS="$QUICK_OPTS --time-limit 0.01 --include-first-iter"
175+ local GAUGE_ARGS1=$(echo "$GAUGE_ARGS" | sed -e 's/\//\\\//g')
176+ MATCH="-p /$GAUGE_ARGS1/"
185177 fi
186− else
187− # Slow but more accurate mode
188− SPEED_OPTIONS="--min-duration 0"
189178 fi
179+ echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file
180+ $target_prog -l $MATCH \
181+ | grep "^All" \
182+ | while read -r name; do bin/bench-exec-one.sh "$name"; done
190183 else
191− # large stream size, always super quick
192− GAUGE_ARGS="$GAUGE_ARGS $bench_name/o-1-space"
193− SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"
184+ if test "$LONG" -ne 0
185+ then
186+ MATCH="$target_name/o-1-space"
187+ else
188+ MATCH="$GAUGE_ARGS"
189+ fi
190+ echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file
191+ # XXX We may have to use "sort | awk" to keep only benchmark names with
192+ # shortest prefix e.g. "a/b/c" and "a/b", we should only keep "a/b"
193+ # otherwise benchmarks will run multiple times.
194+ $target_prog -l \
195+ | grep "^$target_name" \
196+ | grep "^$MATCH" \
197+ | sort | paste -sd "," - | awk 'BEGIN {FS=","} {t="XU987"; for(i=1;i<=NF;i++) if (substr($i,1,length(t)) != t) {print $i; t=$i}}' \
198+ | while read -r name; do bin/bench-exec-one.sh "$name"; done
194199 fi
200+}
195201
196− echo "$SPEED_OPTIONS \
197− --csvraw=$output_file \
198− -v 2 \
199− --measure-with "$SCRIPT_DIR/bench-exec-one.sh" \
200− $GAUGE_ARGS"
202+# $1: package name
203+# $2: component
204+# $3: targets
205+run_bench_targets() {
206+ for i in $3
207+ do
208+ run_bench_target $1 $2 $i
209+ done
201210}
202211
203212run_benches_comparing() {
@@ -217,14 +226,14 @@ run_benches_comparing() {
217226 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
218227
219228 $BUILD_BENCH || die "build failed"
220− run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
229+ run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
221230
222231 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
223232 git checkout "$CANDIDATE" || \
224233 die "Checkout of candidate [$CANDIDATE] commit failed"
225234
226235 $BUILD_BENCH || die "build failed"
227− run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
236+ run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
228237 # XXX reset back to the original commit
229238}
230239
@@ -248,7 +257,7 @@ run_measurements() {
248257
249258 if test "$COMPARE" = "0"
250259 then
251− run_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
260+ run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args
252261 else
253262 run_benches_comparing "$bench_list"
254263 fi
@@ -274,6 +283,8 @@ run_reports() {
274283
275284cd $SCRIPT_DIR/..
276285
286+USE_GAUGE=0
287+export USE_GAUGE
277288USE_GIT_CABAL=1
278289set_common_vars
279290
@@ -323,7 +334,7 @@ do
323334 --long) LONG=1; shift ;;
324335 --graphs) GRAPH=1; shift ;;
325336 --no-measure) MEASURE=0; shift ;;
326− --dev-build) RUNNING_DEVBUILD=1; shift ;;
337+ --dev-build) RUNNING_DEVBUILD=1; shift ;;
327338 --) shift; break ;;
328339 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;
329340 *) break ;;
@@ -409,6 +420,7 @@ then
409420 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"
410421 export QUICK_MODE
411422 export RTS_OPTIONS
423+ export LONG
412424 run_measurements "$TARGETS"
413425fi
414426