@@ -1,686 +0,0 @@
1−#!/usr/bin/env bash
2−
3−# Note that this script is used in the "streamly" package as well as
4−# in "streaming-benchmarks" package. Any changes to the script should be
5−# generic enough so that it works in both the cases.
6−
7−#------------------------------------------------------------------------------
8−
9−set -o pipefail
10−
11−SCRIPT_DIR=$(cd `dirname $0`; pwd)
12−
13−RUNNING_BENCHMARKS=y
14−source $SCRIPT_DIR/build-lib.sh
15−
16−print_help () {
17− echo "Usage: $0 "
18− echo " [--benchmarks <"bench1 bench2 ..." | help>]"
19− echo " [--prefix <benchmark name prefix to match>"
20− echo " [--fields <"field1 field2 ..." | help>]"
21− echo " [--sort-by-name]"
22− echo " [--compare]"
23− echo " [--diff-style <absolute|percent|multiples>]"
24− echo " [--diff-cutoff-percent <percent-value>]"
25− echo " [--graphs]"
26− echo " [--silent]"
27− echo " [--no-measure]"
28− echo " [--append]"
29− echo " [--long]"
30− echo " [--slow]"
31− echo " [--quick]"
32− echo " [--raw]"
33− echo " [--dev-build]"
34− echo " [--use-nix]"
35− echo " [--with-compiler <compiler exe name>]"
36− echo " [--cabal-build-options <options>]"
37− echo " [--rtsopts <opts>]"
38− #echo " [--commit-compare] [--base <commit>] [--candidate <commit>]"
39− #echo " -- <gauge options or benchmarks>"
40− echo
41− echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
42− echo "--compare: compare the specified benchmarks with each other"
43−
44− echo "--diff-cutoff-percent: Diff percentage used for benchmark selection."
45− echo "This applies only to the second column of the report and makes sense"
46− echo "only while comparing benchmarks."
47− echo "A positive cutoff value selects only regressions, whereas, a negative"
48− echo "cutoff value selects only improvements."
49−
50− echo "--fields: measurement fields to report, use 'help' for a list"
51− echo "--graphs: Generate graphical reports"
52− echo "--no-measure: Don't run benchmarks, run reports from previous results"
53− echo "--append: Don't overwrite previous results, append for comparison"
54− echo "--long: Use much longer stream size for infinite stream benchmarks"
55− echo "--slow: Slightly more accurate results at the expense of speed"
56− echo "--quick: Faster results, useful for longer benchmarks"
57− echo "--raw: Run the benchmarks but don't report them. This is useful when"
58− echo " you only want to work with the csv files generated."
59− echo "--cabal-build-options: Pass any cabal build options to be used for build"
60− echo " e.g. --cabal-build-options \"--flag dev\""
61− echo
62− echo "When specific space complexity group is chosen then (and only then) "
63− echo "RTS memory restrictions are used accordingly. For example, "
64− echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space "
65− echo "restricts Heap/Stack space for O(1) characterstics"
66− echo
67− #echo "When using --commit-compare, by default comparative chart of HEAD^ vs HEAD"
68− #echo "commit is generated, in the 'charts' directory."
69− #echo "Use --base and --candidate to select the commits to compare."
70− #echo
71− #echo "Any arguments after a '--' are passed directly to gauge"
72− exit
73−}
74−
75−#-----------------------------------------------------------------------------
76−# Reporting utility functions
77−#-----------------------------------------------------------------------------
78−
79−list_comparisons () {
80− echo "Comparison groups:"
81− for i in $COMPARISONS
82− do
83− echo -n "$i ["
84− eval "echo -n \$$i"
85− echo "]"
86− done
87− echo
88−}
89−
90−build_report_prog() {
91− local prog_path=$BENCH_REPORT_DIR/bin/bench-report
92−
93− hash -r
94− if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"
95− then
96− echo "Building bench-report executables"
97− BUILD_ONCE=1
98− pushd $BENCH_REPORT_DIR
99− local cmd
100− cmd="$CABAL_EXECUTABLE install --installdir bin bench-report"
101− if test "$USE_NIX" -eq 0
102− then
103− $cmd || die "bench-report build failed"
104− else
105− nix-shell --run "$cmd" || die "bench-report build failed"
106− fi
107− popd
108− elif test ! -x "$prog_path"
109− then
110− return 1
111− fi
112− return 0
113−}
114−
115−build_report_progs() {
116− if test "$RAW" = "0"
117− then
118− build_report_prog || exit 1
119− local prog
120− prog=$BENCH_REPORT_DIR/bin/bench-report
121− test -x $prog || die "Cannot find bench-report executable"
122− echo "Using bench-report executable [$prog]"
123− fi
124−}
125−
126−# We run the benchmarks in isolation in a separate process so that different
127−# benchmarks do not interfere with other. To enable that we need to pass the
128−# benchmark exe path to gauge as an argument. Unfortunately it cannot find its
129−# own path currently.
130−
131−# The path is dependent on the architecture and cabal version.
132−
133−bench_output_file() {
134− local bench_name=$1
135− echo "charts/$bench_name/results.csv"
136−}
137−
138−#------------------------------------------------------------------------------
139−# Speed options
140−#------------------------------------------------------------------------------
141−
142−# tasty-bench does not like an option set twice
143−set_super_quick_mode () {
144− echo -n super_quick
145−}
146−
147−# For certain long benchmarks if the user has not requested super quick
148−# mode we anyway use a slightly quicker mode.
149−use_quicker_mode () {
150− if test "$QUICK_MODE" -eq 0
151− then
152− echo quicker
153− fi
154−}
155−
156−#------------------------------------------------------------------------------
157−# Determine options from benchmark name
158−#------------------------------------------------------------------------------
159−
160−# Global environment passed:
161−# BENCH_EXEC_PATH: the benchmark executable
162−# RTS_OPTIONS: additional RTS options
163−# QUICK_MODE: whether we are in quick mode
164−# USE_GAUGE: whether to use gauge or tasty-bench
165−# LONG: whether to use a large stream size
166−
167−# $1: bench name
168−bench_exec_one() {
169− local BENCH_NAME_ORIG
170− BENCH_NAME_ORIG="$1"
171− shift
172−
173− local SUPER_QUICK_OPTIONS
174− local QUICKER_OPTIONS
175− if test "$USE_GAUGE" -eq 0
176− then
177− SUPER_QUICK_OPTIONS="--stdev 1000000"
178− QUICKER_OPTIONS="--stdev 100"
179− else
180− # Do not keep time limit as 0 otherwise GC stats may remain 0 in some cases.
181− SUPER_QUICK_OPTIONS="--quick --min-duration 0 --time-limit 0.01 --include-first-iter"
182− QUICKER_OPTIONS="--min-samples 3 --time-limit 1"
183− fi
184−
185− local BENCH_NAME0
186− local BENCH_NAME1
187− local BENCH_NAME2
188− local BENCH_NAME
189− # XXX this is a hack to make the "/" separated names used in the functions
190− # determining options based on benchmark name. For tasty-bench the benchmark
191− # names are separated by "." instead of "/" and are prefixed by "All".
192− #
193− # For example, All.Prelude.Serial/x.y.z => Prelude.Serial/x/y/z
194− if test "$USE_GAUGE" -eq 0
195− then
196− # Remove the prefix "All."
197− BENCH_NAME0=$(echo $BENCH_NAME_ORIG | sed -e s/^All\.//)
198−
199− # Module names could contain dots e.g. "Prelude.Serial". So we insert
200− # an explicit "/" to separate the module name part and the rest of
201− # the benchmark name. For example, Prelude.Serial/elimination.drain
202− BENCH_NAME1=$(echo $BENCH_NAME0 | cut -f1 -d '/')
203−
204− if test "$BENCH_NAME1" = "$BENCH_NAME0"
205− then
206− # There is no "/" separator
207− BENCH_NAME1=$(echo $BENCH_NAME0 | sed -e 's/\./\//g')
208− BENCH_NAME2=""
209− else
210− BENCH_NAME2=/$(echo $BENCH_NAME0 | cut -f2- -d '/' | sed -e 's/\./\//g')
211− fi
212− BENCH_NAME="${BENCH_NAME1}${BENCH_NAME2}"
213− else
214− BENCH_NAME=$BENCH_NAME_ORIG
215− fi
216−
217− local RTS_OPTIONS1
218− RTS_OPTIONS1="\
219−+RTS -T \
220−$(bench_rts_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME) \
221−$RTS_OPTIONS \
222−-RTS"
223−
224− local QUICK_MODE_TYPE
225− QUICK_MODE_TYPE="\
226− $(if test "$QUICK_MODE" -ne 0; then set_super_quick_mode; fi) \
227− $(bench_speed_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME)"
228−
229− local QUICK_BENCH_OPTIONS
230− for i in $QUICK_MODE_TYPE
231− do
232− case "$i" in
233− super_quick) QUICK_BENCH_OPTIONS="$SUPER_QUICK_OPTIONS"; break ;;
234− quicker) QUICK_BENCH_OPTIONS="$QUICKER_OPTIONS"; break ;;
235− esac
236− done
237−
238− local STREAM_SIZE
239− local STREAM_LEN
240− local STREAM_SIZE_OPT
241− if test "$LONG" -ne 0
242− then
243− STREAM_SIZE=10000000
244− STREAM_LEN=$(env LC_ALL=en_US.UTF-8 printf "--stream-size %'.f\n" $STREAM_SIZE)
245− STREAM_SIZE_OPT="--stream-size $STREAM_SIZE"
246− fi
247−
248− echo "$BENCH_NAME_ORIG: \
249− $RTS_OPTIONS1 \
250− $STREAM_LEN \
251− $QUICK_BENCH_OPTIONS" \
252− "$@"
253−
254− #------------------------------------------------------------------------------
255− # Run benchmark with options and collect results
256− #------------------------------------------------------------------------------
257−
258− local output_file
259− output_file=$(bench_output_file $(basename $BENCH_EXEC_PATH))
260− mkdir -p `dirname $output_file`
261− rm -f ${output_file}.tmp
262−
263− local BENCH_NAME_ESC
264− if test $USE_GAUGE -eq 0
265− then
266− # Escape "\" and double quotes in benchmark names
267− BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')
268− echo "$BENCH_NAME_ESC"
269− $BENCH_EXEC_PATH \
270− -j 1 \
271− $RTS_OPTIONS1 \
272− $STREAM_SIZE_OPT \
273− $QUICK_BENCH_OPTIONS \
274− "$@" \
275− --csv=${output_file}.tmp \
276− -p '$0 == "'"$BENCH_NAME_ESC"'"' || die "Benchmark execution failed."
277−
278− # Convert cpuTime field from picoseconds to seconds
279− awk --version 2>&1 | grep -q "GNU Awk" \
280− || die "Need GNU awk. [$(which awk)] is not GNU awk."
281− tail -n +2 ${output_file}.tmp | \
282− awk 'BEGIN {FPAT = "([^,]+)|(\"[^\"]+\")";OFS=","} {$2=$2/1000000000000;print}' \
283− >> $output_file
284− else
285− $BENCH_EXEC_PATH \
286− $RTS_OPTIONS1 \
287− $STREAM_SIZE_OPT \
288− $QUICK_BENCH_OPTIONS \
289− "$@" \
290− --csvraw=${output_file}.tmp \
291− -m exact "$BENCH_NAME" || die "Benchmark execution failed."
292− tail -n +2 ${output_file}.tmp \
293− >> $output_file
294− fi
295−}
296−
297−invoke_gauge () {
298− local target_prog="$1"
299− local target_name="$2"
300− local output_file="$3"
301−
302− local MATCH=""
303− if test "$LONG" -ne 0
304− then
305− MATCH="$target_name/o-1-space"
306− else
307− MATCH="$BENCH_PREFIX"
308− fi
309− echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file
310− # keep only benchmark names with shortest prefix e.g. "a/b/c" and "a/b", we
311− # should only keep "a/b" otherwise benchmarks will run multiple times. why?
312− $target_prog -l \
313− | grep "^$MATCH" \
314− | while read -r name; \
315− do bench_exec_one "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
316− || die "Benchmark execution failed."
317−}
318−
319−invoke_tasty_bench () {
320− local target_prog="$1"
321− local target_name="$2"
322− local output_file="$3"
323−
324− local MATCH=""
325− if test "$LONG" -ne 0
326− then
327− MATCH="-p /$target_name\/o-1-space/"
328− else
329− if test -n "$BENCH_PREFIX"
330− then
331− # escape "/"
332− local escaped_name=$(echo "$BENCH_PREFIX" | sed -e 's/\//\\\//g')
333− MATCH="-p /$escaped_name/"
334− fi
335− fi
336− echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied,maxrss" >> $output_file
337− $target_prog -l $MATCH \
338− | grep "^All" \
339− | while read -r name; \
340− do bench_exec_one "$name" "${GAUGE_ARGS[@]}" || exit 1; done \
341− || die "Benchmark execution failed."
342−}
343−
344−run_bench_target () {
345− local package_name=$1
346− local component=$2
347− local target_name=$3
348−
349− local target_prog
350− if test -z "$BENCHMARK_PACKAGE_VERSION"
351− then
352− echo "Please set BENCHMARK_PACKAGE_VERSION in bench_config"
353− exit 1
354− fi
355− target_prog=$(cabal_target_prog $package_name-$BENCHMARK_PACKAGE_VERSION $component $target_name) || \
356− die "Cannot find executable for target $target_name"
357−
358− echo "Running executable $target_name ..."
359−
360− # Needed by bench_exec_one
361− BENCH_EXEC_PATH=$target_prog
362−
363− local output_file=$(bench_output_file $target_name)
364− mkdir -p `dirname $output_file`
365− if test "$USE_GAUGE" -eq 0
366− then invoke_tasty_bench "$target_prog" "$target_name" "$output_file"
367− else invoke_gauge "$target_prog" "$target_name" "$output_file"
368− fi
369−}
370−
371−# $1: package name
372−# $2: component
373−# $3: targets
374−run_bench_targets() {
375− for i in $3
376− do
377− run_bench_target $1 $2 $i
378− done
379−}
380−
381−run_benches_comparing() {
382− local bench_list=$1
383−
384− if test -z "$CANDIDATE"
385− then
386− CANDIDATE=$(git rev-parse HEAD)
387− fi
388− if test -z "$BASE"
389− then
390− # XXX Should be where the current branch is forked from master
391− BASE="$CANDIDATE^"
392− fi
393− echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"
394− echo "Checking out base commit [$BASE] for benchmarking"
395− # XXX git checkout will overwrite this script itself and the scripts
396− # imported/used by this script.
397− git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"
398−
399− # $BUILD_BENCH || die "build failed"
400− run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS"
401− run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
402−
403− echo "Checking out candidate commit [$CANDIDATE] for benchmarking"
404− git checkout "$CANDIDATE" || \
405− die "Checkout of candidate [$CANDIDATE] commit failed"
406−
407− # $BUILD_BENCH || die "build failed"
408− run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS"
409− run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
410− # XXX reset back to the original commit
411−}
412−
413−backup_output_file() {
414− local bench_name=$1
415− local output_file=$(bench_output_file $bench_name)
416−
417− if test -e $output_file -a "$APPEND" != 1
418− then
419− mv -f -v $output_file ${output_file}.prev
420− fi
421−}
422−
423−run_measurements() {
424− local bench_list=$1
425−
426− for i in $bench_list
427− do
428− backup_output_file $i
429− done
430−
431− if test "$COMMIT_COMPARE" = "0"
432− then
433− run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS"
434− run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args
435− else
436− run_benches_comparing "$bench_list"
437− fi
438−}
439−
440−run_reports() {
441− local prog
442− prog=$BENCH_REPORT_DIR/bin/bench-report
443− test -x $prog || die "Cannot find bench-report executable"
444− test -z "$SILENT" && echo
445−
446− for i in $1
447− do
448− test -z "$SILENT" && echo "Generating reports for ${i}..."
449− $prog \
450− --benchmark $i \
451− $(test "$USE_GAUGE" = 1 && echo "--use-gauge") \
452− $(test "$GRAPH" = 1 && echo "--graphs") \
453− $(test "$SORT_BY_NAME" = 1 && echo "--sort-by-name") \
454− $(test -n "$BENCH_DIFF_STYLE" && echo "--diff-style $BENCH_DIFF_STYLE") \
455− $(test -n "$BENCH_CUTOFF_PERCENT" && echo "--diff-cutoff-percent $BENCH_CUTOFF_PERCENT") \
456− --fields "$FIELDS"
457− done
458−}
459−
460−#-----------------------------------------------------------------------------
461−# Execution starts here
462−#-----------------------------------------------------------------------------
463−
464−USE_GIT_CABAL=1 # This is used by set_common_vars
465−set_common_vars
466−
467−USE_NIX=0
468−COMPARE=0
469−COMMIT_COMPARE=0
470−BASE=
471−CANDIDATE=
472−
473−APPEND=0
474−LONG=0
475−RAW=0
476−SORT_BY_NAME=0
477−GRAPH=0
478−MEASURE=1
479−
480−GAUGE_ARGS=
481−BUILD_ONCE=0
482−
483−CABAL_BUILD_OPTIONS="--flag fusion-plugin --flag limit-build-mem"
484−
485−#-----------------------------------------------------------------------------
486−# Read command line
487−#-----------------------------------------------------------------------------
488−
489−while test -n "$1"
490−do
491− case $1 in
492− -h|--help|help) print_help ;;
493− # options with arguments
494− --benchmarks) shift; TARGETS=$1; shift ;;
495− --targets) shift; TARGETS=$1; shift ;;
496− --prefix) shift; BENCH_PREFIX="$1"; shift ;;
497− --fields) shift; FIELDS=$1; shift ;;
498− --base) shift; BASE=$1; shift ;;
499− --candidate) shift; CANDIDATE=$1; shift ;;
500− --with-compiler) shift; CABAL_WITH_COMPILER=$1; shift ;;
501− --cabal-build-flags) shift; CABAL_BUILD_OPTIONS+=" $1"; shift ;;
502− --cabal-build-options) shift; CABAL_BUILD_OPTIONS+=" $1"; shift ;;
503− --rtsopts) shift; RTS_OPTIONS=$1; shift ;;
504− --config) shift; BENCH_CONFIG_FILE=$1; shift ;;
505− --diff-style) shift; BENCH_DIFF_STYLE=$1; shift ;;
506− --diff-cutoff-percent) shift; BENCH_CUTOFF_PERCENT=$1; shift ;;
507− # flags
508− --slow) SLOW=1; shift ;;
509− --silent) SILENT=1; shift ;;
510− --quick) QUICK_MODE=1; shift ;;
511− --compare) COMPARE=1; shift ;;
512− --commit-compare) COMMIT_COMPARE=1; shift ;;
513− --raw) RAW=1; shift ;;
514− --append) APPEND=1; shift ;;
515− --long) LONG=1; shift ;;
516− --sort-by-name) SORT_BY_NAME=1; shift ;;
517− --graphs) GRAPH=1; shift ;;
518− --no-measure) MEASURE=0; shift ;;
519− # This is used in build-lib.sh to enable dev_build function which is used
520− # in targets.sh to enable or disable some targets.
521− --dev-build) RUNNING_DEVBUILD=1; shift ;;
522− --use-nix) USE_NIX=1; shift ;;
523− --use-gauge) USE_GAUGE=1; shift ;;
524− --) shift; break ;;
525− *) echo "Unknown flags: $*"; echo; print_help ;;
526− esac
527−done
528−GAUGE_ARGS=("$@")
529−
530−if test -z "$BENCH_CONFIG_FILE"
531−then
532− die "Please use --config to specify config file"
533−fi
534−
535−source "$BENCH_CONFIG_FILE" || \
536− die "Failed to source config file $BENCH_CONFIG_FILE"
537−
538−# Defined in $BENCH_CONFIG_FILE
539−bench_config
540−
541−if test -z "$FIELDS"
542−then
543− FIELDS=$DEFAULT_FIELDS
544−fi
545−
546−set_derived_vars
547−
548−#-----------------------------------------------------------------------------
549−# Determine targets
550−#-----------------------------------------------------------------------------
551−
552−only_real_benchmarks () {
553− for i in $TARGETS
554− do
555− local SKIP=0
556− for j in $COMPARISONS
557− do
558− if test $i == $j
559− then
560− SKIP=1
561− fi
562− done
563− if test "$SKIP" -eq 0
564− then
565− echo -n "$i "
566− fi
567− done
568−}
569−
570−# defined in $BENCH_CONFIG_FILE
571−bench_targets
572−
573−if test "$(has_item "$TARGETS" help)" = "help"
574−then
575− list_target_groups
576− list_comparisons
577− list_targets
578− exit
579−fi
580−
581−COMMON_FIELDS="allocated bytescopied cputime maxrss"
582−if test "$USE_GAUGE" -eq 1
583−then
584− ALL_FIELDS="$COMMON_FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"
585−else
586− ALL_FIELDS="$COMMON_FIELDS"
587−fi
588−if test "$(has_item "$FIELDS" help)" = "help"
589−then
590− echo "Supported fields: $ALL_FIELDS"
591− echo "Default fields: $DEFAULT_FIELDS"
592− exit
593−fi
594−
595−if test "$LONG" -ne 0
596−then
597− if test -n "$TARGETS"
598− then
599− echo "Cannot specify benchmarks [$TARGETS] with --long"
600− exit
601− fi
602− TARGETS=$infinite_grp
603−fi
604−
605−DEFAULT_TARGETS="$(all_grp)"
606−TARGETS=$(set_targets)
607−
608−TARGETS_ORIG=$TARGETS
609−TARGETS=$(only_real_benchmarks)
610−
611−test -z "$SILENT" && echo "Using benchmark suites [$TARGETS]"
612−
613−#-----------------------------------------------------------------------------
614−# Build reporting utility
615−#-----------------------------------------------------------------------------
616−
617−# We need to build the report progs first at the current (latest) commit before
618−# checking out any other commit for benchmarking.
619−if test -z "$SILENT"
620−then build_report_progs
621−else silently build_report_progs
622−fi
623−
624−#-----------------------------------------------------------------------------
625−# Build and run targets
626−#-----------------------------------------------------------------------------
627−
628−if test "$USE_GAUGE" -eq 1
629−then
630− BUILD_FLAGS="--flag use-gauge"
631−fi
632−
633−BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks"
634−if test "$MEASURE" = "1"
635−then
636− run_measurements "$TARGETS"
637−fi
638−
639−#-----------------------------------------------------------------------------
640−# Run reports
641−#-----------------------------------------------------------------------------
642−
643−# $1: var name
644−build_comparison_results () {
645− local name
646− local constituents
647−
648− name=$1
649− constituents=$(eval "echo -n \$${name}")
650− mkdir -p "charts/$name"
651− dest_file="charts/$name/results.csv"
652− : > $dest_file
653− for j in $constituents
654− do
655− cat "charts/$j/results.csv" >> $dest_file
656− done
657−}
658−
659−if test "$COMPARE" -eq 1
660−then
661− DYN_CMP_GRP="$(echo "$TARGETS" | sed -e 's/ /_/g')_cmp"
662− eval "$DYN_CMP_GRP=\"$TARGETS\""
663− COMPARISON_REPORTS=$DYN_CMP_GRP
664− build_comparison_results $DYN_CMP_GRP
665−else
666− COMPARISON_REPORTS=""
667−fi
668−
669−for i in $COMPARISONS
670−do
671− if test "$(has_item "$TARGETS_ORIG" $i)" = $i
672− then
673− COMPARISON_REPORTS="$COMPARISON_REPORTS $i"
674− build_comparison_results $i
675− fi
676−done
677−
678−if test "$RAW" = "0"
679−then
680− run_reports "$TARGETS"
681− run_reports "$COMPARISON_REPORTS"
682− if test -n "$DYN_CMP_GRP"
683− then
684− rm -rf "charts/$DYN_CMP_GRP"
685− fi
686−fi