f6d05d3 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Refactor bench.sh to use build-lib.sh

f6d05d3Harendra Kumar6 years ago

bin/bench.sh+80−303

compare with master

unifiedsplit

@@ -1,134 +1,9 @@

1#!/usr/bin/env bash

3−#------------------------------------------------------------------------------

4−# Benchmark groups

5−#------------------------------------------------------------------------------

6−

7−# IMPORTANT NOTE: the names "_grp" and "_cmp" suffixes are special, do

8−# not rename them to something else.

9−

10−base_stream_grp="\

11− Data.Stream.StreamD \

12− Data.Stream.StreamK \

13− Data.Stream.StreamDK"

14−

15−prelude_serial_grp="\

16− Prelude.Serial \

17− Prelude.WSerial \

18− Prelude.ZipSerial"

19−

20−prelude_concurrent_grp="\

21− Prelude.Async \

22− Prelude.WAsync \

23− Prelude.Ahead \

24− Prelude.Parallel \

25− Prelude.ZipAsync"

26−

27−prelude_other_grp="\

28− Prelude.Rate \

29− Prelude.Concurrent \

30− Prelude.Adaptive"

31−

32−array_grp="\

33− Data.Array \

34− Data.Array.Storable.Foreign \

35− Data.Array.Prim \

36− Data.SmallArray \

37− Data.Array.Prim.Pinned"

38−

39−base_parser_grp="Data.Parser.ParserD Data.Parser.ParserK"

40−parser_grp="Data.Fold Data.Parser"

41−

42−#------------------------------------------------------------------------------

43−# Streaming vs non-streaming

44−#------------------------------------------------------------------------------

45−# The "o-1-space" groups of these benchmarks are run with long stream

46−# sizes when --long option is used.

47−

48−infinite_grp="\

49− $prelude_serial_grp \

50− $prelude_concurrent_grp \

51− Prelude.Rate"

52−

53−#------------------------------------------------------------------------------

54−# Comparison groups

55−#------------------------------------------------------------------------------

56−# *_cmp denotes a comparison benchmarks, the benchmarks provided in *_cmp

57−# variables are compared with each other

58−base_stream_cmp="Data.Stream.StreamD Data.Stream.StreamK"

59−serial_wserial_cmp="Prelude.Serial Prelude.WSerial"

60−serial_async_cmp="Prelude.Serial Prelude.Async"

61−concurrent_cmp="Prelude.Async Prelude.WAsync Prelude.Ahead Prelude.Parallel"

62−array_cmp="Memory.Array Data.Array.Prim Data.Array Data.Array.Prim.Pinned"

63−pinned_array_cmp="Memory.Array Data.Array.Prim.Pinned"

64−base_parser_cmp=$base_parser_grp

65−COMPARISONS="\

66− base_stream_cmp \

67− serial_wserial_cmp \

68− serial_async_cmp \

69− concurrent_cmp \

70− array_cmp \

71− pinned_array_cmp \

72− base_parser_cmp"

73−

74−#------------------------------------------------------------------------------

75−# All

76−#------------------------------------------------------------------------------

77−# All high level benchmarks

78−all_grp="\

79− $prelude_serial_grp \

80− $prelude_concurrent_grp \

81− $array_grp \

82− $parser_grp \

83− Unicode.Stream \

84− Data.Unfold \

85− FileSystem.Handle"

86−

87−ALL_BENCH_GROUPS="\

88− all_grp \

89− prelude_serial_grp \

90− prelude_concurrent_grp \

91− array_grp \

92− infinite_grp \

93− base_stream_grp \

94− base_parser_grp"

95−

96−#------------------------------------------------------------------------------

97−# Script

98−#------------------------------------------------------------------------------

99−

100−BENCH_SH_DIR=$(cd `dirname $0`; pwd)

3+SCRIPT_DIR=$(cd `dirname $0`; pwd)

101−

102−list_benches () {

103− echo "Individual benchmarks:"

104− for i in $all_grp

105− do

106− echo "$i"

107− done

108− echo

109−}

110

111−list_bench_groups () {

5+RUNNING_BENCHMARKS=y

112− echo "Benchmark groups:"

6+source $SCRIPT_DIR/build-lib.sh

113− for i in $ALL_BENCH_GROUPS

114− do

115− echo -n "$i ["

116− eval "echo -n \$$i"

117− echo "]"

118− done

119− echo

120−}

121−

122−list_comparisons () {

123− echo "Comparison groups:"

124− for i in $COMPARISONS

125− do

126− echo -n "$i ["

127− eval "echo -n \$$i"

128− echo "]"

129− done

130− echo

131−}

132

133print_help () {

8print_help () {

134 echo "Usage: $0 "

9 echo "Usage: $0 "

@@ -141,7 +16,9 @@ print_help () {

141 echo " [--slow]"

16 echo " [--slow]"

142 echo " [--quick]"

17 echo " [--quick]"

143 echo " [--raw]"

18 echo " [--raw]"

19+ echo " [--dev-build]"

144 echo " [--cabal-build-flags <flag>]"

20 echo " [--cabal-build-flags <flag>]"

21+ echo " [--rtsopts <opts>]"

145 echo " [--compare] [--base <commit>] [--candidate <commit>]"

22 echo " [--compare] [--base <commit>] [--candidate <commit>]"

146 echo " -- <gauge options or benchmarks>"

23 echo " -- <gauge options or benchmarks>"

147 echo

24 echo

@@ -156,6 +33,7 @@ print_help () {

156 echo "--raw: Run the benchmarks but don't report them. This is useful when"

33 echo "--raw: Run the benchmarks but don't report them. This is useful when"

157 echo " you only want to work with the csv files generated."

34 echo " you only want to work with the csv files generated."

158 echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"

35 echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"

36+ echo " e.g. --cabal-build-flags \"--flag fusion-plugin\""

159 echo

37 echo

160 echo "When specific space complexity group is chosen then (and only then) "

38 echo "When specific space complexity group is chosen then (and only then) "

161 echo "RTS memory restrictions are used accordingly. For example, "

39 echo "RTS memory restrictions are used accordingly. For example, "

@@ -170,33 +48,37 @@ print_help () {

170 exit

48 exit

171}

49}

172

173−# $1: message

51+#-----------------------------------------------------------------------------

174−die () {

52+# Reporting utility functions

53+#-----------------------------------------------------------------------------

54+

55+list_comparisons () {

56+ echo "Comparison groups:"

57+ for i in $COMPARISONS

58+ do

175− >&2 echo -e "Error: $1"

59+ echo -n "$i ["

176− exit 1

60+ eval "echo -n \$$i"

61+ echo "]"

62+ done

63+ echo

177}

64}

178

179−set_benchmarks() {

66+# chart is expensive to build and usually not required to be rebuilt,

67+# use master branch as fallback

68+cabal_which_report() {

69+ local path=$(cabal_which streamly-benchmarks $1)

180− if test -z "$BENCHMARKS"

70+ if test -z "$path"

181 then

71 then

182− echo $DEFAULT_BENCHMARKS

72+ cabal_which_builddir dist-newstyle streamly-benchmarks $1

183 else

73 else

184− for i in $(echo $BENCHMARKS)

74+ echo $path

185− do

186− case $i in

187− *_grp) eval "echo -n \$${i}" ;;

188− *_cmp) eval "echo -n \$${i} $i" ;;

189− *) echo -n $i ;;

190− esac

191− echo -n " "

192− done

193 fi

75 fi

194}

76}

195

196find_report_prog() {

78find_report_prog() {

197 local prog_name="chart"

79 local prog_name="chart"

198 hash -r

80 hash -r

199− local prog_path=$($WHICH_COMMAND_REPORT $prog_name)

81+ local prog_path=$(cabal_which_report $prog_name)

200 if test -x "$prog_path"

82 if test -x "$prog_path"

201 then

83 then

202 echo $prog_path

84 echo $prog_path

@@ -207,14 +89,16 @@ find_report_prog() {

207

208build_report_prog() {

90build_report_prog() {

209 local prog_name="chart"

91 local prog_name="chart"

210− local prog_path=$($WHICH_COMMAND_REPORT $prog_name)

92+ local prog_path=$(cabal_which_report $prog_name)

211

212 hash -r

94 hash -r

213 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"

95 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"

214 then

96 then

215− echo "Building bench-graph executables"

97+ echo "Building bench-show executables"

216 BUILD_ONCE=1

98 BUILD_ONCE=1

99+ $CABAL_EXECUTABLE v2-build --flags dev chart \

217− $BUILD_CHART_EXE || die "build failed"

100+ || die "bench-show build failed"

101+

218 elif test ! -x "$prog_path"

102 elif test ! -x "$prog_path"

219 then

103 then

220 return 1

104 return 1

@@ -228,8 +112,8 @@ build_report_progs() {

228 build_report_prog || exit 1

112 build_report_prog || exit 1

229 local prog

113 local prog

230 prog=$(find_report_prog) || \

114 prog=$(find_report_prog) || \

231− die "Cannot find bench-graph executable"

115+ die "Cannot find bench-show executable"

232− echo "Using bench-graph executable [$prog]"

116+ echo "Using bench-show executable [$prog]"

233 fi

117 fi

234}

118}

235

119

@@ -239,42 +123,12 @@ build_report_progs() {

239# own path currently.

123# own path currently.

240

124

241# The path is dependent on the architecture and cabal version.

125# The path is dependent on the architecture and cabal version.

242−# Use this command to find the exe if this script fails with an error:

243−# find .stack-work/ -type f -name "benchmarks"

244−

245−stack_bench_prog () {

246− local bench_name=$1

247− local bench_prog=`stack path --dist-dir`/build/$bench_name/$bench_name

248− if test -x "$bench_prog"

249− then

250− echo $bench_prog

251− else

252− return 1

253− fi

254−}

255−

256−cabal_bench_prog () {

257− local bench_name=$1

258− local bench_prog=`$WHICH_COMMAND $1`

259− if test -x "$bench_prog"

260− then

261− echo $bench_prog

262− else

263− return 1

264− fi

265−}

266

126

267bench_output_file() {

127bench_output_file() {

268 local bench_name=$1

128 local bench_name=$1

269 echo "charts/$bench_name/results.csv"

129 echo "charts/$bench_name/results.csv"

270}

130}

271

131

272−# $1: command

273−function run_verbose() {

274− echo "$*"

275− bash -c "$*"

276−}

277−

278# --min-duration 0 means exactly one iteration per sample. We use a million

132# --min-duration 0 means exactly one iteration per sample. We use a million

279# iterations in the benchmarking code explicitly and do not use the iterations

133# iterations in the benchmarking code explicitly and do not use the iterations

280# done by the benchmarking tool.

134# done by the benchmarking tool.

@@ -288,25 +142,21 @@ function run_verbose() {

288# We can pass --min-samples value from the command line as second argument

142# We can pass --min-samples value from the command line as second argument

289# after the benchmark name in case we want to use more than one sample.

143# after the benchmark name in case we want to use more than one sample.

290

144

291−run_bench () {

145+# $1: bench name

146+# $2: bench executable

147+target_exe_extra_args () {

292 local bench_name=$1

148 local bench_name=$1

293− local bench_exe=$bench_name

294− local output_file=$(bench_output_file $bench_name)

295− local bench_prog

149+ local bench_prog=$2

296− bench_prog=$($GET_BENCH_PROG $bench_exe) || \

297− die "Cannot find benchmark executable for benchmark $bench_name"

298

150

151+ local output_file=$(bench_output_file $bench_name)

299 mkdir -p `dirname $output_file`

152 mkdir -p `dirname $output_file`

300

153

301− echo "Running benchmark $bench_name ..."

302−

303 local QUICK_OPTS="--quick --min-duration 0"

154 local QUICK_OPTS="--quick --min-duration 0"

304 local SPEED_OPTIONS

155 local SPEED_OPTIONS

305 if test "$LONG" -eq 0

156 if test "$LONG" -eq 0

306 then

157 then

307 if test "$SLOW" -eq 0

158 if test "$SLOW" -eq 0

308 then

159 then

309− export QUICK_MODE

310 if test "$QUICK_MODE" -eq 0

160 if test "$QUICK_MODE" -eq 0

311 then

161 then

312 # default mode, not super quick, not slow

162 # default mode, not super quick, not slow

@@ -325,20 +175,11 @@ run_bench () {

325 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"

175 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"

326 fi

176 fi

327

177

328− export BENCH_EXEC_PATH=$bench_prog

329− export RTS_OPTIONS

330− run_verbose $bench_prog $SPEED_OPTIONS \

178+ echo "$SPEED_OPTIONS \

331 --csvraw=$output_file \

179 --csvraw=$output_file \

332 -v 2 \

180 -v 2 \

333− --measure-with "$BENCH_SH_DIR/bench-exec-one.sh" \

181+ --measure-with "$SCRIPT_DIR/bench-exec-one.sh" \

334− $GAUGE_ARGS || die "Benchmarking failed"

182+ $GAUGE_ARGS"

335−}

336−

337−run_benches() {

338− for i in $1

339− do

340− run_bench $i

341− done

342}

183}

343

184

344run_benches_comparing() {

185run_benches_comparing() {

@@ -358,14 +199,14 @@ run_benches_comparing() {

358 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"

199 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"

359

200

360 $BUILD_BENCH || die "build failed"

201 $BUILD_BENCH || die "build failed"

361− run_benches "$bench_list"

202+ run_targets streamly-benchmarks "$bench_list" target_exe_extra_args

362

203

363 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"

204 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"

364 git checkout "$CANDIDATE" || \

205 git checkout "$CANDIDATE" || \

365 die "Checkout of candidate [$CANDIDATE] commit failed"

206 die "Checkout of candidate [$CANDIDATE] commit failed"

366

207

367 $BUILD_BENCH || die "build failed"

208 $BUILD_BENCH || die "build failed"

368− run_benches "$bench_list"

209+ run_targets streamly-benchmarks "$bench_list" target_exe_extra_args

369 # XXX reset back to the original commit

210 # XXX reset back to the original commit

370}

211}

371

212

@@ -389,7 +230,7 @@ run_measurements() {

389

230

390 if test "$COMPARE" = "0"

231 if test "$COMPARE" = "0"

391 then

232 then

392− run_benches "$bench_list"

233+ run_targets streamly-benchmarks "$bench_list" target_exe_extra_args

393 else

234 else

394 run_benches_comparing "$bench_list"

235 run_benches_comparing "$bench_list"

395 fi

236 fi

@@ -413,9 +254,10 @@ run_reports() {

413# Execution starts here

254# Execution starts here

414#-----------------------------------------------------------------------------

255#-----------------------------------------------------------------------------

415

256

416−cd $BENCH_SH_DIR/..

257+cd $SCRIPT_DIR/..

258+

259+set_common_vars

417

260

418−DEFAULT_BENCHMARKS="$all_grp"

419DEFAULT_FIELDS="allocated bytescopied cputime"

261DEFAULT_FIELDS="allocated bytescopied cputime"

420ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"

262ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw"

421FIELDS=$DEFAULT_FIELDS

263FIELDS=$DEFAULT_FIELDS

@@ -425,53 +267,13 @@ BASE=

425CANDIDATE=

267CANDIDATE=

426

268

427APPEND=0

269APPEND=0

428−SLOW=0

429−QUICK_MODE=0

430LONG=0

270LONG=0

431RAW=0

271RAW=0

432GRAPH=0

272GRAPH=0

433MEASURE=1

273MEASURE=1

434

274

435GAUGE_ARGS=

275GAUGE_ARGS=

436−RTS_OPTIONS=

437BUILD_ONCE=0

276BUILD_ONCE=0

438−USE_STACK=0

439−CABAL_BUILD_FLAGS=""

440−

441−GHC_VERSION=$(ghc --numeric-version)

442−

443−CABAL_EXECUTABLE=cabal

444−

445−# Use branch specific builds if git-cabal is present in PATH

446−BUILD_DIR=dist-newstyle

447−if which git-cabal 2>/dev/null

448−then

449− echo "Using git-cabal for branch specific builds"

450− CABAL_EXECUTABLE=git-cabal

451− BUILD_DIR=$(git-cabal show-builddir)

452−fi

453−

454−# $1: builddir

455−# $2: command to find

456−cabal_which_builddir() {

457− find $1 -type f -path "*${GHC_VERSION}/streamly-benchmarks*/$2" 2>/dev/null

458−}

459−

460−cabal_which() {

461− cabal_which_builddir $BUILD_DIR $1

462−}

463−

464−# chart is expensive to build and usually not required to be rebuilt,

465−# use master branch as fallback

466−cabal_which_report() {

467− local path=$(cabal_which $1)

468− if test -z "$path"

469− then

470− cabal_which_builddir dist-newstyle $1

471− else

472− echo $path

473− fi

474−}

475

277

476#-----------------------------------------------------------------------------

278#-----------------------------------------------------------------------------

477# Read command line

279# Read command line

@@ -482,11 +284,13 @@ do

482 case $1 in

284 case $1 in

483 -h|--help|help) print_help ;;

285 -h|--help|help) print_help ;;

484 # options with arguments

286 # options with arguments

485− --benchmarks) shift; BENCHMARKS=$1; shift ;;

287+ --benchmarks) shift; TARGETS=$1; shift ;;

288+ --targets) shift; TARGETS=$1; shift ;;

486 --fields) shift; FIELDS=$1; shift ;;

289 --fields) shift; FIELDS=$1; shift ;;

487 --base) shift; BASE=$1; shift ;;

290 --base) shift; BASE=$1; shift ;;

488 --candidate) shift; CANDIDATE=$1; shift ;;

291 --candidate) shift; CANDIDATE=$1; shift ;;

489− --cabal-build-flags) shift; CABAL_BUILD_FLAGS=$1; shift ;;

292+ --cabal-build-flags) shift; CABAL_BUILD_OPTIONS=$1; shift ;;

293+ --cabal-build-options) shift; CABAL_BUILD_OPTIONS=$1; shift ;;

490 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;

294 --rtsopts) shift; RTS_OPTIONS=$1; shift ;;

491 # flags

295 # flags

492 --slow) SLOW=1; shift ;;

296 --slow) SLOW=1; shift ;;

@@ -497,6 +301,7 @@ do

497 --long) LONG=1; shift ;;

301 --long) LONG=1; shift ;;

498 --graphs) GRAPH=1; shift ;;

302 --graphs) GRAPH=1; shift ;;

499 --no-measure) MEASURE=0; shift ;;

303 --no-measure) MEASURE=0; shift ;;

304+ --dev-build) RUNNING_DEVBUILD=1; shift ;;

500 --) shift; break ;;

305 --) shift; break ;;

501 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;

306 -*|--*) echo "Unknown flags: $*"; echo; print_help ;;

502 *) break ;;

307 *) break ;;

@@ -504,8 +309,12 @@ do

504done

309done

505GAUGE_ARGS=$*

310GAUGE_ARGS=$*

506

311

312+#-----------------------------------------------------------------------------

313+# Determine targets

314+#-----------------------------------------------------------------------------

315+

507only_real_benchmarks () {

316only_real_benchmarks () {

508− for i in $BENCHMARKS

317+ for i in $TARGETS

509 do

318 do

510 local SKIP=0

319 local SKIP=0

511 for j in $COMPARISONS

320 for j in $COMPARISONS

@@ -522,23 +331,14 @@ only_real_benchmarks () {

522 done

331 done

523}

332}

524

333

525−has_item () {

334+# Requires RUNNING_DEVBUILD var

526− for i in $1

335+source $SCRIPT_DIR/targets.sh

527− do

528− if test "$i" = "$2"

529− then

530− echo "$i"

531− break

532− fi

533− done

534−}

535

336

536−BENCHMARKS_ORIG=$BENCHMARKS

537−if test "$(has_item "$BENCHMARKS_ORIG" help)" = "help"

337+if test "$(has_item "$TARGETS" help)" = "help"

538then

338then

539− list_bench_groups

339+ list_target_groups

540 list_comparisons

340 list_comparisons

541− list_benches

341+ list_targets

542 exit

342 exit

543fi

343fi

544

344

@@ -551,40 +351,24 @@ fi

551

351

552if test "$LONG" -ne 0

352if test "$LONG" -ne 0

553then

353then

554− if test -n "$BENCHMARKS"

354+ if test -n "$TARGETS"

555 then

355 then

556− echo "Cannot specify benchmarks [$BENCHMARKS] with --long"

356+ echo "Cannot specify benchmarks [$TARGETS] with --long"

557 exit

357 exit

558 fi

358 fi

559− BENCHMARKS=$infinite_grp

359+ TARGETS=$infinite_grp

560fi

360fi

561

361

562−BENCHMARKS=$(set_benchmarks)

362+DEFAULT_TARGETS="$(all_grp)"

563−BENCHMARKS_ORIG=$BENCHMARKS

363+TARGETS=$(set_targets)

564−BENCHMARKS=$(only_real_benchmarks)

565−EXECUTABLES=$BENCHMARKS

566

364

567−echo "Using benchmark suites [$BENCHMARKS]"

365+TARGETS_ORIG=$TARGETS

366+TARGETS=$(only_real_benchmarks)

568

367

569−if test "$USE_STACK" = "1"

368+echo "Using benchmark suites [$TARGETS]"

570−then

571− WHICH_COMMAND="stack exec which"

572− WHICH_COMMAND_REPORT="stack exec which"

573− BUILD_CHART_EXE="stack build --flag streamly:dev"

574− GET_BENCH_PROG=stack_bench_prog

575− BUILD_BENCH="stack build $STACK_BUILD_FLAGS --bench --no-run-benchmarks"

576−else

577− # XXX cabal issue "cabal v2-exec which" cannot find benchmark/test executables

578− #WHICH_COMMAND="cabal v2-exec which"

579− WHICH_COMMAND=cabal_which

580− WHICH_COMMAND_REPORT=cabal_which_report

581− BUILD_CHART_EXE="$CABAL_EXECUTABLE v2-build --flags dev chart"

582− GET_BENCH_PROG=cabal_bench_prog

583− BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_FLAGS --enable-benchmarks"

584−fi

585

369

586#-----------------------------------------------------------------------------

370#-----------------------------------------------------------------------------

587−# Build stuff

371+# Build reporting utility

588#-----------------------------------------------------------------------------

372#-----------------------------------------------------------------------------

589

373

590# We need to build the report progs first at the current (latest) commit before

374# We need to build the report progs first at the current (latest) commit before

@@ -592,23 +376,16 @@ fi

592build_report_progs

376build_report_progs

593

377

594#-----------------------------------------------------------------------------

378#-----------------------------------------------------------------------------

595−# Run benchmarks

379+# Build and run targets

596#-----------------------------------------------------------------------------

380#-----------------------------------------------------------------------------

597

381

382+BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-benchmarks"

598if test "$MEASURE" = "1"

383if test "$MEASURE" = "1"

599then

384then

600− echo $BUILD_BENCH

601− if test "$USE_STACK" = "1"

602− then

603− $BUILD_BENCH || die "build failed"

604− else

605− for c in $EXECUTABLES

606− do

607− COMPONENTS+="streamly-benchmarks:bench:$c "

385+ run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"

608− done

386+ export QUICK_MODE

609− $BUILD_BENCH $COMPONENTS || die "build failed"

387+ export RTS_OPTIONS

610− fi

611− run_measurements "$BENCHMARKS"

388+ run_measurements "$TARGETS"

612fi

389fi

613

390

614#-----------------------------------------------------------------------------

391#-----------------------------------------------------------------------------

@@ -618,7 +395,7 @@ fi

618COMPARISON_REPORTS=""

395COMPARISON_REPORTS=""

619for i in $COMPARISONS

396for i in $COMPARISONS

620do

397do

621− if test "$(has_item "$BENCHMARKS_ORIG" $i)" = $i

398+ if test "$(has_item "$TARGETS_ORIG" $i)" = $i

622 then

399 then

623 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"

400 COMPARISON_REPORTS="$COMPARISON_REPORTS $i"

624 mkdir -p "charts/$i"

401 mkdir -p "charts/$i"

@@ -634,6 +411,6 @@ done

634

411

635if test "$RAW" = "0"

412if test "$RAW" = "0"

636then

413then

637− run_reports "$BENCHMARKS"

414+ run_reports "$TARGETS"

638 run_reports "$COMPARISON_REPORTS"

415 run_reports "$COMPARISON_REPORTS"

639fi

416fi

bin/build-lib.sh+19−8

@@ -50,9 +50,15 @@ dev_build () {

50all_grp () {

51 { for i in $GROUP_TARGETS

52 do

53− eval "echo \$$i"

53+ for j in $(eval "echo \$$i")

54+ do

55+ echo $j

56+ done

57+ done

58+ for i in $INDIVIDUAL_TARGETS

59+ do

60+ echo $i

54 done

61 done

55− echo $INDIVIDUAL_TARGETS

56 } | sort | uniq

62 } | sort | uniq

57}

63}

@@ -112,7 +118,7 @@ set_targets() {

112# $1: package name

118# $1: package name

113# $2: target

119# $2: target

114cabal_target_prog () {

120cabal_target_prog () {

115− local target_prog=`$WHICH_COMMAND $1 $2`

121+ local target_prog=`cabal_which $1 $2`

116 if test -x "$target_prog"

122 if test -x "$target_prog"

117 then

123 then

118 echo $target_prog

124 echo $target_prog

@@ -182,26 +188,31 @@ run_build () {

182

188

183# $1: package name

189# $1: package name

184# $2: target

190# $2: target

191+# $3: args generator func

185run_target () {

192run_target () {

186 local package_name=$1

193 local package_name=$1

187 local target_name=$2

194 local target_name=$2

188− local target_exe=$target_name

195+ local extra_args=$3

196+

189 local target_prog

197 local target_prog

190− target_prog=$(cabal_target_prog $package_name $target_exe) || \

198+ target_prog=$(cabal_target_prog $package_name $target_name) || \

191 die "Cannot find executable for target $target_name"

199 die "Cannot find executable for target $target_name"

192

200

193 echo "Running executable $target_name ..."

201 echo "Running executable $target_name ..."

194− TARGET_EXE_EXTRA_ARGS="$RTS_OPTIONS"

195

202

196− run_verbose $target_prog $TARGET_EXE_EXTRA_ARGS $TARGET_EXE_ARGS \

203+ # Needed by bench-exec-one.sh

204+ export BENCH_EXEC_PATH=$target_prog

205+

206+ run_verbose $target_prog $($extra_args $target_name $target_prog) \

197 || die "Target exe failed"

207 || die "Target exe failed"

198}

208}

199

209

200# $1: package name

210# $1: package name

201# $2: targets

211# $2: targets

212+# $3: args generator func

202run_targets() {

213run_targets() {

203 for i in $2

214 for i in $2

204 do

215 do

205− run_target $1 $i

216+ run_target $1 $i $3

206 done

217 done

207}

218}

bin/targets.sh+27−3

@@ -28,7 +28,8 @@ prelude_other_grp="\

28 $(test_only $(dev_build Prelude.Rate)) \

29 `bench_only Prelude.Rate` \

30 `test_only Prelude.Fold` \

31− Prelude.Concurrent \

31+ `test_only Prelude.Concurrent` \

32+ $(bench_only $(dev_build Prelude.Concurrent)) \

32 `bench_only Prelude.Adaptive`"

33 `bench_only Prelude.Adaptive`"

34array_grp="\

35array_grp="\

@@ -47,8 +48,8 @@ parser_grp="\

47 Data.Parser"

48 Data.Parser"

49list_grp="\

50list_grp="\

50− Data.List.Base \

51+ `test_only Data.List.Base` \

51− Data.List"

52+ `test_only Data.List`"

53#------------------------------------------------------------------------------

54#------------------------------------------------------------------------------

54# Streaming vs non-streaming

55# Streaming vs non-streaming

@@ -61,6 +62,29 @@ infinite_grp="\

61 $prelude_concurrent_grp \

62 $prelude_concurrent_grp \

62 `bench_only Prelude.Rate`"

63 `bench_only Prelude.Rate`"

65+#------------------------------------------------------------------------------

66+# Benchmark comparison groups

67+#------------------------------------------------------------------------------

68+

69+# *_cmp denotes a comparison benchmark, the benchmarks provided in *_cmp

70+# variables are compared with each other

71+base_stream_cmp="Data.Stream.StreamD Data.Stream.StreamK"

72+serial_wserial_cmp="Prelude.Serial Prelude.WSerial"

73+serial_async_cmp="Prelude.Serial Prelude.Async"

74+concurrent_cmp="Prelude.Async Prelude.WAsync Prelude.Ahead Prelude.Parallel"

75+array_cmp="Memory.Array Data.Array.Prim Data.Array Data.Array.Prim.Pinned"

76+pinned_array_cmp="Memory.Array Data.Array.Prim.Pinned"

77+base_parser_cmp=$base_parser_grp

78+

79+COMPARISONS="\

80+ base_stream_cmp \

81+ serial_wserial_cmp \

82+ serial_async_cmp \

83+ concurrent_cmp \

84+ array_cmp \

85+ pinned_array_cmp \

86+ base_parser_cmp"

87+

64#------------------------------------------------------------------------------

88#------------------------------------------------------------------------------

65# All test/benchmark modules must be in at least one of these

89# All test/benchmark modules must be in at least one of these

66#------------------------------------------------------------------------------

90#------------------------------------------------------------------------------

bin/test.sh+19−2

@@ -75,7 +75,24 @@ echo "Using targets [$TARGETS]"

75# Build targets

76#-----------------------------------------------------------------------------

78−WHICH_COMMAND=cabal_which

78+test_exe_rts_opts () {

79+ case "$1" in

80+ *) echo -n "-K8M -M64M" ;;

81+ esac

82+}

83+

84+# $1: bench name

85+# $2: bench executable

86+target_exe_extra_args () {

87+ local bench_name=$1

88+ local bench_prog=$2

89+

90+ echo "+RTS \

91+ $(test_exe_rts_opts $(basename $bench_prog)) \

92+ $RTS_OPTIONS \

93+ -RTS"

94+}

95+

79BUILD_TEST="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-tests"

96BUILD_TEST="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-tests"

80run_build "$BUILD_TEST" streamly-tests test "$TARGETS"

97run_build "$BUILD_TEST" streamly-tests test "$TARGETS"

@@ -83,4 +100,4 @@ run_build "$BUILD_TEST" streamly-tests test "$TARGETS"

83# Run targets

100# Run targets

84#-----------------------------------------------------------------------------

101#-----------------------------------------------------------------------------

102

86−run_targets streamly-tests "$TARGETS"

103+run_targets streamly-tests "$TARGETS" target_exe_extra_args

test/streamly-tests.cabal+1

@@ -53,6 +53,7 @@ common compile-options

53 -Wredundant-constraints

54 -Wnoncanonical-monad-instances

55 -Rghc-timing

56+ -rtsopts

57 if flag(has-llvm)

58 if flag(has-llvm)

58 ghc-options: -fllvm

59 ghc-options: -fllvm