c2fa796 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Generalize group/comparative benchmarking

c2fa796Harendra Kumar6 years ago

bench.sh+113−59

compare with master

unifiedsplit

@@ -1,38 +1,53 @@

1#!/bin/bash

3+base_grp="\

4+ Data.Stream.StreamD \

5+ Data.Stream.StreamK \

6+ Data.Stream.StreamDK"

3SERIAL_O_1="linear"

7SERIAL_O_1="linear"

4SERIAL_O_n="serial-o-n-heap serial-o-n-stack serial-o-n-space"

8SERIAL_O_n="serial-o-n-heap serial-o-n-stack serial-o-n-space"

5−FOLD_BENCHMARKS="fold-o-1-space fold-o-n-heap"

6UNFOLD_BENCHMARKS="unfold-o-1-space unfold-o-n-space"

9UNFOLD_BENCHMARKS="unfold-o-1-space unfold-o-n-space"

7−

10+serial_grp="\

8−SERIAL_BENCHMARKS="\

9− Data.Stream.StreamD \

10− Data.Stream.StreamK \

11− Data.Stream.StreamDK \

12 $SERIAL_O_1 \

11 $SERIAL_O_1 \

13 $SERIAL_O_n \

12 $SERIAL_O_n \

14− $FOLD_BENCHMARKS"

13+ Data.Fold"

14+

15# parallel benchmark-suite is separated because we run it with a higher

16# heap size limit.

17−CONCURRENT_BENCHMARKS="linear-async linear-rate nested-concurrent parallel concurrent adaptive"

17+concurrent_grp="linear-async linear-rate nested-concurrent parallel concurrent adaptive"

18−ARRAY_BENCHMARKS="Memory.Array Data.Array Data.Prim.Array Data.SmallArray"

18+array_grp="Memory.Array Data.Array Data.Prim.Array Data.SmallArray"

20# XXX We can include SERIAL_O_1 here once "base" also supports --stream-size

21−INFINITE_BENCHMARKS="linear linear-async linear-rate nested-concurrent"

21+infinite_grp="linear linear-async linear-rate nested-concurrent"

22−FINITE_BENCHMARKS="$SERIAL_O_n $ARRAY_BENCHMARKS fileio parallel concurrent adaptive"

22+finite_grp="$SERIAL_O_n $array_grp fileio parallel concurrent adaptive"

24# Benchmarks that take long time per iteration must run fewer iterations to

25# finish in reasonable time.

26QUICK_BENCHMARKS="linear-rate concurrent adaptive fileio"

27−VIRTUAL_BENCHMARKS="array-cmp"

29−ALL_BENCHMARKS="$SERIAL_BENCHMARKS $CONCURRENT_BENCHMARKS $ARRAY_BENCHMARKS $VIRTUAL_BENCHMARKS"

28+# *_cmp denotes a comparison benchmarks, the benchmarks provided in *_cmp

29+# variables are compared with each other

30+array_cmp="Memory.Array Data.Prim.Array Data.Array"

31+base_cmp="Data.Stream.StreamD Data.Stream.StreamK"

32+COMPARISONS="array_cmp base_cmp"

33+

34+all_grp="\

35+ $base_grp \

36+ $serial_grp \

37+ $concurrent_grp \

38+ $array_grp"

39+

40+ALL_BENCH_GROUPS="\

41+ all_grp \

42+ serial_grp \

43+ concurrent_grp \

44+ array_grp \

45+ infinite_grp \

46+ finite_grp"

31# RTS options that go inside +RTS and -RTS while running the benchmark.

48# RTS options that go inside +RTS and -RTS while running the benchmark.

32bench_rts_opts () {

49bench_rts_opts () {

33 case "$1" in

50 case "$1" in

34− "fold-o-1-space") echo -n "-T -K36K -M16M" ;;

35− "fold-o-n-heap") echo -n "-T -K36K -M128M" ;;

36 "unfold-o-1-space") echo -n "-T -K36K -M16M" ;;

51 "unfold-o-1-space") echo -n "-T -K36K -M16M" ;;

37 "unfold-o-n-space") echo -n "-T -K32M -M64M" ;;

52 "unfold-o-n-space") echo -n "-T -K32M -M64M" ;;

38 "linear") echo -n "-T -K36K -M16M" ;;

53 "linear") echo -n "-T -K36K -M16M" ;;

@@ -80,31 +95,57 @@ bench_gauge_opts () {

80}

95}

82list_benches () {

97list_benches () {

83− for i in $ALL_BENCHMARKS

98+ echo "Individual benchmarks:"

99+ for i in $all_grp

100+ do

101+ echo "$i"

102+ done

103+ echo

104+}

105+

106+list_bench_groups () {

107+ echo "Benchmark groups:"

108+ for i in $ALL_BENCH_GROUPS

109+ do

110+ echo -n "$i ["

111+ eval "echo -n \$$i"

112+ echo "]"

113+ done

114+ echo

115+}

116+

117+list_comparisons () {

118+ echo "Comparison groups:"

119+ for i in $COMPARISONS

84 do

120 do

85− echo -n "|$i"

121+ echo -n "$i ["

122+ eval "echo -n \$$i"

123+ echo "]"

86 done

124 done

125+ echo

87}

126}

127

89print_help () {

128print_help () {

90 echo "Usage: $0 "

129 echo "Usage: $0 "

130+ echo " [--benchmarks <"bench1 bench2 ..." | ?>]"

92− echo " [--group-diff]"

93 echo " [--graphs]"

131 echo " [--graphs]"

94 echo " [--no-measure]"

132 echo " [--no-measure]"

95 echo " [--append]"

133 echo " [--append]"

96 echo " [--long]"

134 echo " [--long]"

97 echo " [--slow]"

135 echo " [--slow]"

98 echo " [--quick]"

136 echo " [--quick]"

99− echo " [--compare] [--base commit] [--candidate commit]"

100− echo " [--cabal-build-flags]"

137+ echo " [--cabal-build-flags <flag>]"

138+ echo " [--compare] [--base <commit>] [--candidate <commit>]"

101 echo " -- <gauge options or benchmarks>"

139 echo " -- <gauge options or benchmarks>"

102 echo

140 echo

103− echo "Multiple benchmarks can be specified as a space separated list"

141+ echo "--benchmarks: benchmarks to run, use '?' for list of benchmarks"

104− echo " e.g. --benchmarks \"linear nested\""

142+ echo "--graphs: Generate graphical reports"

105− echo

143+ echo "--no-measure: Don't run benchmarks, run reports from previous results"

106− echo "--group-diff is used to compare groups within a single benchmark"

144+ echo "--append: Don't overwrite previous results, append for comparison"

107− echo " e.g. StreamD vs StreamK in base benchmark."

145+ echo "--long: Use much longer stream size for infinite stream benchmarks"

146+ echo "--slow: Slightly more accurate results at the expense of speed"

147+ echo "--quick: Faster results, useful for longer benchmarks"

148+ echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"

108 echo

149 echo

109 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

150 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

110 echo "commit is generated, in the 'charts' directory."

151 echo "commit is generated, in the 'charts' directory."

@@ -128,13 +169,8 @@ set_benchmarks() {

128 for i in $(echo $BENCHMARKS)

169 for i in $(echo $BENCHMARKS)

129 do

170 do

130 case $i in

171 case $i in

131− ALL) echo -n $ALL_BENCHMARKS ;;

172+ *_grp) eval "echo -n \$${i}" ;;

132− SERIAL) echo -n $SERIAL_BENCHMARKS ;;

173+ *_cmp) eval "echo -n \$${i} $i" ;;

133− CONCURRENT) echo -n $CONCURRENT_BENCHMARKS ;;

134− ARRAY) echo -n $ARRAY_BENCHMARKS ;;

135− INFINITE) echo -n $INFINITE_BENCHMARKS ;;

136− FINITE) echo -n $FINITE_BENCHMARKS ;;

137− array-cmp) echo -n "$ARRAY_BENCHMARKS array-cmp" ;;

138 *) echo -n $i ;;

174 *) echo -n $i ;;

139 esac

175 esac

140 echo -n " "

176 echo -n " "

@@ -360,7 +396,6 @@ run_reports() {

360 do

396 do

361 echo "Generating reports for ${i}..."

397 echo "Generating reports for ${i}..."

362 $prog $(test "$GRAPH" = 1 && echo "--graphs") \

398 $prog $(test "$GRAPH" = 1 && echo "--graphs") \

363− $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \

364 --benchmark $i

399 --benchmark $i

365 done

400 done

366}

401}

@@ -370,7 +405,6 @@ run_reports() {

370#-----------------------------------------------------------------------------

405#-----------------------------------------------------------------------------

371

406

372DEFAULT_BENCHMARKS="linear"

407DEFAULT_BENCHMARKS="linear"

373−GROUP_DIFF=0

374

408

375COMPARE=0

409COMPARE=0

376BASE=

410BASE=

@@ -415,7 +449,6 @@ do

415 --raw) RAW=1; shift ;;

449 --raw) RAW=1; shift ;;

416 --append) APPEND=1; shift ;;

450 --append) APPEND=1; shift ;;

417 --long) LONG=1; shift ;;

451 --long) LONG=1; shift ;;

418− --group-diff) GROUP_DIFF=1; shift ;;

419 --graphs) GRAPH=1; shift ;;

452 --graphs) GRAPH=1; shift ;;

420 --no-measure) MEASURE=0; shift ;;

453 --no-measure) MEASURE=0; shift ;;

421 --) shift; break ;;

454 --) shift; break ;;

@@ -425,17 +458,11 @@ do

425done

458done

426GAUGE_ARGS=$*

459GAUGE_ARGS=$*

427

460

428−BENCHMARKS=$(set_benchmarks)

429−if test "$LONG" -ne 0

430−then

431− BENCHMARKS=$INFINITE_BENCHMARKS

432−fi

433−

434only_real_benchmarks () {

461only_real_benchmarks () {

435 for i in $BENCHMARKS

462 for i in $BENCHMARKS

436 do

463 do

437 local SKIP=0

464 local SKIP=0

438− for j in $VIRTUAL_BENCHMARKS

465+ for j in $COMPARISONS

439 do

466 do

440 if test $i == $j

467 if test $i == $j

441 then

468 then

@@ -456,12 +483,6 @@ proper_executables () {

456 done

483 done

457}

484}

458

485

459−

460−BENCHMARKS_ORIG=$BENCHMARKS

461−BENCHMARKS=$(only_real_benchmarks)

462−EXECUTABLES=$(proper_executables)

463−echo "Using benchmark suites [$BENCHMARKS]"

464−

465has_benchmark () {

486has_benchmark () {

466 for i in $BENCHMARKS_ORIG

487 for i in $BENCHMARKS_ORIG

467 do

488 do

@@ -473,6 +494,32 @@ has_benchmark () {

473 done

494 done

474}

495}

475

496

497+BENCHMARKS_ORIG=$BENCHMARKS

498+if test "$(has_benchmark help)" = "help"

499+then

500+ list_bench_groups

501+ list_comparisons

502+ list_benches

503+ exit

504+fi

505+

506+if test "$LONG" -ne 0

507+then

508+ if test -n "$BENCHMARKS"

509+ then

510+ echo "Cannot specify benchmarks [$BENCHMARKS] with --long"

511+ exit

512+ fi

513+ BENCHMARKS=$infinite_grp

514+fi

515+

516+BENCHMARKS=$(set_benchmarks)

517+BENCHMARKS_ORIG=$BENCHMARKS

518+BENCHMARKS=$(only_real_benchmarks)

519+EXECUTABLES=$(proper_executables)

520+

521+echo "Using benchmark suites [$BENCHMARKS]"

522+

476if test "$USE_STACK" = "1"

523if test "$USE_STACK" = "1"

477then

524then

478 WHICH_COMMAND="stack exec which"

525 WHICH_COMMAND="stack exec which"

@@ -516,18 +563,25 @@ fi

516# Run reports

563# Run reports

517#-----------------------------------------------------------------------------

564#-----------------------------------------------------------------------------

518

565

519−VIRTUAL_REPORTS=""

566+COMPARISON_REPORTS=""

567+for i in $COMPARISONS

568+do

520−if test "$(has_benchmark 'array-cmp')" = "array-cmp"

569+ if test "$(has_benchmark $i)" = $i

521−then

570+ then

522− VIRTUAL_REPORTS="$VIRTUAL_REPORTS array-cmp"

571+ COMPARISON_REPORTS="$COMPARISON_REPORTS $i"

523− mkdir -p "charts/array-cmp"

572+ mkdir -p "charts/$i"

573+ constituents=$(eval "echo -n \$${i}")

574+ dest_file="charts/$i/results.csv"

575+ : > $dest_file

576+ for j in $constituents

577+ do

524− cat "charts/Memory.Array/results.csv" \

578+ cat "charts/$j/results.csv" >> $dest_file

525− "charts/Data.Prim.Array/results.csv" \

579+ done

526− "charts/Data.Array/results.csv" > "charts/array-cmp/results.csv"

527−fi

580+ fi

581+done

528

582

529if test "$RAW" = "0"

583if test "$RAW" = "0"

530then

584then

531 run_reports "$BENCHMARKS"

585 run_reports "$BENCHMARKS"

532− run_reports "$VIRTUAL_REPORTS"

586+ run_reports "$COMPARISON_REPORTS"

533fi

587fi

benchmark/Chart.hs+16−143

@@ -31,26 +31,16 @@ data BenchType

31 | ParserD

32 | ParserK

33 | Parser

34− | Base

35 | FileIO

34 | FileIO

36− | Array

37− | ArrayCmp

38− | UnpinnedArray

39− | SmallArray

40− | PrimArray

41 | Concurrent

35 | Concurrent

42 | Parallel

36 | Parallel

43 | Adaptive

37 | Adaptive

44− | FoldO1Space

38+ | Compare String

45− | FoldOnHeap

46− | UnfoldO1Space

47− | UnfoldOnSpace

48 | Standard String

39 | Standard String

49 deriving Show

40 deriving Show

51data Options = Options

42data Options = Options

52 { genGraphs :: Bool

43 { genGraphs :: Bool

53− , groupDiff :: Bool

54 , benchType :: BenchType

44 , benchType :: BenchType

55 } deriving Show

45 } deriving Show

@@ -60,10 +50,6 @@ setGenGraphs val = do

60 (args, opts) <- get

50 (args, opts) <- get

61 put (args, opts { genGraphs = val })

51 put (args, opts { genGraphs = val })

63−setGroupDiff val = do

64− (args, opts) <- get

65− put (args, opts { groupDiff = val })

66−

67setBenchType val = do

53setBenchType val = do

68 (args, opts) <- get

54 (args, opts) <- get

69 put (args, opts { benchType = val })

55 put (args, opts { benchType = val })

@@ -87,20 +73,11 @@ parseBench = do

87 Just "parserD" -> setBenchType ParserD

73 Just "parserD" -> setBenchType ParserD

88 Just "parserK" -> setBenchType ParserK

74 Just "parserK" -> setBenchType ParserK

89 Just "parser" -> setBenchType Parser

75 Just "parser" -> setBenchType Parser

90− Just "base" -> setBenchType Base

91 Just "fileio" -> setBenchType FileIO

76 Just "fileio" -> setBenchType FileIO

92− Just "array-cmp" -> setBenchType ArrayCmp

93− Just "array" -> setBenchType Array

94− Just "unpinned-array" -> setBenchType UnpinnedArray

95− Just "small-array" -> setBenchType SmallArray

96− Just "prim-array" -> setBenchType PrimArray

97 Just "concurrent" -> setBenchType Concurrent

77 Just "concurrent" -> setBenchType Concurrent

98 Just "parallel" -> setBenchType Parallel

78 Just "parallel" -> setBenchType Parallel

99 Just "adaptive" -> setBenchType Adaptive

79 Just "adaptive" -> setBenchType Adaptive

100− Just "fold-o-1-space" -> setBenchType FoldO1Space

80+ Just str | "_cmp" `isSuffixOf` str -> setBenchType (Compare str)

101− Just "fold-o-n-heap" -> setBenchType FoldOnHeap

102− Just "unfold-o-1-space" -> setBenchType UnfoldO1Space

103− Just "unfold-o-n-space" -> setBenchType UnfoldOnSpace

104 Just str -> setBenchType (Standard str)

81 Just str -> setBenchType (Standard str)

105 Nothing -> do

82 Nothing -> do

106 liftIO $ putStrLn "please provide a benchmark type "

83 liftIO $ putStrLn "please provide a benchmark type "

@@ -119,7 +96,6 @@ parseOptions = do

119 parseOpt opt =

96 parseOpt opt =

120 case opt of

97 case opt of

121 "--graphs" -> setGenGraphs True

98 "--graphs" -> setGenGraphs True

122− "--group-diff" -> setGroupDiff True

123 "--benchmark" -> parseBench

99 "--benchmark" -> parseBench

124 str -> do

100 str -> do

125 liftIO $ putStrLn $ "Unrecognized option " <> str

101 liftIO $ putStrLn $ "Unrecognized option " <> str

@@ -239,71 +215,21 @@ makeGraphs name cfg@Config{..} inputFile =

239-- Arrays

215-- Arrays

240------------------------------------------------------------------------------

216------------------------------------------------------------------------------

241

217

242−showArrayComparisons Options{..} cfg inp out =

218+showComparisons Options{..} cfg inp out =

243− let cfg' = cfg { classifyBenchmark = classifyArray }

219+ let cfg1 = cfg { classifyBenchmark = classifyComparison }

244− in if genGraphs

220+ in if genGraphs

245− then ignoringErr $ graph inp "Arrays Comparison"

221+ then ignoringErr $ graph inp "comparison"

246− cfg' { outputDir = Just out

222+ cfg1 { outputDir = Just out

247 , presentation = Groups Absolute

223 , presentation = Groups Absolute

248 }

224 }

249− else ignoringErr $ report inp Nothing cfg'

225+ else ignoringErr $ report inp Nothing cfg1

250

226

251 where

227 where

252

228

253− classifyArray b

229+ classifyComparison b = Just $

254− -- SmallArray uses a small number of elements therefore cannot be

230+ ( takeWhile (/= '/') b

255− -- compared

231+ , dropWhile (== '/') $ dropWhile (/= '/') b

256− -- | "SmallArray/" `isPrefixOf` b = ("SmallArray",) <$> stripPrefix "SmallArray/" b

232+ )

257− | "Data.Prim.Array/" `isPrefixOf` b = ("Data.Prim.Array",) <$> stripPrefix "Data.Prim.Array/" b

258− | "Data.Array/" `isPrefixOf` b = ("Data.Array",) <$> stripPrefix "Data.Array/" b

259− | "array/" `isPrefixOf` b = ("array",) <$> stripPrefix "array/" b

260− | otherwise = Nothing

261−

262−------------------------------------------------------------------------------

263−-- Reports/Charts for base streams

264−------------------------------------------------------------------------------

265−

266−showStreamDVsK Options{..} cfg inp out =

267− let cfg' = cfg { classifyBenchmark = classifyBase }

268− in if genGraphs

269− then ignoringErr $ graph inp "streamD-vs-streamK"

270− cfg' { outputDir = Just out

271− , presentation = Groups Absolute

272− }

273− else ignoringErr $ report inp Nothing cfg'

274−

275− where

276−

277− classifyBase b

278− | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b

279− | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b

280− | otherwise = Nothing

281−

282−showStreamD Options{..} cfg inp out =

283− let cfg' = cfg { classifyBenchmark = classifyStreamD }

284− in if genGraphs

285− then ignoringErr $ graph inp "streamD"

286− cfg' {outputDir = Just out}

287− else ignoringErr $ report inp Nothing cfg'

288−

289− where

290−

291− classifyStreamD b

292− | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b

293− | otherwise = Nothing

294−

295−showStreamK Options{..} cfg inp out =

296− let cfg' = cfg { classifyBenchmark = classifyStreamK }

297− in if genGraphs

298− then ignoringErr $ graph inp "streamK"

299− cfg' {outputDir = Just out}

300− else ignoringErr $ report inp Nothing cfg'

301−

302− where

303−

304− classifyStreamK b

305− | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b

306− | otherwise = Nothing

307

233

308------------------------------------------------------------------------------

234------------------------------------------------------------------------------

309-- text reports

235-- text reports

@@ -385,30 +311,6 @@ main = do

385 (makeGraphs "fileIO")

311 (makeGraphs "fileIO")

386 "charts/fileio/results.csv"

312 "charts/fileio/results.csv"

387 "charts/fileio"

313 "charts/fileio"

388− Array -> benchShow opts cfg

389− { title = Just "Array" }

390− (makeGraphs "array")

391− "charts/array/results.csv"

392− "charts/array"

393− UnpinnedArray -> benchShow opts cfg

394− { title = Just "Unpinned Array" }

395− (makeGraphs "unpinned-array")

396− "charts/unpinned-array/results.csv"

397− "charts/unpinned-array"

398− SmallArray -> benchShow opts cfg

399− { title = Just "Small Array" }

400− (makeGraphs "small-array")

401− "charts/small-array/results.csv"

402− "charts/small-array"

403− PrimArray -> benchShow opts cfg

404− { title = Just "Prim Array" }

405− (makeGraphs "prim-array")

406− "charts/prim-array/results.csv"

407− "charts/prim-array"

408− ArrayCmp -> showArrayComparisons opts cfg

409− { title = Just "Arrays Comparison" }

410− "charts/array-cmp/results.csv"

411− "charts/array-cmp"

412 Concurrent -> benchShow opts cfg

314 Concurrent -> benchShow opts cfg

413 { title = Just "Concurrent Ops" }

315 { title = Just "Concurrent Ops" }

414 (makeGraphs "Concurrent")

316 (makeGraphs "Concurrent")

@@ -424,39 +326,10 @@ main = do

424 (makeGraphs "adaptive")

326 (makeGraphs "adaptive")

425 "charts/adaptive/results.csv"

327 "charts/adaptive/results.csv"

426 "charts/adaptive"

328 "charts/adaptive"

427− Base -> do

329+ Compare str -> showComparisons opts cfg

428− let cfg' = cfg { title = Just "Base stream" }

429− if groupDiff

430− then showStreamDVsK opts cfg'

431− "charts/base/results.csv"

432− "charts/base"

433− else do

434− showStreamD opts cfg'

435− "charts/base/results.csv"

436− "charts/base"

437− showStreamK opts cfg'

438− "charts/base/results.csv"

439− "charts/base"

440− FoldO1Space -> benchShow opts cfg

441− { title = Just "Fold O(1) Space" }

330+ { title = Just $ str }

442− (makeGraphs "fold-o-1-space")

443− "charts/fold-o-1-space/results.csv"

444− "charts/fold-o-1-space"

445− FoldOnHeap -> benchShow opts cfg

446− { title = Just "Fold O(n) Heap" }

447− (makeGraphs "fold-o-n-heap")

448− "charts/fold-o-n-heap/results.csv"

331+ ("charts/" ++ str ++ "/results.csv")

449− "charts/fold-o-n-heap"

332+ ("charts/" ++ str)

450− UnfoldO1Space -> benchShow opts cfg

451− { title = Just "Unfold O(1) Space" }

452− (makeGraphs "unfold-o-1-space")

453− "charts/unfold-o-1-space/results.csv"

454− "charts/unfold-o-1-space"

455− UnfoldOnSpace -> benchShow opts cfg

456− { title = Just "Unfold O(n) Space" }

457− (makeGraphs "unfold-o-n-space")

458− "charts/unfold-o-n-space/results.csv"

459− "charts/unfold-o-n-space"

460 Standard str -> benchShow opts cfg

333 Standard str -> benchShow opts cfg

461 { title = Just str }

334 { title = Just str }

462 (makeGraphs str)

335 (makeGraphs str)