c2fa796 · streamly · git.iko.soy

git.iko.soy/streamly

Generalize group/comparative benchmarking

commitc2fa79613e1620a786205c5278bbd6e1726092af authorHarendra Kumar <harendra@composewell.com> dateWed 3 Jun 2020, 10:45 (6 years ago) parentf72d007

2 files changed+129−202

compare with master

unifiedsplit

bench.sh+113−59 benchmark/Chart.hs+16−143

bench.sh+113−59

@@ -1,38 +1,53 @@

11#!/bin/bash

3+base_grp="\

4+ Data.Stream.StreamD \

5+ Data.Stream.StreamK \

6+ Data.Stream.StreamDK"

37SERIAL_O_1="linear"

48SERIAL_O_n="serial-o-n-heap serial-o-n-stack serial-o-n-space"

5−FOLD_BENCHMARKS="fold-o-1-space fold-o-n-heap"

69UNFOLD_BENCHMARKS="unfold-o-1-space unfold-o-n-space"

7−

8−SERIAL_BENCHMARKS="\

9− Data.Stream.StreamD \

10− Data.Stream.StreamK \

11− Data.Stream.StreamDK \

10+serial_grp="\

1211 $SERIAL_O_1 \

1312 $SERIAL_O_n \

14− $FOLD_BENCHMARKS"

13+ Data.Fold"

14+

1515# parallel benchmark-suite is separated because we run it with a higher

1616# heap size limit.

17−CONCURRENT_BENCHMARKS="linear-async linear-rate nested-concurrent parallel concurrent adaptive"

18−ARRAY_BENCHMARKS="Memory.Array Data.Array Data.Prim.Array Data.SmallArray"

17+concurrent_grp="linear-async linear-rate nested-concurrent parallel concurrent adaptive"

18+array_grp="Memory.Array Data.Array Data.Prim.Array Data.SmallArray"

1919

2020# XXX We can include SERIAL_O_1 here once "base" also supports --stream-size

21−INFINITE_BENCHMARKS="linear linear-async linear-rate nested-concurrent"

22−FINITE_BENCHMARKS="$SERIAL_O_n $ARRAY_BENCHMARKS fileio parallel concurrent adaptive"

21+infinite_grp="linear linear-async linear-rate nested-concurrent"

22+finite_grp="$SERIAL_O_n $array_grp fileio parallel concurrent adaptive"

2323

2424# Benchmarks that take long time per iteration must run fewer iterations to

2525# finish in reasonable time.

2626QUICK_BENCHMARKS="linear-rate concurrent adaptive fileio"

27−VIRTUAL_BENCHMARKS="array-cmp"

2827

29−ALL_BENCHMARKS="$SERIAL_BENCHMARKS $CONCURRENT_BENCHMARKS $ARRAY_BENCHMARKS $VIRTUAL_BENCHMARKS"

28+# *_cmp denotes a comparison benchmarks, the benchmarks provided in *_cmp

29+# variables are compared with each other

30+array_cmp="Memory.Array Data.Prim.Array Data.Array"

31+base_cmp="Data.Stream.StreamD Data.Stream.StreamK"

32+COMPARISONS="array_cmp base_cmp"

33+

34+all_grp="\

35+ $base_grp \

36+ $serial_grp \

37+ $concurrent_grp \

38+ $array_grp"

39+

40+ALL_BENCH_GROUPS="\

41+ all_grp \

42+ serial_grp \

43+ concurrent_grp \

44+ array_grp \

45+ infinite_grp \

46+ finite_grp"

3047

3148# RTS options that go inside +RTS and -RTS while running the benchmark.

3249bench_rts_opts () {

3350 case "$1" in

34− "fold-o-1-space") echo -n "-T -K36K -M16M" ;;

35− "fold-o-n-heap") echo -n "-T -K36K -M128M" ;;

3651 "unfold-o-1-space") echo -n "-T -K36K -M16M" ;;

3752 "unfold-o-n-space") echo -n "-T -K32M -M64M" ;;

3853 "linear") echo -n "-T -K36K -M16M" ;;

@@ -80,31 +95,57 @@ bench_gauge_opts () {

8095}

8196

8297list_benches () {

83− for i in $ALL_BENCHMARKS

98+ echo "Individual benchmarks:"

99+ for i in $all_grp

100+ do

101+ echo "$i"

102+ done

103+ echo

104+}

105+

106+list_bench_groups () {

107+ echo "Benchmark groups:"

108+ for i in $ALL_BENCH_GROUPS

109+ do

110+ echo -n "$i ["

111+ eval "echo -n \$$i"

112+ echo "]"

113+ done

114+ echo

115+}

116+

117+list_comparisons () {

118+ echo "Comparison groups:"

119+ for i in $COMPARISONS

84120 do

85− echo -n "|$i"

121+ echo -n "$i ["

122+ eval "echo -n \$$i"

123+ echo "]"

86124 done

125+ echo

87126}

88127

89128print_help () {

90129 echo "Usage: $0 "

92− echo " [--group-diff]"

130+ echo " [--benchmarks <"bench1 bench2 ..." | ?>]"

93131 echo " [--graphs]"

94132 echo " [--no-measure]"

95133 echo " [--append]"

96134 echo " [--long]"

97135 echo " [--slow]"

98136 echo " [--quick]"

99− echo " [--compare] [--base commit] [--candidate commit]"

100− echo " [--cabal-build-flags]"

137+ echo " [--cabal-build-flags <flag>]"

138+ echo " [--compare] [--base <commit>] [--candidate <commit>]"

101139 echo " -- <gauge options or benchmarks>"

102140 echo

103− echo "Multiple benchmarks can be specified as a space separated list"

104− echo " e.g. --benchmarks \"linear nested\""

105− echo

106− echo "--group-diff is used to compare groups within a single benchmark"

107− echo " e.g. StreamD vs StreamK in base benchmark."

141+ echo "--benchmarks: benchmarks to run, use '?' for list of benchmarks"

142+ echo "--graphs: Generate graphical reports"

143+ echo "--no-measure: Don't run benchmarks, run reports from previous results"

144+ echo "--append: Don't overwrite previous results, append for comparison"

145+ echo "--long: Use much longer stream size for infinite stream benchmarks"

146+ echo "--slow: Slightly more accurate results at the expense of speed"

147+ echo "--quick: Faster results, useful for longer benchmarks"

148+ echo "--cabal-build-flags: Pass any cabal builds flags to be used for build"

108149 echo

109150 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

110151 echo "commit is generated, in the 'charts' directory."

@@ -128,13 +169,8 @@ set_benchmarks() {

128169 for i in $(echo $BENCHMARKS)

129170 do

130171 case $i in

131− ALL) echo -n $ALL_BENCHMARKS ;;

132− SERIAL) echo -n $SERIAL_BENCHMARKS ;;

133− CONCURRENT) echo -n $CONCURRENT_BENCHMARKS ;;

134− ARRAY) echo -n $ARRAY_BENCHMARKS ;;

135− INFINITE) echo -n $INFINITE_BENCHMARKS ;;

136− FINITE) echo -n $FINITE_BENCHMARKS ;;

137− array-cmp) echo -n "$ARRAY_BENCHMARKS array-cmp" ;;

172+ *_grp) eval "echo -n \$${i}" ;;

173+ *_cmp) eval "echo -n \$${i} $i" ;;

138174 *) echo -n $i ;;

139175 esac

140176 echo -n " "

@@ -360,7 +396,6 @@ run_reports() {

360396 do

361397 echo "Generating reports for ${i}..."

362398 $prog $(test "$GRAPH" = 1 && echo "--graphs") \

363− $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \

364399 --benchmark $i

365400 done

366401}

@@ -370,7 +405,6 @@ run_reports() {

370405#-----------------------------------------------------------------------------

371406

372407DEFAULT_BENCHMARKS="linear"

373−GROUP_DIFF=0

374408

375409COMPARE=0

376410BASE=

@@ -415,7 +449,6 @@ do

415449 --raw) RAW=1; shift ;;

416450 --append) APPEND=1; shift ;;

417451 --long) LONG=1; shift ;;

418− --group-diff) GROUP_DIFF=1; shift ;;

419452 --graphs) GRAPH=1; shift ;;

420453 --no-measure) MEASURE=0; shift ;;

421454 --) shift; break ;;

@@ -425,17 +458,11 @@ do

425458done

426459GAUGE_ARGS=$*

427460

428−BENCHMARKS=$(set_benchmarks)

429−if test "$LONG" -ne 0

430−then

431− BENCHMARKS=$INFINITE_BENCHMARKS

432−fi

433−

434461only_real_benchmarks () {

435462 for i in $BENCHMARKS

436463 do

437464 local SKIP=0

438− for j in $VIRTUAL_BENCHMARKS

465+ for j in $COMPARISONS

439466 do

440467 if test $i == $j

441468 then

@@ -456,12 +483,6 @@ proper_executables () {

456483 done

457484}

458485

459−

460−BENCHMARKS_ORIG=$BENCHMARKS

461−BENCHMARKS=$(only_real_benchmarks)

462−EXECUTABLES=$(proper_executables)

463−echo "Using benchmark suites [$BENCHMARKS]"

464−

465486has_benchmark () {

466487 for i in $BENCHMARKS_ORIG

467488 do

@@ -473,6 +494,32 @@ has_benchmark () {

473494 done

474495}

475496

497+BENCHMARKS_ORIG=$BENCHMARKS

498+if test "$(has_benchmark help)" = "help"

499+then

500+ list_bench_groups

501+ list_comparisons

502+ list_benches

503+ exit

504+fi

505+

506+if test "$LONG" -ne 0

507+then

508+ if test -n "$BENCHMARKS"

509+ then

510+ echo "Cannot specify benchmarks [$BENCHMARKS] with --long"

511+ exit

512+ fi

513+ BENCHMARKS=$infinite_grp

514+fi

515+

516+BENCHMARKS=$(set_benchmarks)

517+BENCHMARKS_ORIG=$BENCHMARKS

518+BENCHMARKS=$(only_real_benchmarks)

519+EXECUTABLES=$(proper_executables)

520+

521+echo "Using benchmark suites [$BENCHMARKS]"

522+

476523if test "$USE_STACK" = "1"

477524then

478525 WHICH_COMMAND="stack exec which"

@@ -516,18 +563,25 @@ fi

516563# Run reports

517564#-----------------------------------------------------------------------------

518565

519−VIRTUAL_REPORTS=""

520−if test "$(has_benchmark 'array-cmp')" = "array-cmp"

521−then

522− VIRTUAL_REPORTS="$VIRTUAL_REPORTS array-cmp"

523− mkdir -p "charts/array-cmp"

524− cat "charts/Memory.Array/results.csv" \

525− "charts/Data.Prim.Array/results.csv" \

526− "charts/Data.Array/results.csv" > "charts/array-cmp/results.csv"

527−fi

566+COMPARISON_REPORTS=""

567+for i in $COMPARISONS

568+do

569+ if test "$(has_benchmark $i)" = $i

570+ then

571+ COMPARISON_REPORTS="$COMPARISON_REPORTS $i"

572+ mkdir -p "charts/$i"

573+ constituents=$(eval "echo -n \$${i}")

574+ dest_file="charts/$i/results.csv"

575+ : > $dest_file

576+ for j in $constituents

577+ do

578+ cat "charts/$j/results.csv" >> $dest_file

579+ done

580+ fi

581+done

528582

529583if test "$RAW" = "0"

530584then

531585 run_reports "$BENCHMARKS"

532− run_reports "$VIRTUAL_REPORTS"

586+ run_reports "$COMPARISON_REPORTS"

533587fi

benchmark/Chart.hs+16−143

@@ -31,26 +31,16 @@ data BenchType

3131 | ParserD

3232 | ParserK

3333 | Parser

34− | Base

3534 | FileIO

36− | Array

37− | ArrayCmp

38− | UnpinnedArray

39− | SmallArray

40− | PrimArray

4135 | Concurrent

4236 | Parallel

4337 | Adaptive

44− | FoldO1Space

45− | FoldOnHeap

46− | UnfoldO1Space

47− | UnfoldOnSpace

38+ | Compare String

4839 | Standard String

4940 deriving Show

5041

5142data Options = Options

5243 { genGraphs :: Bool

53− , groupDiff :: Bool

5444 , benchType :: BenchType

5545 } deriving Show

5646

@@ -60,10 +50,6 @@ setGenGraphs val = do

6050 (args, opts) <- get

6151 put (args, opts { genGraphs = val })

6252

63−setGroupDiff val = do

64− (args, opts) <- get

65− put (args, opts { groupDiff = val })

66−

6753setBenchType val = do

6854 (args, opts) <- get

6955 put (args, opts { benchType = val })

@@ -87,20 +73,11 @@ parseBench = do

8773 Just "parserD" -> setBenchType ParserD

8874 Just "parserK" -> setBenchType ParserK

8975 Just "parser" -> setBenchType Parser

90− Just "base" -> setBenchType Base

9176 Just "fileio" -> setBenchType FileIO

92− Just "array-cmp" -> setBenchType ArrayCmp

93− Just "array" -> setBenchType Array

94− Just "unpinned-array" -> setBenchType UnpinnedArray

95− Just "small-array" -> setBenchType SmallArray

96− Just "prim-array" -> setBenchType PrimArray

9777 Just "concurrent" -> setBenchType Concurrent

9878 Just "parallel" -> setBenchType Parallel

9979 Just "adaptive" -> setBenchType Adaptive

100− Just "fold-o-1-space" -> setBenchType FoldO1Space

101− Just "fold-o-n-heap" -> setBenchType FoldOnHeap

102− Just "unfold-o-1-space" -> setBenchType UnfoldO1Space

103− Just "unfold-o-n-space" -> setBenchType UnfoldOnSpace

80+ Just str | "_cmp" `isSuffixOf` str -> setBenchType (Compare str)

10481 Just str -> setBenchType (Standard str)

10582 Nothing -> do

10683 liftIO $ putStrLn "please provide a benchmark type "

@@ -119,7 +96,6 @@ parseOptions = do

11996 parseOpt opt =

12097 case opt of

12198 "--graphs" -> setGenGraphs True

122− "--group-diff" -> setGroupDiff True

12399 "--benchmark" -> parseBench

124100 str -> do

125101 liftIO $ putStrLn $ "Unrecognized option " <> str

@@ -239,71 +215,21 @@ makeGraphs name cfg@Config{..} inputFile =

239215-- Arrays

240216------------------------------------------------------------------------------

241217

242−showArrayComparisons Options{..} cfg inp out =

243− let cfg' = cfg { classifyBenchmark = classifyArray }

244− in if genGraphs

245− then ignoringErr $ graph inp "Arrays Comparison"

246− cfg' { outputDir = Just out

218+showComparisons Options{..} cfg inp out =

219+ let cfg1 = cfg { classifyBenchmark = classifyComparison }

220+ in if genGraphs

221+ then ignoringErr $ graph inp "comparison"

222+ cfg1 { outputDir = Just out

247223 , presentation = Groups Absolute

248224 }

249− else ignoringErr $ report inp Nothing cfg'

225+ else ignoringErr $ report inp Nothing cfg1

250226

251227 where

252228

253− classifyArray b

254− -- SmallArray uses a small number of elements therefore cannot be

255− -- compared

256− -- | "SmallArray/" `isPrefixOf` b = ("SmallArray",) <$> stripPrefix "SmallArray/" b

257− | "Data.Prim.Array/" `isPrefixOf` b = ("Data.Prim.Array",) <$> stripPrefix "Data.Prim.Array/" b

258− | "Data.Array/" `isPrefixOf` b = ("Data.Array",) <$> stripPrefix "Data.Array/" b

259− | "array/" `isPrefixOf` b = ("array",) <$> stripPrefix "array/" b

260− | otherwise = Nothing

261−

262−------------------------------------------------------------------------------

263−-- Reports/Charts for base streams

264−------------------------------------------------------------------------------

265−

266−showStreamDVsK Options{..} cfg inp out =

267− let cfg' = cfg { classifyBenchmark = classifyBase }

268− in if genGraphs

269− then ignoringErr $ graph inp "streamD-vs-streamK"

270− cfg' { outputDir = Just out

271− , presentation = Groups Absolute

272− }

273− else ignoringErr $ report inp Nothing cfg'

274−

275− where

276−

277− classifyBase b

278− | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b

279− | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b

280− | otherwise = Nothing

281−

282−showStreamD Options{..} cfg inp out =

283− let cfg' = cfg { classifyBenchmark = classifyStreamD }

284− in if genGraphs

285− then ignoringErr $ graph inp "streamD"

286− cfg' {outputDir = Just out}

287− else ignoringErr $ report inp Nothing cfg'

288−

289− where

290−

291− classifyStreamD b

292− | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b

293− | otherwise = Nothing

294−

295−showStreamK Options{..} cfg inp out =

296− let cfg' = cfg { classifyBenchmark = classifyStreamK }

297− in if genGraphs

298− then ignoringErr $ graph inp "streamK"

299− cfg' {outputDir = Just out}

300− else ignoringErr $ report inp Nothing cfg'

301−

302− where

303−

304− classifyStreamK b

305− | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b

306− | otherwise = Nothing

229+ classifyComparison b = Just $

230+ ( takeWhile (/= '/') b

231+ , dropWhile (== '/') $ dropWhile (/= '/') b

232+ )

307233

308234------------------------------------------------------------------------------

309235-- text reports

@@ -385,30 +311,6 @@ main = do

385311 (makeGraphs "fileIO")

386312 "charts/fileio/results.csv"

387313 "charts/fileio"

388− Array -> benchShow opts cfg

389− { title = Just "Array" }

390− (makeGraphs "array")

391− "charts/array/results.csv"

392− "charts/array"

393− UnpinnedArray -> benchShow opts cfg

394− { title = Just "Unpinned Array" }

395− (makeGraphs "unpinned-array")

396− "charts/unpinned-array/results.csv"

397− "charts/unpinned-array"

398− SmallArray -> benchShow opts cfg

399− { title = Just "Small Array" }

400− (makeGraphs "small-array")

401− "charts/small-array/results.csv"

402− "charts/small-array"

403− PrimArray -> benchShow opts cfg

404− { title = Just "Prim Array" }

405− (makeGraphs "prim-array")

406− "charts/prim-array/results.csv"

407− "charts/prim-array"

408− ArrayCmp -> showArrayComparisons opts cfg

409− { title = Just "Arrays Comparison" }

410− "charts/array-cmp/results.csv"

411− "charts/array-cmp"

412314 Concurrent -> benchShow opts cfg

413315 { title = Just "Concurrent Ops" }

414316 (makeGraphs "Concurrent")

@@ -424,39 +326,10 @@ main = do

424326 (makeGraphs "adaptive")

425327 "charts/adaptive/results.csv"

426328 "charts/adaptive"

427− Base -> do

428− let cfg' = cfg { title = Just "Base stream" }

429− if groupDiff

430− then showStreamDVsK opts cfg'

431− "charts/base/results.csv"

432− "charts/base"

433− else do

434− showStreamD opts cfg'

435− "charts/base/results.csv"

436− "charts/base"

437− showStreamK opts cfg'

438− "charts/base/results.csv"

439− "charts/base"

440− FoldO1Space -> benchShow opts cfg

441− { title = Just "Fold O(1) Space" }

442− (makeGraphs "fold-o-1-space")

443− "charts/fold-o-1-space/results.csv"

444− "charts/fold-o-1-space"

445− FoldOnHeap -> benchShow opts cfg

446− { title = Just "Fold O(n) Heap" }

447− (makeGraphs "fold-o-n-heap")

448− "charts/fold-o-n-heap/results.csv"

449− "charts/fold-o-n-heap"

450− UnfoldO1Space -> benchShow opts cfg

451− { title = Just "Unfold O(1) Space" }

452− (makeGraphs "unfold-o-1-space")

453− "charts/unfold-o-1-space/results.csv"

454− "charts/unfold-o-1-space"

455− UnfoldOnSpace -> benchShow opts cfg

456− { title = Just "Unfold O(n) Space" }

457− (makeGraphs "unfold-o-n-space")

458− "charts/unfold-o-n-space/results.csv"

459− "charts/unfold-o-n-space"

329+ Compare str -> showComparisons opts cfg

330+ { title = Just $ str }

331+ ("charts/" ++ str ++ "/results.csv")

332+ ("charts/" ++ str)

460333 Standard str -> benchShow opts cfg

461334 { title = Just str }

462335 (makeGraphs str)