03218ad · streamly · git.iko.soy

benchmark/README.md+25−14

compare with master

unifiedsplit

@@ -1,3 +1,12 @@

1+# Benchmarking

2+

3+## Benchmark Dirvers

4+

5+Two benchmark drivers are supported:

6+

7+* `tasty-bench` (default)

8+* `gauge` (enabled by `--use-gauge` build flag)

9+

1## Building a single benchmark suite

10## Building a single benchmark suite

2

11

3```

12```

@@ -30,15 +39,18 @@ $ cabal build --flag "-opt" ...

30

39

31## Build and run single benchmarks:

40## Build and run single benchmarks:

32

41

42+The benchmark executables are `tasty-bench` executables unless you have

43+passed `--use-gauge` cabal flag when building in which case it is a

44+`gauge` executable.

45+

33−For quick results you may have to use `--quick` and possibly some other

46+For quick results you may have to use a large value for `--stdev` or use

34−`gauge` options, or better use `bench.sh --quick` as described in the

47+`bench.sh --quick` as described in the following sections.

35−following sections.

36

48

37```

49```

38−$ cabal run bench:Prelude.Serial -- --quick

50+$ cabal run bench:Prelude.Serial -- --stdev 100000

39```

51```

40

52

41−`cabal bench` can be used but you cannot pass arguments (like --quick):

53+`cabal bench` can be used but you cannot pass arguments (like --stdev):

42

54

43```

55```

44$ cabal bench Prelude.Serial

56$ cabal bench Prelude.Serial

@@ -104,7 +116,7 @@ $ bench.sh --benchmarks "Prelude.Serial Data.Parser"

104Run all O(1) space complexity benchmarks in `Prelude.Serial` suite:

116Run all O(1) space complexity benchmarks in `Prelude.Serial` suite:

105

117

106```

118```

107−$ bench.sh --benchmarks Prelude.Serial -- Prelude.Serial/o-1-space

119+$ bench.sh --benchmarks Prelude.Serial --prefix Prelude.Serial/o-1-space

108```

120```

109

121

110Anything after a `--` is passed to the benchmark executable,

122Anything after a `--` is passed to the benchmark executable,

@@ -114,13 +126,13 @@ it basically selects all benchmarks starting with

114Run a specific benchmark in `Prelude.Serial` suite:

126Run a specific benchmark in `Prelude.Serial` suite:

115

127

116```

128```

117−$ bench.sh --benchmarks Prelude.Serial -- Prelude.Serial/o-1-space/generation/unfoldr

129+$ bench.sh --benchmarks Prelude.Serial --prefix Prelude.Serial/o-1-space/generation/unfoldr

118```

130```

119

131

120Run a benchmark directly instead of running it through `bench.sh`:

132Run a benchmark directly instead of running it through `bench.sh`:

121

133

122```

134```

123−$ cabal run bench:Prelude.Serial -- --quick Prelude.Serial/o-1-space/generation/unfoldr

135+$ cabal run bench:Prelude.Serial -- Prelude.Serial/o-1-space/generation/unfoldr

124```

136```

125

137

126The options after `--` are the benchmark executable options.

138The options after `--` are the benchmark executable options.

@@ -212,7 +224,7 @@ You can specify the stream size (default is 100000) to be used for

212benchmarking:

224benchmarking:

213

225

214```

226```

215−$ cabal run bench:Prelude.Serial -- --quick --stream-size 1000000

227+$ cabal run bench:Prelude.Serial -- --stream-size 1000000

216```

228```

217

229

218### Unicode input

230### Unicode input

@@ -222,7 +234,7 @@ environment variable:

222

234

223```

235```

224$ export Benchmark_FileSystem_Handle_InputFile=./gutenberg-500.txt

236$ export Benchmark_FileSystem_Handle_InputFile=./gutenberg-500.txt

225−$ cabal run FileSystem.Handle -- --quick FileSystem.Handle/o-1-space/reduce/read/S.splitOnSeq

237+$ cabal run FileSystem.Handle -- FileSystem.Handle/o-1-space/reduce/read/S.splitOnSeq

226```

238```

227

239

228The automatic tests do not test unicode input, this option is useful to specify

240The automatic tests do not test unicode input, this option is useful to specify

@@ -230,10 +242,9 @@ a unicode text file manually.

230

242

231## Benchmarking notes

243## Benchmarking notes

232

244

233−We use gauge instead of criterion for benchmarking. We have fixed

245+We run each benchmark in an isolated process to minimize interference

234−several issues in gauge inherited from criterion. We have added several

246+of benchmarks and to be able to control the RTS memory restrictions per

235−features as well e.g. rusage stats, running benchmarks in an isolated

247+benchmark.

236−process etc which are crucial to our benchmarking analysis process.

237

248

238### Gotchas

249### Gotchas

239

250

benchmark/Streamly/Benchmark/Data/Array/Stream/Foreign.hs+2−2

@@ -193,11 +193,11 @@ moduleName = "Data.Array.Stream.Foreign"

193main :: IO ()

194main = do

195 env <- mkHandleBenchEnv

196− runWithCLIOpts defaultStreamSize (allBenchmarks env)

196+ defaultMain (allBenchmarks env)

197

198 where

199

200− allBenchmarks env _ =

200+ allBenchmarks env =

201 [ bgroup (o_1_space_prefix moduleName)

202 ( o_1_space_read_chunked env

203 ++ o_1_space_copy_toChunks_group_ungroup env

benchmark/Streamly/Benchmark/Data/Parser.hs+3−13

@@ -395,25 +395,15 @@ o_n_heap_serial value =

395-- Driver

396-------------------------------------------------------------------------------

397

398−getArray :: (Int -> [Benchmark]) -> IO [Array.Array Int]

399−#ifndef MIN_VERSION_gauge

400−getArray f = do

401− (value, _) <- parseCLIOpts defaultStreamSize $ bgroup "All" (f 0)

402− IP.toList $ IP.arraysOf 100 $ sourceUnfoldrM value 0

403−#else

404−getArray _ = do

405− (value, _, _) <- parseCLIOpts defaultStreamSize

406− IP.toList $ IP.arraysOf 100 $ sourceUnfoldrM value 0

407−#endif

408−

409main :: IO ()

398main :: IO ()

410main = do

399main = do

411 env <- mkHandleBenchEnv

400 env <- mkHandleBenchEnv

412− arrays <- getArray (allBenchmarks env [])

413− runWithCLIOpts defaultStreamSize (allBenchmarks env arrays)

401+ runWithCLIOptsEnv defaultStreamSize alloc (allBenchmarks env)

414

402

415 where

403 where

416

404

405+ alloc value = IP.toList $ IP.arraysOf 100 $ sourceUnfoldrM value 0

406+

417 allBenchmarks env arrays value =

407 allBenchmarks env arrays value =

418 [ bgroup (o_1_space_prefix moduleName) (o_1_space_serial value)

408 [ bgroup (o_1_space_prefix moduleName) (o_1_space_serial value)

419 , bgroup

409 , bgroup

benchmark/Streamly/Benchmark/Data/Parser/ParserD.hs+3−15

@@ -5,7 +5,6 @@

5-- License : BSD-3-Clause

6-- Maintainer : streamly@composewell.com

7

8−{-# LANGUAGE CPP #-}

9{-# LANGUAGE FlexibleContexts #-}

8{-# LANGUAGE FlexibleContexts #-}

10{-# LANGUAGE ScopedTypeVariables #-}

9{-# LANGUAGE ScopedTypeVariables #-}

11{-# OPTIONS_GHC -fspec-constr-recursive=4 #-}

10{-# OPTIONS_GHC -fspec-constr-recursive=4 #-}

@@ -359,24 +358,13 @@ o_n_space_serial value =

359-- Driver

358-- Driver

360-------------------------------------------------------------------------------

359-------------------------------------------------------------------------------

361

360

362−getArray :: (Int -> [Benchmark]) -> IO [Array.Array Int]

363−#ifndef MIN_VERSION_gauge

364−getArray f = do

365− (value, _) <- parseCLIOpts defaultStreamSize $ bgroup "All" (f 0)

366− IP.toList $ IP.arraysOf 100 $ sourceUnfoldrM value 0

367−#else

368−getArray _ = do

369− (value, _, _) <- parseCLIOpts defaultStreamSize

370− IP.toList $ IP.arraysOf 100 $ sourceUnfoldrM value 0

371−#endif

372−

373main :: IO ()

361main :: IO ()

374−main = do

375− arrays <- getArray (allBenchmarks [])

376− runWithCLIOpts defaultStreamSize (allBenchmarks arrays)

362+main = runWithCLIOptsEnv defaultStreamSize alloc allBenchmarks

377

363

378 where

364 where

379

365

366+ alloc value = IP.toList $ IP.arraysOf 100 $ sourceUnfoldrM value 0

367+

380 allBenchmarks arraysSmall value =

368 allBenchmarks arraysSmall value =

381 [ bgroup (o_1_space_prefix moduleName) (o_1_space_serial value)

369 [ bgroup (o_1_space_prefix moduleName) (o_1_space_serial value)

382 , bgroup (o_1_space_prefix moduleName) (o_1_space_serial_spanning value)

370 , bgroup (o_1_space_prefix moduleName) (o_1_space_serial_spanning value)

benchmark/Streamly/Benchmark/FileSystem/Handle.hs+2−2

@@ -44,11 +44,11 @@ moduleName = "FileSystem.Handle"

44main :: IO ()

45main = do

46 env <- mkHandleBenchEnv

47− runWithCLIOpts defaultStreamSize (allBenchmarks env)

47+ defaultMain (allBenchmarks env)

48

49 where

50

51− allBenchmarks env _ =

51+ allBenchmarks env =

52 [ bgroup (o_1_space_prefix moduleName) $ Prelude.concat

53 [ RO.allBenchmarks env

54 , RW.allBenchmarks env

benchmark/Streamly/Benchmark/Prelude/Concurrent.hs+12−2

@@ -1,3 +1,5 @@

1+{-# LANGUAGE CPP #-}

2+{-# LANGUAGE RankNTypes #-}

1-- |

3-- |

2-- Module : Main

4-- Module : Main

@@ -5,8 +7,6 @@

5-- License : BSD3

7-- License : BSD3

6-- Maintainer : streamly@composewell.com

8-- Maintainer : streamly@composewell.com

7

9

8−{-# LANGUAGE RankNTypes #-}

9−

10import Control.Concurrent

11import Control.Monad (when, replicateM)

12import Streamly.Prelude

@@ -78,7 +78,17 @@ concatGroup buflen threads usec n =

78

79main :: IO ()

80main =

81+#ifdef MIN_VERSION_gauge

82+ defaultMainWith (defaultConfig

83+ { timeLimit = Just 0

84+ , minSamples = Just 1

85+ , minDuration = 0

86+ , includeFirstIter = True

87+ , quickMode = True

88+ })

89+#else

81 defaultMain

90 defaultMain

91+#endif

82

92

83 [ -- bgroup "append/buf-1-threads-10k-0sec" (appendGroup 1 10000 0)

93 [ -- bgroup "append/buf-1-threads-10k-0sec" (appendGroup 1 10000 0)

84 -- , bgroup "append/buf-100-threads-100k-0sec" (appendGroup 100 100000 0)

94 -- , bgroup "append/buf-100-threads-100k-0sec" (appendGroup 100 100000 0)

benchmark/Streamly/Benchmark/Prelude/Rate.hs−1

@@ -1,5 +1,4 @@

1{-# LANGUAGE FlexibleContexts #-}

2−{-# LANGUAGE CPP #-}

3

2

4-- |

3-- |

5-- Module : Main

4-- Module : Main

benchmark/Streamly/Benchmark/Prelude/WAsync.hs+1−3

@@ -5,8 +5,6 @@

5-- License : BSD3

6-- Maintainer : streamly@composewell.com

7

8−{-# LANGUAGE CPP #-}

9−

10import Prelude hiding (mapM)

8import Prelude hiding (mapM)

11

9

12import Streamly.Prelude (fromWAsync, fromSerial, wAsync, maxBuffer, maxThreads)

10import Streamly.Prelude (fromWAsync, fromSerial, wAsync, maxBuffer, maxThreads)

@@ -170,7 +168,7 @@ o_n_space_outerProduct value =

170

168

171main :: IO ()

169main :: IO ()

172main = runWithCLIOpts defaultStreamSize allBenchmarks

170main = runWithCLIOpts defaultStreamSize allBenchmarks

173−

171+

174 where

172 where

175

173

176 allBenchmarks value =

174 allBenchmarks value =

benchmark/Streamly/Benchmark/Prelude/ZipAsync.hs−1

@@ -6,7 +6,6 @@

6-- Maintainer : streamly@composewell.com

7

8{-# LANGUAGE FlexibleContexts #-}

9−{-# LANGUAGE CPP #-}

10

9

11import Streamly.Prelude (fromSerial)

10import Streamly.Prelude (fromSerial)

12import qualified Streamly.Prelude as S

11import qualified Streamly.Prelude as S

benchmark/Streamly/Benchmark/Prelude/ZipSerial.hs+5−5

@@ -60,8 +60,8 @@ zipWith count n =

60 S.drain $

61 S.zipWith

62 (,)

63− (S.fromSerial $ Main.sourceUnfoldrM count n)

63+ (S.fromSerial $ sourceUnfoldrM count n)

64− (S.fromSerial $ Main.sourceUnfoldrM count (n + 1))

64+ (S.fromSerial $ sourceUnfoldrM count (n + 1))

65

66#ifdef INSPECTION

67inspect $ hasNoTypeClasses 'zipWith

@@ -75,8 +75,8 @@ zipWithM count n =

75 S.drain $

76 S.zipWithM

77 (curry return)

78− (Main.sourceUnfoldrM count n)

78+ (sourceUnfoldrM count n)

79− (Main.sourceUnfoldrM count (n + 1))

79+ (sourceUnfoldrM count (n + 1))

80

81#ifdef INSPECTION

82inspect $ hasNoTypeClasses 'zipWithM

@@ -125,7 +125,7 @@ o_1_space_outerProduct value =

125-- passed using the --stream-size option.

126--

127main :: IO ()

128−main = runWithCLIOpts defaultStreamSize allBenchmarks

128+main = runWithCLIOpts defaultStreamSize allBenchmarks

129

130 where

131

benchmark/lib/Streamly/Benchmark/Common.hs+28−26

@@ -16,8 +16,7 @@ module Streamly.Benchmark.Common

16 , o_n_heap_prefix

17 , o_n_stack_prefix

18

19− -- , parseEnvOpts

19+ , runWithCLIOptsEnv

20− , parseCLIOpts

21 , runWithCLIOpts

20 , runWithCLIOpts

22

21

23 , benchIOSink1

22 , benchIOSink1

@@ -33,14 +32,6 @@ module Streamly.Benchmark.Common

33 , mkListString

32 , mkListString

34

33

35 , defaultStreamSize

34 , defaultStreamSize

36− , BenchOpts(..)

37−#ifndef MIN_VERSION_gauge

38− , OptionDescription(..)

39− , includingOptions

40− , lookupOption

41− , defaultMainWithIngredients

42− , parseOptions

43−#endif

44 )

35 )

45where

36where

46

37

@@ -54,10 +45,11 @@ import System.Console.GetOpt

54 (OptDescr(..), ArgDescr(..), ArgOrder(..), getOpt')

45 (OptDescr(..), ArgDescr(..), ArgOrder(..), getOpt')

55import System.Environment (getArgs, lookupEnv, setEnv)

46import System.Environment (getArgs, lookupEnv, setEnv)

56#else

47#else

57−import Data.Proxy

48+import Data.Proxy (Proxy(..))

58−import Test.Tasty.Ingredients.Basic

49+import Test.Tasty.Ingredients.Basic (includingOptions)

59import Test.Tasty.Options

50import Test.Tasty.Options

51+ (IsOption(..), OptionDescription(..), lookupOption, safeRead)

60−import Test.Tasty.Runners

52+import Test.Tasty.Runners (Ingredient, defaultMainWithIngredients, parseOptions)

61#endif

53#endif

62import Control.DeepSeq (NFData(..))

54import Control.DeepSeq (NFData(..))

63import Data.Functor.Identity (Identity, runIdentity)

55import Data.Functor.Identity (Identity, runIdentity)

@@ -230,27 +222,37 @@ instance IsOption BenchOpts where

230 defaultValue = StreamSize defaultStreamSize

222 defaultValue = StreamSize defaultStreamSize

231 parseValue = fmap StreamSize . safeRead

223 parseValue = fmap StreamSize . safeRead

232 optionName = pure "stream-size"

224 optionName = pure "stream-size"

233− optionHelp = pure "StreamSize used in benchmarks"

225+ optionHelp = pure "Size of the stream to be used in benchmarks"

234

226

235parseCLIOpts :: Int -> Benchmark -> IO (Int, [Ingredient])

227parseCLIOpts :: Int -> Benchmark -> IO (Int, [Ingredient])

236−parseCLIOpts cDefSize benches = do

228+parseCLIOpts defStreamSize benches = do

237 let customOpts = [Test.Tasty.Options.Option (Proxy :: Proxy BenchOpts)]

229 let customOpts = [Test.Tasty.Options.Option (Proxy :: Proxy BenchOpts)]

238 ingredients = includingOptions customOpts : benchIngredients

230 ingredients = includingOptions customOpts : benchIngredients

239− opts <- parseOptions ingredients benches

231+ opts <- parseOptions ingredients benches -- (TestGroup "" [])

240 let StreamSize size = lookupOption opts

232 let StreamSize size = lookupOption opts

241− print $ "Stream-Size = " ++ show size

233+ -- putStrLn $ "Stream size: " ++ show size

242− if size == defaultStreamSize -- LONG option is not set

234+ if size == defaultStreamSize

243− then return (cDefSize, ingredients) -- use custom defaut size of Benchmark

235+ then return (defStreamSize, ingredients)

244− else return (size, ingredients) -- LONG option is set use large stream size

236+ else return (size, ingredients)

245#endif

237#endif

246

238

247−runWithCLIOpts :: Int -> (Int -> [Benchmark]) -> IO ()

239+runWithCLIOptsEnv :: Int -> (Int -> IO a) -> (a -> Int -> [Benchmark]) -> IO ()

248−runWithCLIOpts cDefSize f = do

240+runWithCLIOptsEnv defStreamSize alloc mkBench = do

249

241

250#ifdef MIN_VERSION_gauge

242#ifdef MIN_VERSION_gauge

251− (value, cfg, benches) <- parseCLIOpts cDefSize

243+ (value, cfg, benches) <- parseCLIOpts defStreamSize

244+ r <- alloc value

252− value `seq` runMode (mode cfg) cfg benches (f value)

245+ value `seq` runMode (mode cfg) cfg benches (mkBench r value)

253#else

246#else

254− (value, ingredients) <- parseCLIOpts cDefSize $ bgroup "All" (f 0)

247+ (value, ingredients) <-

248+ parseCLIOpts defStreamSize $ bgroup "All" (mkBench undefined 0)

249+ r <- alloc value

255− value `seq` defaultMainWithIngredients ingredients $ bgroup "All" (f value)

250+ value `seq` defaultMainWithIngredients ingredients

251+ $ bgroup "All" (mkBench r value)

256#endif

252#endif

253+

254+runWithCLIOpts :: Int -> (Int -> [Benchmark]) -> IO ()

255+runWithCLIOpts defStreamSize f =

256+ runWithCLIOptsEnv defStreamSize

257+ (const $ return undefined)

258+ (\_ v -> f v)

benchmark/lib/Streamly/Benchmark/Common/Handle.hs+1−1

@@ -106,7 +106,7 @@ getHandles env mkHandles = do

106 }

107

108 -- update

109− writeIORef (href env) $ refHandles

109+ writeIORef (href env) refHandles

110 return $ mkHandles refHandles

111

112mkBenchCommon ::

benchmark/streamly-benchmarks.cabal+3−3

@@ -124,13 +124,13 @@ common bench-depends

124 , directory >= 1.2.2 && < 1.4

125 , ghc-prim >= 0.4 && < 0.8

126

127− if flag(use-gauge)

127+ if flag(use-gauge)

128 build-depends: gauge >= 0.2.4 && < 0.3

129− else

129+ else

130 build-depends: tasty-bench >= 0.2.5 && < 0.3

131 , tasty >= 1.4.1

132 mixins: tasty-bench

133− (Test.Tasty.Bench as Gauge

133+ (Test.Tasty.Bench as Gauge

134 , Test.Tasty.Bench as Gauge.Main

135 )

136

bin/bench-exec-one.sh+76−38

@@ -4,6 +4,8 @@

4# BENCH_EXEC_PATH: the benchmark executable

5# RTS_OPTIONS: additional RTS options

6# QUICK_MODE: whether we are in quick mode

7+# USE_GAUGE: whether to use gauge or tasty-bench

8+# LONG: whether to use a large stream size

7

9

8# $1: message

10# $1: message

9die () {

11die () {

@@ -42,6 +44,9 @@ bench_rts_opts_default () {

42}

44}

43

45

44# Overrides for specific benchmarks

46# Overrides for specific benchmarks

47+# XXX Note: for tasty-bench we replace the "." separator in the benchmark names

48+# with "/" for matching with this. It may not work reliably if the benchmark

49+# name already contains ".".

45bench_rts_opts_specific () {

50bench_rts_opts_specific () {

46 case "$1" in

51 case "$1" in

47 Data.Stream.StreamD/o-n-space/elimination/toList) echo -n "-K2M" ;;

52 Data.Stream.StreamD/o-n-space/elimination/toList) echo -n "-K2M" ;;

@@ -103,43 +108,46 @@ bench_rts_opts_specific () {

103if test "$USE_GAUGE" -eq 0

108if test "$USE_GAUGE" -eq 0

104then

109then

105 SUPER_QUICK_OPTIONS="--stdev 1000000"

110 SUPER_QUICK_OPTIONS="--stdev 1000000"

106− QUICKER_OPTIONS="--stdev 1000"

111+ QUICKER_OPTIONS="--stdev 100"

107else

112else

108 # Do not keep time limit as 0 otherwise GC stats may remain 0 in some cases.

113 # Do not keep time limit as 0 otherwise GC stats may remain 0 in some cases.

109 SUPER_QUICK_OPTIONS="--quick --min-duration 0 --time-limit 0.01 --include-first-iter"

114 SUPER_QUICK_OPTIONS="--quick --min-duration 0 --time-limit 0.01 --include-first-iter"

110 QUICKER_OPTIONS="--min-samples 3 --time-limit 1"

115 QUICKER_OPTIONS="--min-samples 3 --time-limit 1"

111fi

116fi

112

117

118+# tasty-bench does not like an option set twice

119+set_super_quick_mode () {

120+ echo -n super_quick

121+}

122+

113# For certain long benchmarks if the user has not requested super quick

123# For certain long benchmarks if the user has not requested super quick

114# mode we anyway use a slightly quicker mode.

124# mode we anyway use a slightly quicker mode.

115use_quicker_mode () {

125use_quicker_mode () {

116− if test -n "$QUICK_MODE"

126+ if test "$QUICK_MODE" -eq 0

117 then

127 then

118− if test "$QUICK_MODE" -eq 0

128+ echo quicker

119− then

120− echo $QUICKER_OPTIONS

121− fi

122 fi

129 fi

123}

130}

124

131

125bench_exe_quick_opts () {

132bench_exe_quick_opts () {

126 case "$1" in

133 case "$1" in

127− Prelude.Concurrent) echo -n "$SUPER_QUICK_OPTIONS" ;;

134+ Prelude.Concurrent) set_super_quick_mode ;;

128− Prelude.Rate) echo -n "$SUPER_QUICK_OPTIONS" ;;

135+ Prelude.Rate) set_super_quick_mode ;;

129− Prelude.Adaptive) echo -n "$SUPER_QUICK_OPTIONS" ;;

136+ Prelude.Adaptive) set_super_quick_mode;;

130 *) echo -n "" ;;

137 *) echo -n "" ;;

131 esac

138 esac

132}

139}

133

140

141+# XXX Note: for tasty-bench we replace the "." separator in the benchmark names

142+# with "/" for matching with this. It may not work reliably if the benchmark

143+# name already contains ".".

144+

134# Use quick options for benchmarks that take too long

145# Use quick options for benchmarks that take too long

135bench_quick_opts () {

146bench_quick_opts () {

136 case "$1" in

147 case "$1" in

137− Prelude.Parallel/o-n-heap/mapping/mapM)

148+ Prelude.Parallel/o-n-heap/mapping/mapM) set_super_quick_mode ;;

138− echo -n "$SUPER_QUICK_OPTIONS" ;;

139− Prelude.Parallel/o-n-heap/monad-outer-product/*)

149+ Prelude.Parallel/o-n-heap/monad-outer-product/*) set_super_quick_mode ;;

140− echo -n "$SUPER_QUICK_OPTIONS" ;;

141− Prelude.Parallel/o-n-space/monad-outer-product/*)

150+ Prelude.Parallel/o-n-space/monad-outer-product/*) set_super_quick_mode ;;

142− echo -n "$SUPER_QUICK_OPTIONS" ;;

143 Prelude.Parallel/o-n-heap/generation/*) use_quicker_mode ;;

151 Prelude.Parallel/o-n-heap/generation/*) use_quicker_mode ;;

144 Prelude.Parallel/o-n-heap/mapping/*) use_quicker_mode ;;

152 Prelude.Parallel/o-n-heap/mapping/*) use_quicker_mode ;;

145 Prelude.Parallel/o-n-heap/concat-foldable/*) use_quicker_mode ;;

153 Prelude.Parallel/o-n-heap/concat-foldable/*) use_quicker_mode ;;

@@ -163,11 +171,12 @@ bench_output_file() {

163 echo "charts/$bench_name/results.csv"

171 echo "charts/$bench_name/results.csv"

164}

172}

165

173

166−BENCH_NAME_ORIG=""

174+#------------------------------------------------------------------------------

167−for i in "$@"

175+# Determine options from benchmark name

168−do

176+#------------------------------------------------------------------------------

177+

169− BENCH_NAME_ORIG="$i"

178+BENCH_NAME_ORIG="$1"

170−done

179+shift

171

180

172if test "$USE_GAUGE" -eq 0

181if test "$USE_GAUGE" -eq 0

173then

182then

@@ -178,16 +187,8 @@ then

178 BENCH_NAME1=$(echo $BENCH_NAME | cut -f1 -d '/')

187 BENCH_NAME1=$(echo $BENCH_NAME | cut -f1 -d '/')

179 BENCH_NAME2=$(echo $BENCH_NAME | cut -f2- -d '/' | sed -e 's/\./\//g')

188 BENCH_NAME2=$(echo $BENCH_NAME | cut -f2- -d '/' | sed -e 's/\./\//g')

180 BENCH_NAME="$BENCH_NAME1/$BENCH_NAME2"

189 BENCH_NAME="$BENCH_NAME1/$BENCH_NAME2"

181− JOB_OPT=" -j 1"

182else

190else

183 BENCH_NAME=$BENCH_NAME_ORIG

191 BENCH_NAME=$BENCH_NAME_ORIG

184− JOB_OPT=""

185−fi

186−if test "$LONG" -eq 0

187−then

188− SIZE_OPT=""

189−else

190− SIZE_OPT="--stream-size 1000000"

191fi

192fi

192

193

193RTS_OPTIONS=\

194RTS_OPTIONS=\

@@ -198,28 +199,65 @@ $(bench_rts_opts_specific $BENCH_NAME) \

198$RTS_OPTIONS \

199$RTS_OPTIONS \

199-RTS"

200-RTS"

200

201

201−QUICK_BENCH_OPTIONS="\

202+QUICK_MODE_TYPE="\

202−$(if test "$QUICK_MODE" -ne 0; then echo $SUPER_QUICK_OPTIONS; else :; fi)

203+$(if test "$QUICK_MODE" -ne 0; then set_super_quick_mode; fi) \

203$(bench_exe_quick_opts $(basename $BENCH_EXEC_PATH)) \

204$(bench_exe_quick_opts $(basename $BENCH_EXEC_PATH)) \

204$(bench_quick_opts $BENCH_NAME)"

205$(bench_quick_opts $BENCH_NAME)"

205

206

206−output_file=$(bench_output_file $(basename $BENCH_EXEC_PATH))

207+for i in $QUICK_MODE_TYPE

207−mkdir -p `dirname $output_file`

208+do

209+ case "$i" in

210+ super_quick) QUICK_BENCH_OPTIONS="$SUPER_QUICK_OPTIONS"; break ;;

211+ quicker) QUICK_BENCH_OPTIONS="$QUICKER_OPTIONS"; break ;;

212+ esac

213+done

214+

215+if test "$LONG" -ne 0

216+then

217+ STREAM_SIZE=10000000

218+ STREAM_LEN=$(env LC_ALL=en_US.UTF-8 printf "--stream-size %'.f\n" $STREAM_SIZE)

219+ STREAM_SIZE_OPT="--stream-size $STREAM_SIZE"

220+fi

208

221

209echo "$BENCH_NAME: \

222echo "$BENCH_NAME: \

223+$RTS_OPTIONS \

224+$STREAM_LEN \

210−$QUICK_BENCH_OPTIONS \

225+$QUICK_BENCH_OPTIONS" \

211−$RTS_OPTIONS"

226+"$@"

212

227

228+#------------------------------------------------------------------------------

229+# Run benchmark with options and collect results

230+#------------------------------------------------------------------------------

231+

232+output_file=$(bench_output_file $(basename $BENCH_EXEC_PATH))

233+mkdir -p `dirname $output_file`

213rm -f ${output_file}.tmp

234rm -f ${output_file}.tmp

235+

214if test $USE_GAUGE -eq 0

236if test $USE_GAUGE -eq 0

215then

237then

238+ # Escape "\" and double quotes in benchmark names

216 BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')

239 BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')

217− $BENCH_EXEC_PATH $SIZE_OPT $JOB_OPT $RTS_OPTIONS $QUICK_BENCH_OPTIONS --csv=${output_file}.tmp \

240+ $BENCH_EXEC_PATH \

241+ -j 1 \

242+ $RTS_OPTIONS \

243+ $STREAM_SIZE_OPT \

244+ $QUICK_BENCH_OPTIONS \

245+ "$@" \

246+ --csv=${output_file}.tmp \

218 -p '$0 == "'"$BENCH_NAME_ESC"'"'

247 -p '$0 == "'"$BENCH_NAME_ESC"'"'

248+

249+ # Convert cpuTime field from picoseconds to seconds

219 tail -n +2 ${output_file}.tmp | \

250 tail -n +2 ${output_file}.tmp | \

220− awk 'BEGIN {FPAT = "([^,]+)|(\"[^\"]+\")";OFS=","} {$2=$2/1000000000000;print}' >> $output_file

251+ awk 'BEGIN {FPAT = "([^,]+)|(\"[^\"]+\")";OFS=","} {$2=$2/1000000000000;print}' \

252+ >> $output_file

221else

253else

254+ $BENCH_EXEC_PATH \

255+ $RTS_OPTIONS \

256+ $STREAM_SIZE_OPT \

257+ $QUICK_BENCH_OPTIONS \

258+ "$@" \

222− $BENCH_EXEC_PATH $SIZE_OPT $RTS_OPTIONS $QUICK_BENCH_OPTIONS --csvraw=${output_file}.tmp \

259+ --csvraw=${output_file}.tmp \

223 -m exact "$BENCH_NAME"

260 -m exact "$BENCH_NAME"

224− tail -n +2 ${output_file}.tmp >> $output_file

261+ tail -n +2 ${output_file}.tmp \

262+ >> $output_file

225fi

263fi

bin/bench.sh+68−48

@@ -8,6 +8,7 @@ source $SCRIPT_DIR/build-lib.sh

8print_help () {

9 echo "Usage: $0 "

10 echo " [--benchmarks <"bench1 bench2 ..." | help>]"

11+ echo " [--prefix <benchmark name prefix to match>"

11 echo " [--fields <"field1 field2 ..." | help>]"

12 echo " [--fields <"field1 field2 ..." | help>]"

12 echo " [--graphs]"

13 echo " [--graphs]"

13 echo " [--no-measure]"

14 echo " [--no-measure]"

@@ -142,11 +143,58 @@ bench_output_file() {

142 echo "charts/$bench_name/results.csv"

143 echo "charts/$bench_name/results.csv"

143}

144}

144

145

146+invoke_gauge () {

147+ local target_prog="$1"

148+ local target_name="$2"

149+ local output_file="$3"

150+

151+ local MATCH=""

152+ if test "$LONG" -ne 0

153+ then

154+ MATCH="$target_name/o-1-space"

155+ else

156+ MATCH="$BENCH_PREFIX"

157+ fi

158+ echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file

159+ # keep only benchmark names with shortest prefix e.g. "a/b/c" and "a/b", we

160+ # should only keep "a/b" otherwise benchmarks will run multiple times. why?

161+ $target_prog -l \

162+ | grep "^$target_name" \

163+ | grep "^$MATCH" \

164+ | sort \

165+ | awk 'BEGIN {prev="XXX"} {if (substr($0,1,length(prev)) != prev) {print $0; prev=$0}}' \

166+ | while read -r name; \

167+ do bin/bench-exec-one.sh "$name" "${GAUGE_ARGS[@]}"; done

168+}

169+

170+invoke_tasty_bench () {

171+ local target_prog="$1"

172+ local target_name="$2"

173+ local output_file="$3"

174+

175+ local MATCH=""

176+ if test "$LONG" -ne 0

177+ then

178+ MATCH="-p /$target_name\/o-1-space/"

179+ else

180+ if test -n "$BENCH_PREFIX"

181+ then

182+ # escape "/"

183+ local escaped_name=$(echo "$BENCH_PREFIX" | sed -e 's/\//\\\//g')

184+ MATCH="-p /$escaped_name/"

185+ fi

186+ fi

187+ echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file

188+ $target_prog -l $MATCH \

189+ | grep "^All" \

190+ | while read -r name; \

191+ do bin/bench-exec-one.sh "$name" "${GAUGE_ARGS[@]}"; done

192+}

193+

145run_bench_target () {

194run_bench_target () {

146 local package_name=$1

195 local package_name=$1

147 local component=$2

196 local component=$2

148 local target_name=$3

197 local target_name=$3

149− local output_file=$(bench_output_file $target_name)

150

198

151 local target_prog

199 local target_prog

152 target_prog=$(cabal_target_prog $package_name $component $target_name) || \

200 target_prog=$(cabal_target_prog $package_name $component $target_name) || \

@@ -156,46 +204,16 @@ run_bench_target () {

156

204

157 # Needed by bench-exec-one.sh

205 # Needed by bench-exec-one.sh

158 export BENCH_EXEC_PATH=$target_prog

206 export BENCH_EXEC_PATH=$target_prog

159− if test "$LONG" -ne 0

207+ export RTS_OPTIONS

160− then

208+ export QUICK_MODE

161− BENCH_ARGS="-p /$target_name\/o-1-space/"

209+ export USE_GAUGE

162− STREAM_SIZE=10000000

210+ export LONG

163− export STREAM_SIZE

164− fi

165

211

166− local MATCH=""

212+ local output_file=$(bench_output_file $target_name)

213+ mkdir -p `dirname $output_file`

167 if test "$USE_GAUGE" -eq 0

214 if test "$USE_GAUGE" -eq 0

168− then

215+ then invoke_tasty_bench "$target_prog" "$target_name" "$output_file"

169− if test "$LONG" -ne 0

216+ else invoke_gauge "$target_prog" "$target_name" "$output_file"

170− then

171− MATCH="-p /$target_name\/o-1-space/"

172− else

173− if test -n "$GAUGE_ARGS"

174− then

175− local GAUGE_ARGS1=$(echo "$GAUGE_ARGS" | sed -e 's/\//\\\//g')

176− MATCH="-p /$GAUGE_ARGS1/"

177− fi

178− fi

179− echo "Name,cpuTime,2*Stdev (ps),Allocated,bytesCopied" >> $output_file

180− $target_prog -l $MATCH \

181− | grep "^All" \

182− | while read -r name; do bin/bench-exec-one.sh "$name"; done

183− else

184− if test "$LONG" -ne 0

185− then

186− MATCH="$target_name/o-1-space"

187− else

188− MATCH="$GAUGE_ARGS"

189− fi

190− echo "name,iters,time,cycles,cpuTime,utime,stime,maxrss,minflt,majflt,nvcsw,nivcsw,allocated,numGcs,bytesCopied,mutatorWallSeconds,mutatorCpuSeconds,gcWallSeconds,gcCpuSeconds" >> $output_file

191− # XXX We may have to use "sort | awk" to keep only benchmark names with

192− # shortest prefix e.g. "a/b/c" and "a/b", we should only keep "a/b"

193− # otherwise benchmarks will run multiple times.

194− $target_prog -l \

195− | grep "^$target_name" \

196− | grep "^$MATCH" \

197− | sort | paste -sd "," - | awk 'BEGIN {FS=","} {t="XU987"; for(i=1;i<=NF;i++) if (substr($i,1,length(t)) != t) {print $i; t=$i}}' \

198− | while read -r name; do bin/bench-exec-one.sh "$name"; done

199 fi

217 fi

200}

218}

201

219

@@ -284,7 +302,6 @@ run_reports() {

284cd $SCRIPT_DIR/..

302cd $SCRIPT_DIR/..

285

303

286USE_GAUGE=0

304USE_GAUGE=0

287−export USE_GAUGE

288USE_GIT_CABAL=1

305USE_GIT_CABAL=1

289set_common_vars

306set_common_vars

290

307

@@ -318,6 +335,7 @@ do

318 # options with arguments

335 # options with arguments

319 --benchmarks) shift; TARGETS=$1; shift ;;

336 --benchmarks) shift; TARGETS=$1; shift ;;

320 --targets) shift; TARGETS=$1; shift ;;

337 --targets) shift; TARGETS=$1; shift ;;

338+ --prefix) shift; BENCH_PREFIX="$1" shift ;;

321 --fields) shift; FIELDS=$1; shift ;;

339 --fields) shift; FIELDS=$1; shift ;;

322 --base) shift; BASE=$1; shift ;;

340 --base) shift; BASE=$1; shift ;;

323 --candidate) shift; CANDIDATE=$1; shift ;;

341 --candidate) shift; CANDIDATE=$1; shift ;;

@@ -334,13 +352,13 @@ do

334 --long) LONG=1; shift ;;

352 --long) LONG=1; shift ;;

335 --graphs) GRAPH=1; shift ;;

353 --graphs) GRAPH=1; shift ;;

336 --no-measure) MEASURE=0; shift ;;

354 --no-measure) MEASURE=0; shift ;;

337− --dev-build) RUNNING_DEVBUILD=1; shift ;;

355+ --dev-build) RUNNING_DEVBUILD=1; shift ;;

356+ --use-gauge) USE_GAUGE=1; shift ;;

338 --) shift; break ;;

357 --) shift; break ;;

339− -*|--*) echo "Unknown flags: $*"; echo; print_help ;;

358+ *) echo "Unknown flags: $*"; echo; print_help ;;

340− *) break ;;

341 esac

359 esac

342done

360done

343−GAUGE_ARGS=$*

361+GAUGE_ARGS=("$@")

344

362

345set_derived_vars

363set_derived_vars

346

364

@@ -414,13 +432,15 @@ build_report_progs

414# Build and run targets

432# Build and run targets

415#-----------------------------------------------------------------------------

433#-----------------------------------------------------------------------------

416

434

435+if test "$USE_GAUGE" -eq 1

436+then

437+ BUILD_FLAGS="--flag use-gauge"

438+fi

439+

417−BUILD_BENCH="$CABAL_EXECUTABLE v2-build $CABAL_BUILD_OPTIONS --enable-benchmarks"

440+BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks"

418if test "$MEASURE" = "1"

441if test "$MEASURE" = "1"

419then

442then

420 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"

443 run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS"

421− export QUICK_MODE

422− export RTS_OPTIONS

423− export LONG

424 run_measurements "$TARGETS"

444 run_measurements "$TARGETS"

425fi

445fi

426

446