git.iko.soy/streamly
on master help sign in
Improve benchmarking of base streams

1) Fix options parsing in charting program 2) Add a group-diff options to compare StreamD/K 3) Add more benchmarks to StreamD/K

commit628355fad69ce6adb4f27eedb028afaa49c3557e authorHarendra Kumar <harendra.kumar@gmail.com> dateFri 2 Nov 2018, 21:05 (8 years ago) parent58e8195
5 files changed+154−85
compare with master
unifiedsplit
bench.sh+14−4 benchmark/BaseStreams.hs+51−34 benchmark/Chart.hs+51−30 benchmark/StreamDOps.hs+21−5 benchmark/StreamKOps.hs+17−12
bench.sh+14−4
@@ -2,17 +2,21 @@
22
33print_help () {
44 echo "Usage: $0 "
5− echo " [--compare] [--base commit] [--candidate commit]"
65 echo " [--benchmarks <all|linear|linear-async|linear-rate|nested|base>]"
6+ echo " [--group-diff]"
77 echo " [--graphs]"
8− echo " [--slow]"
98 echo " [--no-measure]"
109 echo " [--append] "
10+ echo " [--compare] [--base commit] [--candidate commit]"
11+ echo " [--slow]"
1112 echo " -- <gauge options>"
1213 echo
1314 echo "Multiple benchmarks can be specified as a space separate list"
1415 echo " e.g. --benchmarks \"linear nested\""
1516 echo
17+ echo "--group-diff is used to compare groups within a single benchmark"
18+ echo " e.g. StreamD vs StreamK in base benchmark."
19+ echo
1620 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
1721 echo "commit is generated, in the 'charts' directory."
1822 echo "Use --base and --candidate to select the commits to compare."
@@ -205,7 +209,9 @@ run_reports() {
205209 for i in $1
206210 do
207211 echo "Generating reports for ${i}..."
208− $prog $(test "$GRAPH" = 1 && echo "--graphs") --benchmark $i
212+ $prog $(test "$GRAPH" = 1 && echo "--graphs") \
213+ $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \
214+ --benchmark $i
209215 done
210216}
211217
@@ -215,6 +221,7 @@ run_reports() {
215221
216222DEFAULT_BENCHMARKS="linear"
217223ALL_BENCHMARKS="linear linear-async linear-rate nested base"
224+GROUP_DIFF=0
218225
219226COMPARE=0
220227BASE=
@@ -239,13 +246,16 @@ while test -n "$1"
239246do
240247 case $1 in
241248 -h|--help|help) print_help ;;
249+ # options with arguments
242250 --slow) SPEED_OPTIONS="--min-duration 0"; shift ;;
243− --append) APPEND=1; shift ;;
244251 --benchmarks) shift; BENCHMARKS=$1; shift ;;
245252 --base) shift; BASE=$1; shift ;;
246253 --candidate) shift; CANDIDATE=$1; shift ;;
254+ # flags
247255 --compare) COMPARE=1; shift ;;
248256 --raw) RAW=1; shift ;;
257+ --append) APPEND=1; shift ;;
258+ --group-diff) GROUP_DIFF=1; shift ;;
249259 --graphs) GRAPH=1; shift ;;
250260 --no-measure) MEASURE=0; shift ;;
251261 --) shift; break ;;
benchmark/BaseStreams.hs+51−34
@@ -48,34 +48,47 @@ main =
4848 , benchFold "tail" D.tail D.sourceUnfoldrM
4949 , benchIO "nullTail" D.nullTail D.sourceUnfoldrM
5050 , benchIO "headTail" D.headTail D.sourceUnfoldrM
51+ , benchFold "toList" K.toList K.sourceUnfoldrM
52+ , benchFold "fold" K.foldl K.sourceUnfoldrM
53+ , benchFold "last" K.last K.sourceUnfoldrM
5154 ]
5255 , bgroup "transformation"
53− [ benchIO "scan" (D.scan 1) D.sourceUnfoldrM
54− , benchIO "map" (D.map 1) D.sourceUnfoldrM
55− , benchIO "mapM" (D.mapM 1) D.sourceUnfoldrM
56+ [ benchIO "scan" (D.scan 1) D.sourceUnfoldrM
57+ , benchIO "map" (D.map 1) D.sourceUnfoldrM
58+ , benchIO "fmap" (D.fmap 1) D.sourceUnfoldrM
59+ , benchIO "mapM" (D.mapM 1) D.sourceUnfoldrM
60+ , benchIO "mapMaybe" (D.mapMaybe 1) D.sourceUnfoldrM
61+ , benchIO "mapMaybeM" (D.mapMaybeM 1) D.sourceUnfoldrM
5662 ]
5763 , bgroup "transformationN"
58− [ benchIO "scan" (D.scan 4) D.sourceUnfoldrM
59− , benchIO "map" (D.map 4) D.sourceUnfoldrM
60− , benchIO "mapM" (D.mapM 4) D.sourceUnfoldrM
64+ [ benchIO "scan" (D.scan 4) D.sourceUnfoldrM
65+ , benchIO "map" (D.map 4) D.sourceUnfoldrM
66+ , benchIO "fmap" (D.fmap 4) D.sourceUnfoldrM
67+ , benchIO "mapM" (D.mapM 4) D.sourceUnfoldrM
68+ , benchIO "mapMaybe" (D.mapMaybe 4) D.sourceUnfoldrM
69+ , benchIO "mapMaybeM" (D.mapMaybeM 4) D.sourceUnfoldrM
6170 ]
6271 , bgroup "filtering"
63− [ benchIO "filter-even" (D.filterEven 1) D.sourceUnfoldrM
64− , benchIO "filter-all-out" (D.filterAllOut 1) D.sourceUnfoldrM
65− , benchIO "filter-all-in" (D.filterAllIn 1) D.sourceUnfoldrM
66− , benchIO "take-all" (D.takeAll 1) D.sourceUnfoldrM
67− , benchIO "takeWhile-true" (D.takeWhileTrue 1) D.sourceUnfoldrM
68− , benchIO "drop-all" (D.dropAll 1) D.sourceUnfoldrM
69− , benchIO "dropWhile-true" (D.dropWhileTrue 1) D.sourceUnfoldrM
72+ [ benchIO "filter-even" (D.filterEven 1) D.sourceUnfoldrM
73+ , benchIO "filter-all-out" (D.filterAllOut 1) D.sourceUnfoldrM
74+ , benchIO "filter-all-in" (D.filterAllIn 1) D.sourceUnfoldrM
75+ , benchIO "take-all" (D.takeAll 1) D.sourceUnfoldrM
76+ , benchIO "takeWhile-true" (D.takeWhileTrue 1) D.sourceUnfoldrM
77+ , benchIO "drop-one" (D.dropOne 1) D.sourceUnfoldrM
78+ , benchIO "drop-all" (D.dropAll 1) D.sourceUnfoldrM
79+ , benchIO "dropWhile-true" (D.dropWhileTrue 1) D.sourceUnfoldrM
80+ , benchIO "dropWhile-false" (D.dropWhileFalse 1) D.sourceUnfoldrM
7081 ]
7182 , bgroup "filteringN"
72− [ benchIO "filter-even" (D.filterEven 4) D.sourceUnfoldrM
73− , benchIO "filter-all-out" (D.filterAllOut 4) D.sourceUnfoldrM
74− , benchIO "filter-all-in" (D.filterAllIn 4) D.sourceUnfoldrM
75− , benchIO "take-all" (D.takeAll 4) D.sourceUnfoldrM
76− , benchIO "takeWhile-true" (D.takeWhileTrue 4) D.sourceUnfoldrM
77− , benchIO "drop-all" (D.dropAll 4) D.sourceUnfoldrM
78− , benchIO "dropWhile-true" (D.dropWhileTrue 4) D.sourceUnfoldrM
83+ [ benchIO "filter-even" (D.filterEven 4) D.sourceUnfoldrM
84+ , benchIO "filter-all-out" (D.filterAllOut 4) D.sourceUnfoldrM
85+ , benchIO "filter-all-in" (D.filterAllIn 4) D.sourceUnfoldrM
86+ , benchIO "take-all" (D.takeAll 4) D.sourceUnfoldrM
87+ , benchIO "takeWhile-true" (D.takeWhileTrue 4) D.sourceUnfoldrM
88+ , benchIO "drop-one" (D.dropOne 4) D.sourceUnfoldrM
89+ , benchIO "drop-all" (D.dropAll 4) D.sourceUnfoldrM
90+ , benchIO "dropWhile-true" (D.dropWhileTrue 4) D.sourceUnfoldrM
91+ , benchIO "dropWhile-false" (D.dropWhileFalse 4) D.sourceUnfoldrM
7992 ]
8093 , benchIO "zip" D.zip D.sourceUnfoldrM
8194 , bgroup "composed"
@@ -128,22 +141,26 @@ main =
128141 -- , benchIO "concat" K.concat K.sourceUnfoldrM
129142 ]
130143 , bgroup "filtering"
131− [ benchIO "filter-even" (K.filterEven 1) K.sourceUnfoldrM
132− , benchIO "filter-all-out" (K.filterAllOut 1) K.sourceUnfoldrM
133− , benchIO "filter-all-in" (K.filterAllIn 1) K.sourceUnfoldrM
134− , benchIO "take-all" (K.takeAll 1) K.sourceUnfoldrM
135− , benchIO "takeWhile-true" (K.takeWhileTrue 1) K.sourceUnfoldrM
136− , benchIO "drop-all" (K.dropAll 1) K.sourceUnfoldrM
137− , benchIO "dropWhile-true" (K.dropWhileTrue 1) K.sourceUnfoldrM
144+ [ benchIO "filter-even" (K.filterEven 1) K.sourceUnfoldrM
145+ , benchIO "filter-all-out" (K.filterAllOut 1) K.sourceUnfoldrM
146+ , benchIO "filter-all-in" (K.filterAllIn 1) K.sourceUnfoldrM
147+ , benchIO "take-all" (K.takeAll 1) K.sourceUnfoldrM
148+ , benchIO "takeWhile-true" (K.takeWhileTrue 1) K.sourceUnfoldrM
149+ , benchIO "drop-one" (K.dropOne 1) K.sourceUnfoldrM
150+ , benchIO "drop-all" (K.dropAll 1) K.sourceUnfoldrM
151+ , benchIO "dropWhile-true" (K.dropWhileTrue 1) K.sourceUnfoldrM
152+ , benchIO "dropWhile-false" (K.dropWhileFalse 1) K.sourceUnfoldrM
138153 ]
139154 , bgroup "filteringN"
140− [ benchIO "filter-even" (K.filterEven 4) K.sourceUnfoldrM
141− , benchIO "filter-all-out" (K.filterAllOut 4) K.sourceUnfoldrM
142− , benchIO "filter-all-in" (K.filterAllIn 4) K.sourceUnfoldrM
143− , benchIO "take-all" (K.takeAll 4) K.sourceUnfoldrM
144− , benchIO "takeWhile-true" (K.takeWhileTrue 4) K.sourceUnfoldrM
145− , benchIO "drop-all" (K.dropAll 4) K.sourceUnfoldrM
146− , benchIO "dropWhile-true" (K.dropWhileTrue 4) K.sourceUnfoldrM
155+ [ benchIO "filter-even" (K.filterEven 4) K.sourceUnfoldrM
156+ , benchIO "filter-all-out" (K.filterAllOut 4) K.sourceUnfoldrM
157+ , benchIO "filter-all-in" (K.filterAllIn 4) K.sourceUnfoldrM
158+ , benchIO "take-all" (K.takeAll 4) K.sourceUnfoldrM
159+ , benchIO "takeWhile-true" (K.takeWhileTrue 4) K.sourceUnfoldrM
160+ , benchIO "drop-one" (K.dropOne 4) K.sourceUnfoldrM
161+ , benchIO "drop-all" (K.dropAll 4) K.sourceUnfoldrM
162+ , benchIO "dropWhile-true" (K.dropWhileTrue 4) K.sourceUnfoldrM
163+ , benchIO "dropWhile-false" (K.dropWhileFalse 4) K.sourceUnfoldrM
147164 ]
148165 , benchIO "zip" K.zip K.sourceUnfoldrM
149166 , bgroup "composed"
benchmark/Chart.hs+51−30
@@ -21,18 +21,24 @@ import BenchShow
2121------------------------------------------------------------------------------
2222
2323data BenchType = Linear | LinearAsync | LinearRate | Nested | Base
24+ deriving Show
2425
2526data Options = Options
2627 { genGraphs :: Bool
28+ , groupDiff :: Bool
2729 , benchType :: BenchType
28− }
30+ } deriving Show
2931
30−defaultOptions = Options False Linear
32+defaultOptions = Options False False Linear
3133
3234setGenGraphs val = do
3335 (args, opts) <- get
3436 put (args, opts { genGraphs = val })
3537
38+setGroupDiff val = do
39+ (args, opts) <- get
40+ put (args, opts { groupDiff = val })
41+
3642setBenchType val = do
3743 (args, opts) <- get
3844 put (args, opts { benchType = val })
@@ -66,15 +72,25 @@ parseOptions :: IO (Maybe Options)
6672parseOptions = do
6773 args <- getArgs
6874 runMaybeT $ flip evalStateT (args, defaultOptions) $ do
69− x <- shift
70− case x of
71− Just "--graphs" -> setGenGraphs True
72− Just "--benchmark" -> parseBench
73− Just str -> do
75+ parseLoop
76+ fmap snd get
77+
78+ where
79+
80+ parseOpt opt =
81+ case opt of
82+ "--graphs" -> setGenGraphs True
83+ "--group-diff" -> setGroupDiff True
84+ "--benchmark" -> parseBench
85+ str -> do
7486 liftIO $ putStrLn $ "Unrecognized option " <> str
7587 mzero
88+
89+ parseLoop = do
90+ next <- shift
91+ case next of
92+ Just opt -> parseOpt opt >> parseLoop
7693 Nothing -> return ()
77− fmap snd get
7894
7995ignoringErr a = catch a (\(ErrorCall err :: ErrorCall) ->
8096 putStrLn $ "Failed with error:\n" <> err <> "\nSkipping.")
@@ -169,13 +185,27 @@ makeLinearRateGraphs cfg inputFile = do
169185 return ()
170186
171187------------------------------------------------------------------------------
172−-- Charts for base streams
188+-- Reports/Charts for base streams
173189------------------------------------------------------------------------------
174190
175−makeBaseGraphs :: Config -> String -> IO ()
176−makeBaseGraphs cfg inputFile = do
177− putStrLn "Not implemented"
178− return ()
191+classifyBase b
192+ | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b
193+ | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b
194+ | otherwise = Nothing
195+
196+showStreamDVsK Options{..} cfg inp out =
197+ let cfg' = cfg { classifyBenchmark = classifyBase }
198+ in if genGraphs
199+ then ignoringErr $ graph inp "streamD-vs-streamK"
200+ cfg' {outputDir = Just out}
201+ else ignoringErr $ report inp Nothing cfg'
202+
203+showBaseStreams Options{..} cfg inp out =
204+ let cfg' = cfg { classifyBenchmark = classifyBase }
205+ in if genGraphs
206+ then ignoringErr $ graph inp "streamD"
207+ cfg' {outputDir = Just out}
208+ else ignoringErr $ report inp Nothing cfg'
179209
180210------------------------------------------------------------------------------
181211-- text reports
@@ -195,20 +225,6 @@ benchShow Options{..} cfg func inp out =
195225 then func cfg {outputDir = Just out} inp
196226 else ignoringErr $ report inp Nothing cfg
197227
198−showStreamDVsK Options{..} cfg func inp out =
199− let cfg' = cfg { classifyBenchmark = classify }
200− in if genGraphs
201− then ignoringErr $ graph inp "streamD-vs-streamK"
202− cfg' {outputDir = Just out}
203− else ignoringErr $ report inp Nothing cfg'
204−
205− where
206−
207− classify b
208− | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b
209− | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b
210− | otherwise = Nothing
211−
212228main :: IO ()
213229main = do
214230 let cfg = defaultConfig
@@ -235,6 +251,11 @@ main = do
235251 Nested -> benchShow opts cfg makeNestedGraphs
236252 "charts/nested/results.csv"
237253 "charts/nested"
238− Base -> showStreamDVsK opts cfg makeBaseGraphs
239− "charts/base/results.csv"
240− "charts/base"
254+ Base ->
255+ if groupDiff
256+ then showStreamDVsK opts cfg
257+ "charts/base/results.csv"
258+ "charts/base"
259+ else showBaseStreams opts cfg
260+ "charts/base/results.csv"
261+ "charts/base"
benchmark/StreamDOps.hs+21−5
@@ -14,7 +14,7 @@ import Data.Maybe (isJust)
1414import Prelude
1515 (Monad, Int, (+), ($), (.), return, (>), even, (<=),
1616 subtract, undefined, Maybe(..), not, mapM_, (>>=),
17− maxBound)
17+ maxBound, fmap, odd)
1818
1919import qualified Streamly.Streams.StreamD as S
2020
@@ -138,31 +138,47 @@ composeN n f =
138138
139139{-# INLINE scan #-}
140140{-# INLINE map #-}
141+{-# INLINE fmap #-}
141142{-# INLINE mapM #-}
143+{-# INLINE mapMaybe #-}
142144{-# INLINE filterEven #-}
143145{-# INLINE filterAllOut #-}
144146{-# INLINE filterAllIn #-}
145147{-# INLINE takeOne #-}
146148{-# INLINE takeAll #-}
147149{-# INLINE takeWhileTrue #-}
150+{-# INLINE takeWhileMTrue #-}
151+{-# INLINE dropOne #-}
148152{-# INLINE dropAll #-}
149153{-# INLINE dropWhileTrue #-}
150−scan, map, mapM, filterEven, filterAllOut,
151− filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropWhileTrue
154+{-# INLINE dropWhileMTrue #-}
155+{-# INLINE dropWhileFalse #-}
156+scan, map, fmap, mapM, mapMaybe, mapMaybeM, filterEven, filterAllOut,
157+ filterAllIn, takeOne, takeAll, takeWhileTrue, takeWhileMTrue, dropOne,
158+ dropAll, dropWhileTrue, dropWhileMTrue, dropWhileFalse
152159 :: Monad m
153160 => Int -> Stream m Int -> m ()
154161
155162scan n = composeN n $ S.scanl' (+) 0
163+fmap n = composeN n $ Prelude.fmap (+1)
156164map n = composeN n $ S.map (+1)
157165mapM n = composeN n $ S.mapM return
166+mapMaybe n = composeN n $ S.mapMaybe
167+ (\x -> if Prelude.odd x then Nothing else Just x)
168+mapMaybeM n = composeN n $ S.mapMaybeM
169+ (\x -> if Prelude.odd x then return Nothing else return $ Just x)
158170filterEven n = composeN n $ S.filter even
159171filterAllOut n = composeN n $ S.filter (> maxValue)
160172filterAllIn n = composeN n $ S.filter (<= maxValue)
161173takeOne n = composeN n $ S.take 1
162174takeAll n = composeN n $ S.take maxValue
163175takeWhileTrue n = composeN n $ S.takeWhile (<= maxValue)
164−dropAll n = composeN n $ S.drop maxValue
165−dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue)
176+takeWhileMTrue n = composeN n $ S.takeWhileM (return . (<= maxValue))
177+dropOne n = composeN n $ S.drop 1
178+dropAll n = composeN n $ S.drop maxValue
179+dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue)
180+dropWhileMTrue n = composeN n $ S.dropWhileM (return . (<= maxValue))
181+dropWhileFalse n = composeN n $ S.dropWhile (<= 1)
166182
167183-------------------------------------------------------------------------------
168184-- Zipping and concat
benchmark/StreamKOps.hs+17−12
@@ -162,27 +162,32 @@ composeN n f =
162162{-# INLINE takeOne #-}
163163{-# INLINE takeAll #-}
164164{-# INLINE takeWhileTrue #-}
165+{-# INLINE dropOne #-}
165166{-# INLINE dropAll #-}
166167{-# INLINE dropWhileTrue #-}
168+{-# INLINE dropWhileFalse #-}
167169scan, map, filterEven, filterAllOut,
168− filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropWhileTrue
170+ filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropOne,
171+ dropWhileTrue, dropWhileFalse
169172 :: Monad m
170173 => Int -> Stream m Int -> m ()
171174
172175{-# INLINE mapM #-}
173176mapM :: S.MonadAsync m => Int -> Stream m Int -> m ()
174177
175−scan n = composeN n $ S.scanl' (+) 0
176−map n = composeN n $ fmap (+1)
177−mapM n = composeN n $ S.mapM return
178−filterEven n = composeN n $ S.filter even
179−filterAllOut n = composeN n $ S.filter (> maxValue)
180−filterAllIn n = composeN n $ S.filter (<= maxValue)
181−takeOne n = composeN n $ S.take 1
182−takeAll n = composeN n $ S.take maxValue
183−takeWhileTrue n = composeN n $ S.takeWhile (<= maxValue)
184−dropAll n = composeN n $ S.drop maxValue
185−dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue)
178+scan n = composeN n $ S.scanl' (+) 0
179+map n = composeN n $ fmap (+1)
180+mapM n = composeN n $ S.mapM return
181+filterEven n = composeN n $ S.filter even
182+filterAllOut n = composeN n $ S.filter (> maxValue)
183+filterAllIn n = composeN n $ S.filter (<= maxValue)
184+takeOne n = composeN n $ S.take 1
185+takeAll n = composeN n $ S.take maxValue
186+takeWhileTrue n = composeN n $ S.takeWhile (<= maxValue)
187+dropOne n = composeN n $ S.drop 1
188+dropAll n = composeN n $ S.drop maxValue
189+dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue)
190+dropWhileFalse n = composeN n $ S.dropWhile (<= 1)
186191
187192-------------------------------------------------------------------------------
188193-- Zipping and concat