diff --git a/bench.sh b/bench.sh index f1e908443..0112ef14e 100755 --- a/bench.sh +++ b/bench.sh @@ -2,17 +2,21 @@ print_help () { echo "Usage: $0 " - echo " [--compare] [--base commit] [--candidate commit]" echo " [--benchmarks ]" + echo " [--group-diff]" echo " [--graphs]" - echo " [--slow]" echo " [--no-measure]" echo " [--append] " + echo " [--compare] [--base commit] [--candidate commit]" + echo " [--slow]" echo " -- " echo echo "Multiple benchmarks can be specified as a space separate list" echo " e.g. --benchmarks \"linear nested\"" echo + echo "--group-diff is used to compare groups within a single benchmark" + echo " e.g. StreamD vs StreamK in base benchmark." + echo echo "When using --compare, by default comparative chart of HEAD^ vs HEAD" echo "commit is generated, in the 'charts' directory." echo "Use --base and --candidate to select the commits to compare." @@ -205,7 +209,9 @@ run_reports() { for i in $1 do echo "Generating reports for ${i}..." - $prog $(test "$GRAPH" = 1 && echo "--graphs") --benchmark $i + $prog $(test "$GRAPH" = 1 && echo "--graphs") \ + $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \ + --benchmark $i done } @@ -215,6 +221,7 @@ run_reports() { DEFAULT_BENCHMARKS="linear" ALL_BENCHMARKS="linear linear-async linear-rate nested base" +GROUP_DIFF=0 COMPARE=0 BASE= @@ -239,13 +246,16 @@ while test -n "$1" do case $1 in -h|--help|help) print_help ;; + # options with arguments --slow) SPEED_OPTIONS="--min-duration 0"; shift ;; - --append) APPEND=1; shift ;; --benchmarks) shift; BENCHMARKS=$1; shift ;; --base) shift; BASE=$1; shift ;; --candidate) shift; CANDIDATE=$1; shift ;; + # flags --compare) COMPARE=1; shift ;; --raw) RAW=1; shift ;; + --append) APPEND=1; shift ;; + --group-diff) GROUP_DIFF=1; shift ;; --graphs) GRAPH=1; shift ;; --no-measure) MEASURE=0; shift ;; --) shift; break ;; diff --git a/benchmark/BaseStreams.hs b/benchmark/BaseStreams.hs index 4cf249594..0834cf2ba 100644 --- a/benchmark/BaseStreams.hs +++ b/benchmark/BaseStreams.hs @@ -48,34 +48,47 @@ main = , benchFold "tail" D.tail D.sourceUnfoldrM , benchIO "nullTail" D.nullTail D.sourceUnfoldrM , benchIO "headTail" D.headTail D.sourceUnfoldrM + , benchFold "toList" K.toList K.sourceUnfoldrM + , benchFold "fold" K.foldl K.sourceUnfoldrM + , benchFold "last" K.last K.sourceUnfoldrM ] , bgroup "transformation" - [ benchIO "scan" (D.scan 1) D.sourceUnfoldrM - , benchIO "map" (D.map 1) D.sourceUnfoldrM - , benchIO "mapM" (D.mapM 1) D.sourceUnfoldrM + [ benchIO "scan" (D.scan 1) D.sourceUnfoldrM + , benchIO "map" (D.map 1) D.sourceUnfoldrM + , benchIO "fmap" (D.fmap 1) D.sourceUnfoldrM + , benchIO "mapM" (D.mapM 1) D.sourceUnfoldrM + , benchIO "mapMaybe" (D.mapMaybe 1) D.sourceUnfoldrM + , benchIO "mapMaybeM" (D.mapMaybeM 1) D.sourceUnfoldrM ] , bgroup "transformationN" - [ benchIO "scan" (D.scan 4) D.sourceUnfoldrM - , benchIO "map" (D.map 4) D.sourceUnfoldrM - , benchIO "mapM" (D.mapM 4) D.sourceUnfoldrM + [ benchIO "scan" (D.scan 4) D.sourceUnfoldrM + , benchIO "map" (D.map 4) D.sourceUnfoldrM + , benchIO "fmap" (D.fmap 4) D.sourceUnfoldrM + , benchIO "mapM" (D.mapM 4) D.sourceUnfoldrM + , benchIO "mapMaybe" (D.mapMaybe 4) D.sourceUnfoldrM + , benchIO "mapMaybeM" (D.mapMaybeM 4) D.sourceUnfoldrM ] , bgroup "filtering" - [ benchIO "filter-even" (D.filterEven 1) D.sourceUnfoldrM - , benchIO "filter-all-out" (D.filterAllOut 1) D.sourceUnfoldrM - , benchIO "filter-all-in" (D.filterAllIn 1) D.sourceUnfoldrM - , benchIO "take-all" (D.takeAll 1) D.sourceUnfoldrM - , benchIO "takeWhile-true" (D.takeWhileTrue 1) D.sourceUnfoldrM - , benchIO "drop-all" (D.dropAll 1) D.sourceUnfoldrM - , benchIO "dropWhile-true" (D.dropWhileTrue 1) D.sourceUnfoldrM + [ benchIO "filter-even" (D.filterEven 1) D.sourceUnfoldrM + , benchIO "filter-all-out" (D.filterAllOut 1) D.sourceUnfoldrM + , benchIO "filter-all-in" (D.filterAllIn 1) D.sourceUnfoldrM + , benchIO "take-all" (D.takeAll 1) D.sourceUnfoldrM + , benchIO "takeWhile-true" (D.takeWhileTrue 1) D.sourceUnfoldrM + , benchIO "drop-one" (D.dropOne 1) D.sourceUnfoldrM + , benchIO "drop-all" (D.dropAll 1) D.sourceUnfoldrM + , benchIO "dropWhile-true" (D.dropWhileTrue 1) D.sourceUnfoldrM + , benchIO "dropWhile-false" (D.dropWhileFalse 1) D.sourceUnfoldrM ] , bgroup "filteringN" - [ benchIO "filter-even" (D.filterEven 4) D.sourceUnfoldrM - , benchIO "filter-all-out" (D.filterAllOut 4) D.sourceUnfoldrM - , benchIO "filter-all-in" (D.filterAllIn 4) D.sourceUnfoldrM - , benchIO "take-all" (D.takeAll 4) D.sourceUnfoldrM - , benchIO "takeWhile-true" (D.takeWhileTrue 4) D.sourceUnfoldrM - , benchIO "drop-all" (D.dropAll 4) D.sourceUnfoldrM - , benchIO "dropWhile-true" (D.dropWhileTrue 4) D.sourceUnfoldrM + [ benchIO "filter-even" (D.filterEven 4) D.sourceUnfoldrM + , benchIO "filter-all-out" (D.filterAllOut 4) D.sourceUnfoldrM + , benchIO "filter-all-in" (D.filterAllIn 4) D.sourceUnfoldrM + , benchIO "take-all" (D.takeAll 4) D.sourceUnfoldrM + , benchIO "takeWhile-true" (D.takeWhileTrue 4) D.sourceUnfoldrM + , benchIO "drop-one" (D.dropOne 4) D.sourceUnfoldrM + , benchIO "drop-all" (D.dropAll 4) D.sourceUnfoldrM + , benchIO "dropWhile-true" (D.dropWhileTrue 4) D.sourceUnfoldrM + , benchIO "dropWhile-false" (D.dropWhileFalse 4) D.sourceUnfoldrM ] , benchIO "zip" D.zip D.sourceUnfoldrM , bgroup "composed" @@ -128,22 +141,26 @@ main = -- , benchIO "concat" K.concat K.sourceUnfoldrM ] , bgroup "filtering" - [ benchIO "filter-even" (K.filterEven 1) K.sourceUnfoldrM - , benchIO "filter-all-out" (K.filterAllOut 1) K.sourceUnfoldrM - , benchIO "filter-all-in" (K.filterAllIn 1) K.sourceUnfoldrM - , benchIO "take-all" (K.takeAll 1) K.sourceUnfoldrM - , benchIO "takeWhile-true" (K.takeWhileTrue 1) K.sourceUnfoldrM - , benchIO "drop-all" (K.dropAll 1) K.sourceUnfoldrM - , benchIO "dropWhile-true" (K.dropWhileTrue 1) K.sourceUnfoldrM + [ benchIO "filter-even" (K.filterEven 1) K.sourceUnfoldrM + , benchIO "filter-all-out" (K.filterAllOut 1) K.sourceUnfoldrM + , benchIO "filter-all-in" (K.filterAllIn 1) K.sourceUnfoldrM + , benchIO "take-all" (K.takeAll 1) K.sourceUnfoldrM + , benchIO "takeWhile-true" (K.takeWhileTrue 1) K.sourceUnfoldrM + , benchIO "drop-one" (K.dropOne 1) K.sourceUnfoldrM + , benchIO "drop-all" (K.dropAll 1) K.sourceUnfoldrM + , benchIO "dropWhile-true" (K.dropWhileTrue 1) K.sourceUnfoldrM + , benchIO "dropWhile-false" (K.dropWhileFalse 1) K.sourceUnfoldrM ] , bgroup "filteringN" - [ benchIO "filter-even" (K.filterEven 4) K.sourceUnfoldrM - , benchIO "filter-all-out" (K.filterAllOut 4) K.sourceUnfoldrM - , benchIO "filter-all-in" (K.filterAllIn 4) K.sourceUnfoldrM - , benchIO "take-all" (K.takeAll 4) K.sourceUnfoldrM - , benchIO "takeWhile-true" (K.takeWhileTrue 4) K.sourceUnfoldrM - , benchIO "drop-all" (K.dropAll 4) K.sourceUnfoldrM - , benchIO "dropWhile-true" (K.dropWhileTrue 4) K.sourceUnfoldrM + [ benchIO "filter-even" (K.filterEven 4) K.sourceUnfoldrM + , benchIO "filter-all-out" (K.filterAllOut 4) K.sourceUnfoldrM + , benchIO "filter-all-in" (K.filterAllIn 4) K.sourceUnfoldrM + , benchIO "take-all" (K.takeAll 4) K.sourceUnfoldrM + , benchIO "takeWhile-true" (K.takeWhileTrue 4) K.sourceUnfoldrM + , benchIO "drop-one" (K.dropOne 4) K.sourceUnfoldrM + , benchIO "drop-all" (K.dropAll 4) K.sourceUnfoldrM + , benchIO "dropWhile-true" (K.dropWhileTrue 4) K.sourceUnfoldrM + , benchIO "dropWhile-false" (K.dropWhileFalse 4) K.sourceUnfoldrM ] , benchIO "zip" K.zip K.sourceUnfoldrM , bgroup "composed" diff --git a/benchmark/Chart.hs b/benchmark/Chart.hs index 79bd77386..01df4bbd4 100644 --- a/benchmark/Chart.hs +++ b/benchmark/Chart.hs @@ -21,18 +21,24 @@ import BenchShow ------------------------------------------------------------------------------ data BenchType = Linear | LinearAsync | LinearRate | Nested | Base + deriving Show data Options = Options { genGraphs :: Bool + , groupDiff :: Bool , benchType :: BenchType - } + } deriving Show -defaultOptions = Options False Linear +defaultOptions = Options False False Linear setGenGraphs val = do (args, opts) <- get put (args, opts { genGraphs = val }) +setGroupDiff val = do + (args, opts) <- get + put (args, opts { groupDiff = val }) + setBenchType val = do (args, opts) <- get put (args, opts { benchType = val }) @@ -66,15 +72,25 @@ parseOptions :: IO (Maybe Options) parseOptions = do args <- getArgs runMaybeT $ flip evalStateT (args, defaultOptions) $ do - x <- shift - case x of - Just "--graphs" -> setGenGraphs True - Just "--benchmark" -> parseBench - Just str -> do + parseLoop + fmap snd get + + where + + parseOpt opt = + case opt of + "--graphs" -> setGenGraphs True + "--group-diff" -> setGroupDiff True + "--benchmark" -> parseBench + str -> do liftIO $ putStrLn $ "Unrecognized option " <> str mzero + + parseLoop = do + next <- shift + case next of + Just opt -> parseOpt opt >> parseLoop Nothing -> return () - fmap snd get ignoringErr a = catch a (\(ErrorCall err :: ErrorCall) -> putStrLn $ "Failed with error:\n" <> err <> "\nSkipping.") @@ -169,13 +185,27 @@ makeLinearRateGraphs cfg inputFile = do return () ------------------------------------------------------------------------------ --- Charts for base streams +-- Reports/Charts for base streams ------------------------------------------------------------------------------ -makeBaseGraphs :: Config -> String -> IO () -makeBaseGraphs cfg inputFile = do - putStrLn "Not implemented" - return () +classifyBase b + | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b + | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b + | otherwise = Nothing + +showStreamDVsK Options{..} cfg inp out = + let cfg' = cfg { classifyBenchmark = classifyBase } + in if genGraphs + then ignoringErr $ graph inp "streamD-vs-streamK" + cfg' {outputDir = Just out} + else ignoringErr $ report inp Nothing cfg' + +showBaseStreams Options{..} cfg inp out = + let cfg' = cfg { classifyBenchmark = classifyBase } + in if genGraphs + then ignoringErr $ graph inp "streamD" + cfg' {outputDir = Just out} + else ignoringErr $ report inp Nothing cfg' ------------------------------------------------------------------------------ -- text reports @@ -195,20 +225,6 @@ benchShow Options{..} cfg func inp out = then func cfg {outputDir = Just out} inp else ignoringErr $ report inp Nothing cfg -showStreamDVsK Options{..} cfg func inp out = - let cfg' = cfg { classifyBenchmark = classify } - in if genGraphs - then ignoringErr $ graph inp "streamD-vs-streamK" - cfg' {outputDir = Just out} - else ignoringErr $ report inp Nothing cfg' - - where - - classify b - | "streamD/" `isPrefixOf` b = ("streamD",) <$> stripPrefix "streamD/" b - | "streamK/" `isPrefixOf` b = ("streamK",) <$> stripPrefix "streamK/" b - | otherwise = Nothing - main :: IO () main = do let cfg = defaultConfig @@ -235,6 +251,11 @@ main = do Nested -> benchShow opts cfg makeNestedGraphs "charts/nested/results.csv" "charts/nested" - Base -> showStreamDVsK opts cfg makeBaseGraphs - "charts/base/results.csv" - "charts/base" + Base -> + if groupDiff + then showStreamDVsK opts cfg + "charts/base/results.csv" + "charts/base" + else showBaseStreams opts cfg + "charts/base/results.csv" + "charts/base" diff --git a/benchmark/StreamDOps.hs b/benchmark/StreamDOps.hs index 0009ab5af..c3e3a9ef0 100644 --- a/benchmark/StreamDOps.hs +++ b/benchmark/StreamDOps.hs @@ -14,7 +14,7 @@ import Data.Maybe (isJust) import Prelude (Monad, Int, (+), ($), (.), return, (>), even, (<=), subtract, undefined, Maybe(..), not, mapM_, (>>=), - maxBound) + maxBound, fmap, odd) import qualified Streamly.Streams.StreamD as S @@ -138,31 +138,47 @@ composeN n f = {-# INLINE scan #-} {-# INLINE map #-} +{-# INLINE fmap #-} {-# INLINE mapM #-} +{-# INLINE mapMaybe #-} {-# INLINE filterEven #-} {-# INLINE filterAllOut #-} {-# INLINE filterAllIn #-} {-# INLINE takeOne #-} {-# INLINE takeAll #-} {-# INLINE takeWhileTrue #-} +{-# INLINE takeWhileMTrue #-} +{-# INLINE dropOne #-} {-# INLINE dropAll #-} {-# INLINE dropWhileTrue #-} -scan, map, mapM, filterEven, filterAllOut, - filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropWhileTrue +{-# INLINE dropWhileMTrue #-} +{-# INLINE dropWhileFalse #-} +scan, map, fmap, mapM, mapMaybe, mapMaybeM, filterEven, filterAllOut, + filterAllIn, takeOne, takeAll, takeWhileTrue, takeWhileMTrue, dropOne, + dropAll, dropWhileTrue, dropWhileMTrue, dropWhileFalse :: Monad m => Int -> Stream m Int -> m () scan n = composeN n $ S.scanl' (+) 0 +fmap n = composeN n $ Prelude.fmap (+1) map n = composeN n $ S.map (+1) mapM n = composeN n $ S.mapM return +mapMaybe n = composeN n $ S.mapMaybe + (\x -> if Prelude.odd x then Nothing else Just x) +mapMaybeM n = composeN n $ S.mapMaybeM + (\x -> if Prelude.odd x then return Nothing else return $ Just x) filterEven n = composeN n $ S.filter even filterAllOut n = composeN n $ S.filter (> maxValue) filterAllIn n = composeN n $ S.filter (<= maxValue) takeOne n = composeN n $ S.take 1 takeAll n = composeN n $ S.take maxValue takeWhileTrue n = composeN n $ S.takeWhile (<= maxValue) -dropAll n = composeN n $ S.drop maxValue -dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue) +takeWhileMTrue n = composeN n $ S.takeWhileM (return . (<= maxValue)) +dropOne n = composeN n $ S.drop 1 +dropAll n = composeN n $ S.drop maxValue +dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue) +dropWhileMTrue n = composeN n $ S.dropWhileM (return . (<= maxValue)) +dropWhileFalse n = composeN n $ S.dropWhile (<= 1) ------------------------------------------------------------------------------- -- Zipping and concat diff --git a/benchmark/StreamKOps.hs b/benchmark/StreamKOps.hs index 8577de472..e242819d4 100644 --- a/benchmark/StreamKOps.hs +++ b/benchmark/StreamKOps.hs @@ -162,27 +162,32 @@ composeN n f = {-# INLINE takeOne #-} {-# INLINE takeAll #-} {-# INLINE takeWhileTrue #-} +{-# INLINE dropOne #-} {-# INLINE dropAll #-} {-# INLINE dropWhileTrue #-} +{-# INLINE dropWhileFalse #-} scan, map, filterEven, filterAllOut, - filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropWhileTrue + filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropOne, + dropWhileTrue, dropWhileFalse :: Monad m => Int -> Stream m Int -> m () {-# INLINE mapM #-} mapM :: S.MonadAsync m => Int -> Stream m Int -> m () -scan n = composeN n $ S.scanl' (+) 0 -map n = composeN n $ fmap (+1) -mapM n = composeN n $ S.mapM return -filterEven n = composeN n $ S.filter even -filterAllOut n = composeN n $ S.filter (> maxValue) -filterAllIn n = composeN n $ S.filter (<= maxValue) -takeOne n = composeN n $ S.take 1 -takeAll n = composeN n $ S.take maxValue -takeWhileTrue n = composeN n $ S.takeWhile (<= maxValue) -dropAll n = composeN n $ S.drop maxValue -dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue) +scan n = composeN n $ S.scanl' (+) 0 +map n = composeN n $ fmap (+1) +mapM n = composeN n $ S.mapM return +filterEven n = composeN n $ S.filter even +filterAllOut n = composeN n $ S.filter (> maxValue) +filterAllIn n = composeN n $ S.filter (<= maxValue) +takeOne n = composeN n $ S.take 1 +takeAll n = composeN n $ S.take maxValue +takeWhileTrue n = composeN n $ S.takeWhile (<= maxValue) +dropOne n = composeN n $ S.drop 1 +dropAll n = composeN n $ S.drop maxValue +dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue) +dropWhileFalse n = composeN n $ S.dropWhile (<= 1) ------------------------------------------------------------------------------- -- Zipping and concat