on master help sign in
use stack and heap limits on benchmarks

To detect problems early and to classify operations based on their memory usage and scalabaility.

commitd6993b76b9af12e6f6e14e345b2aa851f724d80d authorHarendra Kumar <harendra.kumar@gmail.com> dateFri 14 Feb 2020, 17:46 (7 years ago) parent4f4964a
6 files changed+76−38
benchmark/BaseStreams.hs+21 benchmark/Linear.hs+19−10 benchmark/Nested.hs+1 benchmark/NestedConcurrent.hs+1 benchmark/StreamDKOps.hs+2−3 streamly.cabal+32−25
benchmark/BaseStreams.hs+21
@@ -41,6 +41,7 @@ benchFold :: NFData b
4141 => String -> (t IO Int -> IO b) -> (Int -> t IO Int) -> Benchmark
4242benchFold name f src = bench name $ nfIO $ randomRIO (1,1) >>= f . src
4343
44+#ifdef DEVBUILD
4445-- | Takes a source, and uses it with a default drain/fold method.
4546{-# INLINE benchD #-}
4647benchD :: String -> (Int -> D.Stream IO Int) -> Benchmark
@@ -49,6 +50,7 @@ benchD name f = bench name $ nfIO $ randomRIO (1,1) >>= D.toNull . f
4950{-# INLINE benchK #-}
5051benchK :: String -> (Int -> K.Stream IO Int) -> Benchmark
5152benchK name f = bench name $ nfIO $ randomRIO (1,1) >>= K.toNull . f
53+#endif
5254
5355{-
5456_benchId :: NFData b => String -> (Ops.Stream m Int -> Identity b) -> Benchmark
@@ -75,10 +77,14 @@ main =
7577 [ benchIO "toNull" D.toNull D.sourceUnfoldrM
7678 , benchIO "mapM_" D.mapM_ D.sourceUnfoldrM
7779 , benchIO "uncons" D.uncons D.sourceUnfoldrM
80+#ifdef DEVBUILD
81+ -- XXX these consume too much stack space, need to fix or segregate in
82+ -- another benchmark.
7883 , benchFold "tail" D.tail D.sourceUnfoldrM
7984 , benchIO "nullTail" D.nullTail D.sourceUnfoldrM
8085 , benchIO "headTail" D.headTail D.sourceUnfoldrM
8186 , benchFold "toList" D.toList D.sourceUnfoldrM
87+#endif
8288 , benchFold "foldl'" D.foldl D.sourceUnfoldrM
8389 , benchFold "last" D.last D.sourceUnfoldrM
8490 ]
@@ -186,6 +192,9 @@ main =
186192 , benchIO "filter-scan" (D.filterScan 4) D.sourceUnfoldrM
187193 , benchIO "filter-map" (D.filterMap 4) D.sourceUnfoldrM
188194 ]
195+#ifdef DEVBUILD
196+ -- XXX these consume too much stack space, need to fix or segregate in
197+ -- another benchmark.
189198 , bgroup "iterated"
190199 [ benchD "mapM" D.iterateMapM
191200 , benchD "scan(1/10)" D.iterateScan
@@ -197,6 +206,7 @@ main =
197206 , benchD "iterateM" D.iterateM
198207
199208 ]
209+#endif
200210 ]
201211 , bgroup "list"
202212 [ bgroup "elimination"
@@ -228,10 +238,14 @@ main =
228238 , benchIO "mapM_" K.mapM_ K.sourceUnfoldrM
229239 , benchIO "uncons" K.uncons K.sourceUnfoldrM
230240 , benchFold "init" K.init K.sourceUnfoldrM
241+#ifdef DEVBUILD
242+ -- XXX these consume too much stack space, need to fix or segregate in
243+ -- another benchmark.
231244 , benchFold "tail" K.tail K.sourceUnfoldrM
232245 , benchIO "nullTail" K.nullTail K.sourceUnfoldrM
233246 , benchIO "headTail" K.headTail K.sourceUnfoldrM
234247 , benchFold "toList" K.toList K.sourceUnfoldrM
248+#endif
235249 , benchFold "foldl'" K.foldl K.sourceUnfoldrM
236250 , benchFold "last" K.last K.sourceUnfoldrM
237251 ]
@@ -260,7 +274,10 @@ main =
260274 (K.sourceUnfoldrMN (K.value `div` 4))
261275 , benchIO "intersperse" (K.intersperse 1) (K.sourceUnfoldrMN K.value2)
262276 , benchIO "interspersePure" (K.intersperse 1) (K.sourceUnfoldrN K.value2)
277+#ifdef DEVBUILD
278+ -- XXX this consumes too much heap
263279 , benchIO "foldlS" (K.foldlS 1) K.sourceUnfoldrM
280+#endif
264281 ]
265282 , bgroup "transformationX4"
266283 [ benchIO "scan" (K.scan 4) K.sourceUnfoldrM
@@ -332,6 +349,9 @@ main =
332349 , benchIO "filter-scan" (K.filterScan 4) K.sourceUnfoldrM
333350 , benchIO "filter-map" (K.filterMap 4) K.sourceUnfoldrM
334351 ]
352+#ifdef DEVBUILD
353+ -- XXX these consume too much stack space, need to fix or segregate in
354+ -- another benchmark.
335355 , bgroup "iterated"
336356 [ benchK "mapM" K.iterateMapM
337357 , benchK "scan(1/10)" K.iterateScan
@@ -341,6 +361,7 @@ main =
341361 , benchK "dropWhileFalse(1/10)" K.iterateDropWhileFalse
342362 , benchK "dropWhileTrue" K.iterateDropWhileTrue
343363 ]
364+#endif
344365 ]
345366 , bgroup "streamDK"
346367 [ bgroup "generation"
benchmark/Linear.hs+19−10
@@ -434,16 +434,6 @@ main = do
434434 , benchIOSink value "filter-scanl1" (Ops.filterScanl1 4)
435435 , benchIOSink value "filter-map" (Ops.filterMap value 4)
436436 ]
437− , bgroup "iterated"
438− [ benchIOSrc serially "mapM" Ops.iterateMapM
439− , benchIOSrc serially "scan(1/100)" Ops.iterateScan
440− , benchIOSrc serially "scanl1(1/100)" Ops.iterateScanl1
441− , benchIOSrc serially "filterEven" Ops.iterateFilterEven
442− , benchIOSrc serially "takeAll" (Ops.iterateTakeAll value)
443− , benchIOSrc serially "dropOne" Ops.iterateDropOne
444− , benchIOSrc serially "dropWhileFalse" (Ops.iterateDropWhileFalse value)
445− , benchIOSrc serially "dropWhileTrue" (Ops.iterateDropWhileTrue value)
446− ]
447437 ]
448438 , bgroup "wSerially"
449439 [ bgroup "transformation"
@@ -460,11 +450,24 @@ main = do
460450 --
461451 -- These are also the operations that programmers should be aware of and
462452 -- should avoid using in a streaming application.
453+
454+ -- XXX stack dominant (upto 1M), segregate?
455+ , bgroup "iterated"
456+ [ benchIOSrc serially "mapM" Ops.iterateMapM
457+ , benchIOSrc serially "scan(1/100)" Ops.iterateScan
458+ , benchIOSrc serially "scanl1(1/100)" Ops.iterateScanl1
459+ , benchIOSrc serially "filterEven" Ops.iterateFilterEven
460+ , benchIOSrc serially "takeAll" (Ops.iterateTakeAll value)
461+ , benchIOSrc serially "dropOne" Ops.iterateDropOne
462+ , benchIOSrc serially "dropWhileFalse" (Ops.iterateDropWhileFalse value)
463+ , benchIOSrc serially "dropWhileTrue" (Ops.iterateDropWhileTrue value)
464+ ]
463465 , bgroup "buffered"
464466 [ -- Inherently non-streaming operations
465467
466468 -- Right folds for reducing are inherently non-streaming as the
467469 -- expression needs to be fully built before it can be reduced.
470+ -- XXX Stack dominant (up to 4MB), segregate?
468471 benchIOSink bufValue "foldrM/reduce/IO" Ops.foldrMReduce
469472 , benchIdentitySink bufValue "foldrM/reduce/Identity" Ops.foldrMReduce
470473
@@ -476,12 +479,15 @@ main = do
476479 , benchIdentitySink bufValue "foldlM'/build/Identity" Ops.foldlM'Build
477480
478481 -- accumulation due to strictness of IO monad
482+ -- XXX Stack dominant, segregate?
479483 , benchIOSink bufValue "foldrM/build/IO" Ops.foldrMBuild
480484 , benchPureSinkIO bufValue "traversable/mapM" Ops.traversableMapM
481485
482486 -- Converting the stream to a list or pure stream
487+ -- XXX Stack dominant, segregate?
483488 , benchIOSink bufValue "toList" Ops.toList
484489 , benchIOSink bufValue "toListRev" Ops.toListRev
490+
485491 , benchIOSink bufValue "toStream" (S.fold IP.toStream)
486492 , benchIOSink bufValue "toStreamRev" (S.fold IP.toStreamRev)
487493
@@ -506,9 +512,12 @@ main = do
506512 nf Ops.showInstanceList (mkList bufValue))
507513
508514 -- XXX streaming operations that can potentially be fixed
515+
516+ -- XXX These consume a lot of stack, fix or segregate
509517 , benchPureSink bufValue "foldable/sum" Ops.foldableSum
510518 , benchIOSink bufValue "tail" Ops.tail
511519 , benchIOSink bufValue "nullHeadTail" Ops.nullHeadTail
520+
512521 , benchIOSrc1 "concatUnfoldInterleaveRepl (x/4,4)"
513522 (Ops.concatUnfoldInterleaveRepl4xN bufValue)
514523 , benchIOSrc1 "concatUnfoldRoundrobinRepl (x/4,4)"
benchmark/Nested.hs+1
@@ -35,6 +35,7 @@ main = do
3535 , benchIO "toNull" $ Ops.toNull linearCount serially
3636 , benchIO "toNull3" $ Ops.toNull3 linearCount serially
3737 -- , benchIO "toList" $ Ops.toList linearCount serially
38+ -- XXX takes too much stack space
3839 , benchIO "toListSome" $ Ops.toListSome linearCount serially
3940 , benchIO "filterAllOut" $ Ops.filterAllOut linearCount serially
4041 , benchIO "filterAllIn" $ Ops.filterAllIn linearCount serially
benchmark/NestedConcurrent.hs+1
@@ -43,6 +43,7 @@ main = do
4343 , benchIO "toNull" $ Ops.toNull linearCount aheadly
4444 , benchIO "toNull3" $ Ops.toNull3 linearCount aheadly
4545 -- , benchIO "toList" $ Ops.toList linearCount aheadly
46+ -- XXX consumes too much stack space
4647 , benchIO "toListSome" $ Ops.toListSome linearCount aheadly
4748 , benchIO "filterAllOut" $ Ops.filterAllOut linearCount aheadly
4849 , benchIO "filterAllIn" $ Ops.filterAllIn linearCount aheadly
benchmark/StreamDKOps.hs+2−3
@@ -13,9 +13,8 @@ module StreamDKOps where
1313-- import Control.Monad (when)
1414-- import Data.Maybe (isJust)
1515import Prelude
16− (Monad, Int, (+), ($), (.), return, even, (>), (<=), div,
17− subtract, undefined, Maybe(..), not, (>>=),
18− maxBound, flip, (<$>), (<*>), round, (/), (**), (<))
16+ (Monad, Int, (+), (.), return, undefined, Maybe(..), round, (/),
17+ (**), (>))
1918import qualified Prelude as P
2019-- import qualified Data.List as List
2120
streamly.cabal+32−25
@@ -262,7 +262,7 @@ common exe-options
262262-- XXX dependencies should be separated under bench-depends
263263common bench-options
264264 import: compile-options, optimization-options
265− ghc-options: -with-rtsopts "-T -M256M"
265+ ghc-options: -with-rtsopts "-T -K32K -M16M"
266266 if flag(fusion-plugin) && !impl(ghcjs) && !impl(ghc < 8.6)
267267 ghc-options: -fplugin Fusion.Plugin
268268 build-depends:
@@ -273,7 +273,7 @@ common bench-options-threaded
273273 import: compile-options, optimization-options
274274 -- -threaded and -N2 is important because some GC and space leak issues
275275 -- trigger only with these options.
276− ghc-options: -threaded -with-rtsopts "-T -N2 -M256M"
276+ ghc-options: -threaded -with-rtsopts "-T -N2 -K32K -M16M"
277277 if flag(fusion-plugin) && !impl(ghcjs) && !impl(ghc < 8.6)
278278 ghc-options: -fplugin Fusion.Plugin
279279 build-depends:
@@ -670,6 +670,9 @@ benchmark linear
670670 import: bench-options
671671 type: exitcode-stdio-1.0
672672 hs-source-dirs: benchmark
673+ -- XXX heap/stack limits can be reduced once we split out the buffered
674+ -- benchmarks into a separate suite
675+ ghc-options: -with-rtsopts "-T -K4M -M128M"
673676 main-is: Linear.hs
674677 other-modules: Streamly.Benchmark.Prelude, Common
675678 build-depends:
@@ -689,6 +692,7 @@ benchmark nested
689692 import: bench-options
690693 type: exitcode-stdio-1.0
691694 hs-source-dirs: benchmark
695+ ghc-options: -with-rtsopts "-T -K256K -M16M"
692696 main-is: Nested.hs
693697 other-modules: NestedOps, Common
694698 build-depends:
@@ -705,6 +709,7 @@ benchmark nested-unfold
705709 import: bench-options
706710 type: exitcode-stdio-1.0
707711 hs-source-dirs: benchmark
712+ ghc-options: -with-rtsopts "-T -K64K -M16M"
708713 main-is: NestedUnfold.hs
709714 other-modules: NestedUnfoldOps, Common
710715 build-depends:
@@ -721,6 +726,7 @@ benchmark unpinned-array
721726 import: bench-options
722727 type: exitcode-stdio-1.0
723728 hs-source-dirs: benchmark
729+ ghc-options: -with-rtsopts "-T -K1K -M128M"
724730 main-is: Streamly/Benchmark/Data/Array.hs
725731 other-modules: Streamly.Benchmark.Data.ArrayOps
726732 build-depends:
@@ -737,6 +743,7 @@ benchmark prim-array
737743 import: bench-options
738744 type: exitcode-stdio-1.0
739745 hs-source-dirs: benchmark
746+ ghc-options: -with-rtsopts "-T -K64K -M32M"
740747 main-is: Streamly/Benchmark/Data/Prim/Array.hs
741748 other-modules: Streamly.Benchmark.Data.Prim.ArrayOps
742749 build-depends:
@@ -753,6 +760,7 @@ benchmark small-array
753760 import: bench-options
754761 type: exitcode-stdio-1.0
755762 hs-source-dirs: benchmark
763+ ghc-options: -with-rtsopts "-T -K128K -M16M"
756764 main-is: Streamly/Benchmark/Data/SmallArray.hs
757765 other-modules: Streamly.Benchmark.Data.SmallArrayOps
758766 build-depends:
@@ -769,6 +777,7 @@ benchmark array
769777 import: bench-options
770778 type: exitcode-stdio-1.0
771779 hs-source-dirs: benchmark
780+ ghc-options: -with-rtsopts "-T -K64K -M128M"
772781 main-is: Array.hs
773782 other-modules: ArrayOps
774783 build-depends:
@@ -809,6 +818,7 @@ benchmark linear-async
809818 import: bench-options-threaded
810819 type: exitcode-stdio-1.0
811820 hs-source-dirs: benchmark
821+ ghc-options: -with-rtsopts "-T -N2 -K64K -M16M"
812822 main-is: LinearAsync.hs
813823 other-modules: Streamly.Benchmark.Prelude, Common
814824 build-depends:
@@ -828,6 +838,8 @@ benchmark nested-concurrent
828838 import: bench-options-threaded
829839 type: exitcode-stdio-1.0
830840 hs-source-dirs: benchmark
841+ -- XXX this can be lowered once we split out the finite benchmarks
842+ ghc-options: -with-rtsopts "-T -N2 -K256K -M128M"
831843 main-is: NestedConcurrent.hs
832844 other-modules: NestedOps, Common
833845 build-depends:
@@ -844,6 +856,7 @@ benchmark parallel
844856 import: bench-options-threaded
845857 type: exitcode-stdio-1.0
846858 hs-source-dirs: benchmark
859+ ghc-options: -with-rtsopts "-T -N2 -K128K -M256M"
847860 main-is: Parallel.hs
848861 other-modules: Streamly.Benchmark.Prelude, NestedOps, Common
849862 build-depends:
@@ -883,14 +896,11 @@ benchmark concurrent
883896 type: exitcode-stdio-1.0
884897 hs-source-dirs: benchmark
885898 main-is: Concurrent.hs
886− if flag(dev)
887− buildable: True
888− build-depends:
889− streamly
890− , base >= 4.8 && < 5
891− , gauge >= 0.2.4 && < 0.3
892− else
893− buildable: False
899+ ghc-options: -with-rtsopts "-T -N2 -K256K -M384M"
900+ build-depends:
901+ streamly
902+ , base >= 4.8 && < 5
903+ , gauge >= 0.2.4 && < 0.3
894904
895905-------------------------------------------------------------------------------
896906-- Internal benchmarks
@@ -900,21 +910,22 @@ benchmark base
900910 import: bench-options
901911 type: exitcode-stdio-1.0
902912 hs-source-dirs: benchmark
913+ if flag(dev)
914+ cpp-options: -DDEVBUILD
915+ ghc-options: -with-rtsopts "-T -K2M -M16M"
916+ else
917+ ghc-options: -with-rtsopts "-T -K128K -M16M"
903918 main-is: BaseStreams.hs
904919 other-modules: StreamDOps
905920 , StreamKOps
906921 , StreamDKOps
907922
908− if flag(dev)
909− buildable: True
910− build-depends:
923+ build-depends:
911924 streamly
912925 , base >= 4.8 && < 5
913926 , deepseq >= 1.4.1 && < 1.5
914927 , random >= 1.0 && < 2.0
915928 , gauge >= 0.2.4 && < 0.3
916− else
917− buildable: False
918929
919930executable nano-bench
920931 import: bench-options
@@ -931,20 +942,16 @@ executable nano-bench
931942 buildable: False
932943
933944benchmark adaptive
934− import: bench-options
945+ import: bench-options-threaded
935946 type: exitcode-stdio-1.0
936947 hs-source-dirs: benchmark
937948 main-is: Adaptive.hs
938949 default-language: Haskell2010
939− if flag(dev)
940− buildable: True
941− build-depends:
942− streamly
943− , base >= 4.8 && < 5
944− , gauge >= 0.2.4 && < 0.3
945− , random >= 1.0 && < 2.0
946− else
947− buildable: False
950+ build-depends:
951+ streamly
952+ , base >= 4.8 && < 5
953+ , gauge >= 0.2.4 && < 0.3
954+ , random >= 1.0 && < 2.0
948955
949956executable chart
950957 default-language: Haskell2010