on master help sign in
Concurrent concatMap, foldr/build fusion, benchmarks

* Implement a stream flavor specific (concurrent/wSerial etc.) concatMap (concatMapBy). Now concatMap has the same capabilities that the monad bind of different streams has. In fact the monad bind is implemented using concatMap * Use foldr/build fusion for concatMap (currently disabled) and for map/fmap and mapM (enabled) * Add benchmarks for bind/concatMap performance

commit1949eb90afd6f932c426fa1b979f0ae07128c496 authorHarendra Kumar <harendra.kumar@gmail.com> dateMon 27 May 2019, 23:29 (7 years ago) parent7a232eb
17 files changed+1134−382
benchmark/BaseStreams.hs+61−2 benchmark/LinearOps.hs+5−2 benchmark/Nested.hs+6 benchmark/NestedOps.hs+35−19 benchmark/StreamDOps.hs+71−9 benchmark/StreamKOps.hs+124−8 src/Streamly/Streams/Ahead.hs+9−11 src/Streamly/Streams/Async.hs+8 src/Streamly/Streams/Instances.hs+2 src/Streamly/Streams/Parallel.hs+8−10 src/Streamly/Streams/Prelude.hs+20 src/Streamly/Streams/Serial.hs+6 src/Streamly/Streams/StreamD.hs+1−63 src/Streamly/Streams/StreamD/Type.hs+106−1 src/Streamly/Streams/StreamK.hs+38−138 src/Streamly/Streams/StreamK/Type.hs+534−19 streamly.cabal+100−100
benchmark/BaseStreams.hs+61−2
@@ -21,6 +21,20 @@ import qualified StreamKOps as K
2121benchIO :: String -> (a IO Int -> IO ()) -> (Int -> a IO Int) -> Benchmark
2222benchIO name run f = bench name $ nfIO $ randomRIO (1,1) >>= run . f
2323
24+{-# INLINE _benchIOSrcK #-}
25+_benchIOSrcK
26+ :: String
27+ -> (Int -> K.Stream IO Int)
28+ -> Benchmark
29+_benchIOSrcK name f = bench name $ nfIO $ randomRIO (1,1) >>= K.toNull . f
30+
31+{-# INLINE _benchIOSrcD #-}
32+_benchIOSrcD
33+ :: String
34+ -> (Int -> D.Stream IO Int)
35+ -> Benchmark
36+_benchIOSrcD name f = bench name $ nfIO $ randomRIO (1,1) >>= D.toNull . f
37+
2438benchFold :: NFData b
2539 => String -> (t IO Int -> IO b) -> (Int -> t IO Int) -> Benchmark
2640benchFold name f src = bench name $ nfIO $ randomRIO (1,1) >>= f . src
@@ -39,6 +53,10 @@ _benchId :: NFData b => String -> (Ops.Stream m Int -> Identity b) -> Benchmark
3953_benchId name f = bench name $ nf (runIdentity . f) (Ops.source 10)
4054-}
4155
56+{-# INLINE benchPure #-}
57+benchPure :: String -> ([Int] -> [Int]) -> (Int -> [Int]) -> Benchmark
58+benchPure name run f = bench name $ nfIO $ randomRIO (1,1) >>= return . run . f
59+
4260main :: IO ()
4361main =
4462 defaultMain
@@ -62,6 +80,18 @@ main =
6280 , benchFold "foldl'" D.foldl D.sourceUnfoldrM
6381 , benchFold "last" D.last D.sourceUnfoldrM
6482 ]
83+ , bgroup "nested"
84+ [ benchIO "toNullAp" D.toNullApNested (D.sourceUnfoldrMN D.value2)
85+ , benchIO "toNull" D.toNullNested (D.sourceUnfoldrMN D.value2)
86+ , benchIO "toNull3" D.toNullNested3 (D.sourceUnfoldrMN D.value3)
87+ , benchIO "filterAllIn" D.filterAllInNested (D.sourceUnfoldrMN K.value2)
88+ , benchIO "filterAllOut" D.filterAllOutNested (D.sourceUnfoldrMN K.value2)
89+ , benchIO "toNullApPure" D.toNullApNested (D.sourceUnfoldrN K.value2)
90+ , benchIO "toNullPure" D.toNullNested (D.sourceUnfoldrN K.value2)
91+ , benchIO "toNull3Pure" D.toNullNested3 (D.sourceUnfoldrN K.value3)
92+ , benchIO "filterAllInPure" D.filterAllInNested (D.sourceUnfoldrN K.value2)
93+ , benchIO "filterAllOutPure" D.filterAllOutNested (D.sourceUnfoldrN K.value2)
94+ ]
6595 , bgroup "transformation"
6696 [ benchIO "scan" (D.scan 1) D.sourceUnfoldrM
6797 , benchIO "map" (D.map 1) D.sourceUnfoldrM
@@ -69,6 +99,9 @@ main =
6999 , benchIO "mapM" (D.mapM 1) D.sourceUnfoldrM
70100 , benchIO "mapMaybe" (D.mapMaybe 1) D.sourceUnfoldrM
71101 , benchIO "mapMaybeM" (D.mapMaybeM 1) D.sourceUnfoldrM
102+ , benchIO "concatMap" (D.concatMap 1) (D.sourceUnfoldrMN D.value2)
103+ , benchIO "concatMapPure" (D.concatMap 1) (D.sourceUnfoldrN D.value2)
104+ -- , benchIO "foldrS" (D.foldrS 1) D.sourceUnfoldrM
72105 -- This has horrible performance, never finishes
73106 -- , benchIO "foldlS" (D.foldlS 1) D.sourceUnfoldrM
74107 ]
@@ -79,6 +112,7 @@ main =
79112 , benchIO "mapM" (D.mapM 4) D.sourceUnfoldrM
80113 , benchIO "mapMaybe" (D.mapMaybe 4) D.sourceUnfoldrM
81114 , benchIO "mapMaybeM" (D.mapMaybeM 4) D.sourceUnfoldrM
115+ -- , benchIO "concatMap" (D.concatMap 4) (D.sourceUnfoldrMN D.value16)
82116 ]
83117 , bgroup "filtering"
84118 [ benchIO "filter-even" (D.filterEven 1) D.sourceUnfoldrM
@@ -153,6 +187,15 @@ main =
153187 , benchD "dropWhileTrue" D.iterateDropWhileTrue
154188 ]
155189 ]
190+ , bgroup "list"
191+ [ bgroup "nested"
192+ [ benchPure "toNullAp" K.toNullApNestedList (K.sourceUnfoldrList K.value2)
193+ , benchPure "toNull" K.toNullNestedList (K.sourceUnfoldrList K.value2)
194+ , benchPure "toNull3" K.toNullNestedList3 (K.sourceUnfoldrList K.value3)
195+ , benchPure "filterAllIn" K.filterAllInNestedList (K.sourceUnfoldrList K.value2)
196+ , benchPure "filterAllOut" K.filterAllOutNestedList (K.sourceUnfoldrList K.value2)
197+ ]
198+ ]
156199 , bgroup "streamK"
157200 [ bgroup "generation"
158201 [ benchIO "unfoldr" K.toNull K.sourceUnfoldr
@@ -178,12 +221,27 @@ main =
178221 , benchFold "foldl'" K.foldl K.sourceUnfoldrM
179222 , benchFold "last" K.last K.sourceUnfoldrM
180223 ]
224+ , bgroup "nested"
225+ [ benchIO "toNullAp" K.toNullApNested (K.sourceUnfoldrMN K.value2)
226+ , benchIO "toNull" K.toNullNested (K.sourceUnfoldrMN K.value2)
227+ , benchIO "toNull3" K.toNullNested3 (K.sourceUnfoldrMN K.value3)
228+ , benchIO "filterAllIn" K.filterAllInNested (K.sourceUnfoldrMN K.value2)
229+ , benchIO "filterAllOut" K.filterAllOutNested (K.sourceUnfoldrMN K.value2)
230+ , benchIO "toNullApPure" K.toNullApNested (K.sourceUnfoldrN K.value2)
231+ , benchIO "toNullPure" K.toNullNested (K.sourceUnfoldrN K.value2)
232+ , benchIO "toNull3Pure" K.toNullNested3 (K.sourceUnfoldrN K.value3)
233+ , benchIO "filterAllInPure" K.filterAllInNested (K.sourceUnfoldrN K.value2)
234+ , benchIO "filterAllOutPure" K.filterAllOutNested (K.sourceUnfoldrN K.value2)
235+ ]
181236 , bgroup "transformation"
182237 [ benchIO "scan" (K.scan 1) K.sourceUnfoldrM
183238 , benchIO "map" (K.map 1) K.sourceUnfoldrM
184239 , benchIO "fmap" (K.fmap 1) K.sourceUnfoldrM
185240 , benchIO "mapM" (K.mapM 1) K.sourceUnfoldrM
186− -- , benchIO "concat" K.concat K.sourceUnfoldrM
241+ , benchIO "mapMSerial" (K.mapMSerial 1) K.sourceUnfoldrM
242+ -- , benchIOSrcK "concatMap" K.concatMap
243+ , benchIO "concatMap" (K.concatMap 1) (K.sourceUnfoldrMN K.value2)
244+ , benchIO "concatMapPure" (K.concatMap 1) (K.sourceUnfoldrN K.value2)
187245 , benchIO "foldlS" (K.foldlS 1) K.sourceUnfoldrM
188246 ]
189247 , bgroup "transformationX4"
@@ -191,7 +249,8 @@ main =
191249 , benchIO "map" (K.map 4) K.sourceUnfoldrM
192250 , benchIO "fmap" (K.fmap 4) K.sourceUnfoldrM
193251 , benchIO "mapM" (K.mapM 4) K.sourceUnfoldrM
194− -- , benchIO "concat" K.concat K.sourceUnfoldrM
252+ , benchIO "mapMSerial" (K.mapMSerial 4) K.sourceUnfoldrM
253+ -- , benchIO "concatMap" (K.concatMap 4) (K.sourceUnfoldrMN K.value16)
195254 ]
196255 , bgroup "filtering"
197256 [ benchIO "filter-even" (K.filterEven 1) K.sourceUnfoldrM
benchmark/LinearOps.hs+5−2
@@ -21,7 +21,7 @@ import Prelude
2121 (Monad, Int, (+), ($), (.), return, fmap, even, (>), (<=), (==), (>=),
2222 subtract, undefined, Maybe(..), odd, Bool, not, (>>=), mapM_, curry,
2323 maxBound, div, IO, compare, Double, fromIntegral, Integer, (<$>),
24− (<*>), flip)
24+ (<*>), flip, (**), (/))
2525import qualified Prelude as P
2626import qualified Data.Foldable as F
2727import qualified GHC.Exts as GHC
@@ -31,13 +31,14 @@ import GHC.Generics (Generic)
3131import qualified Streamly as S hiding (foldMapWith, runStream)
3232import qualified Streamly.Prelude as S
3333
34−value, maxValue :: Int
34+value, maxValue, value2 :: Int
3535#ifdef LINEAR_ASYNC
3636value = 10000
3737#else
3838value = 100000
3939#endif
4040maxValue = value + 1
41+value2 = P.round (P.fromIntegral value**(1/2::P.Double)) -- double nested loop
4142
4243-------------------------------------------------------------------------------
4344-- Benchmark ops
@@ -505,6 +506,8 @@ cmpBy :: (Monad m, P.Ord a) => Stream m a -> m P.Ordering
505506cmpBy src = S.cmpBy P.compare src src
506507
507508concatStreamLen, maxNested :: Int
509+-- concatStreamLen = value2
510+-- maxNested = value2
508511concatStreamLen = 1
509512maxNested = 100000
510513
benchmark/Nested.hs+6
@@ -25,6 +25,7 @@ main =
2525 [ bgroup "serially"
2626 [ benchIO "toNullAp" $ Ops.toNullAp serially
2727 , benchIO "toNull" $ Ops.toNull serially
28+ , benchIO "toNull3" $ Ops.toNull3 serially
2829 , benchIO "toList" $ Ops.toList serially
2930 -- , benchIO "toListSome" $ Ops.toListSome serially
3031 , benchIO "filterAllOut" $ Ops.filterAllOut serially
@@ -36,6 +37,7 @@ main =
3637 , bgroup "wSerially"
3738 [ benchIO "toNullAp" $ Ops.toNullAp wSerially
3839 , benchIO "toNull" $ Ops.toNull wSerially
40+ , benchIO "toNull3" $ Ops.toNull3 wSerially
3941 , benchIO "toList" $ Ops.toList wSerially
4042 -- , benchIO "toListSome" $ Ops.toListSome wSerially
4143 , benchIO "filterAllOut" $ Ops.filterAllOut wSerially
@@ -47,6 +49,7 @@ main =
4749 , bgroup "aheadly"
4850 [ benchIO "toNullAp" $ Ops.toNullAp aheadly
4951 , benchIO "toNull" $ Ops.toNull aheadly
52+ , benchIO "toNull3" $ Ops.toNull3 aheadly
5053 , benchIO "toList" $ Ops.toList aheadly
5154 -- , benchIO "toListSome" $ Ops.toListSome aheadly
5255 , benchIO "filterAllOut" $ Ops.filterAllOut aheadly
@@ -58,6 +61,7 @@ main =
5861 , bgroup "asyncly"
5962 [ benchIO "toNullAp" $ Ops.toNullAp asyncly
6063 , benchIO "toNull" $ Ops.toNull asyncly
64+ , benchIO "toNull3" $ Ops.toNull3 asyncly
6165 , benchIO "toList" $ Ops.toList asyncly
6266 -- , benchIO "toListSome" $ Ops.toListSome asyncly
6367 , benchIO "filterAllOut" $ Ops.filterAllOut asyncly
@@ -69,6 +73,7 @@ main =
6973 , bgroup "wAsyncly"
7074 [ benchIO "toNullAp" $ Ops.toNullAp wAsyncly
7175 , benchIO "toNull" $ Ops.toNull wAsyncly
76+ , benchIO "toNull3" $ Ops.toNull3 wAsyncly
7277 , benchIO "toList" $ Ops.toList wAsyncly
7378 -- , benchIO "toListSome" $ Ops.toListSome wAsyncly
7479 , benchIO "filterAllOut" $ Ops.filterAllOut wAsyncly
@@ -80,6 +85,7 @@ main =
8085 , bgroup "parallely"
8186 [ benchIO "toNullAp" $ Ops.toNullAp parallely
8287 , benchIO "toNull" $ Ops.toNull parallely
88+ , benchIO "toNull3" $ Ops.toNull3 parallely
8389 , benchIO "toList" $ Ops.toList parallely
8490 --, benchIO "toListSome" $ Ops.toListSome parallely
8591 , benchIO "filterAllOut" $ Ops.filterAllOut parallely
benchmark/NestedOps.hs+35−19
@@ -16,11 +16,17 @@ import GHC.Exception (ErrorCall)
1616import qualified Streamly as S hiding (runStream)
1717import qualified Streamly.Prelude as S
1818
19−sumCount :: Int
20−sumCount = 1000000
19+linearCount :: Int
20+linearCount = 100000
2121
22−prodCount :: Int
23−prodCount = 100
22+-- double nested loop
23+nestedCount2 :: Int
24+-- nestedCount2 = round (fromIntegral linearCount**(1/2::Double))
25+nestedCount2 = 100
26+
27+-- triple nested loop
28+nestedCount3 :: Int
29+nestedCount3 = round (fromIntegral linearCount**(1/3::Double))
2430
2531-------------------------------------------------------------------------------
2632-- Stream generation and elimination
@@ -67,24 +73,34 @@ toNullAp
6773 :: (S.IsStream t, S.MonadAsync m, Monad (t m))
6874 => (t m Int -> S.SerialT m Int) -> Int -> m ()
6975toNullAp t start = runStream . t $
70− (+) <$> source start prodCount <*> source start prodCount
76+ (+) <$> source start nestedCount2 <*> source start nestedCount2
7177
7278{-# INLINE toNull #-}
7379toNull
7480 :: (S.IsStream t, S.MonadAsync m, Monad (t m))
7581 => (t m Int -> S.SerialT m Int) -> Int -> m ()
7682toNull t start = runStream . t $ do
77− x <- source start prodCount
78− y <- source start prodCount
83+ x <- source start nestedCount2
84+ y <- source start nestedCount2
7985 return $ x + y
8086
87+{-# INLINE toNull3 #-}
88+toNull3
89+ :: (S.IsStream t, S.MonadAsync m, Monad (t m))
90+ => (t m Int -> S.SerialT m Int) -> Int -> m ()
91+toNull3 t start = runStream . t $ do
92+ x <- source start nestedCount3
93+ y <- source start nestedCount3
94+ z <- source start nestedCount3
95+ return $ x + y + z
96+
8197{-# INLINE toList #-}
8298toList
8399 :: (S.IsStream t, S.MonadAsync m, Monad (t m))
84100 => (t m Int -> S.SerialT m Int) -> Int -> m [Int]
85101toList t start = runToList . t $ do
86− x <- source start prodCount
87− y <- source start prodCount
102+ x <- source start nestedCount2
103+ y <- source start nestedCount2
88104 return $ x + y
89105
90106{-# INLINE toListSome #-}
@@ -93,8 +109,8 @@ toListSome
93109 => (t m Int -> S.SerialT m Int) -> Int -> m [Int]
94110toListSome t start =
95111 runToList . t $ S.take 1000 $ do
96− x <- source start prodCount
97− y <- source start prodCount
112+ x <- source start nestedCount2
113+ y <- source start nestedCount2
98114 return $ x + y
99115
100116{-# INLINE filterAllOut #-}
@@ -102,8 +118,8 @@ filterAllOut
102118 :: (S.IsStream t, S.MonadAsync m, Monad (t m))
103119 => (t m Int -> S.SerialT m Int) -> Int -> m ()
104120filterAllOut t start = runStream . t $ do
105− x <- source start prodCount
106− y <- source start prodCount
121+ x <- source start nestedCount2
122+ y <- source start nestedCount2
107123 let s = x + y
108124 if s < 0
109125 then return s
@@ -114,8 +130,8 @@ filterAllIn
114130 :: (S.IsStream t, S.MonadAsync m, Monad (t m))
115131 => (t m Int -> S.SerialT m Int) -> Int -> m ()
116132filterAllIn t start = runStream . t $ do
117− x <- source start prodCount
118− y <- source start prodCount
133+ x <- source start nestedCount2
134+ y <- source start nestedCount2
119135 let s = x + y
120136 if s > 0
121137 then return s
@@ -126,8 +142,8 @@ filterSome
126142 :: (S.IsStream t, S.MonadAsync m, Monad (t m))
127143 => (t m Int -> S.SerialT m Int) -> Int -> m ()
128144filterSome t start = runStream . t $ do
129− x <- source start prodCount
130− y <- source start prodCount
145+ x <- source start nestedCount2
146+ y <- source start nestedCount2
131147 let s = x + y
132148 if s > 1100000
133149 then return s
@@ -139,8 +155,8 @@ breakAfterSome
139155 => (t IO Int -> S.SerialT IO Int) -> Int -> IO ()
140156breakAfterSome t start = do
141157 (_ :: Either ErrorCall ()) <- try $ runStream . t $ do
142− x <- source start prodCount
143− y <- source start prodCount
158+ x <- source start nestedCount2
159+ y <- source start nestedCount2
144160 let s = x + y
145161 if s > 1100000
146162 then error "break"
benchmark/StreamDOps.hs+71−9
@@ -15,14 +15,19 @@ import Data.Maybe (isJust)
1515import Prelude
1616 (Monad, Int, (+), ($), (.), return, (>), even, (<=), div,
1717 subtract, undefined, Maybe(..), not, (>>=),
18− maxBound, fmap, odd, (==), flip)
18+ maxBound, fmap, odd, (==), flip, (<$>), (<*>), round, (/), (**), (<))
1919import qualified Prelude as P
2020
2121import qualified Streamly.Streams.StreamD as S
2222
23−value, maxValue :: Int
23+-- We try to keep the total number of iterations same irrespective of nesting
24+-- of the loops so that the overhead is easy to compare.
25+value, value2, value3, value16, maxValue :: Int
2426value = 100000
25−maxValue = value + 1000
27+value2 = round (P.fromIntegral value**(1/2::P.Double)) -- double nested loop
28+value3 = round (P.fromIntegral value**(1/3::P.Double)) -- triple nested loop
29+value16 = round (P.fromIntegral value**(1/16::P.Double)) -- triple nested loop
30+maxValue = value
2631
2732-------------------------------------------------------------------------------
2833-- Stream generation and elimination
@@ -39,6 +44,15 @@ sourceUnfoldr n = S.unfoldr step n
3944 then Nothing
4045 else Just (cnt, cnt + 1)
4146
47+{-# INLINE sourceUnfoldrN #-}
48+sourceUnfoldrN :: Monad m => Int -> Int -> Stream m Int
49+sourceUnfoldrN m n = S.unfoldr step n
50+ where
51+ step cnt =
52+ if cnt > n + m
53+ then Nothing
54+ else Just (cnt, cnt + 1)
55+
4256{-# INLINE sourceUnfoldrMN #-}
4357sourceUnfoldrMN :: Monad m => Int -> Int -> Stream m Int
4458sourceUnfoldrMN m n = S.unfoldrM step n
@@ -160,10 +174,13 @@ composeN n f =
160174{-# INLINE dropWhileTrue #-}
161175{-# INLINE dropWhileMTrue #-}
162176{-# INLINE dropWhileFalse #-}
177+{-# INLINE foldrS #-}
163178{-# INLINE foldlS #-}
179+{-# INLINE concatMap #-}
164180scan, map, fmap, mapM, mapMaybe, mapMaybeM, filterEven, filterAllOut,
165181 filterAllIn, takeOne, takeAll, takeWhileTrue, takeWhileMTrue, dropOne,
166− dropAll, dropWhileTrue, dropWhileMTrue, dropWhileFalse, foldlS
182+ dropAll, dropWhileTrue, dropWhileMTrue, dropWhileFalse, foldrS, foldlS,
183+ concatMap
167184 :: Monad m
168185 => Int -> Stream m Int -> m ()
169186
@@ -187,7 +204,9 @@ dropAll n = composeN n $ S.drop maxValue
187204dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue)
188205dropWhileMTrue n = composeN n $ S.dropWhileM (return . (<= maxValue))
189206dropWhileFalse n = composeN n $ S.dropWhile (> maxValue)
207+foldrS n = composeN n $ S.foldrS S.cons S.nil
190208foldlS n = composeN n $ S.foldlS (flip S.cons) S.nil
209+concatMap n = composeN n $ (\s -> S.concatMap (\_ -> s) s)
191210
192211-------------------------------------------------------------------------------
193212-- Iteration
@@ -245,11 +264,6 @@ cmpBy src = S.cmpBy P.compare src src
245264zip :: Monad m => Stream m Int -> m ()
246265zip src = transform $ S.zipWith (,) src src
247266
248−{-
249−{-# INLINE concat #-}
250−concat _n = return ()
251−-}
252−
253267-------------------------------------------------------------------------------
254268-- Mixed Composition
255269-------------------------------------------------------------------------------
@@ -278,3 +292,51 @@ filterDrop n = composeN n $ S.drop 1 . S.filter (<= maxValue)
278292filterTake n = composeN n $ S.take maxValue . S.filter (<= maxValue)
279293filterScan n = composeN n $ S.scanl' (+) 0 . S.filter (<= maxBound)
280294filterMap n = composeN n $ S.map (subtract 1) . S.filter (<= maxValue)
295+
296+-------------------------------------------------------------------------------
297+-- Nested Composition
298+-------------------------------------------------------------------------------
299+
300+{-# INLINE toNullApNested #-}
301+toNullApNested :: Monad m => Stream m Int -> m ()
302+toNullApNested s = runStream $ do
303+ (+) <$> s <*> s
304+
305+{-# INLINE toNullNested #-}
306+toNullNested :: Monad m => Stream m Int -> m ()
307+toNullNested s = runStream $ do
308+ x <- s
309+ y <- s
310+ return $ x + y
311+
312+{-# INLINE toNullNested3 #-}
313+toNullNested3 :: Monad m => Stream m Int -> m ()
314+toNullNested3 s = runStream $ do
315+ x <- s
316+ y <- s
317+ z <- s
318+ return $ x + y + z
319+
320+{-# INLINE filterAllOutNested #-}
321+filterAllOutNested
322+ :: Monad m
323+ => Stream m Int -> m ()
324+filterAllOutNested str = runStream $ do
325+ x <- str
326+ y <- str
327+ let s = x + y
328+ if s < 0
329+ then return s
330+ else S.nil
331+
332+{-# INLINE filterAllInNested #-}
333+filterAllInNested
334+ :: Monad m
335+ => Stream m Int -> m ()
336+filterAllInNested str = runStream $ do
337+ x <- str
338+ y <- str
339+ let s = x + y
340+ if s > 0
341+ then return s
342+ else S.nil
benchmark/StreamKOps.hs+124−8
@@ -15,16 +15,20 @@ import Data.Maybe (isJust)
1515import Prelude
1616 (Monad, Int, (+), ($), (.), return, even, (>), (<=), div,
1717 subtract, undefined, Maybe(..), not, (>>=),
18− maxBound, flip)
18+ maxBound, flip, (<$>), (<*>), round, (/), (**), (<))
1919import qualified Prelude as P
20+import qualified Data.List as List
2021
2122import qualified Streamly.Streams.StreamK as S
2223import qualified Streamly.Streams.Prelude as SP
2324import qualified Streamly.SVar as S
2425
25−value, maxValue :: Int
26+value, value2, value3, value16, maxValue :: Int
2627value = 100000
27−maxValue = value + 1000
28+value2 = round (P.fromIntegral value**(1/2::P.Double)) -- double nested loop
29+value3 = round (P.fromIntegral value**(1/3::P.Double)) -- triple nested loop
30+value16 = round (P.fromIntegral value**(1/16::P.Double)) -- triple nested loop
31+maxValue = value
2832
2933-------------------------------------------------------------------------------
3034-- Benchmark ops
@@ -35,8 +39,7 @@ maxValue = value + 1000
3539{-# INLINE nullTail #-}
3640{-# INLINE headTail #-}
3741{-# INLINE zip #-}
38−{-# INLINE concat #-}
39−toNull, uncons, nullTail, headTail, zip, concat
42+toNull, uncons, nullTail, headTail, zip
4043 :: Monad m
4144 => Stream m Int -> m ()
4245
@@ -62,6 +65,15 @@ sourceUnfoldr n = S.unfoldr step n
6265 then Nothing
6366 else Just (cnt, cnt + 1)
6467
68+{-# INLINE sourceUnfoldrN #-}
69+sourceUnfoldrN :: Int -> Int -> Stream m Int
70+sourceUnfoldrN m n = S.unfoldr step n
71+ where
72+ step cnt =
73+ if cnt > n + m
74+ then Nothing
75+ else Just (cnt, cnt + 1)
76+
6577{-# INLINE sourceUnfoldrM #-}
6678sourceUnfoldrM :: S.MonadAsync m => Int -> Stream m Int
6779sourceUnfoldrM n = S.unfoldrM step n
@@ -115,6 +127,7 @@ source = sourceUnfoldrM
115127{-# INLINE runStream #-}
116128runStream :: Monad m => Stream m a -> m ()
117129runStream = S.drain
130+-- runStream = S.mapM_ (\_ -> return ())
118131
119132{-# INLINE mapM_ #-}
120133mapM_ :: Monad m => Stream m a -> m ()
@@ -183,19 +196,22 @@ composeN n f =
183196{-# INLINE dropWhileTrue #-}
184197{-# INLINE dropWhileFalse #-}
185198{-# INLINE foldlS #-}
199+{-# INLINE concatMap #-}
186200scan, map, fmap, filterEven, filterAllOut,
187201 filterAllIn, takeOne, takeAll, takeWhileTrue, dropAll, dropOne,
188− dropWhileTrue, dropWhileFalse, foldlS
202+ dropWhileTrue, dropWhileFalse, foldlS, concatMap
189203 :: Monad m
190204 => Int -> Stream m Int -> m ()
191205
192206{-# INLINE mapM #-}
193−mapM :: S.MonadAsync m => Int -> Stream m Int -> m ()
207+{-# INLINE mapMSerial #-}
208+mapM, mapMSerial :: S.MonadAsync m => Int -> Stream m Int -> m ()
194209
195210scan n = composeN n $ S.scanl' (+) 0
196211map n = composeN n $ P.fmap (+1)
197212fmap n = composeN n $ P.fmap (+1)
198213mapM n = composeN n $ S.mapM return
214+mapMSerial n = composeN n $ S.mapMSerial return
199215filterEven n = composeN n $ S.filter even
200216filterAllOut n = composeN n $ S.filter (> maxValue)
201217filterAllIn n = composeN n $ S.filter (<= maxValue)
@@ -207,6 +223,7 @@ dropAll n = composeN n $ S.drop maxValue
207223dropWhileTrue n = composeN n $ S.dropWhile (<= maxValue)
208224dropWhileFalse n = composeN n $ S.dropWhile (<= 1)
209225foldlS n = composeN n $ S.foldlS (flip S.cons) S.nil
226+concatMap n = composeN n $ (\s -> S.concatMap (\_ -> s) s)
210227
211228-------------------------------------------------------------------------------
212229-- Iteration
@@ -253,7 +270,6 @@ iterateDropWhileTrue = iterateSource (S.dropWhile (<= maxValue)) maxIters
253270-------------------------------------------------------------------------------
254271
255272zip src = transform $ S.zipWith (,) src src
256−concat _n = return ()
257273
258274-------------------------------------------------------------------------------
259275-- Mixed Composition
@@ -283,3 +299,103 @@ filterDrop n = composeN n $ S.drop 1 . S.filter (<= maxValue)
283299filterTake n = composeN n $ S.take maxValue . S.filter (<= maxValue)
284300filterScan n = composeN n $ S.scanl' (+) 0 . S.filter (<= maxBound)
285301filterMap n = composeN n $ S.map (subtract 1) . S.filter (<= maxValue)
302+
303+-------------------------------------------------------------------------------
304+-- Nested Composition
305+-------------------------------------------------------------------------------
306+
307+{-# INLINE toNullApNested #-}
308+toNullApNested :: Monad m => Stream m Int -> m ()
309+toNullApNested s = runStream $ do
310+ (+) <$> s <*> s
311+
312+{-# INLINE toNullNested #-}
313+toNullNested :: Monad m => Stream m Int -> m ()
314+toNullNested s = runStream $ do
315+ x <- s
316+ y <- s
317+ return $ x + y
318+
319+{-# INLINE toNullNested3 #-}
320+toNullNested3 :: Monad m => Stream m Int -> m ()
321+toNullNested3 s = runStream $ do
322+ x <- s
323+ y <- s
324+ z <- s
325+ return $ x + y + z
326+
327+{-# INLINE filterAllOutNested #-}
328+filterAllOutNested
329+ :: Monad m
330+ => Stream m Int -> m ()
331+filterAllOutNested str = runStream $ do
332+ x <- str
333+ y <- str
334+ let s = x + y
335+ if s < 0
336+ then return s
337+ else S.nil
338+
339+{-# INLINE filterAllInNested #-}
340+filterAllInNested
341+ :: Monad m
342+ => Stream m Int -> m ()
343+filterAllInNested str = runStream $ do
344+ x <- str
345+ y <- str
346+ let s = x + y
347+ if s > 0
348+ then return s
349+ else S.nil
350+
351+-------------------------------------------------------------------------------
352+-- Nested Composition Pure lists
353+-------------------------------------------------------------------------------
354+
355+{-# INLINE sourceUnfoldrList #-}
356+sourceUnfoldrList :: Int -> Int -> [Int]
357+sourceUnfoldrList maxval n = List.unfoldr step n
358+ where
359+ step cnt =
360+ if cnt > n + maxval
361+ then Nothing
362+ else Just (cnt, cnt + 1)
363+
364+{-# INLINE toNullApNestedList #-}
365+toNullApNestedList :: [Int] -> [Int]
366+toNullApNestedList s = (+) <$> s <*> s
367+
368+{-# INLINE toNullNestedList #-}
369+toNullNestedList :: [Int] -> [Int]
370+toNullNestedList s = do
371+ x <- s
372+ y <- s
373+ return $ x + y
374+
375+{-# INLINE toNullNestedList3 #-}
376+toNullNestedList3 :: [Int] -> [Int]
377+toNullNestedList3 s = do
378+ x <- s
379+ y <- s
380+ z <- s
381+ return $ x + y + z
382+
383+{-# INLINE filterAllOutNestedList #-}
384+filterAllOutNestedList :: [Int] -> [Int]
385+filterAllOutNestedList str = do
386+ x <- str
387+ y <- str
388+ let s = x + y
389+ if s < 0
390+ then return s
391+ else []
392+
393+{-# INLINE filterAllInNestedList #-}
394+filterAllInNestedList :: [Int] -> [Int]
395+filterAllInNestedList str = do
396+ x <- str
397+ y <- str
398+ let s = x + y
399+ if s > 0
400+ then return s
401+ else []
src/Streamly/Streams/Ahead.hs+9−11
@@ -677,23 +677,21 @@ instance MonadAsync m => Monoid (AheadT m a) where
677677-- Monad
678678------------------------------------------------------------------------------
679679
680−{-# INLINE bindAhead #-}
681−{-# SPECIALIZE bindAhead :: AheadT IO a -> (a -> AheadT IO b) -> AheadT IO b #-}
682−bindAhead :: MonadAsync m => AheadT m a -> (a -> AheadT m b) -> AheadT m b
683−bindAhead m f = fromStream $ K.bindWith ahead (K.adapt m) (\a -> K.adapt $ f a)
680+{-# INLINE concatMapAhead #-}
681+{-# SPECIALIZE concatMapAhead :: (a -> AheadT IO b) -> AheadT IO a -> AheadT IO b #-}
682+concatMapAhead :: MonadAsync m => (a -> AheadT m b) -> AheadT m a -> AheadT m b
683+concatMapAhead f m = fromStream $
684+ K.concatMapBy ahead (\a -> K.adapt $ f a) (K.adapt m)
684685
685686instance MonadAsync m => Monad (AheadT m) where
686687 return = pure
687− (>>=) = bindAhead
688−
689−{-# INLINE apAhead #-}
690−{-# SPECIALIZE apAhead :: AheadT IO (a -> b) -> AheadT IO a -> AheadT IO b #-}
691−apAhead :: MonadAsync m => AheadT m (a -> b) -> AheadT m a -> AheadT m b
692−apAhead mf m = ap (K.adapt mf) (K.adapt m)
688+ {-# INLINE (>>=) #-}
689+ (>>=) = flip concatMapAhead
693690
694691instance (Monad m, MonadAsync m) => Applicative (AheadT m) where
695692 pure = AheadT . K.yield
696− (<*>) = apAhead
693+ {-# INLINE (<*>) #-}
694+ (<*>) = ap
697695
698696------------------------------------------------------------------------------
699697-- Other instances
src/Streamly/Streams/Async.hs+8
@@ -684,11 +684,15 @@ instance MonadAsync m => Monoid (AsyncT m a) where
684684-- Monad
685685------------------------------------------------------------------------------
686686
687+-- GHC: if we change the implementation of bindWith with arguments in a
688+-- different order we see a significant performance degradation (~2x).
687689{-# INLINE bindAsync #-}
688690{-# SPECIALIZE bindAsync :: AsyncT IO a -> (a -> AsyncT IO b) -> AsyncT IO b #-}
689691bindAsync :: MonadAsync m => AsyncT m a -> (a -> AsyncT m b) -> AsyncT m b
690692bindAsync m f = fromStream $ K.bindWith async (adapt m) (\a -> adapt $ f a)
691693
694+-- GHC: if we specify arguments in the definition of (>>=) we see a significant
695+-- performance degradation (~2x).
692696instance MonadAsync m => Monad (AsyncT m) where
693697 return = pure
694698 (>>=) = bindAsync
@@ -819,11 +823,15 @@ instance MonadAsync m => Monoid (WAsyncT m a) where
819823-- Monad
820824------------------------------------------------------------------------------
821825
826+-- GHC: if we change the implementation of bindWith with arguments in a
827+-- different order we see a significant performance degradation (~2x).
822828{-# INLINE bindWAsync #-}
823829{-# SPECIALIZE bindWAsync :: WAsyncT IO a -> (a -> WAsyncT IO b) -> WAsyncT IO b #-}
824830bindWAsync :: MonadAsync m => WAsyncT m a -> (a -> WAsyncT m b) -> WAsyncT m b
825831bindWAsync m f = fromStream $ K.bindWith wAsync (adapt m) (\a -> adapt $ f a)
826832
833+-- GHC: if we specify arguments in the definition of (>>=) we see a significant
834+-- performance degradation (~2x).
827835instance MonadAsync m => Monad (WAsyncT m) where
828836 return = pure
829837 (>>=) = bindWAsync
src/Streamly/Streams/Instances.hs+2
@@ -9,7 +9,9 @@
99
1010#define MONAD_APPLICATIVE_INSTANCE(STREAM,CONSTRAINT) \
1111instance (Monad m CONSTRAINT) => Applicative (STREAM m) where { \
12+ {-# INLINE pure #-}; \
1213 pure = STREAM . K.yield; \
14+ {-# INLINE (<*>) #-}; \
1315 (<*>) = ap }
1416
1517#define MONAD_COMMON_INSTANCES(STREAM,CONSTRAINT) \
src/Streamly/Streams/Parallel.hs+8−10
@@ -134,9 +134,9 @@ mkParallel m = do
134134-- Stream to stream concurrent function application
135135------------------------------------------------------------------------------
136136
137−{-# INLINE applyWith #-}
138−applyWith :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
139−applyWith f m = mkStream $ \st yld sng stp -> do
137+{-# INLINE applyParallel #-}
138+applyParallel :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
139+applyParallel f m = mkStream $ \st yld sng stp -> do
140140 sv <- newParallelVar (adaptState st)
141141 pushWorkerPar sv (runOne st{streamVar = Just sv} (toStream m))
142142 foldStream st yld sng stp $ f $ fromSVar sv
@@ -145,9 +145,9 @@ applyWith f m = mkStream $ \st yld sng stp -> do
145145-- Stream runner concurrent function application
146146------------------------------------------------------------------------------
147147
148−{-# INLINE runWith #-}
149−runWith :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
150−runWith f m = do
148+{-# INLINE foldParallel #-}
149+foldParallel :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
150+foldParallel f m = do
151151 sv <- newParallelVar defState
152152 pushWorkerPar sv (runOne defState{streamVar = Just sv} $ toStream m)
153153 f $ fromSVar sv
@@ -179,7 +179,7 @@ infixl 1 |&.
179179-- @since 0.3.0
180180{-# INLINE (|$) #-}
181181(|$) :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
182−f |$ x = applyWith f x
182+f |$ x = applyParallel f x
183183
184184-- | Parallel reverse function application operator for streams; just like the
185185-- regular reverse function application operator '&' except that it is
@@ -214,7 +214,7 @@ x |& f = f |$ x
214214-- @since 0.3.0
215215{-# INLINE (|$.) #-}
216216(|$.) :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
217−f |$. x = runWith f x
217+f |$. x = foldParallel f x
218218
219219-- | Parallel reverse function application operator for applying a run or fold
220220-- functions to a stream. Just like '|$.' except that the operands are reversed.
@@ -380,8 +380,6 @@ instance MonadAsync m => Monad (ParallelT m) where
380380 return = pure
381381 (>>=) = bindParallel
382382
383−-- XXX Specialize the applicative instance
384−
385383------------------------------------------------------------------------------
386384-- Other instances
387385------------------------------------------------------------------------------
src/Streamly/Streams/Prelude.hs+20
@@ -51,6 +51,10 @@ module Streamly.Streams.Prelude
5151 , eqBy
5252 , cmpBy
5353
54+ -- * Nesting
55+ , K.concatMapBy
56+ , K.concatMap
57+
5458 -- * Fold Utilities
5559 , foldWith
5660 , foldMapWith
@@ -243,6 +247,22 @@ cmpBy f m1 m2 = D.cmpBy f (D.toStreamD m1) (D.toStreamD m2)
243247-- Fold Utilities
244248------------------------------------------------------------------------------
245249
250+{-
251+-- XXX do we have facilities in Foldable to fold any Foldable in this manner?
252+--
253+-- | Perform a pair wise bottom up hierarchical fold of elements in the
254+-- container using the given function as the merge function.
255+--
256+-- This will perform a balanced merge sort if the merge function is
257+-- 'mergeBy compare'.
258+--
259+-- @since 0.7.0
260+{-# INLINABLE foldbWith #-}
261+foldbWith :: IsStream t
262+ => (t m a -> t m a -> t m a) -> SerialT Identity (t m a) -> t m a
263+foldbWith f = K.foldb f K.nil
264+-}
265+
246266-- | A variant of 'Data.Foldable.fold' that allows you to fold a 'Foldable'
247267-- container of streams using the specified stream sum operation.
248268--
src/Streamly/Streams/Serial.hs+6
@@ -166,8 +166,13 @@ instance IsStream SerialT where
166166
167167instance Monad m => Monad (SerialT m) where
168168 return = pure
169+ {-# INLINE (>>=) #-}
169170 (>>=) = K.bindWith K.serial
170171
172+ -- StreamD based implementation
173+ -- return = SerialT . D.fromStreamD . D.yield
174+ -- m >>= f = D.fromStreamD $ D.concatMap (\a -> D.toStreamD (f a)) (D.toStreamD m)
175+
171176------------------------------------------------------------------------------
172177-- Other instances
173178------------------------------------------------------------------------------
@@ -327,6 +332,7 @@ instance Monoid (WSerialT m a) where
327332
328333instance Monad m => Monad (WSerialT m) where
329334 return = pure
335+ {-# INLINE (>>=) #-}
330336 (>>=) = K.bindWith wSerial
331337
332338------------------------------------------------------------------------------
src/Streamly/Streams/StreamD.hs+1−63
@@ -88,6 +88,7 @@ module Streamly.Streams.StreamD
8888
8989 -- * Elimination
9090 -- ** General Folds
91+ , foldrS
9192 , foldrT
9293 , foldrM
9394 , foldrMx
@@ -492,24 +493,6 @@ enumerateFromThenToFractional from next to =
492493-- Generation by Conversion
493494-------------------------------------------------------------------------------
494495
495−-- | Create a singleton 'Stream' from a pure value.
496−{-# INLINE_NORMAL yield #-}
497−yield :: Monad m => a -> Stream m a
498−yield x = Stream (\_ s -> return $ step undefined s) True
499− where
500− {-# INLINE_LATE step #-}
501− step _ True = Yield x False
502− step _ False = Stop
503−
504−-- | Create a singleton 'Stream' from a monadic action.
505−{-# INLINE_NORMAL yieldM #-}
506−yieldM :: Monad m => m a -> Stream m a
507−yieldM m = Stream step True
508− where
509− {-# INLINE_LATE step #-}
510− step _ True = m >>= \x -> return $ Yield x False
511− step _ False = return Stop
512−
513496{-# INLINE_NORMAL fromIndicesM #-}
514497fromIndicesM :: Monad m => (Int -> m a) -> Stream m a
515498fromIndicesM gen = Stream step 0
@@ -871,51 +854,6 @@ reverse' m =
871854 $ toStreamK
872855 $ A.fromStreamDArraysOf A.defaultChunkSize m
873856
874−------------------------------------------------------------------------------
875−-- concatMap
876−------------------------------------------------------------------------------
877−
878−{-# INLINE_NORMAL concatMapM #-}
879−concatMapM :: Monad m => (a -> m (Stream m b)) -> Stream m a -> Stream m b
880−concatMapM f (Stream step state) = Stream step' (Left state)
881− where
882− {-# INLINE_LATE step' #-}
883− step' gst (Left st) = do
884− r <- step (adaptState gst) st
885− case r of
886− Yield a s -> do
887− b_stream <- f a
888− return $ Skip (Right (b_stream, s))
889− Skip s -> return $ Skip (Left s)
890− Stop -> return Stop
891−
892− -- XXX flattenArrays is 5x faster than "concatMap fromArray". if somehow we
893− -- can get inner_step to inline and fuse here we can perhaps get the same
894− -- performance using "concatMap fromArray".
895− --
896− -- XXX using the pattern synonym "Stream" causes a major performance issue
897− -- here even if the synonym does not include an adaptState call. Need to
898− -- find out why. Is that something to be fixed in GHC?
899− step' gst (Right (UnStream inner_step inner_st, st)) = do
900− r <- inner_step (adaptState gst) inner_st
901− case r of
902− Yield b inner_s ->
903− return $ Yield b (Right (Stream inner_step inner_s, st))
904− Skip inner_s ->
905− return $ Skip (Right (Stream inner_step inner_s, st))
906− Stop -> return $ Skip (Left st)
907−
908−{-# INLINE concatMap #-}
909−concatMap :: Monad m => (a -> Stream m b) -> Stream m a -> Stream m b
910−concatMap f = concatMapM (return . f)
911−
912−-- XXX The idea behind this rule is to rewrite any calls to "concatMap
913−-- fromArray" automatically to flattenArrays which is much faster. However, we
914−-- need an INLINE_EARLY on concatMap for this rule to fire. But if we use
915−-- INLINE_EARLY on concatMap or fromArray then direct uses of
916−-- "concatMap fromArray" (without the RULE) become much slower, this means
917−-- "concatMap f" in general would become slower. Need to find a solution to
918−-- this.
919857
920858------------------------------------------------------------------------------
921859-- Grouping/Splitting
src/Streamly/Streams/StreamD/Type.hs+106−1
@@ -37,11 +37,16 @@ module Streamly.Streams.StreamD.Type
3737 , toStreamK
3838 , map
3939 , mapM
40+ , yield
41+ , yieldM
42+ , concatMap
43+ , concatMapM
4044
4145 , foldrT
4246 , foldrM
4347 , foldrMx
4448 , foldr
49+ , foldrS
4550
4651 , foldl'
4752 , foldlM'
@@ -59,9 +64,10 @@ module Streamly.Streams.StreamD.Type
5964where
6065
6166import Control.Applicative (liftA2)
67+import Control.Monad (ap)
6268import Control.Monad.Trans (lift, MonadTrans)
6369import GHC.Types (SPEC(..))
64−import Prelude hiding (map, mapM, foldr, take)
70+import Prelude hiding (map, mapM, foldr, take, concatMap)
6571
6672import Streamly.SVar (State(..), adaptState, defState)
6773import Streamly.Fold.Types (Fold(..))
@@ -155,6 +161,78 @@ instance Monad m => Functor (Stream m) where
155161 {-# INLINE fmap #-}
156162 fmap = map
157163
164+------------------------------------------------------------------------------
165+-- concatMap
166+------------------------------------------------------------------------------
167+
168+{-# INLINE_NORMAL concatMapM #-}
169+concatMapM :: Monad m => (a -> m (Stream m b)) -> Stream m a -> Stream m b
170+concatMapM f (Stream step state) = Stream step' (Left state)
171+ where
172+ {-# INLINE_LATE step' #-}
173+ step' gst (Left st) = do
174+ r <- step (adaptState gst) st
175+ case r of
176+ Yield a s -> do
177+ b_stream <- f a
178+ return $ Skip (Right (b_stream, s))
179+ Skip s -> return $ Skip (Left s)
180+ Stop -> return Stop
181+
182+ -- XXX flattenArrays is 5x faster than "concatMap fromArray". if somehow we
183+ -- can get inner_step to inline and fuse here we can perhaps get the same
184+ -- performance using "concatMap fromArray".
185+ --
186+ -- XXX using the pattern synonym "Stream" causes a major performance issue
187+ -- here even if the synonym does not include an adaptState call. Need to
188+ -- find out why. Is that something to be fixed in GHC?
189+ step' gst (Right (UnStream inner_step inner_st, st)) = do
190+ r <- inner_step (adaptState gst) inner_st
191+ case r of
192+ Yield b inner_s ->
193+ return $ Yield b (Right (Stream inner_step inner_s, st))
194+ Skip inner_s ->
195+ return $ Skip (Right (Stream inner_step inner_s, st))
196+ Stop -> return $ Skip (Left st)
197+
198+{-# INLINE concatMap #-}
199+concatMap :: Monad m => (a -> Stream m b) -> Stream m a -> Stream m b
200+concatMap f = concatMapM (return . f)
201+
202+-- XXX The idea behind this rule is to rewrite any calls to "concatMap
203+-- fromArray" automatically to flattenArrays which is much faster. However, we
204+-- need an INLINE_EARLY on concatMap for this rule to fire. But if we use
205+-- INLINE_EARLY on concatMap or fromArray then direct uses of
206+-- "concatMap fromArray" (without the RULE) become much slower, this means
207+-- "concatMap f" in general would become slower. Need to find a solution to
208+-- this.
209+--
210+-- {-# RULES "concatMap Array.toStreamD"
211+-- concatMap Array.toStreamD = Array.flattenArray #-}
212+
213+-- | Create a singleton 'Stream' from a pure value.
214+{-# INLINE_NORMAL yield #-}
215+yield :: Monad m => a -> Stream m a
216+yield x = Stream (\_ s -> return $ step undefined s) True
217+ where
218+ {-# INLINE_LATE step #-}
219+ step _ True = Yield x False
220+ step _ False = Stop
221+
222+instance Monad m => Applicative (Stream m) where
223+ {-# INLINE pure #-}
224+ pure = yield
225+ {-# INLINE (<*>) #-}
226+ (<*>) = ap
227+
228+-- NOTE: even though concatMap for StreamD is 4x faster compared to StreamK,
229+-- the monad instance does not seem to be significantly faster.
230+instance Monad m => Monad (Stream m) where
231+ {-# INLINE return #-}
232+ return = pure
233+ {-# INLINE (>>=) #-}
234+ (>>=) = flip concatMap
235+
158236-- XXX Use of SPEC constructor in folds causes 2x performance degradation in
159237-- one shot operations, but helps immensely in operations composed of multiple
160238-- combinators or the same combinator many times. There seems to be an
@@ -208,6 +286,33 @@ foldrMx fstep final convert (Stream step state) = convert $ go SPEC state
208286foldr :: Monad m => (a -> b -> b) -> b -> Stream m a -> m b
209287foldr f z = foldrM (\a b -> liftA2 f (return a) b) (return z)
210288
289+-- | Create a singleton 'Stream' from a monadic action.
290+{-# INLINE_NORMAL yieldM #-}
291+yieldM :: Monad m => m a -> Stream m a
292+yieldM m = Stream step True
293+ where
294+ {-# INLINE_LATE step #-}
295+ step _ True = m >>= \x -> return $ Yield x False
296+ step _ False = return Stop
297+
298+-- this performs horribly, should not be used
299+{-# INLINE_NORMAL foldrS #-}
300+foldrS
301+ :: Monad m
302+ => (a -> Stream m b -> Stream m b)
303+ -> Stream m b
304+ -> Stream m a
305+ -> Stream m b
306+foldrS f final (Stream step state) = go SPEC state
307+ where
308+ go !_ st = do
309+ -- defState??
310+ r <- yieldM $ step defState st
311+ case r of
312+ Yield x s -> f x (go SPEC s)
313+ Skip s -> go SPEC s
314+ Stop -> final
315+
211316-- Right fold to some transformer (T) monad. This can be useful to implement
212317-- stateless combinators like map, filtering, insertions, takeWhile, dropWhile.
213318--
src/Streamly/Streams/StreamK.hs+38−138
@@ -71,12 +71,19 @@ module Streamly.Streams.StreamK
7171 , fromList
7272 , fromStreamK
7373
74+ -- * foldr/build
75+ , foldrS
76+ , foldrSM
77+ , buildS
78+ , buildM
79+ , augmentS
80+ , augmentSM
81+
7482 -- * Elimination
7583 -- ** General Folds
7684 , foldr
7785 , foldr1
7886 , foldrM
79− , foldrS
8087 , foldrT
8188
8289 , foldl'
@@ -129,6 +136,7 @@ module Streamly.Streams.StreamK
129136 -- ** Mapping
130137 , map
131138 , mapM
139+ , mapMSerial
132140 , sequence
133141
134142 -- ** Inserting
@@ -152,6 +160,11 @@ module Streamly.Streams.StreamK
152160 , mergeBy
153161 , mergeByM
154162
163+ -- ** Nesting
164+ , concatMapBy
165+ , concatMap
166+ , bindWith
167+
155168 -- ** Transformation comprehensions
156169 , the
157170
@@ -160,7 +173,6 @@ module Streamly.Streams.StreamK
160173
161174 -- * Utilities
162175 , consMStream
163− , bindWith
164176 , withLocal
165177
166178 -- * Deprecated
@@ -176,54 +188,12 @@ import Prelude
176188 hiding (foldl, foldr, last, map, mapM, mapM_, repeat, sequence,
177189 take, filter, all, any, takeWhile, drop, dropWhile, minimum,
178190 maximum, elem, notElem, null, head, tail, init, zipWith, lookup,
179− foldr1, (!!), replicate, reverse)
191+ foldr1, (!!), replicate, reverse, concatMap)
180192import qualified Prelude
181193
182194import Streamly.SVar
183195import Streamly.Streams.StreamK.Type
184196
185−-- | Detach a stream from an SVar
186−{-# INLINE unShare #-}
187−unShare :: IsStream t => t m a -> t m a
188−unShare x = mkStream $ \st yld sng stp ->
189− foldStream st yld sng stp x
190−
191−------------------------------------------------------------------------------
192−-- Construction
193−------------------------------------------------------------------------------
194−
195−infixr 5 `cons`
196−
197−-- faster than consM because there is no bind.
198−-- | Construct a stream by adding a pure value at the head of an existing
199−-- stream. For serial streams this is the same as @(return a) \`consM` r@ but
200−-- more efficient. For concurrent streams this is not concurrent whereas
201−-- 'consM' is concurrent. For example:
202−--
203−-- @
204−-- > toList $ 1 \`cons` 2 \`cons` 3 \`cons` nil
205−-- [1,2,3]
206−-- @
207−--
208−-- @since 0.1.0
209−{-# INLINE cons #-}
210−cons :: IsStream t => a -> t m a -> t m a
211−cons a r = mkStream $ \_ yld _ _ -> yld a r
212−
213−infixr 5 .:
214−
215−-- | Operator equivalent of 'cons'.
216−--
217−-- @
218−-- > toList $ 1 .: 2 .: 3 .: nil
219−-- [1,2,3]
220−-- @
221−--
222−-- @since 0.1.1
223−{-# INLINE (.:) #-}
224−(.:) :: IsStream t => a -> t m a -> t m a
225−(.:) = cons
226−
227197-------------------------------------------------------------------------------
228198-- Deconstruction
229199-------------------------------------------------------------------------------
@@ -240,50 +210,29 @@ uncons m =
240210-- Generation
241211-------------------------------------------------------------------------------
242212
243−{-# INLINE_NORMAL build #-}
244−build :: IsStream t => forall a. (forall b. (a -> b -> b) -> b -> b) -> t m a
245−build g = g cons nil
246−
247−{-# INLINE_NORMAL _augment #-}
248−_augment
249− :: IsStream t
250− => forall a. (forall b. (a -> b -> b) -> b -> b) -> t m a -> t m a
251−_augment g xs = g cons xs
252−
253−{-# INLINE_NORMAL _buildM #-}
254−_buildM
255− :: (IsStream t, MonadAsync m)
256− => forall a. ((m a -> t m a -> t m a) -> t m a -> t m a) -> t m a
257−_buildM g = g consM nil
258−
259213{-# INLINE unfoldr #-}
260214unfoldr :: IsStream t => (b -> Maybe (a, b)) -> b -> t m a
261−unfoldr step b0 = build $ \cns nl ->
262− let go s =
263− case step s of
264− Just (a, b) -> a `cns` go b
265− Nothing -> nl
266− in go b0
215+unfoldr step = go
216+ where
217+ go s = buildS $ \yld stp ->
218+ case step s of
219+ Just (a, b) -> yld a (go b)
220+ Nothing -> stp
267221
268222{-# INLINE unfoldrM #-}
269223unfoldrM :: (IsStream t, MonadAsync m) => (b -> m (Maybe (a, b))) -> b -> t m a
270224unfoldrM step = go
271225 where
272− go s = mkStream $ \st yld sng stp -> do
273− mayb <- step s
274− case mayb of
275− Nothing -> stp
276− Just (a, b) ->
277− foldStreamShared st yld sng stp $ return a |: go b
226+ go s = sharedM $ \yld _ stp -> do
227+ r <- step s
228+ case r of
229+ Just (a, b) -> yld a (go b)
230+ Nothing -> stp
278231
279232-------------------------------------------------------------------------------
280233-- Special generation
281234-------------------------------------------------------------------------------
282235
283−{-# INLINE yield #-}
284−yield :: IsStream t => a -> t m a
285−yield a = mkStream $ \_ _ single _ -> single a
286−
287236-- | Same as yieldM
288237--
289238-- @since 0.2.0
@@ -348,36 +297,7 @@ fromStreamK = fromStream
348297-- | Lazy right associative fold.
349298{-# INLINE foldr #-}
350299foldr :: (IsStream t, Monad m) => (a -> b -> b) -> b -> t m a -> m b
351−foldr step acc m = go m
352− where
353− go m1 =
354− let stop = return acc
355− single a = return (step a acc)
356− yieldk a r = go r >>= \b -> return (step a b)
357− in foldStream defState yieldk single stop m1
358−
359−-- | Lazy right fold with a monadic step function.
360−{-# INLINE foldrM #-}
361−foldrM :: IsStream t => (a -> m b -> m b) -> m b -> t m a -> m b
362−foldrM step acc m = go m
363− where
364− go m1 =
365− let stop = acc
366− single a = step a acc
367− yieldk a r = step a (go r)
368− in foldStream defState yieldk single stop m1
369−
370−-- | Lazy right associative fold to a stream.
371−{-# INLINE foldrS #-}
372−foldrS :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b
373−foldrS step final m = go m
374− where
375− go m1 = mkStream $ \st yld sng stp ->
376− let run x = foldStream st yld sng stp x
377− stop = run final
378− single a = run $ step a final
379− yieldk a r = run $ step a (go r)
380− in foldStream (adaptState st) yieldk single stop m1
300+foldr step acc = foldrM (\x xs -> xs >>= \b -> return (step x b)) (return acc)
381301
382302-- | Right associative fold to an arbitrary transformer monad.
383303{-# INLINE foldrT #-}
@@ -489,11 +409,19 @@ foldlT step begin m = go begin m
489409-- Specialized folds
490410------------------------------------------------------------------------------
491411
412+-- XXX use foldrM to implement folds where possible
413+-- XXX This (commented) definition of drain and mapM_ perform much better on
414+-- some benchmarks but worse on others. Need to investigate why, may there is
415+-- an optimization opportunity that we can exploit.
416+-- drain = foldrM (\_ xs -> return () >> xs) (return ())
417+
492418-- |
493419-- > drain = foldl' (\_ _ -> ()) ()
494420-- > drain = mapM_ (\_ -> return ())
495421{-# INLINE drain #-}
496422drain :: (Monad m, IsStream t) => t m a -> m ()
423+drain = foldrM (\_ xs -> xs) (return ())
424+{-
497425drain = go
498426 where
499427 go m1 =
@@ -501,9 +429,11 @@ drain = go
501429 single _ = return ()
502430 yieldk _ r = go r
503431 in foldStream defState yieldk single stop m1
432+-}
504433
505434{-# INLINE null #-}
506435null :: (IsStream t, Monad m) => t m a -> m Bool
436+-- null = foldrM (\_ _ -> return True) (return False)
507437null m =
508438 let stop = return True
509439 single _ = return False
@@ -512,6 +442,7 @@ null m =
512442
513443{-# INLINE head #-}
514444head :: (IsStream t, Monad m) => t m a -> m (Maybe a)
445+-- head = foldrM (\x _ -> return $ Just x) (return Nothing)
515446head m =
516447 let stop = return Nothing
517448 single a = return (Just a)
@@ -836,17 +767,6 @@ dropWhile p m = go m
836767-- Mapping
837768-------------------------------------------------------------------------------
838769
839−-- Be careful when modifying this, this uses a consM (|:) deliberately to allow
840−-- other stream types to overload it.
841−{-# INLINE mapM #-}
842−mapM :: (IsStream t, MonadAsync m) => (a -> m b) -> t m a -> t m b
843−mapM f m = go m
844− where
845− go m1 = mkStream $ \st yld sng stp ->
846− let single a = f a >>= sng
847− yieldk a r = foldStreamShared st yld sng stp $ f a |: go r
848− in foldStream (adaptState st) yieldk single stp m1
849−
850770-- Be careful when modifying this, this uses a consM (|:) deliberately to allow
851771-- other stream types to overload it.
852772{-# INLINE sequence #-}
@@ -1019,26 +939,6 @@ the m = do
1019939 | otherwise = return Nothing
1020940 in foldStream defState yieldk single (return $ Just h) m1
1021941
1022−-------------------------------------------------------------------------------
1023−-- Bind utility
1024−-------------------------------------------------------------------------------
1025−
1026−{-# INLINE bindWith #-}
1027−bindWith
1028− :: IsStream t
1029− => (forall c. t m c -> t m c -> t m c)
1030− -> t m a
1031− -> (a -> t m b)
1032− -> t m b
1033−bindWith par m1 f = go m1
1034− where
1035− go m =
1036− mkStream $ \st yld sng stp ->
1037− let foldShared = foldStreamShared st yld sng stp
1038− single a = foldShared $ unShare (f a)
1039− yieldk a r = foldShared $ unShare (f a) `par` go r
1040− in foldStream (adaptState st) yieldk single stp m
1041−
1042942------------------------------------------------------------------------------
1043943-- Alternative & MonadPlus
1044944------------------------------------------------------------------------------
src/Streamly/Streams/StreamK/Type.hs+534−19
@@ -6,6 +6,7 @@
66{-# LANGUAGE InstanceSigs #-}
77{-# LANGUAGE MultiParamTypeClasses #-}
88{-# LANGUAGE PatternSynonyms #-}
9+{-# LANGUAGE KindSignatures #-}
910{-# LANGUAGE ViewPatterns #-}
1011{-# LANGUAGE RankNTypes #-}
1112{-# LANGUAGE UndecidableInstances #-} -- XXX
@@ -45,23 +46,45 @@ module Streamly.Streams.StreamK.Type
4546 , foldStreamShared
4647 , foldStreamSVar
4748
49+ -- * foldr/build
50+ , foldrM
51+ , foldrS
52+ , foldrSM
53+ , buildS
54+ , buildM
55+ , buildSM
56+ , sharedM
57+ , augmentS
58+ , augmentSM
59+
4860 -- instances
61+ , cons
62+ , (.:)
4963 , consMStream
64+ , consMBy
65+ , yieldM
66+ , yield
5067
5168 , nil
69+ , conjoin
5270 , serial
5371 , map
54− , yieldM
72+ , mapM
73+ , mapMSerial
74+ , unShare
75+ , concatMapBy
76+ , concatMap
77+ , bindWith
5578
5679 , Streaming -- deprecated
5780 )
5881where
5982
60−import Control.Monad (void)
83+import Control.Monad (void, ap, (>=>))
6184import Control.Monad.IO.Class (MonadIO(liftIO))
6285import Control.Monad.Trans.Class (MonadTrans(lift))
6386import Data.Semigroup (Semigroup(..))
64−import Prelude hiding (map)
87+import Prelude hiding (map, mapM, concatMap, foldr)
6588
6689import Streamly.SVar
6790
@@ -247,6 +270,68 @@ consK k r = mkStream $ \_ yld _ _ -> k (\x -> yld x r)
247270
248271-- XXX Build a stream from a repeating callback function.
249272
273+------------------------------------------------------------------------------
274+-- Construction
275+------------------------------------------------------------------------------
276+
277+infixr 5 `cons`
278+
279+-- faster than consM because there is no bind.
280+-- | Construct a stream by adding a pure value at the head of an existing
281+-- stream. For serial streams this is the same as @(return a) \`consM` r@ but
282+-- more efficient. For concurrent streams this is not concurrent whereas
283+-- 'consM' is concurrent. For example:
284+--
285+-- @
286+-- > toList $ 1 \`cons` 2 \`cons` 3 \`cons` nil
287+-- [1,2,3]
288+-- @
289+--
290+-- @since 0.1.0
291+{-# INLINE_NORMAL cons #-}
292+cons :: IsStream t => a -> t m a -> t m a
293+cons a r = mkStream $ \_ yld _ _ -> yld a r
294+
295+infixr 5 .:
296+
297+-- | Operator equivalent of 'cons'.
298+--
299+-- @
300+-- > toList $ 1 .: 2 .: 3 .: nil
301+-- [1,2,3]
302+-- @
303+--
304+-- @since 0.1.1
305+{-# INLINE (.:) #-}
306+(.:) :: IsStream t => a -> t m a -> t m a
307+(.:) = cons
308+
309+-- | An empty stream.
310+--
311+-- @
312+-- > toList nil
313+-- []
314+-- @
315+--
316+-- @since 0.1.0
317+{-# INLINE_NORMAL nil #-}
318+nil :: IsStream t => t m a
319+nil = mkStream $ \_ _ _ stp -> stp
320+
321+{-# INLINE_NORMAL yield #-}
322+yield :: IsStream t => a -> t m a
323+yield a = mkStream $ \_ _ single _ -> single a
324+
325+{-# INLINE_NORMAL yieldM #-}
326+yieldM :: (Monad m, IsStream t) => m a -> t m a
327+yieldM m = fromStream $ mkStream $ \_ _ single _ -> m >>= single
328+
329+-- XXX specialize to IO?
330+{-# INLINE consMBy #-}
331+consMBy :: (IsStream t, MonadAsync m) => (t m a -> t m a -> t m a)
332+ -> m a -> t m a -> t m a
333+consMBy f m r = (fromStream $ yieldM m) `f` r
334+
250335------------------------------------------------------------------------------
251336-- Folding a stream
252337------------------------------------------------------------------------------
@@ -345,6 +430,299 @@ instance IsStream Stream where
345430 (|:) :: Monad m => m a -> Stream m a -> Stream m a
346431 (|:) = consMStream
347432
433+-------------------------------------------------------------------------------
434+-- foldr/build fusion
435+-------------------------------------------------------------------------------
436+
437+-- XXX perhaps we can just use foldrSM/buildM everywhere as they are more
438+-- general and cover foldrS/buildS as well.
439+
440+-- | The function 'f' decides how to reconstruct the stream. We could
441+-- reconstruct using a shared state (SVar) or without sharing the state.
442+--
443+{-# INLINE foldrSWith #-}
444+foldrSWith :: IsStream t
445+ => (forall r. State Stream m b
446+ -> (b -> t m b -> m r)
447+ -> (b -> m r)
448+ -> m r
449+ -> t m b
450+ -> m r)
451+ -> (a -> t m b -> t m b) -> t m b -> t m a -> t m b
452+foldrSWith f step final m = go m
453+ where
454+ go m1 = mkStream $ \st yld sng stp ->
455+ let run x = f st yld sng stp x
456+ stop = run final
457+ single a = run $ step a final
458+ yieldk a r = run $ step a (go r)
459+ -- XXX if type a and b are the same we do not need adaptState, can we
460+ -- save some perf with that?
461+ -- XXX since we are using adaptState anyway here we can use
462+ -- foldStreamShared instead, will that save some perf?
463+ in foldStream (adaptState st) yieldk single stop m1
464+
465+-- XXX we can use rewrite rules just for foldrSWith, if the function f is the
466+-- same we can rewrite it.
467+
468+-- | Fold sharing the SVar state within the reconstructed stream
469+{-# INLINE_NORMAL foldrSShared #-}
470+foldrSShared :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b
471+foldrSShared = foldrSWith foldStreamShared
472+
473+-- XXX consM is a typeclass method, therefore rewritten already. Instead maybe
474+-- we can make consM polymorphic using rewrite rules.
475+-- {-# RULES "foldrSShared/id" foldrSShared consM nil = \x -> x #-}
476+{-# RULES "foldrSShared/nil"
477+ forall k z. foldrSShared k z nil = z #-}
478+{-# RULES "foldrSShared/single"
479+ forall k z x. foldrSShared k z (yield x) = k x z #-}
480+-- {-# RULES "foldrSShared/app" [1]
481+-- forall ys. foldrSShared consM ys = \xs -> xs `conjoin` ys #-}
482+
483+-- | Lazy right associative fold to a stream.
484+{-# INLINE_NORMAL foldrS #-}
485+foldrS :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b
486+foldrS = foldrSWith foldStream
487+
488+{-# RULES "foldrS/id" foldrS cons nil = \x -> x #-}
489+{-# RULES "foldrS/nil" forall k z. foldrS k z nil = z #-}
490+-- See notes in GHC.Base about this rule
491+-- {-# RULES "foldr/cons"
492+-- forall k z x xs. foldrS k z (x `cons` xs) = k x (foldrS k z xs) #-}
493+{-# RULES "foldrS/single" forall k z x. foldrS k z (yield x) = k x z #-}
494+-- {-# RULES "foldrS/app" [1]
495+-- forall ys. foldrS cons ys = \xs -> xs `conjoin` ys #-}
496+
497+-------------------------------------------------------------------------------
498+-- foldrS with monadic cons i.e. consM
499+-------------------------------------------------------------------------------
500+
501+{-# INLINE foldrSMWith #-}
502+foldrSMWith :: (IsStream t, Monad m)
503+ => (forall r. State Stream m b
504+ -> (b -> t m b -> m r)
505+ -> (b -> m r)
506+ -> m r
507+ -> t m b
508+ -> m r)
509+ -> (m a -> t m b -> t m b) -> t m b -> t m a -> t m b
510+foldrSMWith f step final m = go m
511+ where
512+ go m1 = mkStream $ \st yld sng stp ->
513+ let run x = f st yld sng stp x
514+ stop = run final
515+ single a = run $ step (return a) final
516+ yieldk a r = run $ step (return a) (go r)
517+ in foldStream (adaptState st) yieldk single stop m1
518+
519+{-# INLINE_NORMAL foldrSM #-}
520+foldrSM :: (IsStream t, Monad m)
521+ => (m a -> t m b -> t m b) -> t m b -> t m a -> t m b
522+foldrSM = foldrSMWith foldStream
523+
524+-- {-# RULES "foldrSM/id" foldrSM consM nil = \x -> x #-}
525+{-# RULES "foldrSM/nil" forall k z. foldrSM k z nil = z #-}
526+{-# RULES "foldrSM/single" forall k z x. foldrSM k z (yieldM x) = k x z #-}
527+-- {-# RULES "foldrSM/app" [1]
528+-- forall ys. foldrSM consM ys = \xs -> xs `conjoin` ys #-}
529+
530+-- Like foldrSM but sharing the SVar state within the recostructed stream.
531+{-# INLINE_NORMAL foldrSMShared #-}
532+foldrSMShared :: (IsStream t, Monad m)
533+ => (m a -> t m b -> t m b) -> t m b -> t m a -> t m b
534+foldrSMShared = foldrSMWith foldStreamShared
535+
536+-- {-# RULES "foldrSM/id" foldrSM consM nil = \x -> x #-}
537+{-# RULES "foldrSMShared/nil"
538+ forall k z. foldrSMShared k z nil = z #-}
539+{-# RULES "foldrSMShared/single"
540+ forall k z x. foldrSMShared k z (yieldM x) = k x z #-}
541+-- {-# RULES "foldrSM/app" [1]
542+-- forall ys. foldrSM consM ys = \xs -> xs `conjoin` ys #-}
543+
544+-------------------------------------------------------------------------------
545+-- build
546+-------------------------------------------------------------------------------
547+
548+-- build a stream by applying cons and nil to a build function
549+{-# INLINE_NORMAL buildS #-}
550+buildS :: IsStream t => ((a -> t m a -> t m a) -> t m a -> t m a) -> t m a
551+buildS g = g cons nil
552+
553+{-# RULES "foldrS/buildS"
554+ forall k z (g :: (a -> t m a -> t m a) -> t m a -> t m a).
555+ foldrS k z (buildS g) = g k z #-}
556+
557+{-# RULES "foldrS/cons/buildS"
558+ forall k z x (g :: (a -> t m a -> t m a) -> t m a -> t m a).
559+ foldrS k z (x `cons` buildS g) = k x (g k z) #-}
560+
561+{-# RULES "foldrSShared/buildS"
562+ forall k z (g :: (a -> t m a -> t m a) -> t m a -> t m a).
563+ foldrSShared k z (buildS g) = g k z #-}
564+
565+{-# RULES "foldrSShared/cons/buildS"
566+ forall k z x (g :: (a -> t m a -> t m a) -> t m a -> t m a).
567+ foldrSShared k z (x `cons` buildS g) = k x (g k z) #-}
568+
569+-- build a stream by applying consM and nil to a build function
570+{-# INLINE_NORMAL buildSM #-}
571+buildSM :: (IsStream t, MonadAsync m)
572+ => ((m a -> t m a -> t m a) -> t m a -> t m a) -> t m a
573+buildSM g = g consM nil
574+
575+{-# RULES "foldrSM/buildSM"
576+ forall k z (g :: (m a -> t m a -> t m a) -> t m a -> t m a).
577+ foldrSM k z (buildSM g) = g k z #-}
578+
579+{-# RULES "foldrSMShared/buildSM"
580+ forall k z (g :: (m a -> t m a -> t m a) -> t m a -> t m a).
581+ foldrSMShared k z (buildSM g) = g k z #-}
582+
583+-- Disabled because this may not fire as consM is a class Op
584+{-
585+{-# RULES "foldrS/consM/buildSM"
586+ forall k z x (g :: (m a -> t m a -> t m a) -> t m a -> t m a)
587+ . foldrSM k z (x `consM` buildSM g)
588+ = k x (g k z)
589+#-}
590+-}
591+
592+-- Build using monadic build functions (continuations) instead of
593+-- reconstructing a stream.
594+{-# INLINE_NORMAL buildM #-}
595+buildM :: (IsStream t, MonadAsync m)
596+ => (forall r. (a -> t m a -> m r)
597+ -> (a -> m r)
598+ -> m r
599+ -> m r
600+ )
601+ -> t m a
602+buildM g = mkStream $ \st yld sng stp ->
603+ g (\a r -> foldStream st yld sng stp (return a `consM` r)) sng stp
604+
605+-- | Like 'buildM' but shares the SVar state across computations.
606+{-# INLINE_NORMAL sharedM #-}
607+sharedM :: (IsStream t, MonadAsync m)
608+ => (forall r. (a -> t m a -> m r)
609+ -> (a -> m r)
610+ -> m r
611+ -> m r
612+ )
613+ -> t m a
614+sharedM g = mkStream $ \st yld sng stp ->
615+ g (\a r -> foldStreamShared st yld sng stp (return a `consM` r)) sng stp
616+
617+-------------------------------------------------------------------------------
618+-- augment
619+-------------------------------------------------------------------------------
620+
621+{-# INLINE_NORMAL augmentS #-}
622+augmentS :: IsStream t
623+ => ((a -> t m a -> t m a) -> t m a -> t m a) -> t m a -> t m a
624+augmentS g xs = g cons xs
625+
626+{-# RULES "augmentS/nil"
627+ forall (g :: (a -> t m a -> t m a) -> t m a -> t m a)
628+ . augmentS g nil = buildS g
629+#-}
630+
631+{-# RULES "foldrS/augmentS"
632+ forall k z xs (g :: (a -> t m a -> t m a) -> t m a -> t m a)
633+ . foldrS k z (augmentS g xs) = g k (foldrS k z xs)
634+#-}
635+
636+{-# RULES "augmentS/buildS"
637+ forall (g :: (a -> t m a -> t m a) -> t m a -> t m a)
638+ (h :: (a -> t m a -> t m a) -> t m a -> t m a)
639+ . augmentS g (buildS h) = buildS (\c n -> g c (h c n))
640+#-}
641+
642+{-# INLINE_NORMAL augmentSM #-}
643+augmentSM :: (IsStream t, MonadAsync m)
644+ => ((m a -> t m a -> t m a) -> t m a -> t m a) -> t m a -> t m a
645+augmentSM g xs = g consM xs
646+
647+{-# RULES "augmentSM/nil"
648+ forall (g :: (m a -> t m a -> t m a) -> t m a -> t m a)
649+ . augmentSM g nil = buildSM g
650+#-}
651+
652+{-# RULES "foldrSM/augmentSM"
653+ forall k z xs (g :: (m a -> t m a -> t m a) -> t m a -> t m a)
654+ . foldrSM k z (augmentSM g xs) = g k (foldrSM k z xs)
655+#-}
656+
657+{-# RULES "augmentSM/buildSM"
658+ forall (g :: (m a -> t m a -> t m a) -> t m a -> t m a)
659+ (h :: (m a -> t m a -> t m a) -> t m a -> t m a)
660+ . augmentSM g (buildSM h) = buildSM (\c n -> g c (h c n))
661+#-}
662+
663+-------------------------------------------------------------------------------
664+-- Experimental foldrM/buildM
665+-------------------------------------------------------------------------------
666+
667+-- | Lazy right fold with a monadic step function.
668+{-# INLINE_NORMAL foldrM #-}
669+foldrM :: IsStream t => (a -> m b -> m b) -> m b -> t m a -> m b
670+foldrM step acc m = go m
671+ where
672+ go m1 =
673+ let stop = acc
674+ single a = step a acc
675+ yieldk a r = step a (go r)
676+ in foldStream defState yieldk single stop m1
677+
678+{-# INLINE_NORMAL foldrMKWith #-}
679+foldrMKWith
680+ :: (State Stream m a
681+ -> (a -> t m a -> m b)
682+ -> (a -> m b)
683+ -> m b
684+ -> t m a
685+ -> m b)
686+ -> (a -> m b -> m b)
687+ -> m b
688+ -> ((a -> t m a -> m b) -> (a -> m b) -> m b -> m b)
689+ -> m b
690+foldrMKWith f step acc g = go g
691+ where
692+ go k =
693+ let stop = acc
694+ single a = step a acc
695+ yieldk a r = step a (go (\yld sng stp -> f defState yld sng stp r))
696+ in k yieldk single stop
697+
698+{-
699+{-# RULES "foldrM/buildS"
700+ forall k z (g :: (a -> t m a -> t m a) -> t m a -> t m a)
701+ . foldrM k z (buildS g)
702+ = g k z
703+#-}
704+-}
705+-- XXX in which case will foldrM/buildM fusion be useful?
706+{-# RULES "foldrM/buildM"
707+ forall step acc (g :: (forall r.
708+ (a -> t m a -> m r)
709+ -> (a -> m r)
710+ -> m r
711+ -> m r
712+ )).
713+ foldrM step acc (buildM g) = foldrMKWith foldStream step acc g
714+#-}
715+
716+{-# RULES "foldrM/sharedM"
717+ forall step acc (g :: (forall r.
718+ (a -> t m a -> m r)
719+ -> (a -> m r)
720+ -> m r
721+ -> m r
722+ )).
723+ foldrM step acc (sharedM g) = foldrMKWith foldStreamShared step acc g
724+#-}
725+
348726------------------------------------------------------------------------------
349727-- Semigroup
350728------------------------------------------------------------------------------
@@ -356,6 +734,8 @@ instance IsStream Stream where
356734-- @since 0.2.0
357735{-# INLINE serial #-}
358736serial :: IsStream t => t m a -> t m a -> t m a
737+serial xs ys = augmentS (\c n -> foldrS c n xs) ys
738+{-
359739serial m1 m2 = go m1
360740 where
361741 go m = mkStream $ \st yld sng stp ->
@@ -363,6 +743,12 @@ serial m1 m2 = go m1
363743 single a = yld a m2
364744 yieldk a r = yld a (go r)
365745 in foldStream st yieldk single stop m
746+-}
747+
748+-- join/merge/append streams depending on consM
749+{-# INLINE conjoin #-}
750+conjoin :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
751+conjoin xs ys = augmentSM (\c n -> foldrSM c n xs) ys
366752
367753instance Semigroup (Stream m a) where
368754 (<>) = serial
@@ -371,18 +757,6 @@ instance Semigroup (Stream m a) where
371757-- Monoid
372758------------------------------------------------------------------------------
373759
374−-- | An empty stream.
375−--
376−-- @
377−-- > toList nil
378−-- []
379−-- @
380−--
381−-- @since 0.1.0
382−{-# INLINE nil #-}
383−nil :: IsStream t => t m a
384−nil = mkStream $ \_ _ _ stp -> stp
385−
386760instance Monoid (Stream m a) where
387761 mempty = nil
388762 mappend = (<>)
@@ -391,8 +765,26 @@ instance Monoid (Stream m a) where
391765-- Functor
392766-------------------------------------------------------------------------------
393767
768+-- Note eta expanded
769+{-# INLINE_LATE mapFB #-}
770+mapFB :: forall (t :: (* -> *) -> * -> *) b m a.
771+ (b -> t m b -> t m b) -> (a -> b) -> a -> t m b -> t m b
772+mapFB c f = \x ys -> c (f x) ys
773+
774+{-# RULES
775+"mapFB/mapFB" forall c f g. mapFB (mapFB c f) g = mapFB c (f . g)
776+"mapFB/id" forall c. mapFB c (\x -> x) = c
777+#-}
778+
394779{-# INLINE map #-}
395780map :: IsStream t => (a -> b) -> t m a -> t m b
781+map f xs = buildS (\c n -> foldrS (mapFB c f) n xs)
782+
783+-- XXX This definition might potentially be more efficient, but the cost in the
784+-- benchmark is dominated by unfoldrM cost so we cannot correctly determine
785+-- differences in the mapping cost. We should perhaps deduct the cost of
786+-- unfoldrM from the benchmarks and then compare.
787+{-
396788map f m = go m
397789 where
398790 go m1 =
@@ -400,6 +792,47 @@ map f m = go m
400792 let single = sng . f
401793 yieldk a r = yld (f a) (go r)
402794 in foldStream (adaptState st) yieldk single stp m1
795+-}
796+
797+{-# INLINE_LATE mapMFB #-}
798+mapMFB :: Monad m => (m b -> t m b -> t m b) -> (a -> m b) -> m a -> t m b -> t m b
799+mapMFB c f = \x ys -> c (x >>= f) ys
800+
801+{-# RULES
802+"mapMFB/mapMFB" forall c f g. mapMFB (mapMFB c f) g = mapMFB c (f >=> g)
803+#-}
804+-- XXX These rules may never fire because pure/return type class rules will
805+-- fire first.
806+{-
807+"mapMFB/pure" forall c. mapMFB c (\x -> pure x) = c
808+"mapMFB/return" forall c. mapMFB c (\x -> return x) = c
809+-}
810+
811+-- Be careful when modifying this, this uses a consM (|:) deliberately to allow
812+-- other stream types to overload it.
813+{-# INLINE mapM #-}
814+mapM :: (IsStream t, MonadAsync m) => (a -> m b) -> t m a -> t m b
815+mapM f = foldrSShared (\x xs -> f x `consM` xs) nil
816+-- See note under map definition above.
817+{-
818+mapM f m = go m
819+ where
820+ go m1 = mkStream $ \st yld sng stp ->
821+ let single a = f a >>= sng
822+ yieldk a r = foldStreamShared st yld sng stp $ f a |: go r
823+ in foldStream (adaptState st) yieldk single stp m1
824+ -}
825+
826+-- This is experimental serial version supporting fusion.
827+--
828+-- XXX what if we do not want to fuse two concurrent mapMs?
829+-- XXX we can combine two concurrent mapM only if the SVar is of the same type
830+-- So for now we use it only for serial streams.
831+-- XXX fusion would be easier for monomoprhic stream types.
832+-- {-# RULES "mapM serial" mapM = mapMSerial #-}
833+{-# INLINE mapMSerial #-}
834+mapMSerial :: MonadAsync m => (a -> m b) -> Stream m a -> Stream m b
835+mapMSerial f xs = buildSM (\c n -> foldrSMShared (mapMFB c f) n xs)
403836
404837-- XXX in fact use the Stream type everywhere and only use polymorphism in the
405838-- high level modules/prelude.
@@ -410,9 +843,91 @@ instance Monad m => Functor (Stream m) where
410843-- Transformers
411844-------------------------------------------------------------------------------
412845
413−{-# INLINE yieldM #-}
414−yieldM :: (Monad m, IsStream t) => m a -> t m a
415−yieldM m = fromStream $ mkStream $ \_ _ single _ -> m >>= single
416−
417846instance MonadTrans Stream where
418847 lift = yieldM
848+
849+-------------------------------------------------------------------------------
850+-- Nesting
851+-------------------------------------------------------------------------------
852+
853+-- | Detach a stream from an SVar
854+{-# INLINE unShare #-}
855+unShare :: IsStream t => t m a -> t m a
856+unShare x = mkStream $ \st yld sng stp ->
857+ foldStream st yld sng stp x
858+
859+-- XXX This is just concatMapBy with arguments flipped. We need to keep this
860+-- instead of using a concatMap style definition because the bind
861+-- implementation in Async and WAsync streams show significant perf degradation
862+-- if the argument order is changed.
863+{-# INLINE bindWith #-}
864+bindWith
865+ :: IsStream t
866+ => (forall c. t m c -> t m c -> t m c)
867+ -> t m a
868+ -> (a -> t m b)
869+ -> t m b
870+bindWith par m1 f = go m1
871+ where
872+ go m =
873+ mkStream $ \st yld sng stp ->
874+ let foldShared = foldStreamShared st yld sng stp
875+ single a = foldShared $ unShare (f a)
876+ yieldk a r = foldShared $ unShare (f a) `par` go r
877+ in foldStream (adaptState st) yieldk single stp m
878+
879+-- XXX express in terms of foldrS?
880+-- XXX can we use a different stream type for the generated stream being
881+-- falttened so that we can combine them differently and keep the resulting
882+-- stream different?
883+-- XXX do we need specialize to IO?
884+-- XXX can we optimize when c and a are same, by removing the forall using
885+-- rewrite rules with type applications?
886+{-# INLINE concatMapBy #-}
887+concatMapBy
888+ :: IsStream t
889+ => (forall c. t m c -> t m c -> t m c)
890+ -> (a -> t m b)
891+ -> t m a
892+ -> t m b
893+concatMapBy par f xs = bindWith par xs f
894+
895+{-# INLINE concatMap #-}
896+concatMap :: IsStream t => (a -> t m b) -> t m a -> t m b
897+concatMap f m = fromStream $
898+ concatMapBy serial
899+ (\a -> adapt $ toStream $ f a)
900+ (adapt $ toStream m)
901+
902+{-
903+-- Fused version.
904+-- XXX This fuses but when the stream is nil this performs poorly.
905+-- The filterAllOut benchmark degrades. Need to investigate and fix that.
906+{-# INLINE concatMap #-}
907+concatMap :: IsStream t => (a -> t m b) -> t m a -> t m b
908+concatMap f xs = buildS
909+ (\c n -> foldrS (\x b -> foldrS c b (f x)) n xs)
910+
911+-- Stream polymorphic concatMap implementation
912+-- XXX need to use buildSM/foldrSMShared for parallel behavior
913+-- XXX unShare seems to degrade the fused performance
914+{-# INLINE_EARLY concatMap_ #-}
915+concatMap_ :: IsStream t => (a -> t m b) -> t m a -> t m b
916+concatMap_ f xs = buildS
917+ (\c n -> foldrSShared (\x b -> foldrSShared c b (unShare $ f x)) n xs)
918+-}
919+
920+instance Monad m => Applicative (Stream m) where
921+ {-# INLINE pure #-}
922+ pure = yield
923+ {-# INLINE (<*>) #-}
924+ (<*>) = ap
925+
926+-- NOTE: even though concatMap for StreamD is 3x faster compared to StreamK,
927+-- the monad instance of StreamD is slower than StreamK after foldr/build
928+-- fusion.
929+instance Monad m => Monad (Stream m) where
930+ {-# INLINE return #-}
931+ return = pure
932+ {-# INLINE (>>=) #-}
933+ (>>=) = flip concatMap
streamly.cabal+100−100
@@ -536,40 +536,40 @@ benchmark linear-async
536536 if impl(ghc < 8.0)
537537 build-depends:
538538 transformers >= 0.4 && < 0.6
539−
540−benchmark linear-rate
541− type: exitcode-stdio-1.0
542− hs-source-dirs: benchmark
543− main-is: LinearRate.hs
544− other-modules: LinearOps
545− default-language: Haskell2010
546− ghc-options: -O2 -Wall -fspec-constr-recursive=10 -with-rtsopts "-T"
547−
548− if flag(has-llvm)
549− ghc-options: -fllvm
550−
551− if flag(dev)
552− ghc-options: -Wmissed-specialisations
553− -Wall-missed-specialisations
554− -fno-ignore-asserts
555− if impl(ghc >= 8.0)
556− ghc-options: -Wcompat
557− -Wunrecognised-warning-flags
558− -Widentities
559− -Wincomplete-record-updates
560− -Wincomplete-uni-patterns
561− -Wredundant-constraints
562− -Wnoncanonical-monad-instances
563− -Wnoncanonical-monadfail-instances
564− build-depends:
565− streamly
566− , base >= 4.8 && < 5
567− , deepseq >= 1.4.3 && < 1.5
568− , random >= 1.0 && < 2.0
569− , gauge >= 0.2.4 && < 0.3
570− if impl(ghc < 8.0)
571− build-depends:
572− transformers >= 0.4 && < 0.6
539+--
540+-- benchmark linear-rate
541+-- type: exitcode-stdio-1.0
542+-- hs-source-dirs: benchmark
543+-- main-is: LinearRate.hs
544+-- other-modules: LinearOps
545+-- default-language: Haskell2010
546+-- ghc-options: -O2 -Wall -fspec-constr-recursive=10 -with-rtsopts "-T"
547+--
548+-- if flag(has-llvm)
549+-- ghc-options: -fllvm
550+--
551+-- if flag(dev)
552+-- ghc-options: -Wmissed-specialisations
553+-- -Wall-missed-specialisations
554+-- -fno-ignore-asserts
555+-- if impl(ghc >= 8.0)
556+-- ghc-options: -Wcompat
557+-- -Wunrecognised-warning-flags
558+-- -Widentities
559+-- -Wincomplete-record-updates
560+-- -Wincomplete-uni-patterns
561+-- -Wredundant-constraints
562+-- -Wnoncanonical-monad-instances
563+-- -Wnoncanonical-monadfail-instances
564+-- build-depends:
565+-- streamly
566+-- , base >= 4.8 && < 5
567+-- , deepseq >= 1.4.3 && < 1.5
568+-- , random >= 1.0 && < 2.0
569+-- , gauge >= 0.2.4 && < 0.3
570+-- if impl(ghc < 8.0)
571+-- build-depends:
572+-- transformers >= 0.4 && < 0.6
573573
574574benchmark nested
575575 type: exitcode-stdio-1.0
@@ -606,72 +606,72 @@ benchmark nested
606606 build-depends:
607607 transformers >= 0.4 && < 0.6
608608
609−benchmark array
610− type: exitcode-stdio-1.0
611− hs-source-dirs: benchmark
612− main-is: Array.hs
613− other-modules: ArrayOps
614− default-language: Haskell2010
615− ghc-options: -O2 -Wall -with-rtsopts "-T" -fspec-constr-recursive=10
616−
617− if flag(has-llvm)
618− ghc-options: -fllvm
619−
620− if flag(dev)
621− cpp-options: -DDEVBUILD
622− ghc-options: -Wmissed-specialisations
623− -Wall-missed-specialisations
624− -fno-ignore-asserts
625− if impl(ghc >= 8.0)
626− ghc-options: -Wcompat
627− -Wunrecognised-warning-flags
628− -Widentities
629− -Wincomplete-record-updates
630− -Wincomplete-uni-patterns
631− -Wredundant-constraints
632− -Wnoncanonical-monad-instances
633− -Wnoncanonical-monadfail-instances
634− build-depends:
635− streamly
636− , base >= 4.8 && < 5
637− , deepseq >= 1.4.3 && < 1.5
638− , random >= 1.0 && < 2.0
639− , gauge >= 0.2.4 && < 0.3
640− if impl(ghc < 8.0)
641− build-depends:
642− transformers >= 0.4 && < 0.6
643−
644−benchmark fileio
645− type: exitcode-stdio-1.0
646− hs-source-dirs: benchmark
647− main-is: FileIO.hs
648− default-language: Haskell2010
649− ghc-options: -O2 -Wall -fspec-constr-recursive=10 -with-rtsopts "-T"
650−
651− if flag(has-llvm)
652− ghc-options: -fllvm
653−
654− if flag(dev)
655− cpp-options: -DDEVBUILD
656− ghc-options: -Wmissed-specialisations
657− -Wall-missed-specialisations
658− -fno-ignore-asserts
659− if impl(ghc >= 8.0)
660− ghc-options: -Wcompat
661− -Wunrecognised-warning-flags
662− -Widentities
663− -Wincomplete-record-updates
664− -Wincomplete-uni-patterns
665− -Wredundant-constraints
666− -Wnoncanonical-monad-instances
667− -Wnoncanonical-monadfail-instances
668− build-depends:
669− streamly
670− , base >= 4.8 && < 5
671− , gauge >= 0.2.4 && < 0.3
672− , typed-process >= 0.2.3 && < 0.3
673− , deepseq >= 1.4.3 && < 1.5
674−
609+-- benchmark array
610+-- type: exitcode-stdio-1.0
611+-- hs-source-dirs: benchmark
612+-- main-is: Array.hs
613+-- other-modules: ArrayOps
614+-- default-language: Haskell2010
615+-- ghc-options: -O2 -Wall -with-rtsopts "-T" -fspec-constr-recursive=10
616+--
617+-- if flag(has-llvm)
618+-- ghc-options: -fllvm
619+--
620+-- if flag(dev)
621+-- cpp-options: -DDEVBUILD
622+-- ghc-options: -Wmissed-specialisations
623+-- -Wall-missed-specialisations
624+-- -fno-ignore-asserts
625+-- if impl(ghc >= 8.0)
626+-- ghc-options: -Wcompat
627+-- -Wunrecognised-warning-flags
628+-- -Widentities
629+-- -Wincomplete-record-updates
630+-- -Wincomplete-uni-patterns
631+-- -Wredundant-constraints
632+-- -Wnoncanonical-monad-instances
633+-- -Wnoncanonical-monadfail-instances
634+-- build-depends:
635+-- streamly
636+-- , base >= 4.8 && < 5
637+-- , deepseq >= 1.4.3 && < 1.5
638+-- , random >= 1.0 && < 2.0
639+-- , gauge >= 0.2.4 && < 0.3
640+-- if impl(ghc < 8.0)
641+-- build-depends:
642+-- transformers >= 0.4 && < 0.6
643+--
644+-- benchmark fileio
645+-- type: exitcode-stdio-1.0
646+-- hs-source-dirs: benchmark
647+-- main-is: FileIO.hs
648+-- default-language: Haskell2010
649+-- ghc-options: -O2 -Wall -fspec-constr-recursive=10 -with-rtsopts "-T"
650+--
651+-- if flag(has-llvm)
652+-- ghc-options: -fllvm
653+--
654+-- if flag(dev)
655+-- cpp-options: -DDEVBUILD
656+-- ghc-options: -Wmissed-specialisations
657+-- -Wall-missed-specialisations
658+-- -fno-ignore-asserts
659+-- if impl(ghc >= 8.0)
660+-- ghc-options: -Wcompat
661+-- -Wunrecognised-warning-flags
662+-- -Widentities
663+-- -Wincomplete-record-updates
664+-- -Wincomplete-uni-patterns
665+-- -Wredundant-constraints
666+-- -Wnoncanonical-monad-instances
667+-- -Wnoncanonical-monadfail-instances
668+-- build-depends:
669+-- streamly
670+-- , base >= 4.8 && < 5
671+-- , gauge >= 0.2.4 && < 0.3
672+-- , typed-process >= 0.2.3 && < 0.3
673+-- , deepseq >= 1.4.3 && < 1.5
674+--
675675-------------------------------------------------------------------------------
676676-- Internal benchmarks for unexposed modules
677677-------------------------------------------------------------------------------