on master help sign in
Refactor StreamD/IsStream for terminating folds

* IsStream: * Add foldMany * Update haddock docs, with fold equivalents. Now with terminating folds the fold behvaior is equivalent to these folds. * Add skeletons for foldSequence/foldIterate/parseSequence * Implement combinators directly in terms of foldMany/foldMany1 StreamD: * export foldMany/foldMany1 * Remove groupsOf/splitBy/splitSuffixBy/splitSuffixWith, we can use foldMany/foldMany1 directly instead. * Make some stylistic changes to code StreamD/Types: * Remove groupsOf * Remove GroupConsume state from foldMany/foldMany1 * Add benchmarks for foldMany

commit82efd3a5bc35bb572b2486f63ce9210cd7b53c6b authorHarendra Kumar <harendra@composewell.com> dateThu 10 Dec 2020, 19:11 (6 years ago) parentc191f9c
6 files changed+388−257
benchmark/Streamly/Benchmark/FileSystem/Handle/Read.hs+29−32 benchmark/Streamly/Benchmark/Prelude/Serial/Split.hs+21−7 benchmark/Streamly/Benchmark/Prelude/Serial/Transformation2.hs+21−2 src/Streamly/Internal/Data/Stream/IsStream.hs+145−24 src/Streamly/Internal/Data/Stream/StreamD.hs+152−166 src/Streamly/Internal/Data/Stream/StreamD/Type.hs+20−26
benchmark/Streamly/Benchmark/FileSystem/Handle/Read.hs+29−32
@@ -32,18 +32,17 @@ import GHC.Magic (noinline)
3232#endif
3333import System.IO (Handle)
3434
35−import qualified Streamly.Data.Fold as FL
36−import qualified Streamly.Unicode.Stream as SS
3735import qualified Streamly.FileSystem.Handle as FH
38−import qualified Streamly.Internal.Data.Parser as PR
39−import qualified Streamly.Internal.Data.Stream.StreamD as D
40−import qualified Streamly.Internal.Unicode.Stream as IUS
41−import qualified Streamly.Internal.FileSystem.Handle as IFH
4236import qualified Streamly.Internal.Data.Array.Storable.Foreign as A
4337import qualified Streamly.Internal.Data.Array.Storable.Foreign.Types as AT
44−import qualified Streamly.Internal.Memory.ArrayStream as AS
38+import qualified Streamly.Internal.Data.Fold as FL
39+import qualified Streamly.Internal.Data.Parser as PR
4540import qualified Streamly.Internal.Data.Stream.IsStream as IP
41+import qualified Streamly.Internal.FileSystem.Handle as IFH
42+import qualified Streamly.Internal.Memory.ArrayStream as AS
43+import qualified Streamly.Internal.Unicode.Stream as IUS
4644import qualified Streamly.Prelude as S
45+import qualified Streamly.Unicode.Stream as SS
4746
4847import Gauge hiding (env)
4948import Prelude hiding (last, length)
@@ -52,6 +51,7 @@ import Streamly.Benchmark.Common.Handle
5251#ifdef INSPECTION
5352import Streamly.Internal.Data.Stream.StreamD.Type (Step(..), GroupState)
5453
54+import qualified Streamly.Internal.Data.Stream.StreamD as D
5555import qualified Streamly.Internal.Data.Unfold as IUF
5656
5757import Test.Inspection
@@ -309,9 +309,14 @@ o_1_space_reduce_toBytes env =
309309chunksOfSum :: Int -> Handle -> IO Int
310310chunksOfSum n inh = S.length $ S.chunksOf n FL.sum (S.unfold FH.read inh)
311311
312+foldManyChunksOfSum :: Int -> Handle -> IO Int
313+foldManyChunksOfSum n inh =
314+ S.length $ IP.foldMany (FL.ltake n FL.sum) (S.unfold FH.read inh)
315+
312316parseManyChunksOfSum :: Int -> Handle -> IO Int
313317parseManyChunksOfSum n inh =
314− S.length $ IP.parseMany (PR.take n FL.sum) (S.unfold FH.read inh)
318+ S.length
319+ $ IP.parseMany (PR.fromFold $ FL.ltake n FL.sum) (S.unfold FH.read inh)
315320
316321-- XXX investigate why we need an INLINE in this case (GHC)
317322-- Even though allocations remain the same in both cases inlining improves time
@@ -333,25 +338,6 @@ inspect $ 'chunksOf `hasNoType` ''IUF.ConcatState -- FH.read/UF.concat
333338inspect $ 'chunksOf `hasNoType` ''A.ReadUState -- FH.read/A.read
334339#endif
335340
336−-- This is to make sure that the concatMap in FH.read, groupsOf and foldlM'
337−-- together can fuse.
338−--
339−-- | Slice in chunks of size n and get the count of chunks.
340−_chunksOfD :: Int -> Handle -> IO Int
341−_chunksOfD n inh =
342− D.foldlM' (\i _ -> return $ i + 1) (return 0)
343− $ D.groupsOf n (AT.writeNUnsafe n)
344− $ D.fromStreamK (S.unfold FH.read inh)
345−
346−#ifdef INSPECTION
347−inspect $ hasNoTypeClasses '_chunksOfD
348−inspect $ '_chunksOfD `hasNoType` ''Step
349−inspect $ '_chunksOfD `hasNoType` ''GroupState
350−inspect $ '_chunksOfD `hasNoType` ''AT.ArrayUnsafe -- AT.writeNUnsafe
351−inspect $ '_chunksOfD `hasNoType` ''IUF.ConcatState -- FH.read/UF.concat
352−inspect $ '_chunksOfD `hasNoType` ''A.ReadUState -- FH.read/A.read
353−#endif
354−
355341o_1_space_reduce_read_grouped :: BenchEnv -> [Benchmark]
356342o_1_space_reduce_read_grouped env =
357343 [ bgroup "reduce/read/chunks"
@@ -363,11 +349,22 @@ o_1_space_reduce_read_grouped env =
363349
364350 -- XXX investigate why we need inline/noinline in these cases (GHC)
365351 -- Chunk using parsers
366− , mkBenchSmall ("S.parseMany (PR.take " ++ show (bigSize env) ++ " FL.sum)")
367− env $ \inh _ ->
368− noinline parseManyChunksOfSum (bigSize env) inh
369− , mkBench "S.parseMany (PR.take 1 FL.sum)" env $ \inh _ ->
370− inline parseManyChunksOfSum 1 inh
352+ , mkBenchSmall
353+ ("S.foldMany (FL.take " ++ show (bigSize env) ++ " FL.sum)")
354+ env
355+ $ \inh _ -> noinline foldManyChunksOfSum (bigSize env) inh
356+ , mkBench
357+ "S.foldMany (FL.take 1 FL.sum)"
358+ env
359+ $ \inh _ -> inline foldManyChunksOfSum 1 inh
360+ , mkBenchSmall
361+ ("S.parseMany (FL.take " ++ show (bigSize env) ++ " FL.sum)")
362+ env
363+ $ \inh _ -> noinline parseManyChunksOfSum (bigSize env) inh
364+ , mkBench
365+ "S.parseMany (FL.take 1 FL.sum)"
366+ env
367+ $ \inh _ -> inline parseManyChunksOfSum 1 inh
371368
372369 -- folding chunks to arrays
373370 , mkBenchSmall "S.arraysOf 1" env $ \inh _ ->
benchmark/Streamly/Benchmark/Prelude/Serial/Split.hs+21−7
@@ -24,13 +24,13 @@ import Data.Char (ord)
2424import Data.Word (Word8)
2525import System.IO (Handle)
2626
27−import qualified Streamly.Data.Fold as FL
2827import qualified Streamly.FileSystem.Handle as FH
29−import qualified Streamly.Internal.Data.Parser as PR
30−import qualified Streamly.Internal.Unicode.Stream as IUS
31−import qualified Streamly.Internal.FileSystem.Handle as IFH
3228import qualified Streamly.Internal.Data.Array.Storable.Foreign as A
29+import qualified Streamly.Internal.Data.Fold as FL
30+import qualified Streamly.Internal.Data.Parser as PR
3331import qualified Streamly.Internal.Data.Stream.IsStream as IP
32+import qualified Streamly.Internal.FileSystem.Handle as IFH
33+import qualified Streamly.Internal.Unicode.Stream as IUS
3434import qualified Streamly.Prelude as S
3535
3636import Gauge hiding (env)
@@ -95,11 +95,23 @@ inspect $ 'splitWithSuffix `hasNoType` ''IUF.ConcatState -- FH.read/UF.concat
9595inspect $ 'splitWithSuffix `hasNoType` ''A.ReadUState -- FH.read/A.read
9696#endif
9797
98+-- | Split on line feed.
99+foldManySepBy :: Handle -> IO Int
100+foldManySepBy inh =
101+ (S.length
102+ $ IP.foldMany
103+ (FL.sliceSepBy (== lf) FL.drain)
104+ (S.unfold FH.read inh)
105+ ) -- >>= print
106+
98107-- | Split on line feed.
99108parseManySepBy :: Handle -> IO Int
100109parseManySepBy inh =
101− (S.length $ IP.parseMany (PR.sliceSepBy (== lf) FL.drain)
102− (S.unfold FH.read inh)) -- >>= print
110+ (S.length
111+ $ IP.parseMany
112+ (PR.fromFold $ FL.sliceSepBy (== lf) FL.drain)
113+ (S.unfold FH.read inh)
114+ ) -- >>= print
103115
104116-- | Words by space
105117wordsBy :: Handle -> IO Int
@@ -146,7 +158,9 @@ splitOnSuffixSeq str inh =
146158o_1_space_reduce_read_split :: BenchEnv -> [Benchmark]
147159o_1_space_reduce_read_split env =
148160 [ bgroup "split"
149− [ mkBench "S.parseMany (PR.sliceSepBy (== lf) FL.drain)" env
161+ [ mkBench "S.foldMany (FL.sliceSepBy (== lf) FL.drain)" env
162+ $ \inh _ -> foldManySepBy inh
163+ , mkBench "S.parseMany (FL.sliceSepBy (== lf) FL.drain)" env
150164 $ \inh _ -> parseManySepBy inh
151165 , mkBench "S.wordsBy isSpace FL.drain" env $ \inh _ ->
152166 wordsBy inh
benchmark/Streamly/Benchmark/Prelude/Serial/Transformation2.hs+21−2
@@ -17,11 +17,12 @@ module Serial.Transformation2 (benchmarks) where
1717import Control.DeepSeq (NFData(..))
1818import Control.Monad (when)
1919import Control.Monad.IO.Class (MonadIO(..))
20+import Data.Monoid (Sum(..))
2021import GHC.Generics (Generic)
2122
22−import qualified Streamly.Prelude as S
23−import qualified Streamly.Internal.Data.Stream.IsStream as Internal
2423import qualified Streamly.Internal.Data.Fold as FL
24+import qualified Streamly.Internal.Data.Stream.IsStream as Internal
25+import qualified Streamly.Prelude as S
2526
2627import Gauge
2728import Streamly.Prelude (SerialT, serially)
@@ -131,6 +132,22 @@ groupsByRollingEq :: MonadIO m => SerialT m Int -> m ()
131132groupsByRollingEq =
132133 S.drain . S.groupsByRolling (==) FL.drain
133134
135+{-# INLINE foldMany #-}
136+foldMany :: Monad m => SerialT m Int -> m ()
137+foldMany =
138+ S.drain
139+ . S.map getSum
140+ . Internal.foldMany (FL.ltake 2 FL.mconcat)
141+ . S.map Sum
142+
143+{-# INLINE _foldIterate #-}
144+_foldIterate :: Monad m => SerialT m Int -> m ()
145+_foldIterate =
146+ S.drain
147+ . S.map getSum
148+ . Internal.foldIterate (FL.ltake 2 . FL.sconcat) (Sum 0)
149+ . S.map Sum
150+
134151o_1_space_grouping :: Int -> [Benchmark]
135152o_1_space_grouping value =
136153 -- Buffering operations using heap proportional to group/window sizes.
@@ -140,6 +157,8 @@ o_1_space_grouping value =
140157 , benchIOSink value "groupsByEq" groupsByEq
141158 , benchIOSink value "groupsByRollingLT" groupsByRollingLT
142159 , benchIOSink value "groupsByRollingEq" groupsByRollingEq
160+ , benchIOSink value "foldMany" foldMany
161+ -- , benchIOSink value "foldIterate" foldIterate
143162 ]
144163 ]
145164
src/Streamly/Internal/Data/Stream/IsStream.hs+145−24
@@ -136,10 +136,22 @@ module Streamly.Internal.Data.Stream.IsStream
136136 , foldlM'
137137
138138 -- ** Composable Left Folds
139+ -- | See "Streamly.Internal.Data.Fold"
139140 , fold
141+ , foldMany
142+ , foldSequence
143+ , foldIterate
144+
145+ -- ** Parsers
146+ -- | See "Streamly.Internal.Data.Parser"
140147 , parse
141148 , parseK
142149 , parseD
150+ , parseMany
151+ , parseManyD
152+ , parseManyTill
153+ , parseSequence
154+ , parseIterate
143155
144156 -- ** Concurrent Folds
145157 , foldAsync
@@ -319,12 +331,6 @@ module Streamly.Internal.Data.Stream.IsStream
319331 , reverse
320332 , reverse'
321333
322− -- ** Parsing
323− , parseMany
324− , parseManyD
325− , parseManyTill
326− , parseIterate
327−
328334 -- ** Trimming
329335 , take
330336 -- , takeGE
@@ -643,8 +649,7 @@ import Streamly.Internal.Data.IORef.Prim (Prim, IORef)
643649import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))
644650
645651import qualified Streamly.Internal.Data.Array.Storable.Foreign as A
646−import qualified Streamly.Data.Fold as FL
647−import qualified Streamly.Internal.Data.Fold.Types as FL
652+import qualified Streamly.Internal.Data.Fold as FL
648653import qualified Streamly.Internal.Data.Stream.Prelude as P
649654import qualified Streamly.Internal.Data.Stream.StreamK as K
650655import qualified Streamly.Internal.Data.Stream.StreamD as D
@@ -1410,7 +1415,10 @@ foldlM' step begin m = S.foldlM' step begin $ toStreamS m
14101415-- Running a Fold
14111416------------------------------------------------------------------------------
14121417
1413−-- | Fold a stream using the supplied left fold.
1418+-- | Fold a stream using the supplied left 'Fold' and reducing the resulting
1419+-- expression strictly at each step. The behavior is similar to 'foldl''. A
1420+-- 'Fold' can terminate early without consuming the full stream. See the
1421+-- documentation of individual 'Fold's for termination behavior.
14141422--
14151423-- >>> S.fold FL.sum (S.enumerateFromTo 1 100)
14161424-- 5050
@@ -1461,6 +1469,7 @@ parse = parseD . PRK.fromParserK
14611469
14621470-- |
14631471-- > drain = mapM_ (\_ -> return ())
1472+-- > drain = fold Fold.drain
14641473--
14651474-- Run a stream, discarding the results. By default it interprets the stream
14661475-- as 'SerialT', to run other types of streams use the type adapting
@@ -1483,6 +1492,7 @@ runStream = drain
14831492
14841493-- |
14851494-- > drainN n = drain . take n
1495+-- > drainN n = fold (Fold.ltake n Fold.drain)
14861496--
14871497-- Run maximum up to @n@ iterations of a stream.
14881498--
@@ -1504,6 +1514,7 @@ runN = drainN
15041514
15051515-- |
15061516-- > drainWhile p = drain . takeWhile p
1517+-- > drainWhile p = fold (Fold.sliceSepBy (not . p) Fold.drain)
15071518--
15081519-- Run a stream as long as the predicate holds true.
15091520--
@@ -1525,6 +1536,8 @@ runWhile = drainWhile
15251536
15261537-- | Determine whether the stream is empty.
15271538--
1539+-- > null = fold Fold.null
1540+--
15281541-- @since 0.1.1
15291542{-# INLINE null #-}
15301543null :: Monad m => SerialT m a -> m Bool
@@ -1533,6 +1546,7 @@ null = S.null . toStreamS
15331546-- | Extract the first element of the stream, if any.
15341547--
15351548-- > head = (!! 0)
1549+-- > head = fold Fold.head
15361550--
15371551-- @since 0.1.0
15381552{-# INLINE head #-}
@@ -1568,6 +1582,7 @@ init m = K.init (K.adapt m)
15681582-- | Extract the last element of the stream, if any.
15691583--
15701584-- > last xs = xs !! (length xs - 1)
1585+-- > last = fold Fold.last
15711586--
15721587-- @since 0.1.1
15731588{-# INLINE last #-}
@@ -1576,6 +1591,8 @@ last m = S.last $ toStreamS m
15761591
15771592-- | Determine whether an element is present in the stream.
15781593--
1594+-- > elem = fold Fold.elem
1595+--
15791596-- @since 0.1.0
15801597{-# INLINE elem #-}
15811598elem :: (Monad m, Eq a) => a -> SerialT m a -> m Bool
@@ -1583,6 +1600,8 @@ elem e m = S.elem e (toStreamS m)
15831600
15841601-- | Determine whether an element is not present in the stream.
15851602--
1603+-- > notElem = fold Fold.notElem
1604+--
15861605-- @since 0.1.0
15871606{-# INLINE notElem #-}
15881607notElem :: (Monad m, Eq a) => a -> SerialT m a -> m Bool
@@ -1590,6 +1609,8 @@ notElem e m = S.notElem e (toStreamS m)
15901609
15911610-- | Determine the length of the stream.
15921611--
1612+-- > notElem = fold Fold.length
1613+--
15931614-- @since 0.1.0
15941615{-# INLINE length #-}
15951616length :: Monad m => SerialT m a -> m Int
@@ -1597,6 +1618,8 @@ length = foldl' (\n _ -> n + 1) 0
15971618
15981619-- | Determine whether all elements of a stream satisfy a predicate.
15991620--
1621+-- > all = fold Fold.all
1622+--
16001623-- @since 0.1.0
16011624{-# INLINE all #-}
16021625all :: Monad m => (a -> Bool) -> SerialT m a -> m Bool
@@ -1604,6 +1627,8 @@ all p m = S.all p (toStreamS m)
16041627
16051628-- | Determine whether any of the elements of a stream satisfy a predicate.
16061629--
1630+-- > any = fold Fold.any
1631+--
16071632-- @since 0.1.0
16081633{-# INLINE any #-}
16091634any :: Monad m => (a -> Bool) -> SerialT m a -> m Bool
@@ -1611,6 +1636,8 @@ any p m = S.any p (toStreamS m)
16111636
16121637-- | Determines if all elements of a boolean stream are True.
16131638--
1639+-- > and = fold Fold.and
1640+--
16141641-- @since 0.5.0
16151642{-# INLINE and #-}
16161643and :: Monad m => SerialT m Bool -> m Bool
@@ -1618,6 +1645,8 @@ and = all (==True)
16181645
16191646-- | Determines whether at least one element of a boolean stream is True.
16201647--
1648+-- > or = fold Fold.or
1649+--
16211650-- @since 0.5.0
16221651{-# INLINE or #-}
16231652or :: Monad m => SerialT m Bool -> m Bool
@@ -1627,6 +1656,8 @@ or = any (==True)
16271656-- the stream is empty. Note that this is not numerically stable for floating
16281657-- point numbers.
16291658--
1659+-- > sum = fold Fold.sum
1660+--
16301661-- @since 0.1.0
16311662{-# INLINE sum #-}
16321663sum :: (Monad m, Num a) => SerialT m a -> m a
@@ -1635,6 +1666,8 @@ sum = foldl' (+) 0
16351666-- | Determine the product of all elements of a stream of numbers. Returns @1@
16361667-- when the stream is empty.
16371668--
1669+-- > product = fold Fold.product
1670+--
16381671-- @since 0.1.1
16391672{-# INLINE product #-}
16401673product :: (Monad m, Num a) => SerialT m a -> m a
@@ -1642,6 +1675,8 @@ product = foldl' (*) 1
16421675
16431676-- | Fold a stream of monoid elements by appending them.
16441677--
1678+-- > mconcat = fold Fold.mconcat
1679+--
16451680-- /Internal/
16461681{-# INLINE mconcat #-}
16471682mconcat :: (Monad m, Monoid a) => SerialT m a -> m a
@@ -1650,6 +1685,7 @@ mconcat = foldr mappend mempty
16501685-- |
16511686-- @
16521687-- minimum = 'minimumBy' compare
1688+-- minimum = fold Fold.minimum
16531689-- @
16541690--
16551691-- Determine the minimum element in a stream.
@@ -1662,6 +1698,8 @@ minimum m = S.minimum (toStreamS m)
16621698-- | Determine the minimum element in a stream using the supplied comparison
16631699-- function.
16641700--
1701+-- > minimumBy = fold Fold.minimumBy
1702+--
16651703-- @since 0.6.0
16661704{-# INLINE minimumBy #-}
16671705minimumBy :: Monad m => (a -> a -> Ordering) -> SerialT m a -> m (Maybe a)
@@ -1670,6 +1708,7 @@ minimumBy cmp m = S.minimumBy cmp (toStreamS m)
16701708-- |
16711709-- @
16721710-- maximum = 'maximumBy' compare
1711+-- maximum = fold Fold.maximum
16731712-- @
16741713--
16751714-- Determine the maximum element in a stream.
@@ -1682,6 +1721,8 @@ maximum = P.maximum
16821721-- | Determine the maximum element in a stream using the supplied comparison
16831722-- function.
16841723--
1724+-- > maximumBy = fold Fold.maximumBy
1725+--
16851726-- @since 0.6.0
16861727{-# INLINE maximumBy #-}
16871728maximumBy :: Monad m => (a -> a -> Ordering) -> SerialT m a -> m (Maybe a)
@@ -1698,6 +1739,7 @@ m !! i = toStreamS m S.!! i
16981739-- first pair where the key equals the given value @a@.
16991740--
17001741-- > lookup = snd <$> find ((==) . fst)
1742+-- > lookup = fold Fold.lookup
17011743--
17021744-- @since 0.5.0
17031745{-# INLINE lookup #-}
@@ -1707,6 +1749,7 @@ lookup a m = S.lookup a (toStreamS m)
17071749-- | Like 'findM' but with a non-monadic predicate.
17081750--
17091751-- > find p = findM (return . p)
1752+-- > find = fold Fold.find
17101753--
17111754-- @since 0.5.0
17121755{-# INLINE find #-}
@@ -1715,6 +1758,8 @@ find p m = S.find p (toStreamS m)
17151758
17161759-- | Returns the first element that satisfies the given predicate.
17171760--
1761+-- > findM = fold Fold.findM
1762+--
17181763-- @since 0.6.0
17191764{-# INLINE findM #-}
17201765findM :: Monad m => (a -> m Bool) -> SerialT m a -> m (Maybe a)
@@ -1723,6 +1768,8 @@ findM p m = S.findM p (toStreamS m)
17231768-- | Find all the indices where the element in the stream satisfies the given
17241769-- predicate.
17251770--
1771+-- > findIndices = fold Fold.findIndices
1772+--
17261773-- @since 0.5.0
17271774{-# INLINE findIndices #-}
17281775findIndices :: (IsStream t, Monad m) => (a -> Bool) -> t m a -> t m Int
@@ -1730,6 +1777,8 @@ findIndices p m = fromStreamS $ S.findIndices p (toStreamS m)
17301777
17311778-- | Returns the first index that satisfies the given predicate.
17321779--
1780+-- > findIndex = fold Fold.findIndex
1781+--
17331782-- @since 0.5.0
17341783{-# INLINE findIndex #-}
17351784findIndex :: Monad m => (a -> Bool) -> SerialT m a -> m (Maybe Int)
@@ -1738,10 +1787,12 @@ findIndex p = head . findIndices p
17381787-- | Find all the indices where the value of the element in the stream is equal
17391788-- to the given value.
17401789--
1790+-- > elemIndices a = findIndices (== a)
1791+--
17411792-- @since 0.5.0
17421793{-# INLINE elemIndices #-}
17431794elemIndices :: (IsStream t, Eq a, Monad m) => a -> t m a -> t m Int
1744−elemIndices a = findIndices (==a)
1795+elemIndices a = findIndices (== a)
17451796
17461797-- | Returns the first index where a given value is found in the stream.
17471798--
@@ -3636,7 +3687,7 @@ iterateMapLeftsWith
36363687iterateMapLeftsWith combine f = iterateMapWith combine (either f (const K.nil))
36373688
36383689------------------------------------------------------------------------------
3639−-- Parsing
3690+-- Folding/Parsing chunks in a stream
36403691------------------------------------------------------------------------------
36413692
36423693-- Splitting operations that take a predicate and a Fold can be
@@ -3645,6 +3696,67 @@ iterateMapLeftsWith combine f = iterateMapWith combine (either f (const K.nil))
36453696--
36463697-- XXX We need takeGE/takeBetween to implement "some" using "many".
36473698
3699+-- | Apply a 'Fold' repeatedly on a stream and emit the parsed values in the
3700+-- output stream.
3701+--
3702+-- This is the streaming dual of the 'Streamly.Internal.Data.Fold.many'
3703+-- parse combinator.
3704+--
3705+-- >>> f = Fold.ltake 2 Fold.sum
3706+-- >>> Stream.toList $ Stream.foldMany f $ Stream.fromList [1..10]
3707+-- > [3,7,11,15,19]
3708+--
3709+-- >>> f = Fold.sliceEndWith Fold.toList
3710+-- >>> Stream.toList $ Stream.foldMany f $ Stream.fromList "hello\nworld"
3711+-- > ["hello\n","world"]
3712+--
3713+-- /Internal/
3714+--
3715+{-# INLINE foldMany #-}
3716+foldMany
3717+ :: (IsStream t, Monad m)
3718+ => Fold m a b
3719+ -> t m a
3720+ -> t m b
3721+foldMany f m = D.fromStreamD $ D.foldMany f (D.toStreamD m)
3722+
3723+-- | Apply a stream of folds to an input stream and emit the results in the
3724+-- output stream.
3725+--
3726+-- /Internal/
3727+--
3728+{-# INLINE foldSequence #-}
3729+foldSequence
3730+ :: -- (IsStream t, Monad m) =>
3731+ t m (Fold m a b)
3732+ -> t m a
3733+ -> t m b
3734+foldSequence _f _m = undefined
3735+
3736+-- | Iterate a fold generator on a stream. The initial value @b@ is used to
3737+-- generate the first fold, the fold is applied on the stream and the result of
3738+-- the fold is used to generate the next fold and so on.
3739+--
3740+-- >>> f x = Fold.ltake 2 (Fold.mconcatTo x)
3741+-- >>> s = Stream.map Sum $ Stream.fromList [1..10]
3742+-- >>> Stream.toList $ Stream.map getSum $ Stream.foldIterate f 0 s
3743+-- > [3,10,21,36,55,55]
3744+--
3745+-- This is the streaming equivalent of monad like sequenced application of
3746+-- folds where next fold is dependent on the previous fold.
3747+--
3748+-- /Internal/
3749+--
3750+{-# INLINE foldIterate #-}
3751+foldIterate
3752+ :: -- (IsStream t, Monad m) =>
3753+ (b -> Fold m a b)
3754+ -> b
3755+ -> t m a
3756+ -> t m b
3757+foldIterate _f _i _m = undefined
3758+-- D.fromStreamD $ D.foldIterate f i (D.toStreamD m)
3759+
36483760-- | Apply a 'Parser' repeatedly on a stream and emit the parsed values in the
36493761-- output stream.
36503762--
@@ -3657,7 +3769,7 @@ iterateMapLeftsWith combine f = iterateMapWith combine (either f (const K.nil))
36573769-- >>> S.toList $ S.parseMany (PR.line FL.toList) $ S.fromList "hello\nworld"
36583770-- > ["hello\n","world"]
36593771--
3660−-- /Internal
3772+-- /Internal/
36613773--
36623774{-# INLINE parseMany #-}
36633775parseMany
@@ -3677,6 +3789,19 @@ parseManyD
36773789parseManyD p m =
36783790 D.fromStreamD $ D.parseMany p (D.toStreamD m)
36793791
3792+-- | Apply a stream of parsers to an input stream and emit the results in the
3793+-- output stream.
3794+--
3795+-- /Internal/
3796+--
3797+{-# INLINE parseSequence #-}
3798+parseSequence
3799+ :: -- (IsStream t, Monad m) =>
3800+ t m (Parser m a b)
3801+ -> t m a
3802+ -> t m b
3803+parseSequence _f _m = undefined
3804+
36803805-- | @parseManyTill collect test stream@ tries the parser @test@ on the input,
36813806-- if @test@ fails it backtracks and tries @collect@, after @collect@ succeeds
36823807-- @test@ is tried again and so on. The parser stops when @test@ succeeds. The
@@ -3786,7 +3911,7 @@ groupScan split fold m = undefined
37863911chunksOf
37873912 :: (IsStream t, Monad m)
37883913 => Int -> Fold m a b -> t m a -> t m b
3789−chunksOf n f m = D.fromStreamD $ D.groupsOf n f (D.toStreamD m)
3914+chunksOf n f = foldMany (FL.ltake n f)
37903915
37913916-- |
37923917--
@@ -3831,8 +3956,6 @@ intervalsOf n f xs =
38313956-- N-ary APIs
38323957------------------------------------------------------------------------------
38333958
3834−-- XXX We should probably change the order of the comparision and update the
3835−-- docs accordingly.
38363959-- | @groupsBy cmp f $ S.fromList [a,b,c,...]@ assigns the element @a@ to the
38373960-- first group, if @b \`cmp` a@ is 'True' then @b@ is also assigned to the same
38383961-- group. If @c \`cmp` a@ is 'True' then @c@ is also assigned to the same
@@ -3945,8 +4068,7 @@ groups = groupsBy (==)
39454068splitOn
39464069 :: (IsStream t, Monad m)
39474070 => (a -> Bool) -> Fold m a b -> t m a -> t m b
3948−splitOn predicate f m =
3949− D.fromStreamD $ D.splitBy predicate f (D.toStreamD m)
4071+splitOn predicate f = foldMany (FL.sliceSepBy predicate f)
39504072
39514073-- | Like 'splitOn' but the separator is considered as suffixed to the segments
39524074-- in the stream. A missing suffix at the end is allowed. A separator at the
@@ -3998,7 +4120,7 @@ splitOnSuffix
39984120 :: (IsStream t, Monad m)
39994121 => (a -> Bool) -> Fold m a b -> t m a -> t m b
40004122splitOnSuffix predicate f m =
4001− D.fromStreamD $ D.splitSuffixBy predicate f (D.toStreamD m)
4123+ D.fromStreamD $ D.foldMany1 (FL.sliceSepBy predicate f) (D.toStreamD m)
40024124
40034125-- | Like 'splitOn' after stripping leading, trailing, and repeated separators.
40044126-- Therefore, @".a..b."@ with '.' as the separator would be parsed as
@@ -4068,7 +4190,7 @@ splitWithSuffix
40684190 :: (IsStream t, Monad m)
40694191 => (a -> Bool) -> Fold m a b -> t m a -> t m b
40704192splitWithSuffix predicate f m =
4071− D.fromStreamD $ D.splitSuffixWith predicate f (D.toStreamD m)
4193+ D.fromStreamD $ D.foldMany1 (FL.sliceEndWith predicate f) (D.toStreamD m)
40724194
40734195------------------------------------------------------------------------------
40744196-- Split on a delimiter sequence
@@ -4758,13 +4880,12 @@ classifySessionsBy tick tmout reset ejectPred
47584880 -- better performance.
47594881 --
47604882 let curTime = max sessionEventTime timestamp
4761− extractOld v =
4762− case v of
4763− Nothing -> initial
4764− Just (Tuple' _ acc) -> return acc
47654883 mOld = Map.lookup key sessionKeyValueMap
47664884
4767− fs <- extractOld mOld
4885+ fs <-
4886+ case mOld of
4887+ Nothing -> initial
4888+ Just (Tuple' _ acc) -> return acc
47684889 res <- step fs value
47694890 case res of
47704891 FL.Done fb -> do
src/Streamly/Internal/Data/Stream/StreamD.hs+152−166
@@ -103,6 +103,8 @@ module Streamly.Internal.Data.Stream.StreamD
103103 , foldlx'
104104 , foldlMx'
105105 , foldOnce
106+ , foldMany
107+ , foldMany1
106108
107109 , parselMx'
108110 , parseMany
@@ -157,16 +159,12 @@ module Streamly.Internal.Data.Stream.StreamD
157159 , interpose
158160
159161 -- ** Grouping
160− , groupsOf
161162 , groupsOf2
162163 , groupsBy
163164 , groupsRollingBy
164165
165166 -- ** Splitting
166− , splitBy
167− , splitSuffixBy
168167 , wordsBy
169− , splitSuffixWith
170168
171169 , splitOnSeq
172170 , splitOnSuffixSeq
@@ -1519,11 +1517,6 @@ reverse' m =
15191517-- Grouping/Splitting
15201518------------------------------------------------------------------------------
15211519
1522−{-# INLINE_NORMAL splitSuffixWith #-}
1523−splitSuffixWith :: Monad m
1524− => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b
1525−splitSuffixWith predicate f = foldMany1 (FL.sliceEndWith predicate f)
1526−
15271520{-# INLINE_NORMAL groupsBy #-}
15281521groupsBy :: Monad m
15291522 => (a -> a -> Bool)
@@ -1540,11 +1533,11 @@ groupsBy cmp f (Stream step state) = Stream (stepOuter f) (Just state, Nothing)
15401533 case res of
15411534 Yield x s -> do
15421535 fs <- initial
1543− sfs <- fstep fs x
1544− case sfs of
1536+ r <- fstep fs x
1537+ case r of
15451538 FL.Partial fs1 -> go SPEC x s fs1
1546− FL.Done fb -> return $ Yield fb (Just s, Just x)
1547− Skip s -> return $ Skip $ (Just s, Nothing)
1539+ FL.Done b -> return $ Yield b (Just s, Just x)
1540+ Skip s -> return $ Skip (Just s, Nothing)
15481541 Stop -> return Stop
15491542
15501543 where
@@ -1555,19 +1548,19 @@ groupsBy cmp f (Stream step state) = Stream (stepOuter f) (Just state, Nothing)
15551548 Yield x s -> do
15561549 if cmp x prev
15571550 then do
1558− sfs <- fstep acc x
1559− case sfs of
1551+ r <- fstep acc x
1552+ case r of
15601553 FL.Partial fs1 -> go SPEC prev s fs1
1561− FL.Done fb -> return $ Yield fb (Just s, Just x)
1554+ FL.Done b -> return $ Yield b (Just s, Just x)
15621555 else done acc >>= \r -> return $ Yield r (Just s, Just x)
15631556 Skip s -> go SPEC prev s acc
15641557 Stop -> done acc >>= \r -> return $ Yield r (Nothing, Nothing)
15651558 stepOuter (Fold fstep initial done) gst (Just st, Just prev) = do
15661559 fs <- initial
1567− sfs <- fstep fs prev
1568− case sfs of
1560+ r <- fstep fs prev
1561+ case r of
15691562 FL.Partial fs1 -> go SPEC st fs1
1570− FL.Done fb -> return $ Yield fb (Just st, Nothing)
1563+ FL.Done b -> return $ Yield b (Just st, Nothing)
15711564
15721565 where
15731566
@@ -1578,10 +1571,10 @@ groupsBy cmp f (Stream step state) = Stream (stepOuter f) (Just state, Nothing)
15781571 Yield x s -> do
15791572 if cmp x prev
15801573 then do
1581− sfs <- fstep acc x
1582− case sfs of
1574+ r <- fstep acc x
1575+ case r of
15831576 FL.Partial fs1 -> go SPEC s fs1
1584− FL.Done fb -> return $ Yield fb (Just s, Just x)
1577+ FL.Done b -> return $ Yield b (Just s, Just x)
15851578 else done acc >>= \r -> return $ Yield r (Just s, Just x)
15861579 Skip s -> go SPEC s acc
15871580 Stop -> done acc >>= \r -> return $ Yield r (Nothing, Nothing)
@@ -1604,10 +1597,10 @@ groupsRollingBy cmp f (Stream step state) =
16041597 case res of
16051598 Yield x s -> do
16061599 fs <- initial
1607− sfs <- fstep fs x
1608− case sfs of
1600+ r <- fstep fs x
1601+ case r of
16091602 FL.Partial fs1 -> go SPEC x s fs1
1610− FL.Done fb -> return $ Yield fb (Just s, Just x)
1603+ FL.Done b -> return $ Yield b (Just s, Just x)
16111604 Skip s -> return $ Skip $ (Just s, Nothing)
16121605 Stop -> return Stop
16131606
@@ -1619,19 +1612,19 @@ groupsRollingBy cmp f (Stream step state) =
16191612 Yield x s -> do
16201613 if cmp prev x
16211614 then do
1622− sfs <- fstep acc x
1623− case sfs of
1615+ r <- fstep acc x
1616+ case r of
16241617 FL.Partial fs1 -> go SPEC x s fs1
1625− FL.Done fb -> return $ Yield fb (Just s, Just x)
1618+ FL.Done b -> return $ Yield b (Just s, Just x)
16261619 else done acc >>= \r -> return $ Yield r (Just s, Just x)
16271620 Skip s -> go SPEC prev s acc
16281621 Stop -> done acc >>= \r -> return $ Yield r (Nothing, Nothing)
16291622 stepOuter (Fold fstep initial done) gst (Just st, Just prev') = do
16301623 fs <- initial
1631− sfs <- fstep fs prev'
1632− case sfs of
1624+ r <- fstep fs prev'
1625+ case r of
16331626 FL.Partial fs1 -> go SPEC prev' st fs1
1634− FL.Done fb -> return $ Yield fb (Just st, Nothing)
1627+ FL.Done b -> return $ Yield b (Just st, Nothing)
16351628
16361629 where
16371630
@@ -1641,24 +1634,15 @@ groupsRollingBy cmp f (Stream step state) =
16411634 Yield x s -> do
16421635 if cmp prevv x
16431636 then do
1644− sfs <- fstep acc x
1645− case sfs of
1637+ r <- fstep acc x
1638+ case r of
16461639 FL.Partial fs1 -> go SPEC x s fs1
1647− FL.Done fb -> return $ Yield fb (Just s, Just x)
1640+ FL.Done b -> return $ Yield b (Just s, Just x)
16481641 else done acc >>= \r -> return $ Yield r (Just s, Just x)
16491642 Skip s -> go SPEC prevv s acc
16501643 Stop -> done acc >>= \r -> return $ Yield r (Nothing, Nothing)
16511644 stepOuter _ _ (Nothing, _) = return Stop
16521645
1653−{-# INLINE_NORMAL splitBy #-}
1654−splitBy :: Monad m => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b
1655−splitBy predicate f = foldMany (FL.sliceSepBy predicate f)
1656−
1657−{-# INLINE_NORMAL splitSuffixBy #-}
1658−splitSuffixBy :: Monad m
1659− => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b
1660−splitSuffixBy predicate f = foldMany1 (FL.sliceSepBy predicate f)
1661−
16621646data WordsByState s = WordsByJust s | WordsByNothing
16631647
16641648{-# INLINE_NORMAL wordsBy #-}
@@ -1677,10 +1661,10 @@ wordsBy predicate f (Stream step state) =
16771661 then return $ Skip (WordsByJust s)
16781662 else do
16791663 fs <- initial
1680− sfs <- fstep fs x
1681− case sfs of
1664+ r <- fstep fs x
1665+ case r of
16821666 FL.Partial fs1 -> go SPEC s fs1
1683− FL.Done fb -> return $ Yield fb (WordsByJust s)
1667+ FL.Done b -> return $ Yield b (WordsByJust s)
16841668
16851669 Skip s -> return $ Skip $ WordsByJust s
16861670 Stop -> return Stop
@@ -1694,10 +1678,10 @@ wordsBy predicate f (Stream step state) =
16941678 if predicate x
16951679 then done acc >>= \r -> return $ Yield r (WordsByJust s)
16961680 else do
1697− sfs <- fstep acc x
1698− case sfs of
1681+ r <- fstep acc x
1682+ case r of
16991683 FL.Partial fs1 -> go SPEC s fs1
1700− FL.Done fb -> return $ Yield fb (WordsByJust s)
1684+ FL.Done b -> return $ Yield b (WordsByJust s)
17011685 Skip s -> go SPEC s acc
17021686 Stop -> done acc >>= \r -> return $ Yield r WordsByNothing
17031687
@@ -1808,13 +1792,13 @@ splitOnSeq patArr (Fold fstep initial done) (Stream step state) =
18081792 res <- step (adaptState gst) st
18091793 case res of
18101794 Yield x s -> do
1811− acc <- initial
1812− sfs <- fstep acc x
1813− case sfs of
1814− FL.Partial acc1 -> do
1815− r <- done acc1
1816− skip $ SplitOnSeqYield r (SplitOnSeqEmpty s)
1817− FL.Done fb -> skip $ SplitOnSeqYield fb (SplitOnSeqEmpty s)
1795+ fs <- initial
1796+ r <- fstep fs x
1797+ case r of
1798+ FL.Partial fs1 -> do
1799+ b <- done fs1
1800+ skip $ SplitOnSeqYield b (SplitOnSeqEmpty s)
1801+ FL.Done b -> skip $ SplitOnSeqYield b (SplitOnSeqEmpty s)
18181802
18191803 Skip s -> return $ Skip (SplitOnSeqEmpty s)
18201804 Stop -> return Stop
@@ -1839,12 +1823,12 @@ splitOnSeq patArr (Fold fstep initial done) (Stream step state) =
18391823 fs1 <- initial
18401824 return $ Skip $ SplitOnSeqYield r (SplitOnSeqSingle fs1 s pat)
18411825 else do
1842− sfs <- fstep fs x
1843− case sfs of
1844− FL.Partial fs1 -> skip $ (SplitOnSeqSingle fs1 s pat)
1845− FL.Done fb -> do
1826+ r <- fstep fs x
1827+ case r of
1828+ FL.Partial fs1 -> skip $ SplitOnSeqSingle fs1 s pat
1829+ FL.Done b -> do
18461830 fs1 <- initial
1847− skip $ SplitOnSeqYield fb (SplitOnSeqSingle fs1 s pat)
1831+ skip $ SplitOnSeqYield b (SplitOnSeqSingle fs1 s pat)
18481832 Skip s -> return $ Skip $ SplitOnSeqSingle fs s pat
18491833 Stop -> do
18501834 r <- done fs
@@ -1859,13 +1843,13 @@ splitOnSeq patArr (Fold fstep initial done) (Stream step state) =
18591843 skip $ SplitOnSeqYield r SplitOnSeqDone
18601844 stepOuter _ (SplitOnSeqWordDone n fs wrd) = do
18611845 let old = elemMask .&. (wrd `shiftR` (elemBits * (n - 1)))
1862− sfs <- fstep fs (toEnum $ fromIntegral old)
1863− case sfs of
1846+ r <- fstep fs (toEnum $ fromIntegral old)
1847+ case r of
18641848 FL.Partial fs1 -> skip $ SplitOnSeqWordDone (n - 1) fs1 wrd
1865− FL.Done r -> do
1849+ FL.Done b -> do
18661850 fs1 <- initial
18671851 let next = SplitOnSeqWordDone (n - 1) fs1 wrd
1868− skip $ SplitOnSeqYield r next
1852+ skip $ SplitOnSeqYield b next
18691853
18701854 stepOuter gst (SplitOnSeqWordInit st0) =
18711855 go SPEC 0 0 st0
@@ -1910,18 +1894,18 @@ splitOnSeq patArr (Fold fstep initial done) (Stream step state) =
19101894 let wrd1 = addToWord wrd x
19111895 old = (wordMask .&. wrd)
19121896 `shiftR` (elemBits * (patLen - 1))
1913− sfs <- fstep fs (toEnum $ fromIntegral old)
1914− case sfs of
1897+ r <- fstep fs (toEnum $ fromIntegral old)
1898+ case r of
19151899 FL.Partial fs1 -> do
19161900 if wrd1 .&. wordMask == wordPat
19171901 then do
1918− r <- done fs1
1902+ b <- done fs1
19191903 let next = SplitOnSeqWordInit s
1920− skip $ SplitOnSeqYield r next
1904+ skip $ SplitOnSeqYield b next
19211905 else go SPEC wrd1 s fs1
1922− FL.Done r ->
1906+ FL.Done b ->
19231907 let next = SplitOnSeqWordInit s
1924− in skip $ SplitOnSeqYield r next
1908+ in skip $ SplitOnSeqYield b next
19251909 Skip s -> go SPEC wrd s fs
19261910 Stop -> skip $ SplitOnSeqWordDone patLen fs wrd
19271911
@@ -1963,16 +1947,16 @@ splitOnSeq patArr (Fold fstep initial done) (Stream step state) =
19631947 Yield x s -> do
19641948 old <- liftIO $ peek rh
19651949 let cksum1 = deltaCksum cksum old x
1966− sfs <- fstep fs old
1967− case sfs of
1950+ r <- fstep fs old
1951+ case r of
19681952 FL.Partial fs1 -> do
19691953 rh1 <- liftIO (RB.unsafeInsert rb rh x)
1970− if (cksum1 == patHash)
1954+ if cksum1 == patHash
19711955 then skip $ SplitOnSeqKRCheck fs1 s rb rh1
19721956 else go SPEC fs1 s rh1 cksum1
1973− FL.Done r ->
1957+ FL.Done b ->
19741958 let next = SplitOnSeqKRInit 0 s rb (RB.startOf rb)
1975− in skip $ SplitOnSeqYield r next
1959+ in skip $ SplitOnSeqYield b next
19761960 Skip s -> go SPEC fs s rh cksum
19771961 Stop -> skip $ SplitOnSeqKRDone patLen fs rb rh
19781962
@@ -2015,13 +1999,13 @@ splitOnSeq patArr (Fold fstep initial done) (Stream step state) =
20151999 stepOuter _ (SplitOnSeqKRDone n fs rb rh) = do
20162000 old <- liftIO $ peek rh
20172001 let rh1 = RB.advance rb rh
2018− sfs <- fstep fs old
2019− case sfs of
2002+ r <- fstep fs old
2003+ case r of
20202004 FL.Partial fs1 -> skip $ SplitOnSeqKRDone (n - 1) fs1 rb rh1
2021− FL.Done r -> do
2005+ FL.Done b -> do
20222006 fs1 <- initial
20232007 let next = SplitOnSeqKRDone (n - 1) fs1 rb rh1
2024− skip $ SplitOnSeqYield r next
2008+ skip $ SplitOnSeqYield b next
20252009
20262010{-# ANN type SplitOnSuffixSeqState Fuse #-}
20272011data SplitOnSuffixSeqState rb rh ck w fs s b x =
@@ -2078,22 +2062,22 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
20782062 processYieldSingle pat x s fs =
20792063 if pat == x
20802064 then do
2081− sfs <- if withSep then fstep fs x else return $ FL.Partial fs
2082− case sfs of
2065+ r <- if withSep then fstep fs x else return $ FL.Partial fs
2066+ case r of
20832067 FL.Partial fs1 -> do
2084− r <- done fs1
2068+ b <- done fs1
20852069 let next = SplitOnSuffixSeqSingleInit s pat
2086− skip $ SplitOnSuffixSeqYield r next
2087− FL.Done r ->
2070+ skip $ SplitOnSuffixSeqYield b next
2071+ FL.Done b ->
20882072 let next = SplitOnSuffixSeqSingleInit s pat
2089− in skip $ SplitOnSuffixSeqYield r next
2073+ in skip $ SplitOnSuffixSeqYield b next
20902074 else do
2091− sfs <- fstep fs x
2092− case sfs of
2075+ r <- fstep fs x
2076+ case r of
20932077 FL.Partial fs1 -> skip $ SplitOnSuffixSeqSingle fs1 s pat
2094− FL.Done r ->
2078+ FL.Done b ->
20952079 let next = SplitOnSuffixSeqSingleInit s pat
2096− in skip $ SplitOnSuffixSeqYield r next
2080+ in skip $ SplitOnSuffixSeqYield b next
20972081
20982082 -- For Rabin-Karp search
20992083 k = 2891336453 :: Word32
@@ -2135,13 +2119,13 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
21352119 case res of
21362120 Yield x s -> do
21372121 acc <- initial
2138− sfs <- fstep acc x
2139− case sfs of
2140− FL.Partial acc1 -> do
2141− r <- done acc1
2142− skip $ SplitOnSuffixSeqYield r (SplitOnSuffixSeqEmpty s)
2143− FL.Done r ->
2144− skip $ SplitOnSuffixSeqYield r (SplitOnSuffixSeqEmpty s)
2122+ r <- fstep acc x
2123+ case r of
2124+ FL.Partial fs -> do
2125+ b <- done fs
2126+ skip $ SplitOnSuffixSeqYield b (SplitOnSuffixSeqEmpty s)
2127+ FL.Done b ->
2128+ skip $ SplitOnSuffixSeqYield b (SplitOnSuffixSeqEmpty s)
21452129 Skip s -> skip (SplitOnSuffixSeqEmpty s)
21462130 Stop -> return Stop
21472131
@@ -2180,13 +2164,13 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
21802164 skip $ SplitOnSuffixSeqYield r SplitOnSuffixSeqDone
21812165 stepOuter _ (SplitOnSuffixSeqWordDone n fs wrd) = do
21822166 let old = elemMask .&. (wrd `shiftR` (elemBits * (n - 1)))
2183− sfs <- fstep fs (toEnum $ fromIntegral old)
2184− case sfs of
2167+ r <- fstep fs (toEnum $ fromIntegral old)
2168+ case r of
21852169 FL.Partial fs1 -> skip $ SplitOnSuffixSeqWordDone (n - 1) fs1 wrd
2186− FL.Done r -> do
2170+ FL.Done b -> do
21872171 fs1 <- initial
21882172 let next = SplitOnSuffixSeqWordDone (n - 1) fs1 wrd
2189− skip $ SplitOnSuffixSeqYield r next
2173+ skip $ SplitOnSuffixSeqYield b next
21902174
21912175 stepOuter gst (SplitOnSuffixSeqWordInit st0) = do
21922176 res <- step (adaptState gst) st0
@@ -2194,12 +2178,12 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
21942178 Yield x s -> do
21952179 fs <- initial
21962180 let wrd = addToWord 0 x
2197− sfs <- if withSep then fstep fs x else return $ FL.Partial fs
2198− case sfs of
2181+ r <- if withSep then fstep fs x else return $ FL.Partial fs
2182+ case r of
21992183 FL.Partial fs1 -> go SPEC 1 wrd s fs1
2200− FL.Done r -> do
2184+ FL.Done b -> do
22012185 let next = SplitOnSuffixSeqWordInit s
2202− in skip $ SplitOnSuffixSeqYield r next
2186+ in skip $ SplitOnSuffixSeqYield b next
22032187 Skip s -> skip (SplitOnSuffixSeqWordInit s)
22042188 Stop -> return Stop
22052189
@@ -2211,20 +2195,20 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
22112195 case res of
22122196 Yield x s -> do
22132197 let wrd1 = addToWord wrd x
2214− sfs <- if withSep then fstep fs x else return $ FL.Partial fs
2215− case sfs of
2198+ r <- if withSep then fstep fs x else return $ FL.Partial fs
2199+ case r of
22162200 FL.Partial fs1 ->
22172201 if idx /= maxIndex
22182202 then go SPEC (idx + 1) wrd1 s fs1
22192203 else if wrd1 .&. wordMask /= wordPat
22202204 then skip $ SplitOnSuffixSeqWordLoop wrd1 s fs1
22212205 else do
2222− r <- done fs
2206+ b <- done fs
22232207 let next = SplitOnSuffixSeqWordInit s
2224− skip $ SplitOnSuffixSeqYield r next
2225− FL.Done r ->
2208+ skip $ SplitOnSuffixSeqYield b next
2209+ FL.Done b ->
22262210 let next = SplitOnSuffixSeqWordInit s
2227− in skip $ SplitOnSuffixSeqYield r next
2211+ in skip $ SplitOnSuffixSeqYield b next
22282212 Skip s -> go SPEC idx wrd s fs
22292213 Stop -> skip $ SplitOnSuffixSeqWordDone idx fs wrd
22302214
@@ -2241,21 +2225,21 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
22412225 let wrd1 = addToWord wrd x
22422226 old = (wordMask .&. wrd)
22432227 `shiftR` (elemBits * (patLen - 1))
2244− sfs <-
2228+ r <-
22452229 if withSep
22462230 then fstep fs x
22472231 else fstep fs (toEnum $ fromIntegral old)
2248− case sfs of
2232+ case r of
22492233 FL.Partial fs1 ->
22502234 if wrd1 .&. wordMask == wordPat
22512235 then do
2252− r <- done fs1
2236+ b <- done fs1
22532237 let next = SplitOnSuffixSeqWordInit s
2254− skip $ SplitOnSuffixSeqYield r next
2238+ skip $ SplitOnSuffixSeqYield b next
22552239 else go SPEC wrd1 s fs1
2256− FL.Done r ->
2240+ FL.Done b ->
22572241 let next = SplitOnSuffixSeqWordInit s
2258− in skip $ SplitOnSuffixSeqYield r next
2242+ in skip $ SplitOnSuffixSeqYield b next
22592243 Skip s -> go SPEC wrd s fs
22602244 Stop ->
22612245 if wrd .&. wordMask == wordPat
@@ -2276,12 +2260,13 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
22762260 Yield x s -> do
22772261 rh1 <- liftIO $ RB.unsafeInsert rb rh0 x
22782262 fs <- initial
2279− sfs <- if withSep then fstep fs x else return $ FL.Partial fs
2280− case sfs of
2281− FL.Partial fs1 -> skip $ SplitOnSuffixSeqKRInit1 fs1 s rb rh1
2282− FL.Done r ->
2263+ r <- if withSep then fstep fs x else return $ FL.Partial fs
2264+ case r of
2265+ FL.Partial fs1 ->
2266+ skip $ SplitOnSuffixSeqKRInit1 fs1 s rb rh1
2267+ FL.Done b ->
22832268 let next = SplitOnSuffixSeqKRInit 0 s rb (RB.startOf rb)
2284− in skip $ SplitOnSuffixSeqYield r next
2269+ in skip $ SplitOnSuffixSeqYield b next
22852270 Skip s -> skip $ SplitOnSuffixSeqKRInit idx0 s rb rh0
22862271 Stop -> return Stop
22872272
@@ -2295,8 +2280,8 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
22952280 case res of
22962281 Yield x s -> do
22972282 rh1 <- liftIO (RB.unsafeInsert rb rh x)
2298− sfs <- if withSep then fstep fs x else return $ FL.Partial fs
2299− case sfs of
2283+ r <- if withSep then fstep fs x else return $ FL.Partial fs
2284+ case r of
23002285 FL.Partial fs1 ->
23012286 if idx /= maxIndex
23022287 then go SPEC (idx + 1) rh1 s fs1
@@ -2305,10 +2290,12 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
23052290 !ringHash = fold addCksum 0 rb
23062291 in if ringHash == patHash
23072292 then SplitOnSuffixSeqKRCheck fs1 s rb rh1
2308− else SplitOnSuffixSeqKRLoop fs1 s rb rh1 ringHash
2309− FL.Done r ->
2310− let next = SplitOnSuffixSeqKRInit 0 st rb (RB.startOf rb)
2311− in skip $ SplitOnSuffixSeqYield r next
2293+ else SplitOnSuffixSeqKRLoop
2294+ fs1 s rb rh1 ringHash
2295+ FL.Done b ->
2296+ let next = SplitOnSuffixSeqKRInit
2297+ 0 st rb (RB.startOf rb)
2298+ in skip $ SplitOnSuffixSeqYield b next
23122299 Skip s -> go SPEC idx rh s fs
23132300 Stop -> do
23142301 -- do not issue a blank segment when we end at pattern
@@ -2332,15 +2319,16 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
23322319 old <- liftIO $ peek rh
23332320 rh1 <- liftIO (RB.unsafeInsert rb rh x)
23342321 let cksum1 = deltaCksum cksum old x
2335− sfs <- if withSep then fstep fs x else fstep fs old
2336− case sfs of
2322+ r <- if withSep then fstep fs x else fstep fs old
2323+ case r of
23372324 FL.Partial fs1 ->
23382325 if (cksum1 /= patHash)
23392326 then go SPEC fs1 s rh1 cksum1
23402327 else skip $ SplitOnSuffixSeqKRCheck fs1 s rb rh1
2341− FL.Done r ->
2342− let next = SplitOnSuffixSeqKRInit 0 st rb (RB.startOf rb)
2343− in skip $ SplitOnSuffixSeqYield r next
2328+ FL.Done b ->
2329+ let next = SplitOnSuffixSeqKRInit
2330+ 0 st rb (RB.startOf rb)
2331+ in skip $ SplitOnSuffixSeqYield b next
23442332 Skip s -> go SPEC fs s rh cksum
23452333 Stop ->
23462334 if RB.unsafeEqArray rb rh patArr
@@ -2365,13 +2353,13 @@ splitOnSuffixSeq withSep patArr (Fold fstep initial done) (Stream step state) =
23652353 stepOuter _ (SplitOnSuffixSeqKRDone n fs rb rh) = do
23662354 old <- liftIO $ peek rh
23672355 let rh1 = RB.advance rb rh
2368− sfs <- fstep fs old
2369− case sfs of
2356+ r <- fstep fs old
2357+ case r of
23702358 FL.Partial fs1 -> skip $ SplitOnSuffixSeqKRDone (n - 1) fs1 rb rh1
2371− FL.Done r -> do
2359+ FL.Done b -> do
23722360 fs1 <- initial
23732361 let next = SplitOnSuffixSeqKRDone (n - 1) fs1 rb rh1
2374− skip $ SplitOnSuffixSeqYield r next
2362+ skip $ SplitOnSuffixSeqYield b next
23752363
23762364{-# ANN type SplitState Fuse #-}
23772365data SplitState s arr
@@ -3765,12 +3753,10 @@ scanlMx' fstep begin done s =
37653753 (begin >>= \x -> x `seq` done x) `consM` postscanlMx' fstep begin done s
37663754
37673755
3768−data PostScanState fsM s a = PostScan s !fsM
3756+data PostScanState s f = PostScan s !f
37693757
3770−-- XXX PostScanWith is can be eliminated if we abstract Yield
37713758{-# INLINE_NORMAL postscanOnce #-}
3772−postscanOnce :: Monad m
3773− => FL.Fold m a b -> Stream m a -> Stream m b
3759+postscanOnce :: Monad m => FL.Fold m a b -> Stream m a -> Stream m b
37743760postscanOnce (FL.Fold fstep begin done) (Stream step state) =
37753761 Stream step' (PostScan state begin)
37763762
@@ -3780,15 +3766,14 @@ postscanOnce (FL.Fold fstep begin done) (Stream step state) =
37803766 step' gst (PostScan st acc) = do
37813767 r <- step (adaptState gst) st
37823768 case r of
3783− -- XXX Move Yield to a common function
37843769 Yield x s -> do
37853770 old <- acc
3786− y <- fstep old x
3787− case y of
3788− FL.Partial sres -> do
3789− !v <- done sres
3790− return $ Yield v $ PostScan s (return sres)
3791− FL.Done _ -> return $ Stop
3771+ res <- fstep old x
3772+ case res of
3773+ FL.Partial fs -> do
3774+ !v <- done fs
3775+ return $ Yield v $ PostScan s (return fs)
3776+ FL.Done _ -> return Stop
37923777 Skip s -> return $ Skip $ PostScan s acc
37933778 Stop -> return Stop
37943779
@@ -4027,17 +4012,17 @@ tap (Fold fstep initial extract) (Stream step state) = Stream step' TapInit
40274012 step' gst (Tapping acc st) = do
40284013 r <- step gst st
40294014 case r of
4030− -- XXX Abstract Yield?
40314015 Yield x s -> do
4032− acc1 <- fstep acc x
4016+ res <- fstep acc x
40334017 return
4034− $ case acc1 of
4035− FL.Partial sres -> Yield x (Tapping sres s)
4036− FL.Done _ -> Yield x (TapDone s)
4018+ $ Yield x
4019+ $ case res of
4020+ FL.Partial fs -> Tapping fs s
4021+ FL.Done _ -> TapDone s
40374022 Skip s -> return $ Skip (Tapping acc s)
40384023 Stop -> do
40394024 void $ extract acc
4040− return $ Stop
4025+ return Stop
40414026 step' gst (TapDone st) = do
40424027 r <- step gst st
40434028 return
@@ -4067,21 +4052,22 @@ tapOffsetEvery offset n (Fold fstep initial extract) (Stream step state) =
40674052 step' gst (TapOffTapping acc st count) = do
40684053 r <- step gst st
40694054 case r of
4070− Yield x s ->
4071− if count <= 0
4072− -- XXX Abstract the then branch?
4073− then do
4074− acc1 <- fstep acc x
4075− return
4076− $ case acc1 of
4077− FL.Partial sres ->
4078− Yield x $ TapOffTapping sres s (n - 1)
4079− FL.Done _ -> Yield x (TapOffDone s)
4080− else return $ Yield x $ TapOffTapping acc s (count - 1)
4055+ Yield x s -> do
4056+ next <-
4057+ if count <= 0
4058+ then do
4059+ res <- fstep acc x
4060+ return
4061+ $ case res of
4062+ FL.Partial sres ->
4063+ TapOffTapping sres s (n - 1)
4064+ FL.Done _ -> TapOffDone s
4065+ else return $ TapOffTapping acc s (count - 1)
4066+ return $ Yield x next
40814067 Skip s -> return $ Skip (TapOffTapping acc s count)
40824068 Stop -> do
40834069 void $ extract acc
4084− return $ Stop
4070+ return Stop
40854071 step' gst (TapOffDone st) = do
40864072 r <- step gst st
40874073 return
src/Streamly/Internal/Data/Stream/StreamD/Type.hs+20−26
@@ -48,7 +48,6 @@ module Streamly.Internal.Data.Stream.StreamD.Type
4848 , GroupState (..) -- for inspection testing
4949 , foldMany
5050 , foldMany1
51− , groupsOf
5251 , groupsOf2
5352 )
5453where
@@ -66,8 +65,8 @@ import Fusion.Plugin.Types (Fuse(..))
6665import Streamly.Internal.Data.SVar (State(..), adaptState, defState)
6766import Streamly.Internal.Data.Fold.Types (Fold(..), Fold2(..))
6867
69−import qualified Streamly.Internal.Data.Stream.StreamK as K
7068import qualified Streamly.Internal.Data.Fold.Types as FL
69+import qualified Streamly.Internal.Data.Stream.StreamK as K
7170
7271------------------------------------------------------------------------------
7372-- The direct style stream type
@@ -589,12 +588,10 @@ take n (Stream step state) = n `seq` Stream step' (state, 0)
589588{-# ANN type GroupState Fuse #-}
590589data GroupState s fs b a
591590 = GroupStart s
592− | GroupConsume s fs a
593591 | GroupBuffer s fs
594592 | GroupYield b (GroupState s fs b a)
595593 | GroupFinish
596594
597−-- XXX Remove GroupConsume
598595{-# INLINE_NORMAL foldMany #-}
599596foldMany :: Monad m => Fold m a b -> Stream m a -> Stream m b
600597foldMany (Fold fstep initial extract) (Stream step state) =
@@ -607,15 +604,16 @@ foldMany (Fold fstep initial extract) (Stream step state) =
607604 -- fs = fold state
608605 fs <- initial
609606 return $ Skip (GroupBuffer st fs)
610− step' _ (GroupConsume st fs x) = do
611− fs' <- fstep fs x
612− case fs' of
613− FL.Done b -> return $ Skip (GroupYield b (GroupStart st))
614− FL.Partial ps -> return $ Skip (GroupBuffer st ps)
615607 step' gst (GroupBuffer st fs) = do
616608 r <- step (adaptState gst) st
617609 case r of
618− Yield x s -> return $ Skip $ GroupConsume s fs x
610+ Yield x s -> do
611+ res <- fstep fs x
612+ return
613+ $ Skip
614+ $ case res of
615+ FL.Done b -> GroupYield b (GroupStart s)
616+ FL.Partial ps -> GroupBuffer s ps
619617 Skip s -> return $ Skip (GroupBuffer s fs)
620618 Stop -> do
621619 b <- extract fs
@@ -630,25 +628,26 @@ foldMany1 (Fold fstep initial extract) (Stream step state) =
630628
631629 where
632630
631+ {-# INLINE consume #-}
632+ consume x s fs = do
633+ res <- fstep fs x
634+ return
635+ $ Skip
636+ $ case res of
637+ FL.Done b -> GroupYield b (GroupStart s)
638+ FL.Partial ps -> GroupBuffer s ps
639+
633640 {-# INLINE_LATE step' #-}
634641 step' gst (GroupStart st) = do
635− -- fs = fold state
636642 r <- step (adaptState gst) st
637643 case r of
638− Yield x s -> do
639− fi <- initial
640− return $ Skip $ GroupConsume s fi x
644+ Yield x s -> initial >>= consume x s
641645 Skip s -> return $ Skip (GroupStart s)
642− Stop -> return $ Stop
643− step' _ (GroupConsume st fs x) = do
644− fs' <- fstep fs x
645− case fs' of
646− FL.Done b -> return $ Skip (GroupYield b (GroupStart st))
647− FL.Partial ps -> return $ Skip (GroupBuffer st ps)
646+ Stop -> return Stop
648647 step' gst (GroupBuffer st fs) = do
649648 r <- step (adaptState gst) st
650649 case r of
651− Yield x s -> return $ Skip $ GroupConsume s fs x
650+ Yield x s -> consume x s fs
652651 Skip s -> return $ Skip (GroupBuffer s fs)
653652 Stop -> do
654653 b <- extract fs
@@ -656,11 +655,6 @@ foldMany1 (Fold fstep initial extract) (Stream step state) =
656655 step' _ (GroupYield b next) = return $ Yield b next
657656 step' _ GroupFinish = return Stop
658657
659−-- XXX Investigate performance
660−{-# INLINE groupsOf #-}
661−groupsOf :: Monad m => Int -> Fold m a b -> Stream m a -> Stream m b
662−groupsOf n fld = foldMany (FL.ltake n fld)
663−
664658data GroupState2 s fs
665659 = GroupStart2 s
666660 | GroupBuffer2 s fs Int