@@ -1,4 +1,5 @@
1{-# LANGUAGE CPP #-}
1{-# LANGUAGE CPP #-}
2+{-# LANGUAGE RankNTypes #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE FlexibleContexts #-}
4#if __GLASGOW_HASKELL__ >= 800
5#if __GLASGOW_HASKELL__ >= 800
@@ -109,39 +110,49 @@ module Streamly.Prelude
110 -- * Elimination
111 -- * Elimination
112− -- ** Primitives
113+ -- ** Deconstruction
113 -- | It is easy to express all the folds in terms of the 'uncons' primitive,
114 -- | It is easy to express all the folds in terms of the 'uncons' primitive,
114 -- however the specific implementations provided later are generally more
115 -- however the specific implementations provided later are generally more
115− -- efficient. Folds are inherently serial as each step needs to use the
116+ -- efficient.
116− -- result of the previous step.
117+ --
119− -- ** General Folds
122+ -- ** Folding
120−-- | Right and left folds.
123+-- | In imperative terms a fold can be considered as a loop over the stream
121−-- As a simple rule, always use lazy right fold for construction and strict
124+-- that reduces the stream to a single value.
122−-- left fold for reduction. By construction we mean using a constructor as the
125+-- Left and right folds use a fold function @f@ and an identity element @z@
123−-- outermost operation in the fold function, by reduction we mean using a
126+-- (@zero@) to recursively deconstruct a structure and then combine and reduce
124−-- function as the outermost operation in the fold function.
127+-- the values or transform and reconstruct a new container.
126−-- +-----------------------------------+--------------------------------------+
129+-- In general, a right fold is suitable for transforming and reconstructing a
127−-- | Right Fold | Left Fold |
130+-- right associated structure (e.g. cons lists and streamly streams) and a left
128−-- +===================================+======================================+
131+-- fold is suitable for reducing a right associated structure. The behavior of
129−-- | Construction consumes input | Construction consumes all input, |
132+-- right and left folds are described in detail in the individual fold's
130−-- | lazily and streams it in FIFO | and constructs in reverse (LIFO) |
132−-- +-----------------------------------+--------------------------------------+
133−-- | Reduction ends up buffering all | Strict reduction works |
134−-- | input before it can be reduced | incrementally, without buffering. |
135−-- +-----------------------------------+--------------------------------------+
137−-- Almost always, we need lazy construction and strict reduction, therefore,
138−-- strict @foldr@ and lazy @foldl@ are rarely useful. If needed, strict @foldr@
139−-- and lazy @foldl@ can be expressed in terms of the available versions. For
140−-- example, a lazy @foldl@ can be replaced by a strict @foldl@ to reverse the
141−-- structure followed by a @foldr@.
143−-- The following equations may help understand the relation between the two
144−-- folds for lists:
133+-- documentation. To illustrate the two folds for cons lists:
135+-- > foldr :: (a -> b -> b) -> b -> [a] -> b
136+-- > foldr f z [] = z
137+-- > foldr f z (x:xs) = x `f` foldr f z xs
139+-- > foldl :: (b -> a -> b) -> b -> [a] -> b
140+-- > foldl f z [] = z
141+-- > foldl f z (x:xs) = foldl f (z `f` x) xs
143+-- @foldr@ is conceptually equivalent to:
145+-- > foldr f z [] = z
146+-- > foldr f z [x] = f x z
147+-- > foldr f z xs = foldr f (foldr f z (tail xs)) [head xs]
149+-- @foldl@ is conceptually equivalent to:
151+-- > foldl f z [] = z
152+-- > foldl f z [x] = f z x
153+-- > foldl f z xs = foldl f (foldl f z (init xs)) [last xs]
155+-- Left and right folds are duals of each other.
147-- foldr f z xs = foldl (flip f) z (reverse xs)
158-- foldr f z xs = foldl (flip f) z (reverse xs)
@@ -151,42 +162,78 @@ module Streamly.Prelude
151-- More generally:
162-- More generally:
154−-- foldl f z xs = foldr g id xs z where g x k = k . flip f x
155-- foldr f z xs = foldl g id xs z where g k x = k . f x
165-- foldr f z xs = foldl g id xs z where g k x = k . f x
166+-- foldl f z xs = foldr g id xs z where g x k = k . flip f x
170+-- As a general rule, foldr cannot have state and foldl cannot have control.
172+-- NOTE: Folds are inherently serial as each step needs to use the result of
173+-- the previous step. However, it is possible to fold parts of the stream in
174+-- parallel and then combine the results using a monoid.
162 , foldl1'
181 , foldl1'
163 , foldlM'
182 , foldlM'
167− -- ** Run Effects
186+ -- ** Strict Full Folds
187+ -- | Folds that are guaranteed to evaluate the whole stream.
189+ -- -- ** To Summary (Full Folds)
190+ -- -- | Folds that summarize the stream to a single value.
168 , runStream
191 , runStream
197+ -- -- ** To Summary (Maybe) (Full Folds)
198+ -- -- | Folds that summarize a non-empty stream to a 'Just' value and return
199+ -- 'Nothing' for an empty stream.
209+ -- | Folds that generate a lazy structure. Note that the generated
210+ -- structure may not be lazy if the underlying monad is strict.
212+ -- -- ** To Containers (Full Folds)
213+ -- -- | Convert or divert a stream into an output structure, container or
219+ -- | Folds that may terminate before evaluating the whole stream. These
220+ -- folds strictly evaluate the stream until the result is determined.
222+ -- -- ** To Elements (Partial Folds)
170 , runWhile
224 , runWhile
173− -- | Folds that extract selected elements of a stream or their properties.
226+ -- -- | Folds that extract selected elements of a stream or their properties.
180 , findIndex
232 , findIndex
181 , elemIndex
233 , elemIndex
184− -- | Folds that extract selected parts of a stream.
188− -- ** To Boolean
235+ -- -- ** To Boolean (Partial Folds)
189− -- | Folds that summarize the stream to a boolean value.
236+ -- -- | Folds that summarize the stream to a boolean value.
192 , notElem
239 , notElem
@@ -195,46 +242,76 @@ module Streamly.Prelude
199− -- | Folds that summarize the stream to a single value.
204− -- ** To Summary (Maybe)
205− -- | Folds that summarize a non-empty stream to a 'Just' value and return
206− -- 'Nothing' for an empty stream.
214− -- | Convert or divert a stream into an output structure, container or
219 -- * Transformation
245 -- * Transformation
248+ -- | In imperative terms a map operation can be considered as a loop over
249+ -- the stream that transforms the stream into another stream by performing
250+ -- an operation on each element of the stream.
252+ -- 'map' is the least powerful transformation operation with strictest
253+ -- guarantees. A map, (1) is a stateless loop which means that no state is
254+ -- allowed to be carried from one iteration to another, therefore,
255+ -- operations on different elements are guaranteed to not affect each
256+ -- other, (2) is a strictly one-to-one transformation of stream elements
257+ -- which means it guarantees that no elements can be added or removed from
258+ -- the stream, it can merely transform them.
221 -- ** Scanning
264 -- ** Scanning
222− -- | Scans stream all the intermediate reduction steps of the corresponding
265+ --
266+ -- | A scan is more powerful than map. While a 'map' is a stateless loop, a
267+ -- @scan@ is a stateful loop which means that a state can be shared across
268+ -- all the loop iterations, therefore, future iterations can be impacted by
269+ -- the state changes made by the past iterations. A scan yields the state
270+ -- of the loop after each iteration. Like a map, a @postscan@ or @prescan@
271+ -- does not add or remove elements in the stream, it just transforms them.
272+ -- However, a @scan@ adds one extra element to the stream.
274+ -- A left associative scan, also known as a prefix sum, can be thought of
275+ -- as a stream transformation consisting of left folds of all prefixes of a
276+ -- stream. Another way of thinking about it is that it streams all the
277+ -- intermediate values of the accumulator while applying a left fold on the
278+ -- input stream. A right associative scan, on the other hand, can be
279+ -- thought of as a stream consisting of right folds of all the suffixes of
223− -- folds. The following equations hold for lists:
282+ -- The following equations hold for lists:
225 -- > scanl f z xs == map (foldl f z) $ inits xs
284 -- > scanl f z xs == map (foldl f z) $ inits xs
226− -- > scanr f z xs == map (foldr f z) $ tails
285+ -- > scanr f z xs == map (foldr f z) $ tails xs
228− -- We do not provide a right associative scan, it can be recovered from a
287+ -- @
229− -- 'scanl'' as follows:
288+ -- > scanl (+) 0 [1,2,3,4]
292+ -- 0 + 1 + 2 + 3 = 6
293+ -- 0 + 1 + 2 + 3 + 4 = 10
231− -- > scanr f z xs == reverse $ scanl' (flip f) z (reverse xs)
295+ -- > scanr (+) 0 [1,2,3,4]
296+ -- 1 + 2 + 3 + 4 + 0 = 10
297+ -- 2 + 3 + 4 + 0 = 9
233− -- Scan is like a stateful map. If we discard the state, we get the map:
303+ -- Left and right scans are duals:
305+ -- > scanr f z xs == reverse $ scanl (flip f) z (reverse xs)
306+ -- > scanl f z xs == reverse $ scanr (flip f) z (reverse xs)
308+ -- A scan is a stateful map i.e. a combination of map and fold:
235− -- > S.drop 1 $ S.scanl' (\_ x -> f x) z xs == map f xs
310+ -- > map f xs = tail $ scanl (\_ x -> f x) z xs
236−
311+ -- > map f xs = reverse $ head $ scanr (\_ x -> f x) z xs
237− -- > S.postscanl' (\_ x -> f x) z xs == map f xs
312+ --
313+ -- > foldl f z xs = last $ scanl f z xs
314+ -- > foldr f z xs = head $ scanr f z xs
240 , scanlM'
317 , scanlM'
@@ -246,20 +323,18 @@ module Streamly.Prelude
246 , scanl1M'
323 , scanl1M'
249− -- ** Mapping
326+ , indexed
250− -- | Map is a strictly one-to-one transformation of stream elements. It
327+ , indexedR
251− -- cannot add or remove elements from the stream, just transforms them.
258 -- ** Filtering
329 -- ** Filtering
259− -- | Filtering may remove some elements from the stream.
330+ -- | Remove some elements from the stream based on a predicate. In
331+ -- imperative terms a filter over a stream corresponds to a loop with a
332+ -- @continue@ clause for the cases when the predicate fails.
262 , filterM
335 , filterM
337+ -- ** Stateful Filters
264 , takeWhile
339 , takeWhile
265 , takeWhileM
340 , takeWhileM
@@ -269,6 +344,18 @@ module Streamly.Prelude
269 , deleteBy
344 , deleteBy
347+ -- ** Mapping Filters
348+ -- | Mapping along with filtering
353+ -- ** Scanning Filters
354+ -- | Stateful transformation along with filtering
272 -- ** Insertion
359 -- ** Insertion
273 -- | Insertion adds more elements to the stream.
360 -- | Insertion adds more elements to the stream.
@@ -278,17 +365,31 @@ module Streamly.Prelude
278 -- ** Reordering
365 -- ** Reordering
279 , reverse
366 , reverse
281− -- * Hybrid Operations
368+ -- ** Nested Producer Loops
282− -- ** Map and Fold
369+ -- | Map each element to a stream and then flatten the results into a
283− , mapM_
370+ -- single stream. In imperative terms, a 'concatMap' corresponds to nested
284−
371+ -- loops. We loop over the outer stream and then for each element of the
285− -- ** Map and Filter
372+ -- outer stream an inner stream is generated and then we loop over the
286− , mapMaybe
373+ -- inner stream as well to generate a single output stream.
377+ -- ----Stream m a----|-Stream m b-|-Stream m b-|-...-|----Stream m b
381+ -- 'concatMap' can perform filtering by mapping an element to a 'nil'
382+ -- stream. Its a map, therefore it can degenerate to a simple map
383+ -- operation as well:
385+ -- > filter p m = S.concatMap (\x -> if p x then S.yield x else S.nil) m
386+ -- > map f m = S.concatMap (\x -> S.yield (f x)) m
388+ -- Though 'concatMap' is not inherently stateful, it can be combined with a
389+ -- scan to perform stateful operations.
289− -- ** Scan and filter
391+ , concatMapM
290− , findIndices
392+ , concatMap
293 -- * Multi-Stream Operations
394 -- * Multi-Stream Operations
294 -- | New streams can be constructed by appending, merging or zipping
395 -- | New streams can be constructed by appending, merging or zipping
@@ -299,6 +400,12 @@ module Streamly.Prelude
299 -- operation.
400 -- operation.
404+ -- -------Stream m a------|-------Stream m a------|=>----Stream m a---
302 -- >> S.toList $ S.fromList [1,2] \<> S.fromList [3,4]
409 -- >> S.toList $ S.fromList [1,2] \<> S.fromList [3,4]
303 -- [1,2,3,4]
410 -- [1,2,3,4]
304 -- >> S.toList $ fold $ [S.fromList [1,2], S.fromList [3,4]]
411 -- >> S.toList $ fold $ [S.fromList [1,2], S.fromList [3,4]]
@@ -308,6 +415,14 @@ module Streamly.Prelude
308 -- ** Merging
415 -- ** Merging
309 -- | Streams form a commutative semigroup under the merge
416 -- | Streams form a commutative semigroup under the merge
310 -- operation.
417 -- operation.
421+ -- -------Stream m a------|
422+ -- |=>----Stream m a---
423+ -- -------Stream m a------|
312 -- , merge
427 -- , merge
313 , mergeBy
428 , mergeBy
@@ -316,20 +431,20 @@ module Streamly.Prelude
316 , mergeAsyncByM
431 , mergeAsyncByM
318 -- ** Zipping
433 -- ** Zipping
437+ -- -------Stream m a------|
438+ -- |=>----Stream m c---
439+ -- -------Stream m b------|
319 , zipWith
442 , zipWith
320 , zipWithM
443 , zipWithM
321 , Z.zipAsyncWith
444 , Z.zipAsyncWith
322 , Z.zipAsyncWithM
445 , Z.zipAsyncWithM
332− -- ** Folds
447+ -- ** Stream Level Folds
335 , isPrefixOf
450 , isPrefixOf
@@ -340,12 +455,15 @@ module Streamly.Prelude
348import Control.Monad.IO.Class (MonadIO(..))
465import Control.Monad.IO.Class (MonadIO(..))
466+import Control.Monad.Trans (MonadTrans(..))
349import Data.Maybe (isJust, fromJust)
467import Data.Maybe (isJust, fromJust)
350import Prelude
468import Prelude
351 hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr,
469 hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr,
@@ -388,6 +506,12 @@ import qualified Streamly.Streams.Serial as Serial
388-- 'Nothing'. If the stream is non-empty, returns @Just (a, ma)@, where @a@ is
506-- 'Nothing'. If the stream is non-empty, returns @Just (a, ma)@, where @a@ is
389-- the head of the stream and @ma@ its tail.
507-- the head of the stream and @ma@ its tail.
509+-- This is a brute force primitive. Avoid using it as long as possible, use it
510+-- when no other combinator can do the job. This can be used to do pretty much
511+-- anything in an imperative manner, as it just breaks down the stream into
512+-- individual elements and we can loop over them as we deem fit. For example,
513+-- this can be used to convert a streamly stream into other stream types.
391-- @since 0.1.0
515-- @since 0.1.0
392{-# INLINE uncons #-}
516{-# INLINE uncons #-}
393uncons :: (IsStream t, Monad m) => SerialT m a -> m (Maybe (a, t m a))
517uncons :: (IsStream t, Monad m) => SerialT m a -> m (Maybe (a, t m a))
@@ -720,92 +844,158 @@ fromHandle h = go
720-- Elimination by Folding
844-- Elimination by Folding
721------------------------------------------------------------------------------
845------------------------------------------------------------------------------
723−-- | Lazy right fold with a monadic step function. For example, to fold a
847+-- | Right associative/pull fold.
724−-- stream into a list:
726−-- @
849+-- Example, determine if any element is 'odd' in a stream:
727−-- >> S.foldrM (\\x xs -> return (x : xs)) [] $ fromList [1,2,3]
731−-- @since 0.2.0
851+-- >>> S.foldrM (\x xs -> if odd x then return True else xs) (return False) $ S.fromList (2:4:5:undefined)
732−{-# INLINE foldrM #-}
852+-- > True
733−foldrM :: Monad m => (a -> b -> m b) -> b -> SerialT m a -> m b
736−-- | Lazy right associative fold.
738−-- For lists a @foldr@ looks like:
854+-- Let's take a closer look at the @foldr@ definition for lists, as given
742-- foldr f z (x:xs) = x \`f` foldr f z xs
858-- foldr f z (x:xs) = x \`f` foldr f z xs
745−-- The recursive expression is the second argument of the fold step `f`.
861+-- @foldr@ invokes the fold step function @f@ as @f x (foldr f z xs)@. At each
746−-- Therefore, the evaluation of the recursive call depends on `f`. It can
862+-- invocation of @f@ @foldr@ gives us the next element in the input container
863+-- @x@ and a recursive expression @foldr f z xs@ representing the yet unbuilt
864+-- (lazy thunk) part of the output. Therefore, when @f x xs@ is lazy in @xs@
865+-- it can consume the input one element at a time in FIFO order to build a lazy
866+-- output expression. For example,
868+-- > f x remaining = show x : remaining
870+-- @take 2 $ foldr f [] (1:2:undefined)@ would consume the input lazily on
871+-- demand, consuming only first two elements and resulting in ["1", "2"]. @f@
747−-- terminate recursion by not inspecting the second argument based on a
872+-- can terminate recursion by not evaluating the @remaining@ part:
748−-- condition. When expanded fully, it results in the following right associated
873+--
874+-- > f 2 remaining = show 2 : []
875+-- > f x remaining = show x : remaining
877+-- @f@ would terminate recursion whenever it sees element @2@ in the input.
878+-- Therefore, @foldr f [] (1:2:undefined)@ would work as before. If @f a b@ is
879+-- strict in @b@ it would end up consuming the whole input right away and
880+-- expanding the recursive expression @b@ (i.e. @foldr f z xs@) fully before
881+-- it yields an output expression, resulting in the following /right
749−-- expression:
882+-- associated expression/:
752-- foldr f z xs == x1 \`f` (x2 \`f` ...(xn \`f` z))
885-- foldr f z xs == x1 \`f` (x2 \`f` ...(xn \`f` z))
755−-- When `f` is a constructor, we can see that the first deconstruction of this
756−-- expression would be @x1@ on the left and the recursive expression on the
757−-- right. Therefore, we can deconstruct it to access the input elements in the
758−-- first-in-first-out (FIFO) order and consume the reconstructed structure
759−-- lazily. The recursive expression on the right gets evaluated incrementall
760−-- as demanded by the consumer. For example:
888+-- For example,
762−-- @
890+-- > f x remaining = x + remaining
763−-- > S.foldr (:) [] $ S.fromList [1,2,3,4]
891+--
764−-- [1,2,3,4]
892+-- With this definition, @foldr f 0 [1..1000]@, would recurse completely until
765−-- @
893+-- it reaches the recursion terminating case @... `f` (1000 `f` 0)@, and then
894+-- start reducing the whole expression from right to left, therefore, consuming
895+-- the input elements in LIFO order. Thus, such an evaluation would require
896+-- memory proportional to the size of input. Try out @foldr (+) 0 (map (\\x ->
897+-- trace (show x) x) [1..10])@.
899+-- Notice, the order of the arguments to the step function @f a b@. It follows
900+-- the order of @a@ and @b@ in the right associative recursive expression
901+-- generated by expanding @a \`f` b@.
903+-- A right fold is a pull fold, the step function is the puller, it can pull
904+-- more data from the input container by using its second argument in the
905+-- output expression or terminate pulling by not using it. As a corollary:
907+-- 1. a step function which is lazy in its second argument (usually functions
908+-- or constructors that build a lazy structure e.g. @(:)@) can pull lazily on
910+-- 2. a step function strict in its second argument (usually reducers e.g.
911+-- (+)) would end up pulling all of its input and buffer it in memory before
912+-- potentially reducing it.
914+-- A right fold is suitable for lazy reconstructions e.g. transformation,
915+-- mapping, filtering of /right associated input structures/ (e.g. cons lists).
916+-- Whereas a left fold is suitable for reductions (e.g. summing a stream of
917+-- numbers) of right associated structures. Note that these roles will reverse
918+-- for left associated structures (e.g. snoc lists). Most of our observations
919+-- here assume right associated structures, lists being the canonical example.
921+-- 1. A lazy FIFO style pull using a right fold allows pulling a potentially
922+-- /infinite/ input stream lazily, perform transformations on it, and
923+-- reconstruct a new structure without having to buffer the whole structure. In
924+-- contrast, a left fold would buffer the entire structure before the
925+-- reconstructed structure can be consumed.
926+-- 2. Even if buffering the entire input structure is ok, we need to keep in
927+-- mind that a right fold reconstructs structures in a FIFO style, whereas a
928+-- left fold reconstructs in a LIFO style, thereby reversing the order of
930+-- 3. A right fold has termination control and therefore can terminate early
931+-- without going throught the entire input, a left fold cannot terminate
932+-- without consuming all of its input. For example, a right fold
933+-- implementation of 'or' can terminate as soon as it finds the first 'True'
934+-- element, whereas a left fold would necessarily go through the entire input
935+-- irrespective of that.
936+-- 4. Reduction (e.g. using (+) on a stream of numbers) using a right fold
937+-- occurs in a LIFO style, which means that the entire input gets buffered
938+-- before reduction starts. Whereas with a strict left fold reductions occur
939+-- incrementally in FIFO style. Therefore, a strict left fold is more suitable
943+{-# INLINE foldrM #-}
944+foldrM :: Monad m => (a -> m b -> m b) -> m b -> SerialT m a -> m b
947+-- | Right fold to a streaming monad.
767−-- When `f` is a function strict in its second argument, the right side of the
949+-- 'foldrS' can be used to perform stateless stream to stream transformations
768−-- expression gets evaluated as follows:
950+-- like map and filter in general. It can be coupled with a scan to perform
951+-- stateful transformations. However, note that the custom map and filter
952+-- routines can be much more efficient than this due to better stream fusion.
770−-- @
954+-- Find if any element in the stream is 'True':
771−-- foldr f z xs == x1 \`f` tail1
772−-- tail1 == x2 \`f` tail2
773−-- tail2 == x3 \`f` tail3
775−-- tailn == xn \`f` z
778−-- In @foldl'@ we have both the arguments of `f` available at each step,
956+-- >>> S.toList $ S.foldrS (\x xs -> if odd x then return True else xs) (return False) $ (S.fromList (2:4:5:undefined) :: SerialT IO Int)
779−-- therefore, each step can be reduced immediately. However, in @foldr@ the
957+-- > [True]
780−-- second argument to `f` is a recursive call, therefore, it ends up building
781−-- the whole expression in memory before it can be reduced, consuming the whole
782−-- input. This makes @foldr@ much less efficient for reduction compared to
783−-- @foldl'@. For example:
785−-- @
959+-- Map (+2) on odd elements and filter out the even elements:
786−-- > S.foldr (+) 0 $ S.fromList [1,2,3,4]
790−-- When the underlying monad @m@ is strict (e.g. IO), then @foldr@ ends up
961+-- >>> S.toList $ S.foldrS (\x xs -> if odd x then (x + 2) `S.cons` xs else xs) S.nil $ (S.fromList [1..5] :: SerialT IO Int)
791−-- evaluating all of its input because of strict evaluation of the recursive
962+-- > [3,5,7]
794−-- >> S.foldr (\_ _ -> []) [] $ S.fromList (1:undefined)
964+-- 'foldrM' can also be represented in terms of 'foldrS', however, the former
795−-- >*** Exception: Prelude.undefined
965+-- is much more efficient:
797−-- In a lazy monad, we can consume the input lazily, and terminate the fold
967+-- > foldrM f z s = runIdentityT $ foldrS (\x xs -> lift $ f x (runIdentityT xs)) (lift z) s
798−-- by conditionally not inspecting the recursive expression.
800−-- >> runIdentity $ S.foldr (\x rest -> if x == 3 then [] else x : rest) [] $ S.fromList (4:1:3:undefined)
969+-- @since 0.7.0
801−-- >[4,1]
970+{-# INLINE foldrS #-}
971+foldrS :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b
974+-- | Right fold to a transformer monad. This is the most general right fold
975+-- function. 'foldrS' is a special case of 'foldrT', however 'foldrS'
976+-- implementation can be more efficient:
979+-- > foldrM f z s = runIdentityT $ foldrT (\x xs -> lift $ f x (runIdentityT xs)) (lift z) s
803−-- The arguments to the folding function (@a -> b -> b@) are in the head and
981+-- 'foldrT' can be used to translate streamly streams to other transformer
804−-- tail order of the output, @a@ is the head and @b@ is the tail. Remember, in
982+-- monads e.g. to a different streaming type.
805−-- a right fold the zero is on the right, it is the tail end.
983+--
985+{-# INLINE foldrT #-}
986+foldrT :: (IsStream t, Monad m, Monad (s m), MonadTrans s)
987+ => (a -> s m b -> s m b) -> s m b -> t m a -> s m b
988+foldrT f z s = S.foldrT f z (toStreamS s)
990+-- | Note that with this signature we cannot implement a lazy foldr when the
991+-- monad @m@ is strict. In that case it would be strict in its accumulator and
992+-- therefore would necessarily consume all its input. For this reason we have
993+-- deprecated this API. Though this can be useful for lazy monads, but we can
994+-- always achieve the same thing using foldrM directly.
807-- @since 0.1.0
996-- @since 0.1.0
808{-# INLINE foldr #-}
997{-# INLINE foldr #-}
998+{-# DEPRECATED foldr "This is unnecessarily strict for strict monads. Use foldrM instead." #-}
809foldr :: Monad m => (a -> b -> b) -> b -> SerialT m a -> m b
999foldr :: Monad m => (a -> b -> b) -> b -> SerialT m a -> m b
810foldr = P.foldr
1000foldr = P.foldr
@@ -817,6 +1007,7 @@ foldr = P.foldr
818-- @since 0.5.0
1008-- @since 0.5.0
819{-# INLINE foldr1 #-}
1009{-# INLINE foldr1 #-}
1010+{-# DEPRECATED foldr1 "This is unnecessarily strict for strict monads. Use foldrM instead." #-}
820foldr1 :: Monad m => (a -> a -> a) -> SerialT m a -> m (Maybe a)
1011foldr1 :: Monad m => (a -> a -> a) -> SerialT m a -> m (Maybe a)
821foldr1 f m = S.foldr1 f (toStreamS m)
1012foldr1 f m = S.foldr1 f (toStreamS m)
@@ -828,48 +1019,59 @@ foldr1 f m = S.foldr1 f (toStreamS m)
828-- @since 0.2.0
1019-- @since 0.2.0
829{-# INLINE foldx #-}
1020{-# INLINE foldx #-}
830foldx :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b
1021foldx :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b
831−foldx = K.foldx
1022+foldx = P.foldlx'
834-- @since 0.1.0
1025-- @since 0.1.0
835{-# DEPRECATED foldl "Please use foldx instead." #-}
1026{-# DEPRECATED foldl "Please use foldx instead." #-}
836foldl :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b
1027foldl :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b
837−foldl = foldx
1028+foldl = P.foldlx'
839−-- | Strict left associative fold.
1030+-- | Strict left associative/push fold. Note that the observations below about
1031+-- the behavior of a left fold assume that we are working on a right associated
1032+-- structure like cons lists and streamly streams. If we are working on a left
1033+-- associated structure (e.g. snoc lists) the roles of right and left folds
841−-- For lists a @foldl@ looks like:
1036+-- Let's take a closer look at the @foldl@ definition for lists given above:
845-- foldl f z (x:xs) = foldl f (z \`f` x) xs
1039-- foldl f z (x:xs) = foldl f (z \`f` x) xs
1042+-- @foldl@ calls itself recursively, in each call it invokes @f@ as @f z x@
1043+-- providing it with the result accumulated till now @z@ (the state) and the
1044+-- next element from the input container. First call to @f@ is supplied with
1045+-- the initial value of the accumulator @z@ and each subsequent call uses the
1046+-- output of the previous call to @f z x@.
1048+-- >> foldl' (+) 0 [1,2,3]
848-- The recursive call at the head of the output expression is bound to be
1051-- The recursive call at the head of the output expression is bound to be
849−-- evaluated until recursion terminates,
1052+-- evaluated until recursion terminates, therefore, a left fold always
850−-- /deconstructing the whole input container/ and building the following left
1053+-- /consumes the whole input container/. The following would result in an
851−-- associated expression:
1054+-- error, even though the fold is not using the values at all:
853−-- @
1056+-- >> foldl' (\_ _ -> 0) 0 (1:undefined)
854−-- foldl f z xs == (((z \`f` x1) \`f` x2) ...) \`f` xn
1057+-- > *** Exception: Prelude.undefined
857−-- When `f` is a constructor, we can see that the first deconstruction of this
1059+-- As @foldl@ recurses, it builds the left associated expression shown below.
858−-- expression would be the recursive expression on the left and `xn` on the
1060+-- Notice, the order of the arguments to the step function @f b a@. It follows
859−-- right. Therefore, it can access the input elements only in the reverse
1061+-- the left associative recursive expression generated by expanding @b \`f` a@.
860−-- (LIFO) order. For example:
863−-- > S.foldl' (flip (:)) [] $ S.fromList [1,2,3,4]
1064+-- foldl f z xs == (((z \`f` x1) \`f` x2) ...) \`f` xn
867-- The strict left fold @foldl'@ forces the reduction of its argument @z \`f`
1068-- The strict left fold @foldl'@ forces the reduction of its argument @z \`f`
868-- x@ before using it, therefore it never builds the whole expression in
1069-- x@ before using it, therefore it never builds the whole expression in
869-- memory. Thus, @z \`f` x1@ would get reduced to @z1@ and then @z1 \`f` x2@
1070-- memory. Thus, @z \`f` x1@ would get reduced to @z1@ and then @z1 \`f` x2@
870-- would get reduced to @z2@ and so on, incrementally reducing the expression
1071-- would get reduced to @z2@ and so on, incrementally reducing the expression
871−-- as it recurses. However, it evaluates the accumulator only to WHNF, it may
1072+-- from left to right as it recurses, consuming the input in FIFO order. Try
872−-- further help to use a strict data structure as accumulator. For example:
1073+-- out @foldl' (+) 0 (map (\\x -> trace (show x) x) [1..10])@ to see how it
1074+-- works. For example:
875-- > S.foldl' (+) 0 $ S.fromList [1,2,3,4]
1077-- > S.foldl' (+) 0 $ S.fromList [1,2,3,4]
@@ -877,36 +1079,44 @@ foldl = foldx
880−-- 0 + 1
1082+-- 0 + 1 = 1
882−-- ((0 + 1) + 2) + 3
1083+-- 1 + 2 = 3
883−-- (((0 + 1) + 2) + 3) + 4
1084+-- 3 + 3 = 6
886−-- @foldl@ strictly deconstructs the whole input container irrespective of
1088+-- However, @foldl'@ evaluates the accumulator only to WHNF. It may further
887−-- whether it needs it or not:
1089+-- help if the step function uses a strict data structure as accumulator to
1090+-- improve performance and to keep the expression fully reduced at all times
889−-- >> S.foldl' (\acc x -> if x == 3 then acc else x : acc) [] $ S.fromList (4:1:3:undefined)
1093+-- A left fold can also build a new structure instead of reducing one if a
890−-- >*** Exception: Prelude.undefined
1094+-- constructor is used as a fold step. However, it may not be very useful
1095+-- because it will consume the whole input and construct the new structure in
1096+-- memory before we can consume it. Thus the whole structure gets buffered in
1097+-- memory. When the list constructor is used it would build a new list in
1098+-- reverse (LIFO) order:
892−-- However, evaluation of the items contained in the input container is lazy as
1100+-- @
893−-- demanded by the fold step function:
895−-- >> S.foldl' (\acc x -> if x == 3 then acc else x : acc) [] $ S.fromList [4,1,3,undefined]
1101+-- > S.foldl' (flip (:)) [] $ S.fromList [1,2,3,4]
896−-- >[4,1]
1102+-- [4,3,2,1]
898−-- To perform a left fold without consuming all the input one can use @scanl@
1105+-- A left fold is a push fold. The producer pushes its contents to the step
899−-- to stream the intermediate results of the fold and use them lazily.
1106+-- function of the fold. The step function therefore has no control to stop the
1107+-- input, it can only discard it if it does not need it. We can also consider a
1108+-- left fold as a state machine where the state is store in the accumulator,
1109+-- the state can be modified based on new inputs that are pushed to the fold.
901−-- In stateful or event-driven programming, we can consider @z@ as the initial
1111+-- In general, a strict left fold is a reducing fold, whereas a right fold is a
902−-- state and the stream being folded as a stream of events, thus @foldl'@
1112+-- constructing fold. A strict left fold reduces in a FIFO order whereas it
903−-- processes all the events in the stream updating the state on each event and
1113+-- constructs in a LIFO order, and vice-versa for the right fold. See the
904−-- then ultimately returning the final state.
1114+-- documentation of 'foldrM' for a discussion on where a left or right fold is
906−-- The arguments to the folding function (@b -> a -> b@) are in the head and
1117+-- To perform a left fold lazily without having to consume all the input one
907−-- tail order of the output expression, @b@ is the head and @a@ is the tail.
1118+-- can use @scanl@ to stream the intermediate results of the fold and consume
908−-- Remember, in a left fold the zero is on the left, at the head of the
1119+-- the resulting stream lazily.
911-- @since 0.2.0
1121-- @since 0.2.0
912{-# INLINE foldl' #-}
1122{-# INLINE foldl' #-}
@@ -932,13 +1142,13 @@ foldl1' step m = do
932-- @since 0.2.0
1142-- @since 0.2.0
933{-# INLINE foldxM #-}
1143{-# INLINE foldxM #-}
934foldxM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b
1144foldxM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b
935−foldxM = K.foldxM
1145+foldxM = P.foldlMx'
938-- @since 0.1.0
1148-- @since 0.1.0
939{-# DEPRECATED foldlM "Please use foldxM instead." #-}
1149{-# DEPRECATED foldlM "Please use foldxM instead." #-}
940foldlM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b
1150foldlM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b
941−foldlM = foldxM
1151+foldlM = P.foldlMx'
943-- | Like 'foldl'' but with a monadic step function.
1153-- | Like 'foldl'' but with a monadic step function.
@@ -951,7 +1161,10 @@ foldlM' step begin m = S.foldlM' step begin $ toStreamS m
951-- Specialized folds
1161-- Specialized folds
952------------------------------------------------------------------------------
1162------------------------------------------------------------------------------
1165+-- > runStream = mapM_ (\_ -> return ())
954−-- | Run a stream, discarding the results. By default it interprets the stream
1167+-- Run a stream, discarding the results. By default it interprets the stream
955-- as 'SerialT', to run other types of streams use the type adapting
1168-- as 'SerialT', to run other types of streams use the type adapting
956-- combinators for example @runStream . 'asyncly'@.
1169-- combinators for example @runStream . 'asyncly'@.
@@ -996,7 +1209,10 @@ null = K.null
996head :: Monad m => SerialT m a -> m (Maybe a)
1209head :: Monad m => SerialT m a -> m (Maybe a)
997head = K.head
1210head = K.head
1213+-- > tail = fmap (fmap snd) . uncons
999−-- | Extract all but the first element of the stream, if any.
1215+-- Extract all but the first element of the stream, if any.
1001-- @since 0.1.1
1217-- @since 0.1.1
1002{-# INLINE tail #-}
1218{-# INLINE tail #-}
@@ -1190,24 +1406,6 @@ elemIndices a = findIndices (==a)
1190elemIndex :: (Monad m, Eq a) => a -> SerialT m a -> m (Maybe Int)
1406elemIndex :: (Monad m, Eq a) => a -> SerialT m a -> m (Maybe Int)
1191elemIndex a = findIndex (== a)
1407elemIndex a = findIndex (== a)
1193−-- | Map each element to a stream and then flatten the results into a single
1196−-- > concatMap f = concatMapM (return . f)
1199−{-# INLINE concatMap #-}
1200−concatMap ::(IsStream t, Monad m) => (a -> t m b) -> t m a -> t m b
1201−concatMap f m = fromStreamD $ D.concatMap (toStreamD . f) (toStreamD m)
1203−-- | Map each element to a stream using a monadic function and then flatten the
1204−-- results into a single stream.
1207−{-# INLINE concatMapM #-}
1208−concatMapM :: (IsStream t, Monad m) => (a -> m (t m b)) -> t m a -> t m b
1209−concatMapM f m = fromStreamD $ D.concatMapM (fmap toStreamD . f) (toStreamD m)
1211------------------------------------------------------------------------------
1409------------------------------------------------------------------------------
1212-- Substreams
1410-- Substreams
1213------------------------------------------------------------------------------
1411------------------------------------------------------------------------------
@@ -1256,8 +1454,12 @@ stripPrefix m1 m2 = fmap fromStreamD <$>
1256------------------------------------------------------------------------------
1454------------------------------------------------------------------------------
1258-- XXX this can utilize parallel mapping if we implement it as runStream . mapM
1456-- XXX this can utilize parallel mapping if we implement it as runStream . mapM
1458+-- > mapM_ = runStream . mapM
1259−-- | Apply a monadic action to each element of the stream and discard the
1460+-- Apply a monadic action to each element of the stream and discard the output
1260−-- output of the action.
1461+-- of the action. This is not really a pure transformation operation but a
1462+-- transformation followed by fold.
1262-- @since 0.1.0
1464-- @since 0.1.0
1263{-# INLINE mapM_ #-}
1465{-# INLINE mapM_ #-}
@@ -1273,13 +1475,27 @@ mapM_ f m = S.mapM_ f $ toStreamS m
1273-- toList = S.foldr (:) []
1475-- toList = S.foldr (:) []
1276−-- Convert a stream into a list in the underlying monad. Same as:
1478+-- Convert a stream into a list in the underlying monad. The list can be
1479+-- consumed lazily in a lazy monad (e.g. 'Identity'). In a strict monad (e.g.
1480+-- IO) the whole list is generated before it can be consumed.
1278-- @since 0.1.0
1482-- @since 0.1.0
1279{-# INLINE toList #-}
1483{-# INLINE toList #-}
1280toList :: Monad m => SerialT m a -> m [a]
1484toList :: Monad m => SerialT m a -> m [a]
1281toList = P.toList
1485toList = P.toList
1489+-- toRevList = S.foldl' (flip (:)) []
1492+-- Convert a stream into a list in reverse order in the underlying monad.
1495+{-# INLINE toRevList #-}
1496+toRevList :: Monad m => SerialT m a -> m [a]
1497+toRevList = D.toRevList . toStreamD
1285-- toHandle h = S.mapM_ $ hPutStrLn h
1501-- toHandle h = S.mapM_ $ hPutStrLn h
@@ -1301,6 +1517,16 @@ toHandle h m = go m
1301-- Transformation by Folding (Scans)
1517-- Transformation by Folding (Scans)
1302------------------------------------------------------------------------------
1518------------------------------------------------------------------------------
1520+-- XXX It may be useful to have a version of scan where we can keep the
1521+-- accumulator independent of the value emitted. So that we do not necessarily
1522+-- have to keep a value in the accumulator which we are not using. We can pass
1523+-- an extraction function that will take the accumulator and the current value
1524+-- of the element and emit the next value in the stream. That will also make it
1525+-- possible to modify the accumulator after using it. In fact, the step function
1526+-- can return new accumulator and the value to be emitted. The signature would
1527+-- be more like mapAccumL. Or we can change the signature of scanx to
1528+-- accommodate this.
1304-- | Strict left scan with an extraction function. Like 'scanl'', but applies a
1530-- | Strict left scan with an extraction function. Like 'scanl'', but applies a
1305-- user supplied extraction function (the third argument) at each step. This is
1531-- user supplied extraction function (the third argument) at each step. This is
1306-- designed to work with the @foldl@ library. The suffix @x@ is a mnemonic for
1532-- designed to work with the @foldl@ library. The suffix @x@ is a mnemonic for
@@ -1309,7 +1535,7 @@ toHandle h m = go m
1309-- @since 0.2.0
1535-- @since 0.2.0
1310{-# INLINE scanx #-}
1536{-# INLINE scanx #-}
1311scanx :: IsStream t => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b
1537scanx :: IsStream t => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b
1312−scanx = K.scanx
1538+scanx = K.scanx'
1315-- @since 0.1.1
1541-- @since 0.1.1
@@ -1325,7 +1551,8 @@ scan = scanx
1325scanlM' :: (IsStream t, Monad m) => (b -> a -> m b) -> b -> t m a -> t m b
1551scanlM' :: (IsStream t, Monad m) => (b -> a -> m b) -> b -> t m a -> t m b
1326scanlM' step begin m = fromStreamD $ D.scanlM' step begin $ toStreamD m
1552scanlM' step begin m = fromStreamD $ D.scanlM' step begin $ toStreamD m
1328−-- | Strict left scan.
1554+-- | Strict left scan. Like 'map', 'scanl'' too is a one to one transformation,
1555+-- however it adds an extra element.
1331-- > S.toList $ S.scanl' (+) 0 $ fromList [1,2,3,4]
1558-- > S.toList $ S.scanl' (+) 0 $ fromList [1,2,3,4]
@@ -1346,16 +1573,16 @@ scanlM' step begin m = fromStreamD $ D.scanlM' step begin $ toStreamD m
1346-- thus modularizing the stream processing. This can be useful in
1573-- thus modularizing the stream processing. This can be useful in
1347-- stateful or event-driven programming.
1574-- stateful or event-driven programming.
1349−-- Consider the following example, computing the sum and the product of the
1576+-- Consider the following monolothic example, computing the sum and the product
1350−-- elements in a stream in one go using a @foldl'@:
1577+-- of the elements in a stream in one go using a @foldl'@:
1353-- > S.foldl' (\\(s, p) x -> (s + x, p * x)) (0,1) $ S.fromList \[1,2,3,4]
1580-- > S.foldl' (\\(s, p) x -> (s + x, p * x)) (0,1) $ S.fromList \[1,2,3,4]
1354-- (10,24)
1581-- (10,24)
1357−-- Using @scanl'@ we can compute the sum in the first stage and pass it down to
1584+-- Using @scanl'@ we can make it modular by computing the sum in the first
1358−-- the next stage for computing the product:
1585+-- stage and passing it down to the next stage for computing the product:
1361-- > S.foldl' (\\(_, p) (s, x) -> (s, p * x)) (0,1)
1588-- > S.foldl' (\\(_, p) (s, x) -> (s, p * x)) (0,1)
@@ -1609,19 +1836,16 @@ mapMaybeMSerial f m = fromStreamD $ D.mapMaybeM f $ toStreamD m
1609-- XXX to scale this we need to use a slab allocated array backed
1836-- XXX to scale this we need to use a slab allocated array backed
1610-- representation for temporary storage.
1837-- representation for temporary storage.
1840+-- > reverse = S.foldlT (flip S.cons) S.nil
1612−-- | Returns the elements of the stream in reverse order.
1842+-- Returns the elements of the stream in reverse order. The stream must be
1613−-- The stream must be finite.
1843+-- finite.
1615-- @since 0.1.1
1845-- @since 0.1.1
1846+{-# INLINE reverse #-}
1616−reverse :: (IsStream t) => t m a -> t m a
1847+reverse :: (IsStream t, Monad m) => t m a -> t m a
1617−reverse m = go K.nil m
1848+reverse s = fromStreamS $ S.reverse $ toStreamS s
1619− go rev rest = K.mkStream $ \st yld sng stp ->
1620− let runIt x = K.foldStream st yld sng stp x
1622− single a = runIt $ a `K.cons` rev
1623− yieldk a r = runIt $ go (a `K.cons` rev) r
1624− in K.foldStream st yieldk single stop rest
1626------------------------------------------------------------------------------
1850------------------------------------------------------------------------------
1627-- Transformation by Inserting
1851-- Transformation by Inserting
@@ -1682,9 +1906,10 @@ deleteBy cmp x m = fromStreamS $ S.deleteBy cmp x (toStreamS m)
1682------------------------------------------------------------------------------
1906------------------------------------------------------------------------------
1685−-- > indexed = S.zipWith (,) (S.intFrom 0)
1909+-- > indexed = S.postscanl' (\(i, _) x -> (i + 1, x)) (-1,undefined)
1910+-- > indexed = S.zipWith (,) (S.enumerateFrom 0)
1687−-- Pair each element in a stream with its index.
1912+-- Pair each element in a stream with its index, starting from index 0.
1690-- > S.toList $ S.indexed $ S.fromList "hello"
1915-- > S.toList $ S.indexed $ S.fromList "hello"
@@ -1697,14 +1922,15 @@ indexed :: (IsStream t, Monad m) => t m a -> t m (Int, a)
1697indexed = fromStreamD . D.indexed . toStreamD
1922indexed = fromStreamD . D.indexed . toStreamD
1700−-- > indexedR n = S.zipWith (,) (S.intFromThen n (n - 1))
1925+-- > indexedR n = S.postscanl' (\(i, _) x -> (i - 1, x)) (n + 1,undefined)
1926+-- > indexedR n = S.zipWith (,) (S.enumerateFromThen n (n - 1))
1702-- Pair each element in a stream with its index, starting from the
1928-- Pair each element in a stream with its index, starting from the
1703-- given index @n@ and counting down.
1929-- given index @n@ and counting down.
1706-- > S.toList $ S.indexedR 10 $ S.fromList "hello"
1932-- > S.toList $ S.indexedR 10 $ S.fromList "hello"
1707−-- [(9,'h'),(8,'e'),(7,'l'),(6,'l'),(5,'o')]
1933+-- [(10,'h'),(9,'e'),(8,'l'),(7,'l'),(6,'o')]
1710-- @since 0.6.0
1936-- @since 0.6.0
@@ -1847,3 +2073,25 @@ mergeAsyncByM f m1 m2 = K.mkStream $ \st stp sng yld -> do
1847 ma <- mkAsync' st m1
2073 ma <- mkAsync' st m1
1848 mb <- mkAsync' st m2
2074 mb <- mkAsync' st m2
1849 K.foldStream st stp sng yld (K.mergeByM f ma mb)
2075 K.foldStream st stp sng yld (K.mergeByM f ma mb)
2077+------------------------------------------------------------------------------
2079+------------------------------------------------------------------------------
2081+-- | Map each element of the stream to produce a stream and then flatten the
2082+-- results into a single stream.
2084+-- > concatMap f = concatMapM (return . f)
2087+{-# INLINE concatMap #-}
2088+concatMap ::(IsStream t, Monad m) => (a -> t m b) -> t m a -> t m b
2089+concatMap f m = fromStreamD $ D.concatMap (toStreamD . f) (toStreamD m)
2091+-- | Map each element to a stream to produce a stream using a monadic function
2092+-- and then flatten the results into a single stream.
2095+{-# INLINE concatMapM #-}
2096+concatMapM :: (IsStream t, Monad m) => (a -> m (t m b)) -> t m a -> t m b
2097+concatMapM f m = fromStreamD $ D.concatMapM (fmap toStreamD . f) (toStreamD m)