@@ -1,4 +1,5 @@
11{-# LANGUAGE CPP #-}
2+{-# LANGUAGE RankNTypes #-}
23{-# LANGUAGE FlexibleContexts #-}
34
45#if __GLASGOW_HASKELL__ >= 800
@@ -109,39 +110,49 @@ module Streamly.Prelude
109110
110111 -- * Elimination
111112
112− -- ** Primitives
113+ -- ** Deconstruction
113114 -- | It is easy to express all the folds in terms of the 'uncons' primitive,
114115 -- however the specific implementations provided later are generally more
115− -- efficient. Folds are inherently serial as each step needs to use the
116− -- result of the previous step.
116+ -- efficient.
117+ --
117118 , uncons
119+ , tail
120+ , init
118121
119− -- ** General Folds
120−-- | Right and left folds.
121−-- As a simple rule, always use lazy right fold for construction and strict
122−-- left fold for reduction. By construction we mean using a constructor as the
123−-- outermost operation in the fold function, by reduction we mean using a
124−-- function as the outermost operation in the fold function.
125−--
126−-- +-----------------------------------+--------------------------------------+
127−-- | Right Fold | Left Fold |
128−-- +===================================+======================================+
129−-- | Construction consumes input | Construction consumes all input, |
130−-- | lazily and streams it in FIFO | and constructs in reverse (LIFO) |
131−-- | order | order |
132−-- +-----------------------------------+--------------------------------------+
133−-- | Reduction ends up buffering all | Strict reduction works |
134−-- | input before it can be reduced | incrementally, without buffering. |
135−-- +-----------------------------------+--------------------------------------+
136−--
137−-- Almost always, we need lazy construction and strict reduction, therefore,
138−-- strict @foldr@ and lazy @foldl@ are rarely useful. If needed, strict @foldr@
139−-- and lazy @foldl@ can be expressed in terms of the available versions. For
140−-- example, a lazy @foldl@ can be replaced by a strict @foldl@ to reverse the
141−-- structure followed by a @foldr@.
142−--
143−-- The following equations may help understand the relation between the two
144−-- folds for lists:
122+ -- ** Folding
123+-- | In imperative terms a fold can be considered as a loop over the stream
124+-- that reduces the stream to a single value.
125+-- Left and right folds use a fold function @f@ and an identity element @z@
126+-- (@zero@) to recursively deconstruct a structure and then combine and reduce
127+-- the values or transform and reconstruct a new container.
128+--
129+-- In general, a right fold is suitable for transforming and reconstructing a
130+-- right associated structure (e.g. cons lists and streamly streams) and a left
131+-- fold is suitable for reducing a right associated structure. The behavior of
132+-- right and left folds are described in detail in the individual fold's
133+-- documentation. To illustrate the two folds for cons lists:
134+--
135+-- > foldr :: (a -> b -> b) -> b -> [a] -> b
136+-- > foldr f z [] = z
137+-- > foldr f z (x:xs) = x `f` foldr f z xs
138+-- >
139+-- > foldl :: (b -> a -> b) -> b -> [a] -> b
140+-- > foldl f z [] = z
141+-- > foldl f z (x:xs) = foldl f (z `f` x) xs
142+--
143+-- @foldr@ is conceptually equivalent to:
144+--
145+-- > foldr f z [] = z
146+-- > foldr f z [x] = f x z
147+-- > foldr f z xs = foldr f (foldr f z (tail xs)) [head xs]
148+--
149+-- @foldl@ is conceptually equivalent to:
150+--
151+-- > foldl f z [] = z
152+-- > foldl f z [x] = f z x
153+-- > foldl f z xs = foldl f (foldl f z (init xs)) [last xs]
154+--
155+-- Left and right folds are duals of each other.
145156--
146157-- @
147158-- foldr f z xs = foldl (flip f) z (reverse xs)
@@ -151,42 +162,78 @@ module Streamly.Prelude
151162-- More generally:
152163--
153164-- @
154−-- foldl f z xs = foldr g id xs z where g x k = k . flip f x
155165-- foldr f z xs = foldl g id xs z where g k x = k . f x
166+-- foldl f z xs = foldr g id xs z where g x k = k . flip f x
156167-- @
168+--
169+
170+-- As a general rule, foldr cannot have state and foldl cannot have control.
171+
172+-- NOTE: Folds are inherently serial as each step needs to use the result of
173+-- the previous step. However, it is possible to fold parts of the stream in
174+-- parallel and then combine the results using a monoid.
157175
158− , foldr
159− , foldr1
160176 , foldrM
177+ , foldrS
178+ , foldrT
179+
161180 , foldl'
162181 , foldl1'
163182 , foldlM'
164183 , foldx
165184 , foldxM
166185
167− -- ** Run Effects
186+ -- ** Strict Full Folds
187+ -- | Folds that are guaranteed to evaluate the whole stream.
188+
189+ -- -- ** To Summary (Full Folds)
190+ -- -- | Folds that summarize the stream to a single value.
168191 , runStream
192+ , last
193+ , length
194+ , sum
195+ , product
196+
197+ -- -- ** To Summary (Maybe) (Full Folds)
198+ -- -- | Folds that summarize a non-empty stream to a 'Just' value and return
199+ -- 'Nothing' for an empty stream.
200+ , maximumBy
201+ , maximum
202+ , minimumBy
203+ , minimum
204+ , the
205+ , toRevList
206+
207+ -- ** Lazy Folds
208+ --
209+ -- | Folds that generate a lazy structure. Note that the generated
210+ -- structure may not be lazy if the underlying monad is strict.
211+
212+ -- -- ** To Containers (Full Folds)
213+ -- -- | Convert or divert a stream into an output structure, container or
214+ -- sink.
215+ , toList
216+ , toHandle
217+
218+ -- ** Partial Folds
219+ -- | Folds that may terminate before evaluating the whole stream. These
220+ -- folds strictly evaluate the stream until the result is determined.
221+
222+ -- -- ** To Elements (Partial Folds)
169223 , runN
170224 , runWhile
171225
172− -- ** To Elements
173− -- | Folds that extract selected elements of a stream or their properties.
226+ -- -- | Folds that extract selected elements of a stream or their properties.
174227 , (!!)
175228 , head
176− , last
177229 , findM
178230 , find
179231 , lookup
180232 , findIndex
181233 , elemIndex
182234
183− -- ** To Parts
184− -- | Folds that extract selected parts of a stream.
185− , tail
186− , init
187−
188− -- ** To Boolean
189− -- | Folds that summarize the stream to a boolean value.
235+ -- -- ** To Boolean (Partial Folds)
236+ -- -- | Folds that summarize the stream to a boolean value.
190237 , null
191238 , elem
192239 , notElem
@@ -195,46 +242,76 @@ module Streamly.Prelude
195242 , and
196243 , or
197244
198− -- ** To Summary
199− -- | Folds that summarize the stream to a single value.
200− , length
201− , sum
202− , product
203−
204− -- ** To Summary (Maybe)
205− -- | Folds that summarize a non-empty stream to a 'Just' value and return
206− -- 'Nothing' for an empty stream.
207− , maximumBy
208− , maximum
209− , minimumBy
210− , minimum
211− , the
212−
213− -- ** To Containers
214− -- | Convert or divert a stream into an output structure, container or
215− -- sink.
216− , toList
217− , toHandle
218−
219245 -- * Transformation
220246
247+ -- ** Mapping
248+ -- | In imperative terms a map operation can be considered as a loop over
249+ -- the stream that transforms the stream into another stream by performing
250+ -- an operation on each element of the stream.
251+ --
252+ -- 'map' is the least powerful transformation operation with strictest
253+ -- guarantees. A map, (1) is a stateless loop which means that no state is
254+ -- allowed to be carried from one iteration to another, therefore,
255+ -- operations on different elements are guaranteed to not affect each
256+ -- other, (2) is a strictly one-to-one transformation of stream elements
257+ -- which means it guarantees that no elements can be added or removed from
258+ -- the stream, it can merely transform them.
259+ , Serial.map
260+ , sequence
261+ , mapM
262+ , mapM_
263+
221264 -- ** Scanning
222− -- | Scans stream all the intermediate reduction steps of the corresponding
223− -- folds. The following equations hold for lists:
265+ --
266+ -- | A scan is more powerful than map. While a 'map' is a stateless loop, a
267+ -- @scan@ is a stateful loop which means that a state can be shared across
268+ -- all the loop iterations, therefore, future iterations can be impacted by
269+ -- the state changes made by the past iterations. A scan yields the state
270+ -- of the loop after each iteration. Like a map, a @postscan@ or @prescan@
271+ -- does not add or remove elements in the stream, it just transforms them.
272+ -- However, a @scan@ adds one extra element to the stream.
273+ --
274+ -- A left associative scan, also known as a prefix sum, can be thought of
275+ -- as a stream transformation consisting of left folds of all prefixes of a
276+ -- stream. Another way of thinking about it is that it streams all the
277+ -- intermediate values of the accumulator while applying a left fold on the
278+ -- input stream. A right associative scan, on the other hand, can be
279+ -- thought of as a stream consisting of right folds of all the suffixes of
280+ -- a stream.
281+ --
282+ -- The following equations hold for lists:
224283 --
225284 -- > scanl f z xs == map (foldl f z) $ inits xs
226− -- > scanr f z xs == map (foldr f z) $ tails
285+ -- > scanr f z xs == map (foldr f z) $ tails xs
227286 --
228− -- We do not provide a right associative scan, it can be recovered from a
229− -- 'scanl'' as follows:
287+ -- @
288+ -- > scanl (+) 0 [1,2,3,4]
289+ -- 0 = 0
290+ -- 0 + 1 = 1
291+ -- 0 + 1 + 2 = 3
292+ -- 0 + 1 + 2 + 3 = 6
293+ -- 0 + 1 + 2 + 3 + 4 = 10
230294 --
231− -- > scanr f z xs == reverse $ scanl' (flip f) z (reverse xs)
295+ -- > scanr (+) 0 [1,2,3,4]
296+ -- 1 + 2 + 3 + 4 + 0 = 10
297+ -- 2 + 3 + 4 + 0 = 9
298+ -- 3 + 4 + 0 = 7
299+ -- 4 + 0 = 4
300+ -- 0 = 0
301+ -- @
232302 --
233− -- Scan is like a stateful map. If we discard the state, we get the map:
303+ -- Left and right scans are duals:
234304 --
235− -- > S.drop 1 $ S.scanl' (\_ x -> f x) z xs == map f xs
236−
237− -- > S.postscanl' (\_ x -> f x) z xs == map f xs
305+ -- > scanr f z xs == reverse $ scanl (flip f) z (reverse xs)
306+ -- > scanl f z xs == reverse $ scanr (flip f) z (reverse xs)
307+ --
308+ -- A scan is a stateful map i.e. a combination of map and fold:
309+ --
310+ -- > map f xs = tail $ scanl (\_ x -> f x) z xs
311+ -- > map f xs = reverse $ head $ scanr (\_ x -> f x) z xs
312+ --
313+ -- > foldl f z xs = last $ scanl f z xs
314+ -- > foldr f z xs = head $ scanr f z xs
238315
239316 , scanl'
240317 , scanlM'
@@ -246,20 +323,18 @@ module Streamly.Prelude
246323 , scanl1M'
247324 , scanx
248325
249− -- ** Mapping
250− -- | Map is a strictly one-to-one transformation of stream elements. It
251− -- cannot add or remove elements from the stream, just transforms them.
252− , Serial.map
253−
254− -- ** Flattening
255− , sequence
256− , mapM
326+ , indexed
327+ , indexedR
257328
258329 -- ** Filtering
259− -- | Filtering may remove some elements from the stream.
330+ -- | Remove some elements from the stream based on a predicate. In
331+ -- imperative terms a filter over a stream corresponds to a loop with a
332+ -- @continue@ clause for the cases when the predicate fails.
260333
261334 , filter
262335 , filterM
336+
337+ -- ** Stateful Filters
263338 , take
264339 , takeWhile
265340 , takeWhileM
@@ -269,6 +344,18 @@ module Streamly.Prelude
269344 , deleteBy
270345 , uniq
271346
347+ -- ** Mapping Filters
348+ -- | Mapping along with filtering
349+
350+ , mapMaybe
351+ , mapMaybeM
352+
353+ -- ** Scanning Filters
354+ -- | Stateful transformation along with filtering
355+
356+ , findIndices
357+ , elemIndices
358+
272359 -- ** Insertion
273360 -- | Insertion adds more elements to the stream.
274361
@@ -278,17 +365,31 @@ module Streamly.Prelude
278365 -- ** Reordering
279366 , reverse
280367
281− -- * Hybrid Operations
282− -- ** Map and Fold
283− , mapM_
284−
285− -- ** Map and Filter
286− , mapMaybe
287− , mapMaybeM
368+ -- ** Nested Producer Loops
369+ -- | Map each element to a stream and then flatten the results into a
370+ -- single stream. In imperative terms, a 'concatMap' corresponds to nested
371+ -- loops. We loop over the outer stream and then for each element of the
372+ -- outer stream an inner stream is generated and then we loop over the
373+ -- inner stream as well to generate a single output stream.
374+ --
375+ -- @
376+ --
377+ -- ----Stream m a----|-Stream m b-|-Stream m b-|-...-|----Stream m b
378+ --
379+ -- @
380+ --
381+ -- 'concatMap' can perform filtering by mapping an element to a 'nil'
382+ -- stream. Its a map, therefore it can degenerate to a simple map
383+ -- operation as well:
384+ --
385+ -- > filter p m = S.concatMap (\x -> if p x then S.yield x else S.nil) m
386+ -- > map f m = S.concatMap (\x -> S.yield (f x)) m
387+ --
388+ -- Though 'concatMap' is not inherently stateful, it can be combined with a
389+ -- scan to perform stateful operations.
288390
289− -- ** Scan and filter
290− , findIndices
291− , elemIndices
391+ , concatMapM
392+ , concatMap
292393
293394 -- * Multi-Stream Operations
294395 -- | New streams can be constructed by appending, merging or zipping
@@ -299,6 +400,12 @@ module Streamly.Prelude
299400 -- operation.
300401 --
301402 -- @
403+ --
404+ -- -------Stream m a------|-------Stream m a------|=>----Stream m a---
405+ --
406+ -- @
407+ --
408+ -- @
302409 -- >> S.toList $ S.fromList [1,2] \<> S.fromList [3,4]
303410 -- [1,2,3,4]
304411 -- >> S.toList $ fold $ [S.fromList [1,2], S.fromList [3,4]]
@@ -308,6 +415,14 @@ module Streamly.Prelude
308415 -- ** Merging
309416 -- | Streams form a commutative semigroup under the merge
310417 -- operation.
418+ --
419+ -- @
420+ --
421+ -- -------Stream m a------|
422+ -- |=>----Stream m a---
423+ -- -------Stream m a------|
424+ -- @
425+ --
311426
312427 -- , merge
313428 , mergeBy
@@ -316,20 +431,20 @@ module Streamly.Prelude
316431 , mergeAsyncByM
317432
318433 -- ** Zipping
434+ -- |
435+ -- @
436+ --
437+ -- -------Stream m a------|
438+ -- |=>----Stream m c---
439+ -- -------Stream m b------|
440+ -- @
441+ --
319442 , zipWith
320443 , zipWithM
321444 , Z.zipAsyncWith
322445 , Z.zipAsyncWithM
323446
324− -- Special zips
325− , indexed
326− , indexedR
327−
328− -- ** Flattening
329− , concatMapM
330− , concatMap
331−
332− -- ** Folds
447+ -- ** Stream Level Folds
333448 , eqBy
334449 , cmpBy
335450 , isPrefixOf
@@ -340,12 +455,15 @@ module Streamly.Prelude
340455 , K.once
341456 , each
342457 , scan
458+ , foldr
459+ , foldr1
343460 , foldl
344461 , foldlM
345462 )
346463where
347464
348465import Control.Monad.IO.Class (MonadIO(..))
466+import Control.Monad.Trans (MonadTrans(..))
349467import Data.Maybe (isJust, fromJust)
350468import Prelude
351469 hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr,
@@ -388,6 +506,12 @@ import qualified Streamly.Streams.Serial as Serial
388506-- 'Nothing'. If the stream is non-empty, returns @Just (a, ma)@, where @a@ is
389507-- the head of the stream and @ma@ its tail.
390508--
509+-- This is a brute force primitive. Avoid using it as long as possible, use it
510+-- when no other combinator can do the job. This can be used to do pretty much
511+-- anything in an imperative manner, as it just breaks down the stream into
512+-- individual elements and we can loop over them as we deem fit. For example,
513+-- this can be used to convert a streamly stream into other stream types.
514+--
391515-- @since 0.1.0
392516{-# INLINE uncons #-}
393517uncons :: (IsStream t, Monad m) => SerialT m a -> m (Maybe (a, t m a))
@@ -720,92 +844,158 @@ fromHandle h = go
720844-- Elimination by Folding
721845------------------------------------------------------------------------------
722846
723−-- | Lazy right fold with a monadic step function. For example, to fold a
724−-- stream into a list:
847+-- | Right associative/pull fold.
725848--
726−-- @
727−-- >> S.foldrM (\\x xs -> return (x : xs)) [] $ fromList [1,2,3]
728−-- [1,2,3]
729−-- @
849+-- Example, determine if any element is 'odd' in a stream:
730850--
731−-- @since 0.2.0
732−{-# INLINE foldrM #-}
733−foldrM :: Monad m => (a -> b -> m b) -> b -> SerialT m a -> m b
734−foldrM = P.foldrM
735−
736−-- | Lazy right associative fold.
851+-- >>> S.foldrM (\x xs -> if odd x then return True else xs) (return False) $ S.fromList (2:4:5:undefined)
852+-- > True
737853--
738−-- For lists a @foldr@ looks like:
854+-- Let's take a closer look at the @foldr@ definition for lists, as given
855+-- earlier:
739856--
740857-- @
741−-- foldr f z [] = z
742858-- foldr f z (x:xs) = x \`f` foldr f z xs
743859-- @
744860--
745−-- The recursive expression is the second argument of the fold step `f`.
746−-- Therefore, the evaluation of the recursive call depends on `f`. It can
747−-- terminate recursion by not inspecting the second argument based on a
748−-- condition. When expanded fully, it results in the following right associated
749−-- expression:
861+-- @foldr@ invokes the fold step function @f@ as @f x (foldr f z xs)@. At each
862+-- invocation of @f@ @foldr@ gives us the next element in the input container
863+-- @x@ and a recursive expression @foldr f z xs@ representing the yet unbuilt
864+-- (lazy thunk) part of the output. Therefore, when @f x xs@ is lazy in @xs@
865+-- it can consume the input one element at a time in FIFO order to build a lazy
866+-- output expression. For example,
867+--
868+-- > f x remaining = show x : remaining
869+--
870+-- @take 2 $ foldr f [] (1:2:undefined)@ would consume the input lazily on
871+-- demand, consuming only first two elements and resulting in ["1", "2"]. @f@
872+-- can terminate recursion by not evaluating the @remaining@ part:
873+--
874+-- > f 2 remaining = show 2 : []
875+-- > f x remaining = show x : remaining
876+--
877+-- @f@ would terminate recursion whenever it sees element @2@ in the input.
878+-- Therefore, @foldr f [] (1:2:undefined)@ would work as before. If @f a b@ is
879+-- strict in @b@ it would end up consuming the whole input right away and
880+-- expanding the recursive expression @b@ (i.e. @foldr f z xs@) fully before
881+-- it yields an output expression, resulting in the following /right
882+-- associated expression/:
750883--
751884-- @
752885-- foldr f z xs == x1 \`f` (x2 \`f` ...(xn \`f` z))
753886-- @
754887--
755−-- When `f` is a constructor, we can see that the first deconstruction of this
756−-- expression would be @x1@ on the left and the recursive expression on the
757−-- right. Therefore, we can deconstruct it to access the input elements in the
758−-- first-in-first-out (FIFO) order and consume the reconstructed structure
759−-- lazily. The recursive expression on the right gets evaluated incrementall
760−-- as demanded by the consumer. For example:
888+-- For example,
761889--
762−-- @
763−-- > S.foldr (:) [] $ S.fromList [1,2,3,4]
764−-- [1,2,3,4]
765−-- @
890+-- > f x remaining = x + remaining
891+--
892+-- With this definition, @foldr f 0 [1..1000]@, would recurse completely until
893+-- it reaches the recursion terminating case @... `f` (1000 `f` 0)@, and then
894+-- start reducing the whole expression from right to left, therefore, consuming
895+-- the input elements in LIFO order. Thus, such an evaluation would require
896+-- memory proportional to the size of input. Try out @foldr (+) 0 (map (\\x ->
897+-- trace (show x) x) [1..10])@.
898+--
899+-- Notice, the order of the arguments to the step function @f a b@. It follows
900+-- the order of @a@ and @b@ in the right associative recursive expression
901+-- generated by expanding @a \`f` b@.
902+--
903+-- A right fold is a pull fold, the step function is the puller, it can pull
904+-- more data from the input container by using its second argument in the
905+-- output expression or terminate pulling by not using it. As a corollary:
906+--
907+-- 1. a step function which is lazy in its second argument (usually functions
908+-- or constructors that build a lazy structure e.g. @(:)@) can pull lazily on
909+-- demand.
910+-- 2. a step function strict in its second argument (usually reducers e.g.
911+-- (+)) would end up pulling all of its input and buffer it in memory before
912+-- potentially reducing it.
913+--
914+-- A right fold is suitable for lazy reconstructions e.g. transformation,
915+-- mapping, filtering of /right associated input structures/ (e.g. cons lists).
916+-- Whereas a left fold is suitable for reductions (e.g. summing a stream of
917+-- numbers) of right associated structures. Note that these roles will reverse
918+-- for left associated structures (e.g. snoc lists). Most of our observations
919+-- here assume right associated structures, lists being the canonical example.
920+--
921+-- 1. A lazy FIFO style pull using a right fold allows pulling a potentially
922+-- /infinite/ input stream lazily, perform transformations on it, and
923+-- reconstruct a new structure without having to buffer the whole structure. In
924+-- contrast, a left fold would buffer the entire structure before the
925+-- reconstructed structure can be consumed.
926+-- 2. Even if buffering the entire input structure is ok, we need to keep in
927+-- mind that a right fold reconstructs structures in a FIFO style, whereas a
928+-- left fold reconstructs in a LIFO style, thereby reversing the order of
929+-- elements..
930+-- 3. A right fold has termination control and therefore can terminate early
931+-- without going throught the entire input, a left fold cannot terminate
932+-- without consuming all of its input. For example, a right fold
933+-- implementation of 'or' can terminate as soon as it finds the first 'True'
934+-- element, whereas a left fold would necessarily go through the entire input
935+-- irrespective of that.
936+-- 4. Reduction (e.g. using (+) on a stream of numbers) using a right fold
937+-- occurs in a LIFO style, which means that the entire input gets buffered
938+-- before reduction starts. Whereas with a strict left fold reductions occur
939+-- incrementally in FIFO style. Therefore, a strict left fold is more suitable
940+-- for reductions.
941+--
942+-- @since 0.7.0
943+{-# INLINE foldrM #-}
944+foldrM :: Monad m => (a -> m b -> m b) -> m b -> SerialT m a -> m b
945+foldrM = P.foldrM
946+
947+-- | Right fold to a streaming monad.
766948--
767−-- When `f` is a function strict in its second argument, the right side of the
768−-- expression gets evaluated as follows:
949+-- 'foldrS' can be used to perform stateless stream to stream transformations
950+-- like map and filter in general. It can be coupled with a scan to perform
951+-- stateful transformations. However, note that the custom map and filter
952+-- routines can be much more efficient than this due to better stream fusion.
769953--
770−-- @
771−-- foldr f z xs == x1 \`f` tail1
772−-- tail1 == x2 \`f` tail2
773−-- tail2 == x3 \`f` tail3
774−-- ...
775−-- tailn == xn \`f` z
776−-- @
954+-- Find if any element in the stream is 'True':
777955--
778−-- In @foldl'@ we have both the arguments of `f` available at each step,
779−-- therefore, each step can be reduced immediately. However, in @foldr@ the
780−-- second argument to `f` is a recursive call, therefore, it ends up building
781−-- the whole expression in memory before it can be reduced, consuming the whole
782−-- input. This makes @foldr@ much less efficient for reduction compared to
783−-- @foldl'@. For example:
956+-- >>> S.toList $ S.foldrS (\x xs -> if odd x then return True else xs) (return False) $ (S.fromList (2:4:5:undefined) :: SerialT IO Int)
957+-- > [True]
784958--
785−-- @
786−-- > S.foldr (+) 0 $ S.fromList [1,2,3,4]
787−-- 10
788−-- @
959+-- Map (+2) on odd elements and filter out the even elements:
789960--
790−-- When the underlying monad @m@ is strict (e.g. IO), then @foldr@ ends up
791−-- evaluating all of its input because of strict evaluation of the recursive
792−-- call:
961+-- >>> S.toList $ S.foldrS (\x xs -> if odd x then (x + 2) `S.cons` xs else xs) S.nil $ (S.fromList [1..5] :: SerialT IO Int)
962+-- > [3,5,7]
793963--
794−-- >> S.foldr (\_ _ -> []) [] $ S.fromList (1:undefined)
795−-- >*** Exception: Prelude.undefined
964+-- 'foldrM' can also be represented in terms of 'foldrS', however, the former
965+-- is much more efficient:
796966--
797−-- In a lazy monad, we can consume the input lazily, and terminate the fold
798−-- by conditionally not inspecting the recursive expression.
967+-- > foldrM f z s = runIdentityT $ foldrS (\x xs -> lift $ f x (runIdentityT xs)) (lift z) s
799968--
800−-- >> runIdentity $ S.foldr (\x rest -> if x == 3 then [] else x : rest) [] $ S.fromList (4:1:3:undefined)
801−-- >[4,1]
969+-- @since 0.7.0
970+{-# INLINE foldrS #-}
971+foldrS :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b
972+foldrS = K.foldrS
973+
974+-- | Right fold to a transformer monad. This is the most general right fold
975+-- function. 'foldrS' is a special case of 'foldrT', however 'foldrS'
976+-- implementation can be more efficient:
977+--
978+-- > foldrS = foldrT
979+-- > foldrM f z s = runIdentityT $ foldrT (\x xs -> lift $ f x (runIdentityT xs)) (lift z) s
802980--
803−-- The arguments to the folding function (@a -> b -> b@) are in the head and
804−-- tail order of the output, @a@ is the head and @b@ is the tail. Remember, in
805−-- a right fold the zero is on the right, it is the tail end.
981+-- 'foldrT' can be used to translate streamly streams to other transformer
982+-- monads e.g. to a different streaming type.
983+--
984+-- @since 0.7.0
985+{-# INLINE foldrT #-}
986+foldrT :: (IsStream t, Monad m, Monad (s m), MonadTrans s)
987+ => (a -> s m b -> s m b) -> s m b -> t m a -> s m b
988+foldrT f z s = S.foldrT f z (toStreamS s)
989+
990+-- | Note that with this signature we cannot implement a lazy foldr when the
991+-- monad @m@ is strict. In that case it would be strict in its accumulator and
992+-- therefore would necessarily consume all its input. For this reason we have
993+-- deprecated this API. Though this can be useful for lazy monads, but we can
994+-- always achieve the same thing using foldrM directly.
806995--
807996-- @since 0.1.0
808997{-# INLINE foldr #-}
998+{-# DEPRECATED foldr "This is unnecessarily strict for strict monads. Use foldrM instead." #-}
809999foldr :: Monad m => (a -> b -> b) -> b -> SerialT m a -> m b
8101000foldr = P.foldr
8111001
@@ -817,6 +1007,7 @@ foldr = P.foldr
8171007--
8181008-- @since 0.5.0
8191009{-# INLINE foldr1 #-}
1010+{-# DEPRECATED foldr1 "This is unnecessarily strict for strict monads. Use foldrM instead." #-}
8201011foldr1 :: Monad m => (a -> a -> a) -> SerialT m a -> m (Maybe a)
8211012foldr1 f m = S.foldr1 f (toStreamS m)
8221013
@@ -828,48 +1019,59 @@ foldr1 f m = S.foldr1 f (toStreamS m)
8281019-- @since 0.2.0
8291020{-# INLINE foldx #-}
8301021foldx :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b
831−foldx = K.foldx
1022+foldx = P.foldlx'
8321023
8331024-- |
8341025-- @since 0.1.0
8351026{-# DEPRECATED foldl "Please use foldx instead." #-}
8361027foldl :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b
837−foldl = foldx
1028+foldl = P.foldlx'
8381029
839−-- | Strict left associative fold.
1030+-- | Strict left associative/push fold. Note that the observations below about
1031+-- the behavior of a left fold assume that we are working on a right associated
1032+-- structure like cons lists and streamly streams. If we are working on a left
1033+-- associated structure (e.g. snoc lists) the roles of right and left folds
1034+-- would reverse.
8401035--
841−-- For lists a @foldl@ looks like:
1036+-- Let's take a closer look at the @foldl@ definition for lists given above:
8421037--
8431038-- @
844−-- foldl f z [] = z
8451039-- foldl f z (x:xs) = foldl f (z \`f` x) xs
8461040-- @
8471041--
1042+-- @foldl@ calls itself recursively, in each call it invokes @f@ as @f z x@
1043+-- providing it with the result accumulated till now @z@ (the state) and the
1044+-- next element from the input container. First call to @f@ is supplied with
1045+-- the initial value of the accumulator @z@ and each subsequent call uses the
1046+-- output of the previous call to @f z x@.
1047+--
1048+-- >> foldl' (+) 0 [1,2,3]
1049+-- > 6
1050+--
8481051-- The recursive call at the head of the output expression is bound to be
849−-- evaluated until recursion terminates,
850−-- /deconstructing the whole input container/ and building the following left
851−-- associated expression:
1052+-- evaluated until recursion terminates, therefore, a left fold always
1053+-- /consumes the whole input container/. The following would result in an
1054+-- error, even though the fold is not using the values at all:
8521055--
853−-- @
854−-- foldl f z xs == (((z \`f` x1) \`f` x2) ...) \`f` xn
855−-- @
1056+-- >> foldl' (\_ _ -> 0) 0 (1:undefined)
1057+-- > *** Exception: Prelude.undefined
8561058--
857−-- When `f` is a constructor, we can see that the first deconstruction of this
858−-- expression would be the recursive expression on the left and `xn` on the
859−-- right. Therefore, it can access the input elements only in the reverse
860−-- (LIFO) order. For example:
1059+-- As @foldl@ recurses, it builds the left associated expression shown below.
1060+-- Notice, the order of the arguments to the step function @f b a@. It follows
1061+-- the left associative recursive expression generated by expanding @b \`f` a@.
8611062--
8621063-- @
863−-- > S.foldl' (flip (:)) [] $ S.fromList [1,2,3,4]
864−-- [4,3,2,1]
1064+-- foldl f z xs == (((z \`f` x1) \`f` x2) ...) \`f` xn
8651065-- @
8661066--
1067+--
8671068-- The strict left fold @foldl'@ forces the reduction of its argument @z \`f`
8681069-- x@ before using it, therefore it never builds the whole expression in
8691070-- memory. Thus, @z \`f` x1@ would get reduced to @z1@ and then @z1 \`f` x2@
8701071-- would get reduced to @z2@ and so on, incrementally reducing the expression
871−-- as it recurses. However, it evaluates the accumulator only to WHNF, it may
872−-- further help to use a strict data structure as accumulator. For example:
1072+-- from left to right as it recurses, consuming the input in FIFO order. Try
1073+-- out @foldl' (+) 0 (map (\\x -> trace (show x) x) [1..10])@ to see how it
1074+-- works. For example:
8731075--
8741076-- @
8751077-- > S.foldl' (+) 0 $ S.fromList [1,2,3,4]
@@ -877,36 +1079,44 @@ foldl = foldx
8771079-- @
8781080--
8791081-- @
880−-- 0 + 1
881−-- (0 + 1) + 2
882−-- ((0 + 1) + 2) + 3
883−-- (((0 + 1) + 2) + 3) + 4
1082+-- 0 + 1 = 1
1083+-- 1 + 2 = 3
1084+-- 3 + 3 = 6
1085+-- 6 + 4 = 10
8841086-- @
8851087--
886−-- @foldl@ strictly deconstructs the whole input container irrespective of
887−-- whether it needs it or not:
1088+-- However, @foldl'@ evaluates the accumulator only to WHNF. It may further
1089+-- help if the step function uses a strict data structure as accumulator to
1090+-- improve performance and to keep the expression fully reduced at all times
1091+-- during the fold.
8881092--
889−-- >> S.foldl' (\acc x -> if x == 3 then acc else x : acc) [] $ S.fromList (4:1:3:undefined)
890−-- >*** Exception: Prelude.undefined
1093+-- A left fold can also build a new structure instead of reducing one if a
1094+-- constructor is used as a fold step. However, it may not be very useful
1095+-- because it will consume the whole input and construct the new structure in
1096+-- memory before we can consume it. Thus the whole structure gets buffered in
1097+-- memory. When the list constructor is used it would build a new list in
1098+-- reverse (LIFO) order:
8911099--
892−-- However, evaluation of the items contained in the input container is lazy as
893−-- demanded by the fold step function:
894−--
895−-- >> S.foldl' (\acc x -> if x == 3 then acc else x : acc) [] $ S.fromList [4,1,3,undefined]
896−-- >[4,1]
1100+-- @
1101+-- > S.foldl' (flip (:)) [] $ S.fromList [1,2,3,4]
1102+-- [4,3,2,1]
1103+-- @
8971104--
898−-- To perform a left fold without consuming all the input one can use @scanl@
899−-- to stream the intermediate results of the fold and use them lazily.
1105+-- A left fold is a push fold. The producer pushes its contents to the step
1106+-- function of the fold. The step function therefore has no control to stop the
1107+-- input, it can only discard it if it does not need it. We can also consider a
1108+-- left fold as a state machine where the state is store in the accumulator,
1109+-- the state can be modified based on new inputs that are pushed to the fold.
9001110--
901−-- In stateful or event-driven programming, we can consider @z@ as the initial
902−-- state and the stream being folded as a stream of events, thus @foldl'@
903−-- processes all the events in the stream updating the state on each event and
904−-- then ultimately returning the final state.
1111+-- In general, a strict left fold is a reducing fold, whereas a right fold is a
1112+-- constructing fold. A strict left fold reduces in a FIFO order whereas it
1113+-- constructs in a LIFO order, and vice-versa for the right fold. See the
1114+-- documentation of 'foldrM' for a discussion on where a left or right fold is
1115+-- suitable.
9051116--
906−-- The arguments to the folding function (@b -> a -> b@) are in the head and
907−-- tail order of the output expression, @b@ is the head and @a@ is the tail.
908−-- Remember, in a left fold the zero is on the left, at the head of the
909−-- expression.
1117+-- To perform a left fold lazily without having to consume all the input one
1118+-- can use @scanl@ to stream the intermediate results of the fold and consume
1119+-- the resulting stream lazily.
9101120--
9111121-- @since 0.2.0
9121122{-# INLINE foldl' #-}
@@ -932,13 +1142,13 @@ foldl1' step m = do
9321142-- @since 0.2.0
9331143{-# INLINE foldxM #-}
9341144foldxM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b
935−foldxM = K.foldxM
1145+foldxM = P.foldlMx'
9361146
9371147-- |
9381148-- @since 0.1.0
9391149{-# DEPRECATED foldlM "Please use foldxM instead." #-}
9401150foldlM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b
941−foldlM = foldxM
1151+foldlM = P.foldlMx'
9421152
9431153-- | Like 'foldl'' but with a monadic step function.
9441154--
@@ -951,7 +1161,10 @@ foldlM' step begin m = S.foldlM' step begin $ toStreamS m
9511161-- Specialized folds
9521162------------------------------------------------------------------------------
9531163
954−-- | Run a stream, discarding the results. By default it interprets the stream
1164+-- |
1165+-- > runStream = mapM_ (\_ -> return ())
1166+--
1167+-- Run a stream, discarding the results. By default it interprets the stream
9551168-- as 'SerialT', to run other types of streams use the type adapting
9561169-- combinators for example @runStream . 'asyncly'@.
9571170--
@@ -996,7 +1209,10 @@ null = K.null
9961209head :: Monad m => SerialT m a -> m (Maybe a)
9971210head = K.head
9981211
999−-- | Extract all but the first element of the stream, if any.
1212+-- |
1213+-- > tail = fmap (fmap snd) . uncons
1214+--
1215+-- Extract all but the first element of the stream, if any.
10001216--
10011217-- @since 0.1.1
10021218{-# INLINE tail #-}
@@ -1190,24 +1406,6 @@ elemIndices a = findIndices (==a)
11901406elemIndex :: (Monad m, Eq a) => a -> SerialT m a -> m (Maybe Int)
11911407elemIndex a = findIndex (== a)
11921408
1193−-- | Map each element to a stream and then flatten the results into a single
1194−-- stream.
1195−--
1196−-- > concatMap f = concatMapM (return . f)
1197−--
1198−-- @since 0.6.0
1199−{-# INLINE concatMap #-}
1200−concatMap ::(IsStream t, Monad m) => (a -> t m b) -> t m a -> t m b
1201−concatMap f m = fromStreamD $ D.concatMap (toStreamD . f) (toStreamD m)
1202−
1203−-- | Map each element to a stream using a monadic function and then flatten the
1204−-- results into a single stream.
1205−--
1206−-- @since 0.6.0
1207−{-# INLINE concatMapM #-}
1208−concatMapM :: (IsStream t, Monad m) => (a -> m (t m b)) -> t m a -> t m b
1209−concatMapM f m = fromStreamD $ D.concatMapM (fmap toStreamD . f) (toStreamD m)
1210−
12111409------------------------------------------------------------------------------
12121410-- Substreams
12131411------------------------------------------------------------------------------
@@ -1256,8 +1454,12 @@ stripPrefix m1 m2 = fmap fromStreamD <$>
12561454------------------------------------------------------------------------------
12571455
12581456-- XXX this can utilize parallel mapping if we implement it as runStream . mapM
1259−-- | Apply a monadic action to each element of the stream and discard the
1260−-- output of the action.
1457+-- |
1458+-- > mapM_ = runStream . mapM
1459+--
1460+-- Apply a monadic action to each element of the stream and discard the output
1461+-- of the action. This is not really a pure transformation operation but a
1462+-- transformation followed by fold.
12611463--
12621464-- @since 0.1.0
12631465{-# INLINE mapM_ #-}
@@ -1273,13 +1475,27 @@ mapM_ f m = S.mapM_ f $ toStreamS m
12731475-- toList = S.foldr (:) []
12741476-- @
12751477--
1276−-- Convert a stream into a list in the underlying monad. Same as:
1478+-- Convert a stream into a list in the underlying monad. The list can be
1479+-- consumed lazily in a lazy monad (e.g. 'Identity'). In a strict monad (e.g.
1480+-- IO) the whole list is generated before it can be consumed.
12771481--
12781482-- @since 0.1.0
12791483{-# INLINE toList #-}
12801484toList :: Monad m => SerialT m a -> m [a]
12811485toList = P.toList
12821486
1487+-- |
1488+-- @
1489+-- toRevList = S.foldl' (flip (:)) []
1490+-- @
1491+--
1492+-- Convert a stream into a list in reverse order in the underlying monad.
1493+--
1494+-- @since 0.7.0
1495+{-# INLINE toRevList #-}
1496+toRevList :: Monad m => SerialT m a -> m [a]
1497+toRevList = D.toRevList . toStreamD
1498+
12831499-- |
12841500-- @
12851501-- toHandle h = S.mapM_ $ hPutStrLn h
@@ -1301,6 +1517,16 @@ toHandle h m = go m
13011517-- Transformation by Folding (Scans)
13021518------------------------------------------------------------------------------
13031519
1520+-- XXX It may be useful to have a version of scan where we can keep the
1521+-- accumulator independent of the value emitted. So that we do not necessarily
1522+-- have to keep a value in the accumulator which we are not using. We can pass
1523+-- an extraction function that will take the accumulator and the current value
1524+-- of the element and emit the next value in the stream. That will also make it
1525+-- possible to modify the accumulator after using it. In fact, the step function
1526+-- can return new accumulator and the value to be emitted. The signature would
1527+-- be more like mapAccumL. Or we can change the signature of scanx to
1528+-- accommodate this.
1529+--
13041530-- | Strict left scan with an extraction function. Like 'scanl'', but applies a
13051531-- user supplied extraction function (the third argument) at each step. This is
13061532-- designed to work with the @foldl@ library. The suffix @x@ is a mnemonic for
@@ -1309,7 +1535,7 @@ toHandle h m = go m
13091535-- @since 0.2.0
13101536{-# INLINE scanx #-}
13111537scanx :: IsStream t => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b
1312−scanx = K.scanx
1538+scanx = K.scanx'
13131539
13141540-- |
13151541-- @since 0.1.1
@@ -1325,7 +1551,8 @@ scan = scanx
13251551scanlM' :: (IsStream t, Monad m) => (b -> a -> m b) -> b -> t m a -> t m b
13261552scanlM' step begin m = fromStreamD $ D.scanlM' step begin $ toStreamD m
13271553
1328−-- | Strict left scan.
1554+-- | Strict left scan. Like 'map', 'scanl'' too is a one to one transformation,
1555+-- however it adds an extra element.
13291556--
13301557-- @
13311558-- > S.toList $ S.scanl' (+) 0 $ fromList [1,2,3,4]
@@ -1346,16 +1573,16 @@ scanlM' step begin m = fromStreamD $ D.scanlM' step begin $ toStreamD m
13461573-- thus modularizing the stream processing. This can be useful in
13471574-- stateful or event-driven programming.
13481575--
1349−-- Consider the following example, computing the sum and the product of the
1350−-- elements in a stream in one go using a @foldl'@:
1576+-- Consider the following monolothic example, computing the sum and the product
1577+-- of the elements in a stream in one go using a @foldl'@:
13511578--
13521579-- @
13531580-- > S.foldl' (\\(s, p) x -> (s + x, p * x)) (0,1) $ S.fromList \[1,2,3,4]
13541581-- (10,24)
13551582-- @
13561583--
1357−-- Using @scanl'@ we can compute the sum in the first stage and pass it down to
1358−-- the next stage for computing the product:
1584+-- Using @scanl'@ we can make it modular by computing the sum in the first
1585+-- stage and passing it down to the next stage for computing the product:
13591586--
13601587-- @
13611588-- > S.foldl' (\\(_, p) (s, x) -> (s, p * x)) (0,1)
@@ -1609,19 +1836,16 @@ mapMaybeMSerial f m = fromStreamD $ D.mapMaybeM f $ toStreamD m
16091836-- XXX to scale this we need to use a slab allocated array backed
16101837-- representation for temporary storage.
16111838--
1612−-- | Returns the elements of the stream in reverse order.
1613−-- The stream must be finite.
1839+-- |
1840+-- > reverse = S.foldlT (flip S.cons) S.nil
1841+--
1842+-- Returns the elements of the stream in reverse order. The stream must be
1843+-- finite.
16141844--
16151845-- @since 0.1.1
1616−reverse :: (IsStream t) => t m a -> t m a
1617−reverse m = go K.nil m
1618− where
1619− go rev rest = K.mkStream $ \st yld sng stp ->
1620− let runIt x = K.foldStream st yld sng stp x
1621− stop = runIt rev
1622− single a = runIt $ a `K.cons` rev
1623− yieldk a r = runIt $ go (a `K.cons` rev) r
1624− in K.foldStream st yieldk single stop rest
1846+{-# INLINE reverse #-}
1847+reverse :: (IsStream t, Monad m) => t m a -> t m a
1848+reverse s = fromStreamS $ S.reverse $ toStreamS s
16251849
16261850------------------------------------------------------------------------------
16271851-- Transformation by Inserting
@@ -1682,9 +1906,10 @@ deleteBy cmp x m = fromStreamS $ S.deleteBy cmp x (toStreamS m)
16821906------------------------------------------------------------------------------
16831907
16841908-- |
1685−-- > indexed = S.zipWith (,) (S.intFrom 0)
1909+-- > indexed = S.postscanl' (\(i, _) x -> (i + 1, x)) (-1,undefined)
1910+-- > indexed = S.zipWith (,) (S.enumerateFrom 0)
16861911--
1687−-- Pair each element in a stream with its index.
1912+-- Pair each element in a stream with its index, starting from index 0.
16881913--
16891914-- @
16901915-- > S.toList $ S.indexed $ S.fromList "hello"
@@ -1697,14 +1922,15 @@ indexed :: (IsStream t, Monad m) => t m a -> t m (Int, a)
16971922indexed = fromStreamD . D.indexed . toStreamD
16981923
16991924-- |
1700−-- > indexedR n = S.zipWith (,) (S.intFromThen n (n - 1))
1925+-- > indexedR n = S.postscanl' (\(i, _) x -> (i - 1, x)) (n + 1,undefined)
1926+-- > indexedR n = S.zipWith (,) (S.enumerateFromThen n (n - 1))
17011927--
17021928-- Pair each element in a stream with its index, starting from the
17031929-- given index @n@ and counting down.
17041930--
17051931-- @
17061932-- > S.toList $ S.indexedR 10 $ S.fromList "hello"
1707−-- [(9,'h'),(8,'e'),(7,'l'),(6,'l'),(5,'o')]
1933+-- [(10,'h'),(9,'e'),(8,'l'),(7,'l'),(6,'o')]
17081934-- @
17091935--
17101936-- @since 0.6.0
@@ -1847,3 +2073,25 @@ mergeAsyncByM f m1 m2 = K.mkStream $ \st stp sng yld -> do
18472073 ma <- mkAsync' st m1
18482074 mb <- mkAsync' st m2
18492075 K.foldStream st stp sng yld (K.mergeByM f ma mb)
2076+
2077+------------------------------------------------------------------------------
2078+-- Nesting
2079+------------------------------------------------------------------------------
2080+
2081+-- | Map each element of the stream to produce a stream and then flatten the
2082+-- results into a single stream.
2083+--
2084+-- > concatMap f = concatMapM (return . f)
2085+--
2086+-- @since 0.6.0
2087+{-# INLINE concatMap #-}
2088+concatMap ::(IsStream t, Monad m) => (a -> t m b) -> t m a -> t m b
2089+concatMap f m = fromStreamD $ D.concatMap (toStreamD . f) (toStreamD m)
2090+
2091+-- | Map each element to a stream to produce a stream using a monadic function
2092+-- and then flatten the results into a single stream.
2093+--
2094+-- @since 0.6.0
2095+{-# INLINE concatMapM #-}
2096+concatMapM :: (IsStream t, Monad m) => (a -> m (t m b)) -> t m a -> t m b
2097+concatMapM f m = fromStreamD $ D.concatMapM (fmap toStreamD . f) (toStreamD m)