git.iko.soy/streamly
on master help sign in
Update Fold module docs and expose some APIs
commit3b1a26cee5e56ef9eab348ad62bcff0fd29052c5 authorHarendra Kumar <harendra@composewell.com> dateSun 7 Mar 2021, 12:35 (6 years ago) parentbadc62d
13 files changed+658−505
compare with master
unifiedsplit
Changelog.md+7−5 benchmark/Streamly/Benchmark/Data/Fold.hs−1 benchmark/Streamly/Benchmark/Prelude/Concurrent.hs+20−17 src/Streamly/Data/Fold.hs+168−179 src/Streamly/Internal/Data/Fold.hs+171−176 src/Streamly/Internal/Data/Fold/Tee.hs+57−25 src/Streamly/Internal/Data/Fold/Type.hs+225−95 src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs+6−3 src/Streamly/Internal/Data/Stream/IsStream/Nesting.hs+1−1 src/Streamly/Internal/Data/Stream/IsStream/Transform.hs+1−1 src/Streamly/Internal/Network/Socket.hs−1 src/Streamly/Prelude.hs+2 test/lib/Streamly/Test/Prelude/Common.hs−1
Changelog.md+7−5
@@ -2,9 +2,11 @@
22
33### Behavioral changes
44
5−* `Streamly.Prelude.fold` can now terminate early without consuming the entire
5+* `Streamly.Prelude.fold` may now terminate early without consuming the entire
66 stream. For example, `fold Fold.head stream` would now terminate immediately
7− after consuming the head element from `stream`.
7+ after consuming the head element from `stream`. This may result in change of
8+ behavior in existing programs if the program relies on the evaluation of the
9+ full stream.
810* Change the associativity of combinators `serial`, `wSerial`,
911 `ahead`, `async`, `wAsync`, `parallel` to be the same as `<>`.
1012* `encodeUtf8`, `decodeUtf8` now replace any invalid character encountered
@@ -26,14 +28,14 @@
2628 require an additional `MonadAsync` constraint. Several other
2729 functions that used these functions also now require the additional
2830 constraint.
29−* Remove `Applicative` instance of folds. Please use `teeWith` or the `Tee` type
30− as an alternative to Fold applicative.
31+* Remove `Applicative` instance of folds. Please use `teeWith` or the `Tee`
32+ type as an alternative to Fold applicative.
3133
3234### Enhancements
3335
3436* New encoding/decoding routines, `encodeUtf8'`, `encodeLatin1'`, `decodeUtf8'`,
3537 are added, these routines fail when they encounter any invalid characters.
36−* `teeWith` is released as a part of `Streamly.Data.Fold`
38+* Several new functions added to `Streamly.Data.Fold`
3739
3840### Bug Fixes
3941
benchmark/Streamly/Benchmark/Data/Fold.hs−1
@@ -19,7 +19,6 @@ import Streamly.Internal.Data.Fold (Fold(..))
1919
2020import qualified Data.Map.Strict as Map
2121import qualified Streamly.Internal.Data.Fold as FL
22−import qualified Streamly.Internal.Data.Fold.Type as FL
2322import qualified Streamly.Internal.Data.Pipe as Pipe
2423import qualified Streamly.Internal.Data.Sink as Sink
2524import qualified Streamly.Internal.Data.Stream.IsStream as IP
benchmark/Streamly/Benchmark/Prelude/Concurrent.hs+20−17
@@ -8,9 +8,12 @@
88
99import Control.Concurrent
1010import Control.Monad (when, replicateM)
11+import Streamly.Prelude
12+ ( IsStream, SerialT, serial, async, asyncly, ahead, aheadly, wAsync
13+ , wAsyncly, parallel, parallely
14+ )
1115
1216import Gauge
13−import Streamly.Prelude hiding (mapM_, replicateM)
1417import qualified Streamly.Prelude as S
1518
1619-------------------------------------------------------------------------------
@@ -28,8 +31,8 @@ append buflen tcount d t =
2831 let work = (\i -> when (d /= 0) (threadDelay d) >> return i)
2932 in S.drain
3033 $ t
31− $ maxBuffer buflen
32− $ maxThreads (-1)
34+ $ S.maxBuffer buflen
35+ $ S.maxThreads (-1)
3336 $ S.fromFoldableM $ fmap work [1..tcount]
3437
3538-- | Run @threads@ concurrently, each producing streams of @elems@ elements
@@ -48,28 +51,28 @@ concated
4851concated buflen threads d elems t =
4952 let work = \i -> S.replicateM i (when (d /= 0) (threadDelay d) >> return i)
5053 in S.drain
51− $ adapt
52− $ maxThreads (-1)
53− $ maxBuffer buflen
54+ $ S.adapt
55+ $ S.maxThreads (-1)
56+ $ S.maxBuffer buflen
5457 $ S.concatMapWith t work
5558 $ S.replicate threads elems
5659
5760appendGroup :: Int -> Int -> Int -> [Benchmark]
58−appendGroup buflen threads delay =
61+appendGroup buflen threads usec =
5962 [ -- bench "serial" $ nfIO $ append buflen threads delay serially
60− bench "ahead" $ nfIO $ append buflen threads delay aheadly
61− , bench "async" $ nfIO $ append buflen threads delay asyncly
62− , bench "wAsync" $ nfIO $ append buflen threads delay wAsyncly
63− , bench "parallel" $ nfIO $ append buflen threads delay parallely
63+ bench "ahead" $ nfIO $ append buflen threads usec aheadly
64+ , bench "async" $ nfIO $ append buflen threads usec asyncly
65+ , bench "wAsync" $ nfIO $ append buflen threads usec wAsyncly
66+ , bench "parallel" $ nfIO $ append buflen threads usec parallely
6467 ]
6568
6669concatGroup :: Int -> Int -> Int -> Int -> [Benchmark]
67−concatGroup buflen threads delay n =
68− [ bench "serial" $ nfIO $ concated buflen threads delay n serial
69− , bench "ahead" $ nfIO $ concated buflen threads delay n ahead
70− , bench "async" $ nfIO $ concated buflen threads delay n async
71− , bench "wAsync" $ nfIO $ concated buflen threads delay n wAsync
72− , bench "parallel" $ nfIO $ concated buflen threads delay n parallel
70+concatGroup buflen threads usec n =
71+ [ bench "serial" $ nfIO $ concated buflen threads usec n serial
72+ , bench "ahead" $ nfIO $ concated buflen threads usec n ahead
73+ , bench "async" $ nfIO $ concated buflen threads usec n async
74+ , bench "wAsync" $ nfIO $ concated buflen threads usec n wAsync
75+ , bench "parallel" $ nfIO $ concated buflen threads usec n parallel
7376 ]
7477
7578main :: IO ()
src/Streamly/Data/Fold.hs+168−179
@@ -1,79 +1,142 @@
11-- |
22-- Module : Streamly.Data.Fold
33-- Copyright : (c) 2019 Composewell Technologies
4−-- License : BSD3
4+-- License : BSD-3-Clause
55-- Maintainer : streamly@composewell.com
66-- Stability : experimental
77-- Portability : GHC
88--
9−-- The 'Fold' type represents an effectful action that absorbs a value into an
10−-- accumulator. For example, a 'sum' 'Fold' represents adding the input to the
11−-- accumulated sum. A fold driver e.g. 'Streamly.Prelude.fold' pushes values
12−-- from a stream to the 'Fold' one at a time, reducing the stream to a single
13−-- value. Therefore, a fold can also be thought of as a sink or a stream
14−-- consumer.
9+-- A 'Fold' is a sink or a consumer of a stream of values. The 'Fold' type
10+-- consists of an accumulator and an effectful action that absorbs a value into
11+-- the accumulator.
1512--
16−-- A 'Fold' is in fact a data representation of a standard left fold ('foldl').
17−-- Unlike the standard left folds, 'Fold's can terminate at any point e.g. the
18−-- 'head' fold would terminate immediately after returning the head element.
13+-- >>> import qualified Streamly.Data.Fold as Fold
14+-- >>> import qualified Streamly.Prelude as Stream
15+--
16+-- For example, a 'sum' Fold represents adding the input to the accumulated
17+-- sum. A fold driver e.g. 'Streamly.Prelude.fold' pushes values from a stream
18+-- to the 'Fold' one at a time, reducing the stream to a single value.
19+--
20+-- >>> Stream.fold Fold.sum $ Stream.fromList [1..100]
21+-- 5050
22+--
23+-- Conceptually, a 'Fold' is a data type that can mimic a strict left fold
24+-- ('Data.List.foldl') as well as lazy right fold ('Prelude.foldr'). The above
25+-- example is similar to a left fold using @(+)@ as the step and @0@ as the
26+-- initial value of the accumulator:
27+--
28+-- >>> Data.List.foldl' (+) 0 [1..100]
29+-- 5050
30+--
31+-- 'Fold's have an early termination capability e.g. the 'head' fold would
32+-- terminate on an infinite stream:
33+--
34+-- >>> Stream.fold Fold.head $ Stream.fromList [1..]
35+-- Just 1
36+--
37+-- The above example is similar to the following right fold:
38+--
39+-- >>> Prelude.foldr (\x _ -> Just x) Nothing [1..]
40+-- Just 1
41+--
42+-- 'Fold's can be combined together using combinators. For example, to create a
43+-- fold that sums first two elements in a stream:
44+--
45+-- >>> sumTwo = Fold.take 2 Fold.sum
46+-- >>> Stream.fold sumTwo $ Stream.fromList [1..100]
47+-- 3
48+--
49+-- Folds can be combined to run in parallel on the same input. For example, to
50+-- compute the average of numbers in a stream without going through the stream
51+-- twice:
52+--
53+-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
54+-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]
55+-- 50.5
56+--
57+-- Folds can be combined so as to partition the input stream over multiple
58+-- folds. For example, to count even and odd numbers in a stream:
1959--
20−-- 'Fold's can be combined efficiently using combinators provided in this
21−-- module; a stream supplied to the combined fold is provided to the
22−-- constituent folds according to the behavior of the combinator. For example,
23−-- the 'tee' combinator distributes the input stream to two folds and then
24−-- combines the resulting fold outputs. Similarly, a 'partition' combinator
25−-- divides the input among constituent folds.
60+-- >>> split n = if even n then Left n else Right n
61+-- >>> stream = Stream.map split $ Stream.fromList [1..100]
62+-- >>> countEven = fmap (("Even " ++) . show) Fold.length
63+-- >>> countOdd = fmap (("Odd " ++) . show) Fold.length
64+-- >>> f = Fold.partition countEven countOdd
65+-- >>> Stream.fold f stream
66+-- ("Even 50","Odd 50")
2667--
27−-- = Accumulators and Terminating Folds
68+-- Terminating folds can be combined to parse the stream serially such that the
69+-- first fold consumes the input until it terminates and the second fold
70+-- consumes the rest of the input until it terminates:
2871--
29−-- Folds in this module can be classified in two categories viz. accumulators
30−-- and terminating folds. Accumulators do not have a terminating condition,
31−-- they run forever and consume the entire stream, for example the 'length'
32−-- fold. Terminating folds have a terminating condition and can terminate
33−-- without consuming the entire stream, for example, the 'head' fold.
72+-- >>> f = Fold.serialWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)
73+-- >>> Stream.fold f $ Stream.fromList "header: hello\n"
74+-- ("header: ","hello\n")
3475--
35−-- = Monoids
76+-- A 'Fold' can be applied repeatedly on a stream to transform it to a stream
77+-- of fold results. To split a stream on newlines:
3678--
37−-- Monoids allow generalized, modular folding. The accumulators in this module
38−-- can be expressed using 'mconcat' and a suitable 'Monoid'. Instead of
39−-- writing folds we can write Monoids and turn them into folds.
79+-- >>> f = Fold.takeEndBy (== '\n') Fold.toList
80+-- >>> Stream.toList $ Stream.foldMany f $ Stream.fromList "Hello there!\nHow are you\n"
81+-- ["Hello there!\n","How are you\n"]
4082--
41−-- = Performance Notes
83+-- Similarly, we can split the input of a fold too:
4284--
43−-- 'Streamly.Prelude' module provides fold functions to directly fold streams
44−-- e.g. Streamly.Prelude/'Streamly.Prelude.sum' serves the same purpose as
45−-- Fold/'sum'. However, the functions in Streamly.Prelude cannot be
46−-- efficiently combined together e.g. we cannot drive the input stream through
47−-- @sum@ and @length@ fold functions simultaneously. Using the 'Fold' type we
48−-- can efficiently split the stream across mutliple folds because it allows the
49−-- compiler to perform stream fusion optimizations.
85+-- >>> Stream.fold (Fold.many f Fold.toList) $ Stream.fromList "Hello there!\nHow are you\n"
86+-- ["Hello there!\n","How are you\n"]
5087--
51−-- = Programmer Notes
88+-- Please see "Streamly.Internal.Data.Fold" for additional @Pre-release@
89+-- functions.
5290--
53−-- > import qualified Streamly.Data.Fold as Fold
91+-- = Folds vs. Streams
5492--
55−-- More, not yet exposed, fold combinators can be found in
56−-- "Streamly.Internal.Data.Fold".
93+-- We can often use streams or folds to achieve the same goal. However, streams
94+-- allow efficient composition of producers (e.g. 'Streamly.Prelude.serial' or
95+-- 'Streamly.Prelude.mergeBy') whereas folds allow efficient composition of
96+-- consumers (e.g. 'serialWith', 'partition' or 'teeWith').
97+--
98+-- Streams are producers, transformations on streams happen on the output side:
99+--
100+-- >>> f = Stream.sum . Stream.map (+1) . Stream.filter odd
101+-- >>> f $ Stream.fromList [1..100]
102+-- 2550
103+--
104+-- Folds are stream consumers with an input stream and an output value, stream
105+-- transformations on folds happen on the input side:
106+--
107+-- >>> f = Fold.filter odd $ Fold.lmap (+1) $ Fold.sum
108+-- >>> Stream.fold f $ Stream.fromList [1..100]
109+-- 2550
110+--
111+-- Notice the composition by @.@ vs @$@ and the order of operations in the
112+-- above examples, the difference is due to output vs input side
113+-- transformations.
57114
58115module Streamly.Data.Fold
59116 (
60117 -- * Fold Type
61− -- |
62− -- A 'Fold' can be run over a stream using the 'Streamly.Prelude.fold'
63− -- combinator:
64− --
65− -- >>> Stream.fold Fold.sum (Stream.enumerateFromTo 1 100)
66− -- 5050
67118
68119 Fold -- (..)
69120
70− -- * Accumulators
71− -- ** Monoids
121+ -- * Constructors
122+ , mkFoldl
123+ , mkFoldlM
124+ , mkFoldr
125+
126+ -- * Folds
127+ -- ** Accumulators
128+ -- | Folds that never terminate, these folds are much like strict left
129+ -- folds. 'mconcat' is the fundamental accumulator. All other accumulators
130+ -- can be expressed in terms of 'mconcat' using a suitable Monoid. Instead
131+ -- of writing folds we could write Monoids and turn them into folds.
132+
133+ -- Monoids
134+ , sconcat
72135 , mconcat
73136 , foldMap
74137 , foldMapM
75138
76− -- ** Reducers
139+ -- Reducers
77140 , drain
78141 , drainBy
79142 , last
@@ -88,22 +151,16 @@ module Streamly.Data.Fold
88151 , variance
89152 , stdDev
90153
91− -- ** Collectors
92− -- | Avoid using these folds in scalable or performance critical
93− -- applications, they buffer all the input in GC memory which can be
94− -- detrimental to performance if the input is large.
95154
155+ -- Collectors
96156 , toList
157+ , toListRev
158+
159+ -- ** Terminating Folds
160+ -- | These are much like lazy right folds.
97161
98− -- * Terminating Folds
99− -- , drainN
100− -- , drainWhile
101− -- , lastN
102− -- , (!!)
103− -- , genericIndex
104162 , index
105163 , head
106− -- , findM
107164 , find
108165 , lookup
109166 , findIndex
@@ -116,143 +173,82 @@ module Streamly.Data.Fold
116173 , and
117174 , or
118175
119− -- * Output Transformations
120− -- | Unlike stream producer types (e.g. @SerialT m a@) which have only
121− -- output side, folds have an input side as well as an output side. In the
122− -- type @Fold m a b@, the input type is @a@ and the output type is @b@.
123− -- Transformations can be applied either on the input side or on the output
124− -- side. The 'Functor' instance of a fold maps on the output of the fold:
176+ -- * Combinators
177+ -- | Combinators are modifiers of folds. In the type @Fold m a b@, @a@ is
178+ -- the input type and @b@ is the output type. Transformations can be
179+ -- applied either on the input side or on the output side. Therefore,
180+ -- combinators are of one of the following general shapes:
181+ --
182+ -- * @... -> Fold m a b -> Fold m c b@ (input transformation)
183+ -- * @... -> Fold m a b -> Fold m a c@ (output transformation)
184+ --
185+ -- Output transformations are also known as covariant transformations, and
186+ -- input transformations are also known as contravariant transformations.
187+ -- The input side transformations are more interesting for folds. Most of
188+ -- the following sections describe the input transformation operations on a
189+ -- fold. The names and signatures of the operations are consistent with
190+ -- corresponding operations in "Streamly.Prelude". When an operation makes
191+ -- sense on both input and output side we use the prefix @l@ (for left) for
192+ -- input side operations and the prefix @r@ (for right) for output side
193+ -- operations.
194+
195+ -- ** Mapping on output
196+ -- | The 'Functor' instance of a fold maps on the output of the fold:
125197 --
126198 -- >>> Stream.fold (fmap show Fold.sum) (Stream.enumerateFromTo 1 100)
127199 -- "5050"
128200 --
129− -- Note: Output transformations are also known as covariant
130− -- transformations.
131− , sequence
132201 , rmapM
133− , mapM
134−
135− {-
136− -- * Input Transformations
137− -- The input side transformations are more interesting for folds. The
138− -- following sections describe the input transformation operations on a
139− -- fold. The names and signatures of the operations are consistent with
140− -- corresponding operations in "Streamly.Prelude".
141− --
142− -- Note: Input transformations are also known as contravariant
143− -- transformations.
144−
145− -- ** Mapping
146− --, transform
147− -- , lmap
148− --, lsequence
149− -- , lmapM
150−
151− -- -- ** Filtering
152− -- , filter
153− -- , filterM
154− -- , ldeleteBy
155− -- , luniq
156−
157− -- ** Mapping Filters
158− , lmapMaybe
159− , lmapMaybeM
160−
161− -- ** Scanning Filters
162− , lfindIndices
163− , lelemIndices
164202
165− -- ** Insertion
166− -- | Insertion adds more elements to the stream.
203+ -- ** Mapping on Input
204+ , lmap
205+ , lmapM
167206
168− , linsertBy
169− , lintersperseM
207+ -- ** Filtering
208+ , filter
209+ , filterM
170210
171− -- ** Reordering
172− , lreverse
173− -}
211+ -- -- ** Mapping Filters
212+ , catMaybes
213+ , mapMaybe
174214
175− {-
176215 -- ** Trimming
177216 , take
178− -- takeByTime
179− , ldrop
180− , ldropWhile
181− , ldropWhileM
182− -}
217+ -- , takeInterval
218+ , takeEndBy_
219+ , takeEndBy
183220
184− -- * Distributing
185− -- |
186− -- The 'Applicative' instance of a distributing 'Fold' distributes one copy
187− -- of the stream to each fold and combines the results using a function.
188− --
189− -- @
190− --
191− -- |-------Fold m a b--------|
192− -- ---stream m a---| |---m (b,c,...)
193− -- |-------Fold m a c--------|
194− -- | |
195− -- ...
196− -- @
197− --
198− -- To compute the average of numbers in a stream without going through the
199− -- stream twice:
200− --
201− -- >>> let avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
202− -- >>> Stream.fold avg (Stream.enumerateFromTo 1.0 100.0)
203− -- 50.5
204− --
205− -- The 'Semigroup' and 'Monoid' instances of a distributing fold distribute
206− -- the input to both the folds and combines the outputs using Monoid or
207− -- Semigroup instances of the output types:
208− --
209− -- >>> import Data.Monoid (Sum(..))
210− -- >>> Stream.fold (Fold.teeWith (<>) Fold.head Fold.last) (fmap Sum $ Stream.enumerateFromTo 1.0 100.0)
211− -- Just (Sum {getSum = 101.0})
212− --
213− -- The 'Num', 'Floating', and 'Fractional' instances work in the same way.
221+ -- ** Serial Append
222+ , serialWith
223+
224+ -- ** Parallel Distribution
225+ -- | For applicative composition using distribution see
226+ -- "Streamly.Internal.Data.Fold.Tee".
214227
215− , tee
216228 , teeWith
229+ , tee
217230 , distribute
218231
219− -- * Partitioning
220− -- |
221− -- Direct items in the input stream to different folds using a binary
232+ -- ** Partitioning
233+ -- | Direct items in the input stream to different folds using a binary
222234 -- fold selector.
223235
224− -- , partitionByM
225− -- , partitionBy
226236 , partition
227237
228− {-
229− -- * Demultiplexing
230− -- | Direct values in the input stream to different folds using an n-ary
231− -- fold selector.
232−
233− , demux
234− -- , demuxWith
235− , demux_
236− -- , demuxWith_
237−
238− -- * Classifying
239− -- | In an input stream of key value pairs fold values for different keys
240− -- in individual output buckets using the given fold.
238+ -- ** Unzipping
239+ , unzip
241240
242− , classify
243− -- , classifyWith
244− -}
241+ -- ** Splitting
242+ , many
243+ , chunksOf
244+ -- , intervalsOf
245245
246− -- * Unzipping
247− , unzip
248− -- These can be expressed using lmap/lmapM and unzip
249− -- , unzipWith
250− -- , unzipWithM
246+ -- ** Nesting
247+ , concatMap
251248
252− -- -- * Nesting
253− -- , concatMap
254− -- , chunksOf
255− -- , duplicate -- experimental
249+ -- * Deprecated
250+ , sequence
251+ , mapM
256252 )
257253where
258254
@@ -265,10 +261,3 @@ import Prelude
265261 span, splitAt, break, mapM)
266262
267263import Streamly.Internal.Data.Fold
268−
269−--
270−-- $setup
271−-- >>> :m
272−-- >>> import Prelude hiding (head, sum, last, length)
273−-- >>> import qualified Streamly.Prelude as Stream
274−-- >>> import qualified Streamly.Data.Fold as Fold
src/Streamly/Internal/Data/Fold.hs+171−176
@@ -7,7 +7,8 @@
77-- Stability : experimental
88-- Portability : GHC
99--
10−-- See "Streamly.Data.Fold" for an overview.
10+-- See "Streamly.Data.Fold" for an overview and
11+-- "Streamly.Internal.Data.Fold.Types" for design notes.
1112--
1213-- IMPORTANT: keep the signatures consistent with the folds in Streamly.Prelude
1314
@@ -17,9 +18,10 @@ module Streamly.Internal.Data.Fold
1718 Step (..)
1819 , Fold (..)
1920
20− -- * Creating
21+ -- * Constructors
2122 , mkFoldl
2223 , mkFoldlM
24+ , mkFoldl1
2325 , mkFoldr
2426 , mkFoldrM
2527 , mkFold
@@ -27,21 +29,21 @@ module Streamly.Internal.Data.Fold
2729 , mkFoldM
2830 , mkFoldM_
2931
30− -- * Generators
32+ -- * Folds
33+ -- ** Identity
3134 , yield
3235 , yieldM
3336
34− -- * Accumulators
35− -- ** Semigroups and Monoids
37+ -- ** Accumulators
38+ -- *** Semigroups and Monoids
3639 , sconcat
3740 , mconcat
3841 , foldMap
3942 , foldMapM
4043
41− -- ** Reducers
44+ -- *** Reducers
4245 , drain
4346 , drainBy
44− , drainBy2
4547 , last
4648 , length
4749 , mean
@@ -52,7 +54,7 @@ module Streamly.Internal.Data.Fold
5254 , rollingHashFirstN
5355 -- , rollingHashLastN
5456
55− -- ** Saturating Reducers
57+ -- *** Saturating Reducers
5658 -- | 'product' terminates if it becomes 0. Other folds can theoretically
5759 -- saturate on bounded types, and therefore terminate, however, they will
5860 -- run forever on unbounded types like Integer/Double.
@@ -63,14 +65,17 @@ module Streamly.Internal.Data.Fold
6365 , minimumBy
6466 , minimum
6567
66− -- ** Collectors
68+ -- *** Collectors
69+ -- | Avoid using these folds in scalable or performance critical
70+ -- applications, they buffer all the input in GC memory which can be
71+ -- detrimental to performance if the input is large.
6772 , toList
6873 , toListRev
6974 -- $toListRev
7075 , toStream
7176 , toStreamRev
7277
73− -- * Terminating Folds
78+ -- ** Terminating Folds
7479 , drainN
7580 -- , lastN
7681 -- , (!!)
@@ -91,26 +96,21 @@ module Streamly.Internal.Data.Fold
9196 , or
9297 -- , the
9398
94− -- * Adapting
99+ -- * Combinators
100+ -- ** Utilities
101+ , with
102+
103+ -- ** Transforming the Monad
95104 , hoist
96105 , generally
97106
98− -- * Running Incrementally
99− , initialize
100− , runStep
101−
102− -- * Output Transformations
103− , sequence
107+ -- ** Mapping on output
104108 , rmapM
105− , mapM
106109
107− -- * Input Transformations
108−
109− -- ** Mapping
110+ -- ** Mapping on Input
110111 , transform
111112 , map
112113 , lmap
113−
114114 --, lsequence
115115 , lmapM
116116 , indexed
@@ -125,63 +125,67 @@ module Streamly.Internal.Data.Fold
125125 -- ** Mapping Filters
126126 , catMaybes
127127 , mapMaybe
128+ -- , mapMaybeM
128129
129130 {-
130131 -- ** Scanning Filters
131− , lfindIndices
132− , lelemIndices
132+ , findIndices
133+ , elemIndices
133134
134135 -- ** Insertion
135136 -- | Insertion adds more elements to the stream.
136137
137− , linsertBy
138− , lintersperseM
138+ , insertBy
139+ , intersperseM
139140
140141 -- ** Reordering
141− , lreverse
142+ , reverse
142143 -}
143144
144145 -- ** Trimming
145146 , take
146147 , takeInterval
148+
147149 -- By elements
148− , takeEndBy_
149150 , takeEndBy
151+ , takeEndBy_
150152 -- , takeEndBySeq
151153 {-
152− , ldrop
153− , ldropWhile
154− , ldropWhileM
154+ , drop
155+ , dropWhile
156+ , dropWhileM
155157 -}
156158
157− -- * Splitting
158−
159− -- Binary
159+ -- ** Serial Append
160+ , serialWith
160161 -- , tail
161162 -- , init
162163 , splitAt -- spanN
163164 -- , splitIn -- sessionN
164165
165− -- * Distributing
166−
167− , tee
166+ -- ** Parallel Distribution
168167 , teeWith
168+ , tee
169169 , teeWithFst
170170 , teeWithMin
171171 , distribute
172172 -- , distributeFst
173173 -- , distributeMin
174174
175− -- * Partitioning
175+ -- ** Parallel Alternative
176+ , shortest
177+ , longest
176178
179+ -- ** Partitioning
177180 , partitionByM
178181 , partitionByFstM
179182 , partitionByMinM
180183 , partitionBy
181184 , partition
182185
183− -- * Demultiplexing
184−
186+ -- ** Demultiplexing
187+ -- | Direct values in the input stream to different folds using an n-ary
188+ -- fold selector.
185189 , demux -- XXX rename this to demux_
186190 , demuxWith
187191 , demuxDefault -- XXX rename this to demux
@@ -189,31 +193,47 @@ module Streamly.Internal.Data.Fold
189193 -- , demuxWithSel
190194 -- , demuxWithMin
191195
192− -- * Classifying
193−
196+ -- ** Classifying
197+ -- | In an input stream of key value pairs fold values for different keys
198+ -- in individual output buckets using the given fold.
194199 , classify
195200 , classifyWith
196201 -- , classifyWithSel
197202 -- , classifyWithMin
198203
199− -- * Unzipping
204+ -- ** Unzipping
200205 , unzip
206+ -- These two can be expressed using lmap/lmapM and unzip
201207 , unzipWith
202208 , unzipWithM
203209 , unzipWithFstM
204210 , unzipWithMinM
205211
206− -- * Nesting
212+ -- ** Zipping
213+ , zipWithM
214+ , zip
215+
216+ -- ** Splitting
207217 , many
208218 , intervalsOf
209219 , chunksOf
210220 , chunksBetween
211− , zipWithM
212− , zip
213221
222+ -- ** Nesting
214223 , concatSequence
215224 , concatMap
225+
226+ -- * Running Partially
227+ , initialize
228+ , runStep
216229 , duplicate
230+
231+ -- * Fold2
232+ , drainBy2
233+
234+ -- * Deprecated
235+ , sequence
236+ , mapM
217237 )
218238where
219239
@@ -228,7 +248,6 @@ import Data.Semigroup (Semigroup((<>)))
228248#endif
229249import Streamly.Internal.Data.Either.Strict
230250 (Either'(..), fromLeft', fromRight', isLeft', isRight')
231−import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)
232251import Streamly.Internal.Data.Pipe.Type (Pipe (..), PipeState(..))
233252import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
234253import Streamly.Internal.Data.Stream.Serial (SerialT)
@@ -270,7 +289,7 @@ hoist f (Fold step initial extract) =
270289
271290-- | Adapt a pure fold to any monad
272291--
273−-- > generally = hoist (return . runIdentity)
292+-- > generally = Fold.hoist (return . runIdentity)
274293--
275294-- /Pre-release/
276295generally :: Monad m => Fold Identity a b -> Fold m a b
@@ -305,7 +324,7 @@ mapM = rmapM
305324-- >>> Stream.fold fld (Stream.enumerateFromTo 1 10)
306325-- [2,4,6,8,10]
307326--
308−-- /Pre-release/
327+-- @since 0.8.0
309328{-# INLINE mapMaybe #-}
310329mapMaybe :: (Monad m) => (a -> Maybe b) -> Fold m b r -> Fold m a r
311330mapMaybe f = map f . filter isJust . map fromJust
@@ -318,7 +337,7 @@ mapMaybe f = map f . filter isJust . map fromJust
318337--
319338-- | Apply a transformation on a 'Fold' using a 'Pipe'.
320339--
321−-- @since 0.7.0
340+-- /Pre-release/
322341{-# INLINE transform #-}
323342transform :: Monad m => Pipe m a b -> Fold m b c -> Fold m a c
324343transform (Pipe pstep1 pstep2 pinitial) (Fold fstep finitial fextract) =
@@ -355,23 +374,6 @@ transform (Pipe pstep1 pstep2 pinitial) (Fold fstep finitial fextract) =
355374
356375 extract (Tuple' _ fs) = fextract fs
357376
358−------------------------------------------------------------------------------
359−-- Utilities
360−------------------------------------------------------------------------------
361−
362−-- | @_Fold1 step@ returns a new 'Fold' using just a step function that has the
363−-- same type for the accumulator and the element. The result type is the
364−-- accumulator type wrapped in 'Maybe'. The initial accumulator is retrieved
365−-- from the 'Foldable', the result is 'None' for empty containers.
366−{-# INLINABLE _Fold1 #-}
367−_Fold1 :: Monad m => (a -> a -> a) -> Fold m a (Maybe a)
368−_Fold1 step = fmap toMaybe $ mkFoldl step_ Nothing'
369−
370− where
371−
372− step_ Nothing' a = Just' a
373− step_ (Just' x) a = Just' $ step x a
374−
375377------------------------------------------------------------------------------
376378-- Left folds
377379------------------------------------------------------------------------------
@@ -382,11 +384,13 @@ _Fold1 step = fmap toMaybe $ mkFoldl step_ Nothing'
382384
383385-- |
384386-- > drainBy f = lmapM f drain
385−-- > drainBy = FL.foldMapM (void . f)
387+-- > drainBy = Fold.foldMapM (void . f)
386388--
387389-- Drain all input after passing it through a monadic function. This is the
388390-- dual of mapM_ on stream producers.
389391--
392+-- See also: "Streamly.Prelude.mapM_"
393+--
390394-- @since 0.7.0
391395{-# INLINABLE drainBy #-}
392396drainBy :: Monad m => (a -> m b) -> Fold m a ()
@@ -401,12 +405,12 @@ drainBy2 f = Fold2 (const (void . f)) (\_ -> return ()) return
401405
402406-- | Extract the last element of the input stream, if any.
403407--
404−-- > last = fmap getLast $ FL.foldMap (Last . Just)
408+-- > last = fmap getLast $ Fold.foldMap (Last . Just)
405409--
406410-- @since 0.7.0
407411{-# INLINABLE last #-}
408412last :: Monad m => Fold m a (Maybe a)
409−last = _Fold1 (\_ x -> x)
413+last = mkFoldl1 (\_ x -> x)
410414
411415------------------------------------------------------------------------------
412416-- To Summary
@@ -416,14 +420,14 @@ last = _Fold1 (\_ x -> x)
416420--
417421-- > genericLength = fmap getSum $ foldMap (Sum . const 1)
418422--
419−-- @since 0.7.0
423+-- /Pre-release/
420424{-# INLINE genericLength #-}
421425genericLength :: (Monad m, Num b) => Fold m a b
422426genericLength = mkFoldl (\n _ -> n + 1) 0
423427
424428-- | Determine the length of the input stream.
425429--
426−-- > length = fmap getSum $ foldMap (Sum . const 1)
430+-- > length = fmap getSum $ Fold.foldMap (Sum . const 1)
427431--
428432-- @since 0.7.0
429433{-# INLINE length #-}
@@ -434,7 +438,7 @@ length = genericLength
434438-- identity (@0@) when the stream is empty. Note that this is not numerically
435439-- stable for floating point numbers.
436440--
437−-- > sum = fmap getSum $ FL.foldMap Sum
441+-- > sum = fmap getSum $ Fold.foldMap Sum
438442--
439443-- @since 0.7.0
440444{-# INLINE sum #-}
@@ -445,7 +449,7 @@ sum = mkFoldl (+) 0
445449-- multiplicative identity (@1@) when the stream is empty. The fold terminates
446450-- when it encounters (@0@) in its input.
447451--
448−-- > product = fmap getProduct $ FL.foldMap Product
452+-- Compare with @Fold.foldMap Product@.
449453--
450454-- @since 0.7.0
451455-- /Since 0.8.0 (Added 'Eq' constraint)/
@@ -470,7 +474,7 @@ product = mkFold_ step (Partial 1)
470474-- @since 0.7.0
471475{-# INLINE maximumBy #-}
472476maximumBy :: Monad m => (a -> a -> Ordering) -> Fold m a (Maybe a)
473−maximumBy cmp = _Fold1 max'
477+maximumBy cmp = mkFoldl1 max'
474478
475479 where
476480
@@ -481,24 +485,24 @@ maximumBy cmp = _Fold1 max'
481485
482486-- |
483487-- @
484−-- maximum = 'maximumBy' compare
488+-- maximum = Fold.maximumBy compare
485489-- @
486490--
487491-- Determine the maximum element in a stream.
488492--
489−-- Compare with @FL.foldMap Max@.
493+-- Compare with @Fold.foldMap Max@.
490494--
491495-- @since 0.7.0
492496{-# INLINE maximum #-}
493497maximum :: (Monad m, Ord a) => Fold m a (Maybe a)
494−maximum = _Fold1 max
498+maximum = mkFoldl1 max
495499
496500-- | Computes the minimum element with respect to the given comparison function
497501--
498502-- @since 0.7.0
499503{-# INLINE minimumBy #-}
500504minimumBy :: Monad m => (a -> a -> Ordering) -> Fold m a (Maybe a)
501−minimumBy cmp = _Fold1 min'
505+minimumBy cmp = mkFoldl1 min'
502506
503507 where
504508
@@ -514,12 +518,12 @@ minimumBy cmp = _Fold1 min'
514518-- minimum = 'minimumBy' compare
515519-- @
516520--
517−-- Compare with @FL.foldMap Min@.
521+-- Compare with @Fold.foldMap Min@.
518522--
519523-- @since 0.7.0
520524{-# INLINE minimum #-}
521525minimum :: (Monad m, Ord a) => Fold m a (Maybe a)
522−minimum = _Fold1 min
526+minimum = mkFoldl1 min
523527
524528------------------------------------------------------------------------------
525529-- To Summary (Statistical)
@@ -603,7 +607,7 @@ defaultSalt = -2578643520546668380
603607
604608-- | Compute an 'Int' sized polynomial rolling hash of a stream.
605609--
606−-- > rollingHash = rollingHashWithSalt defaultSalt
610+-- > rollingHash = Fold.rollingHashWithSalt defaultSalt
607611--
608612-- /Pre-release/
609613{-# INLINABLE rollingHash #-}
@@ -613,7 +617,7 @@ rollingHash = rollingHashWithSalt defaultSalt
613617-- | Compute an 'Int' sized polynomial rolling hash of the first n elements of
614618-- a stream.
615619--
616−-- > rollingHashFirstN = take n rollingHash
620+-- > rollingHashFirstN = Fold.take n Fold.rollingHash
617621--
618622-- /Pre-release/
619623{-# INLINABLE rollingHashFirstN #-}
@@ -626,10 +630,14 @@ rollingHashFirstN n = take n rollingHash
626630
627631-- | Append the elements of an input stream to a provided starting value.
628632--
629−-- > S.fold (FL.sconcat 10) (S.map Sum $ S.enumerateFromTo 1 10)
633+-- >>> Stream.fold (Fold.sconcat 10) (Stream.map Data.Monoid.Sum $ Stream.enumerateFromTo 1 10)
634+-- Sum {getSum = 65}
630635--
631−-- /Pre-release/
636+-- @
637+-- sconcat = Fold.mkFoldl (<>)
638+-- @
632639--
640+-- @since 0.8.0
633641{-# INLINE sconcat #-}
634642sconcat :: (Monad m, Semigroup a) => a -> Fold m a a
635643sconcat = mkFoldl (<>)
@@ -637,7 +645,10 @@ sconcat = mkFoldl (<>)
637645-- | Fold an input stream consisting of monoidal elements using 'mappend'
638646-- and 'mempty'.
639647--
640−-- > S.fold FL.mconcat (S.map Sum $ S.enumerateFromTo 1 10)
648+-- >>> Stream.fold Fold.mconcat (Stream.map Data.Monoid.Sum $ Stream.enumerateFromTo 1 10)
649+-- Sum {getSum = 55}
650+--
651+-- > mconcat = Fold.sconcat mempty
641652--
642653-- @since 0.7.0
643654{-# INLINE mconcat #-}
@@ -650,12 +661,13 @@ mconcat ::
650661mconcat = sconcat mempty
651662
652663-- |
653−-- > foldMap f = map f mconcat
664+-- > foldMap f = Fold.lmap f Fold.mconcat
654665--
655666-- Make a fold from a pure function that folds the output of the function
656667-- using 'mappend' and 'mempty'.
657668--
658−-- > S.fold (FL.foldMap Sum) $ S.enumerateFromTo 1 10
669+-- >>> Stream.fold (Fold.foldMap Data.Monoid.Sum) $ Stream.enumerateFromTo 1 10
670+-- Sum {getSum = 55}
659671--
660672-- @since 0.7.0
661673{-# INLINABLE foldMap #-}
@@ -664,15 +676,16 @@ foldMap :: (Monad m, Monoid b
664676 , Semigroup b
665677#endif
666678 ) => (a -> b) -> Fold m a b
667−foldMap f = map f mconcat
679+foldMap f = lmap f mconcat
668680
669681-- |
670−-- > foldMapM f = lmapM f mconcat
682+-- > foldMapM f = Fold.lmapM f Fold.mconcat
671683--
672684-- Make a fold from a monadic function that folds the output of the function
673685-- using 'mappend' and 'mempty'.
674686--
675−-- > S.fold (FL.foldMapM (return . Sum)) $ S.enumerateFromTo 1 10
687+-- >>> Stream.fold (Fold.foldMapM (return . Data.Monoid.Sum)) $ Stream.enumerateFromTo 1 10
688+-- Sum {getSum = 55}
676689--
677690-- @since 0.7.0
678691{-# INLINABLE foldMapM #-}
@@ -695,10 +708,12 @@ foldMapM act = mkFoldlM step (pure mempty)
695708
696709-- | Buffers the input stream to a list in the reverse order of the input.
697710--
711+-- > toListRev = Fold.mkFoldl (flip (:)) []
712+--
698713-- /Warning!/ working on large lists accumulated as buffers in memory could be
699714-- very inefficient, consider using "Streamly.Array" instead.
700715--
701−-- @since 0.7.0
716+-- @since 0.8.0
702717
703718-- xn : ... : x2 : x1 : []
704719{-# INLINABLE toListRev #-}
@@ -711,6 +726,10 @@ toListRev = mkFoldl (flip (:)) []
711726
712727-- | A fold that drains the first n elements of its input, running the effects
713728-- and discarding the results.
729+--
730+-- > drainN n = Fold.take n Fold.drain
731+--
732+-- /Pre-release/
714733{-# INLINABLE drainN #-}
715734drainN :: Monad m => Int -> Fold m a ()
716735drainN n = take n drain
@@ -735,6 +754,8 @@ genericIndex i = mkFold step (Partial 0) (const Nothing)
735754
736755-- | Lookup the element at the given index.
737756--
757+-- See also: "Streamly.Prelude.!!"
758+--
738759-- @since 0.7.0
739760{-# INLINABLE index #-}
740761index :: Monad m => Int -> Fold m a (Maybe a)
@@ -742,8 +763,6 @@ index = genericIndex
742763
743764-- | Extract the first element of the stream, if any.
744765--
745−-- > head = fmap getFirst $ FL.foldMap (First . Just)
746−--
747766-- @since 0.7.0
748767{-# INLINABLE head #-}
749768head :: Monad m => Fold m a (Maybe a)
@@ -751,11 +770,6 @@ head = mkFold_ (const (Done . Just)) (Partial Nothing)
751770
752771-- | Returns the first element that satisfies the given predicate.
753772--
754−-- @
755−-- find p = fmap getFirst $
756−-- FL.foldMap (\x -> First (if p x then Just x else Nothing))
757−-- @
758−--
759773-- @since 0.7.0
760774{-# INLINABLE find #-}
761775find :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
@@ -771,7 +785,7 @@ find predicate = mkFold step (Partial ()) (const Nothing)
771785-- | In a stream of (key-value) pairs @(a, b)@, return the value @b@ of the
772786-- first pair where the key equals the given value @a@.
773787--
774−-- > lookup = snd <$> find ((==) . fst)
788+-- > lookup = snd <$> Fold.find ((==) . fst)
775789--
776790-- @since 0.7.0
777791{-# INLINABLE lookup #-}
@@ -801,7 +815,7 @@ findIndex predicate = mkFold step (Partial 0) (const Nothing)
801815
802816-- | Returns the first index where a given value is found in the stream.
803817--
804−-- > elemIndex a = findIndex (== a)
818+-- > elemIndex a = Fold.findIndex (== a)
805819--
806820-- @since 0.7.0
807821{-# INLINABLE elemIndex #-}
@@ -814,22 +828,20 @@ elemIndex a = findIndex (a ==)
814828
815829-- | Return 'True' if the input stream is empty.
816830--
817−-- > null = fmap isJust head
831+-- > null = fmap isJust Fold.head
818832--
819833-- @since 0.7.0
820834{-# INLINABLE null #-}
821835null :: Monad m => Fold m a Bool
822836null = mkFold (\() _ -> Done False) (Partial ()) (const True)
823837
824−--
825−-- > any p = map p or
826−-- > any p = fmap getAny . FL.foldMap (Any . p)
827−--
828838-- | Returns 'True' if any of the elements of a stream satisfies a predicate.
829839--
830840-- >>> Stream.fold (Fold.any (== 0)) $ Stream.fromList [1,0,1]
831841-- True
832842--
843+-- > any p = Fold.lmap p Fold.or
844+--
833845-- @since 0.7.0
834846{-# INLINE any #-}
835847any :: Monad m => (a -> Bool) -> Fold m a Bool
@@ -846,22 +858,20 @@ any predicate = mkFold_ step initial
846858
847859-- | Return 'True' if the given element is present in the stream.
848860--
849−-- > elem a = any (== a)
861+-- > elem a = Fold.any (== a)
850862--
851863-- @since 0.7.0
852864{-# INLINABLE elem #-}
853865elem :: (Eq a, Monad m) => a -> Fold m a Bool
854866elem a = any (a ==)
855867
856−--
857−-- > all p = map p and
858−-- > all p = fmap getAll . FL.foldMap (All . p)
859−--
860868-- | Returns 'True' if all elements of a stream satisfy a predicate.
861869--
862870-- >>> Stream.fold (Fold.all (== 0)) $ Stream.fromList [1,0,1]
863871-- False
864872--
873+-- > all p = Fold.lmap p Fold.and
874+--
865875-- @since 0.7.0
866876{-# INLINABLE all #-}
867877all :: Monad m => (a -> Bool) -> Fold m a Bool
@@ -878,7 +888,7 @@ all predicate = mkFold_ step initial
878888
879889-- | Returns 'True' if the given element is not present in the stream.
880890--
881−-- > notElem a = all (/= a)
891+-- > notElem a = Fold.all (/= a)
882892--
883893-- @since 0.7.0
884894{-# INLINABLE notElem #-}
@@ -887,8 +897,7 @@ notElem a = all (a /=)
887897
888898-- | Returns 'True' if all elements are 'True', 'False' otherwise
889899--
890−-- > and = all (== True)
891−-- > and = fmap getAll . FL.foldMap All
900+-- > and = Fold.all (== True)
892901--
893902-- @since 0.7.0
894903{-# INLINE and #-}
@@ -897,8 +906,7 @@ and = all (== True)
897906
898907-- | Returns 'True' if any element is 'True', 'False' otherwise
899908--
900−-- > or = any (== True)
901−-- > or = fmap getAny . FL.foldMap Any
909+-- > or = Fold.any (== True)
902910--
903911-- @since 0.7.0
904912{-# INLINE or #-}
@@ -917,8 +925,6 @@ or = any (== True)
917925-- Binary APIs
918926------------------------------------------------------------------------------
919927
920−-- XXX These would just be applicative compositions of terminating folds.
921−
922928-- | @splitAt n f1 f2@ composes folds @f1@ and @f2@ such that first @n@
923929-- elements of its input are consumed by fold @f1@ and the rest of the stream
924930-- is consumed by fold @f2@.
@@ -943,7 +949,7 @@ or = any (== True)
943949-- >>> splitAt_ 4 [1,2,3]
944950-- ([1,2,3],[])
945951--
946−-- > splitAt n f1 f2 = serialWith (,) (take n f1) f2
952+-- > splitAt n f1 f2 = Fold.serialWith (,) (Fold.take n f1) f2
947953--
948954-- /Internal/
949955
@@ -967,53 +973,22 @@ splitAt n fld = serialWith (,) (take n fld)
967973-- Note: Keep this consistent with S.splitOn. In fact we should eliminate
968974-- S.splitOn in favor of the fold.
969975--
970−-- | Consume the input until it encounters an infixed separator element (i.e.
971−-- when the supplied predicate succeeds), dropping the separator.
972−--
973−-- Repeated applications of 'takeEndBy_' splits the stream on separator
974−-- elements determined by the supplied predicate, separator is considered as
975−-- infixed between two segments, if one side of the separator is missing then
976−-- it is parsed as an empty stream. The supplied 'Fold' is applied on the
977−-- split segments. With '-' representing non-separator elements and '.' as
978−-- separator, repeated 'takeEndBy_' splits the stream as follows:
979−--
980−-- @
981−-- "--.--" => "--" "--"
982−-- "--." => "--" ""
983−-- ".--" => "" "--"
984−-- @
985−--
986−-- Repeated applications of @Fold.takeEndBy_ (== x)@ on the input stream gives us
987−-- an inverse of @Stream.intercalate (Stream.yield x)@
988−--
989−-- > Stream.splitOn pred f = Stream.foldMany (Fold.takeEndBy_ pred f)
990−--
991−-- Let's use the following definition for illustration:
992−--
993−-- >>> splitOn p = Stream.foldMany (Fold.takeEndBy_ pred Fold.toList)
994−-- >>> splitOn' p = Stream.toList . splitOn p . Stream.fromList
995−
996−-- >>> splitOn' (== '.') ""
997−-- [""]
998−--
999−-- >>> splitOn' (== '.') "."
1000−-- ["",""]
976+-- XXX Use Fold.many instead once it is fixed.
1001977--
1002−-- >>> splitOn' (== '.') ".a"
1003−-- ["","a"]
978+-- | Like 'takeEndBy' but drops the element on which the predicate succeeds.
1004979--
1005−-- >>> splitOn' (== '.') "a."
1006−-- ["a",""]
980+-- >>> Stream.fold (Fold.takeEndBy_ (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
981+-- "hello"
1007982--
1008−-- >>> splitOn' (== '.') "a.b"
1009−-- ["a","b"]
983+-- >>> Stream.toList $ Stream.foldMany (Fold.takeEndBy_ (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
984+-- ["hello","there"]
1010985--
1011−-- >>> splitOn' (== '.') "a..b"
1012−-- ["a","","b"]
986+-- > Stream.splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)
1013987--
1014−-- Stops - when the predicate succeeds.
988+-- See 'Streamly.Prelude.splitOnSuffix' for more details on splitting a
989+-- stream using 'takeEndBy_'.
1015990--
1016−-- /Pre-release/
991+-- @since 0.8.0
1017992{-# INLINE takeEndBy_ #-}
1018993takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
1019994takeEndBy_ predicate (Fold fstep finitial fextract) =
@@ -1026,16 +1001,21 @@ takeEndBy_ predicate (Fold fstep finitial fextract) =
10261001 then fstep s a
10271002 else Done <$> fextract s
10281003
1029−-- | Collect stream elements until an element succeeds the predicate. Also take
1030−-- the element on which the predicate succeeded. The succeeding element is
1031−-- treated as a suffix separator which is kept in the output segement.
1004+-- | Take the input, stop when the predicate succeeds taking the succeeding
1005+-- element as well.
10321006--
1033−-- * Stops - when the predicate succeeds.
1007+-- >>> Stream.fold (Fold.takeEndBy (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1008+-- "hello\n"
10341009--
1035−-- > Stream.splitWithSuffix pred f = Stream.foldMany (Fold.takeEndBy pred f)
1010+-- >>> Stream.toList $ Stream.foldMany (Fold.takeEndBy (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1011+-- ["hello\n","there\n"]
10361012--
1037−-- /Pre-release/
1013+-- > Stream.splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)
10381014--
1015+-- See 'Streamly.Prelude.splitWithSuffix' for more details on splitting a
1016+-- stream using 'takeEndBy'.
1017+--
1018+-- @since 0.8.0
10391019{-# INLINE takeEndBy #-}
10401020takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
10411021takeEndBy predicate (Fold fstep finitial fextract) =
@@ -1086,6 +1066,8 @@ breakOn pat f m = undefined
10861066-- >>> Stream.fold (Fold.tee Fold.sum Fold.length) (Stream.enumerateFromTo 1.0 100.0)
10871067-- (5050.0,100)
10881068--
1069+-- > tee = teeWith (,)
1070+--
10891071-- @since 0.7.0
10901072{-# INLINE tee #-}
10911073tee :: Monad m => Fold m a b -> Fold m a c -> Fold m a (b,c)
@@ -1110,6 +1092,8 @@ tee = teeWith (,)
11101092-- >>> Stream.fold (Fold.distribute [Fold.sum, Fold.length]) (Stream.enumerateFromTo 1 5)
11111093-- [15,5]
11121094--
1095+-- > distribute = Prelude.foldr (Fold.teeWith (:)) (Fold.yield [])
1096+--
11131097-- This is the consumer side dual of the producer side 'sequence' operation.
11141098--
11151099-- Stops when all the folds stop.
@@ -1171,7 +1155,7 @@ distribute = foldr (teeWith (:)) (yield [])
11711155--
11721156-- /See also: 'partitionByFstM' and 'partitionByMinM'./
11731157--
1174−-- @since 0.7.0
1158+-- /Pre-release/
11751159{-# INLINE partitionByM #-}
11761160partitionByM :: Monad m
11771161 => (a -> m (Either b c)) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -1270,7 +1254,7 @@ partitionByMinM = undefined
12701254-- :}
12711255-- ("Even 50","Odd 50")
12721256--
1273−-- @since 0.7.0
1257+-- /Pre-release/
12741258{-# INLINE partitionBy #-}
12751259partitionBy :: Monad m
12761260 => (a -> Either b c) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -1346,7 +1330,9 @@ demuxWith f kv = fmap fst $ demuxDefaultWith f kv drain
13461330-- :}
13471331-- fromList [("PRODUCT",8),("SUM",4)]
13481332--
1349−-- @since 0.7.0
1333+-- > demux = demuxWith id
1334+--
1335+-- /Pre-release/
13501336{-# INLINE demux #-}
13511337demux :: (Monad m, Ord k)
13521338 => Map k (Fold m a b) -> Fold m (k, a) (Map k b)
@@ -1476,6 +1462,10 @@ demuxDefaultWith f kv (Fold dstep dinitial dextract) =
14761462 b <- dextract dacc
14771463 return (doneMap, b)
14781464
1465+-- |
1466+-- > demuxDefault = demuxDefaultWith id
1467+--
1468+-- /Pre-release/
14791469{-# INLINE demuxDefault #-}
14801470demuxDefault :: (Monad m, Ord k)
14811471 => Map k (Fold m a b) -> Fold m (k, a) b -> Fold m (k, a) (Map k b, b)
@@ -1555,12 +1545,11 @@ classifyWith f (Fold step1 initial1 extract1) =
15551545-- :}
15561546-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
15571547--
1558−-- @since 0.7.0
1559−--
15601548-- Same as:
15611549--
1562−-- > classify fld = classifyWith fst (map snd fld)
1550+-- > classify fld = Fold.classifyWith fst (map snd fld)
15631551--
1552+-- /Pre-release/
15641553{-# INLINE classify #-}
15651554classify :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)
15661555classify fld = classifyWith fst (map snd fld)
@@ -1571,9 +1560,9 @@ classify fld = classifyWith fst (map snd fld)
15711560
15721561-- | Like 'unzipWith' but with a monadic splitter function.
15731562--
1574−-- -- @unzipWithM k f1 f2 = lmapM k (unzip f1 f2)@
1563+-- @unzipWithM k f1 f2 = lmapM k (unzip f1 f2)@
15751564--
1576−-- @since 0.7.0
1565+-- /Pre-release/
15771566{-# INLINE unzipWithM #-}
15781567unzipWithM :: Monad m
15791568 => (a -> m (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
@@ -1653,11 +1642,11 @@ unzipWithMinM = undefined
16531642-- | Split elements in the input stream into two parts using a pure splitter
16541643-- function, direct each part to a different fold and zip the results.
16551644--
1656−-- @unzipWith f fld1 fld2 = map f (unzip fld1 fld2)@
1645+-- @unzipWith f fld1 fld2 = Fold.lmap f (Fold.unzip fld1 fld2)@
16571646--
16581647-- This fold terminates when both the input folds terminate.
16591648--
1660−-- @since 0.7.0
1649+-- /Pre-release/
16611650{-# INLINE unzipWith #-}
16621651unzipWith :: Monad m
16631652 => (a -> (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
@@ -1674,6 +1663,8 @@ unzipWith f = unzipWithM (return . f)
16741663--
16751664-- @
16761665--
1666+-- > unzip = Fold.unzipWith id
1667+--
16771668-- This is the consumer side dual of the producer side 'zip' operation.
16781669--
16791670-- @since 0.7.0
@@ -1773,6 +1764,8 @@ chunksBetween _low _high _f1 _f2 = undefined
17731764-- /Warning!/ working on large streams accumulated as buffers in memory could
17741765-- be very inefficient, consider using "Streamly.Data.Array" instead.
17751766--
1767+-- > toStream = mkFoldr K.cons K.nil
1768+--
17761769-- /Pre-release/
17771770{-# INLINE toStream #-}
17781771toStream :: Monad m => Fold m a (SerialT Identity a)
@@ -1784,6 +1777,8 @@ toStream = mkFoldr K.cons K.nil
17841777-- | Buffers the input stream to a pure stream in the reverse order of the
17851778-- input.
17861779--
1780+-- > toStreamRev = mkFoldl (flip K.cons) K.nil
1781+--
17871782-- /Warning!/ working on large streams accumulated as buffers in memory could
17881783-- be very inefficient, consider using "Streamly.Data.Array" instead.
17891784--
src/Streamly/Internal/Data/Fold/Tee.hs+57−25
@@ -6,10 +6,35 @@
66-- Stability : experimental
77-- Portability : GHC
88--
9+-- The @Tee@ type is a newtype wrapper over the 'Fold' type providing
10+-- distributing 'Applicative', 'Semigroup', 'Monoid', 'Num', 'Floating' and
11+-- 'Fractional' instances. The input received by the composed 'Tee' is
12+-- replicated and distributed to both the constituent Tee's.
13+--
14+-- For example, to compute the average of numbers in a stream without going
15+-- through the stream twice:
16+--
17+-- >>> import Streamly.Internal.Data.Fold.Tee (mkT, toFold)
18+-- >>> import Streamly.Internal.Data.Fold as Fold
19+--
20+-- >>> avg = (/) <$> (mkT Fold.sum) <*> (mkT $ fmap fromIntegral Fold.length)
21+-- >>> Stream.fold (toFold avg) $ Stream.fromList [1.0..100.0]
22+-- 50.5
23+--
24+-- Similarly, the 'Semigroup' and 'Monoid' instances of 'Tee' distribute the
25+-- input to both the folds and combines the outputs using Monoid or Semigroup
26+-- instances of the output types:
27+--
28+-- >>> import Data.Monoid (Sum(..))
29+-- >>> t = mkT Fold.head <> mkT Fold.last
30+-- >>> Stream.fold (toFold t) (fmap Sum $ Stream.enumerateFromTo 1.0 100.0)
31+-- Just (Sum {getSum = 101.0})
32+--
33+-- The 'Num', 'Floating', and 'Fractional' instances work in the same way.
34+--
935module Streamly.Internal.Data.Fold.Tee
1036 ( Tee(..)
11− , fromFold
12− , toFold
37+ , mkT
1338 )
1439where
1540
@@ -22,44 +47,44 @@ import Streamly.Internal.Data.Fold.Type (Fold)
2247
2348import qualified Streamly.Internal.Data.Fold.Type as Fold
2449
25−-- | The type @Tee m a b@ represents a left fold over an input stream of values
26−-- of type @a@ to a single value of type @b@ in 'Monad' @m@.
27−--
28−-- @Tee@ is a wrapper over 'Fold' that uses 'teeWith' to define the applicative
29−-- instance.
50+-- | @Tee@ is a newtype wrapper over the 'Fold' type providing distributing
51+-- 'Applicative', 'Semigroup', 'Monoid', 'Num', 'Floating' and 'Fractional'
52+-- instances.
3053--
3154-- /Pre-release/
3255newtype Tee m a b =
33− Tee { runTee :: Fold m a b }
56+ Tee { toFold :: Fold m a b }
3457 deriving (Functor)
3558
36−-- | Convert a 'Tee' to 'Fold'.
37−{-# INLINE toFold #-}
38−toFold :: Tee m a b -> Fold m a b
39−toFold = coerce
59+-- | Make a 'Tee' from a 'Fold'.
60+--
61+-- /Pre-release/
62+{-# INLINE mkT #-}
63+mkT :: Fold m a b -> Tee m a b
64+mkT = coerce
4065
41−-- | Convert a 'Fold' to 'Tee'.
42−{-# INLINE fromFold #-}
43−fromFold :: Fold m a b -> Tee m a b
44−fromFold = coerce
4566
46−-- | The 'Tee' resulting from '<*>' distributes its input to both the argument
47−-- 'Tee's and combines their output using function application.
67+-- | '<*>' distributes the input to both the argument 'Tee's and combines their
68+-- outputs using function application.
4869--
4970instance Monad m => Applicative (Tee m a) where
5071
5172 {-# INLINE pure #-}
52− pure a = fromFold (Fold.yield a)
73+ pure a = mkT (Fold.yield a)
5374
5475 {-# INLINE (<*>) #-}
55− (<*>) a b = fromFold (Fold.teeWith ($) (toFold a) (toFold b))
76+ (<*>) a b = mkT (Fold.teeWith ($) (toFold a) (toFold b))
5677
57−-- | Combines the outputs (the type @b@) using their 'Semigroup' instances.
78+-- | '<>' distributes the input to both the argument 'Tee's and combines their
79+-- outputs using the 'Semigroup' instance of the output type.
80+--
5881instance (Semigroup b, Monad m) => Semigroup (Tee m a b) where
5982 {-# INLINE (<>) #-}
6083 (<>) = liftA2 (<>)
6184
62−-- | Combines the outputs (the type @b@) using their 'Monoid' instances.
85+-- | '<>' distributes the input to both the argument 'Tee's and combines their
86+-- outputs using the 'Monoid' instance of the output type.
87+--
6388instance (Semigroup b, Monoid b, Monad m) => Monoid (Tee m a b) where
6489 {-# INLINE mempty #-}
6590 mempty = pure mempty
@@ -67,7 +92,9 @@ instance (Semigroup b, Monoid b, Monad m) => Monoid (Tee m a b) where
6792 {-# INLINE mappend #-}
6893 mappend = (<>)
6994
70−-- | Combines the outputs (type @b@) using their 'Num' instances.
95+-- | Binary 'Num' operations distribute the input to both the argument 'Tee's
96+-- and combine their outputs using the 'Num' instance of the output type.
97+--
7198instance (Monad m, Num b) => Num (Tee m a b) where
7299 {-# INLINE fromInteger #-}
73100 fromInteger = pure . fromInteger
@@ -90,7 +117,10 @@ instance (Monad m, Num b) => Num (Tee m a b) where
90117 {-# INLINE (-) #-}
91118 (-) = liftA2 (-)
92119
93−-- | Combines the outputs (type @b@) using their 'Fractional' instances.
120+-- | Binary 'Fractional' operations distribute the input to both the argument
121+-- 'Tee's and combine their outputs using the 'Fractional' instance of the
122+-- output type.
123+--
94124instance (Monad m, Fractional b) => Fractional (Tee m a b) where
95125 {-# INLINE fromRational #-}
96126 fromRational = pure . fromRational
@@ -101,7 +131,9 @@ instance (Monad m, Fractional b) => Fractional (Tee m a b) where
101131 {-# INLINE (/) #-}
102132 (/) = liftA2 (/)
103133
104−-- | Combines the outputs using their 'Floating' instances.
134+-- | Binary 'Floating' operations distribute the input to both the argument
135+-- 'Tee's and combine their outputs using the 'Floating' instance of the output
136+-- type.
105137instance (Monad m, Floating b) => Floating (Tee m a b) where
106138 {-# INLINE pi #-}
107139 pi = pure pi
src/Streamly/Internal/Data/Fold/Type.hs+225−95
@@ -164,16 +164,41 @@
164164-- can be used for the scan use case, instead of using extract. Extract would
165165-- then be used only for the case when the stream stops before the fold
166166-- completes.
167−
167+--
168+-- = Accumulators and Terminating Folds
169+--
170+-- Folds in this module can be classified in two categories viz. accumulators
171+-- and terminating folds. Accumulators do not have a terminating condition,
172+-- they run forever and consume the entire stream, for example the 'length'
173+-- fold. Terminating folds have a terminating condition and can terminate
174+-- without consuming the entire stream, for example, the 'head' fold.
175+--
176+-- = Monoids
177+--
178+-- Monoids allow generalized, modular folding. The accumulators in this module
179+-- can be expressed using 'mconcat' and a suitable 'Monoid'. Instead of
180+-- writing folds we can write Monoids and turn them into folds.
181+--
182+-- = Performance Notes
183+--
184+-- 'Streamly.Prelude' module provides fold functions to directly fold streams
185+-- e.g. Streamly.Prelude/'Streamly.Prelude.sum' serves the same purpose as
186+-- Fold/'sum'. However, the functions in Streamly.Prelude cannot be
187+-- efficiently combined together e.g. we cannot drive the input stream through
188+-- @sum@ and @length@ fold functions simultaneously. Using the 'Fold' type we
189+-- can efficiently split the stream across multiple folds because it allows the
190+-- compiler to perform stream fusion optimizations.
191+--
168192module Streamly.Internal.Data.Fold.Type
169193 (
170194 -- * Types
171195 Step (..)
172196 , Fold (..)
173197
174− -- * Fold constructors
198+ -- * Constructors
175199 , mkFoldl
176200 , mkFoldlM
201+ , mkFoldl1
177202 , mkFoldr
178203 , mkFoldrM
179204 , mkFold
@@ -181,55 +206,64 @@ module Streamly.Internal.Data.Fold.Type
181206 , mkFoldM
182207 , mkFoldM_
183208
184− -- * Fold2
185− , Fold2 (..)
186− , simplify
187−
188− -- * Basic Folds
209+ -- * Folds
210+ , yield
211+ , yieldM
189212 , drain
190213 , toList
191214
192− -- * Generators
193− , yield
194− , yieldM
215+ -- * Combinators
195216
196− -- * Transformations
217+ -- ** Mapping output
197218 , rmapM
219+
220+ -- ** Mapping Input
198221 , map
199222 , lmap
200223 , lmapM
224+
225+ -- ** Filtering
201226 , filter
202227 , filterM
203228 , catMaybes
229+
230+ -- ** Trimming
204231 , take
205232 , takeInterval
206233
207− -- * Distributing
234+ -- ** Serial Append
235+ , serialWith
236+ , serial_
237+
238+ -- ** Parallel Distribution
239+ , GenericRunner(..)
208240 , teeWith
209241 , teeWithFst
210242 , teeWithMin
243+
244+ -- ** Parallel Alternative
211245 , shortest
212246 , longest
213247
214− -- * Serial Application
215− , serialWith
216− , serial_
217−
218− -- * Nested Application
219− , concatMap
248+ -- ** Splitting
220249 , ManyState
221250 , many
222251 , manyPost
223− , intervalsOf
224252 , chunksOf
225− , chunksOf2
253+ , intervalsOf
254+
255+ -- ** Nesting
256+ , concatMap
226257
258+ -- * Running Partially
227259 , duplicate
228260 , initialize
229261 , runStep
230262
231− -- * Misc
232− , GenericRunner(..) -- Is used in multiple step functions
263+ -- * Fold2
264+ , Fold2 (..)
265+ , simplify
266+ , chunksOf2
233267 )
234268where
235269
@@ -242,6 +276,7 @@ import Control.Monad.Trans.Control (control)
242276import Data.Bifunctor (Bifunctor(..))
243277import Data.Maybe (isJust, fromJust)
244278import Fusion.Plugin.Types (Fuse(..))
279+import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)
245280import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
246281import Streamly.Internal.Data.SVar (MonadAsync)
247282
@@ -255,9 +290,14 @@ import Prelude hiding (concatMap, filter, map, take)
255290-- >>> import qualified Streamly.Internal.Data.Fold as Fold
256291
257292------------------------------------------------------------------------------
258−-- Monadic left folds
293+-- Step of a fold
259294------------------------------------------------------------------------------
260295
296+-- The Step functor around b allows expressing early termination like a right
297+-- fold. Traditional list right folds use function composition and laziness to
298+-- terminate early whereas we use data constructors. It allows stream fusion in
299+-- contrast to the foldr/build fusion when composing with functions.
300+
261301-- | Represents the result of the @step@ of a 'Fold'. 'Partial' returns an
262302-- intermediate state of the fold, the fold step can be called again with the
263303-- state or the driver can use @extract@ on the state to get the result out.
@@ -270,10 +310,7 @@ data Step s b
270310 = Partial !s
271311 | Done !b
272312
273−-- | A bifunctor instance on 'Step'. @first@ maps on the value held by 'Partial'
274−-- and @second@ maps on the result held by 'Done'.
275−--
276−-- /Pre-release/
313+-- | 'first' maps over 'Partial' and 'second' maps over 'Done'.
277314--
278315instance Bifunctor Step where
279316 {-# INLINE bimap #-}
@@ -288,17 +325,19 @@ instance Bifunctor Step where
288325 second _ (Partial x) = Partial x
289326 second f (Done a) = Done (f a)
290327
291−-- | Maps the function over the result held by 'Done'.
328+-- | 'fmap' maps over 'Done'.
329+--
292330-- @
293331-- fmap = 'second'
294332-- @
295333--
296−-- /Pre-release/
297−--
298334instance Functor (Step s) where
299335 {-# INLINE fmap #-}
300336 fmap = second
301337
338+-- | Map a monadic function over the result @b@ in @Step s b@.
339+--
340+-- /Internal/
302341{-# INLINE mapMStep #-}
303342mapMStep :: Applicative m => (a -> m b) -> Step s a -> m (Step s b)
304343mapMStep f res =
@@ -306,25 +345,27 @@ mapMStep f res =
306345 Partial s -> pure $ Partial s
307346 Done b -> Done <$> f b
308347
309−-- The Step functor around b allows expressing early termination like a right
310−-- fold. Traditional list right folds use function composition and laziness to
311−-- terminate early whereas we use data constructors. It allows stream fusion in
312−-- contrast to the foldr/build fusion when composing with functions.
348+------------------------------------------------------------------------------
349+-- The Fold type
350+------------------------------------------------------------------------------
313351
314352-- | The type @Fold m a b@ having constructor @Fold step initial extract@
315−-- represents a left fold over an input stream of values of type @a@ to a
316−-- single value of type @b@ in 'Monad' @m@. The constructor is not exposed via
317−-- exposed modules, smart constructors are provided to create folds.
353+-- represents a fold over an input stream of values of type @a@ to a final
354+-- value of type @b@ in 'Monad' @m@.
318355--
319356-- The fold uses an intermediate state @s@ as accumulator, the type @s@ is
320−-- specific to the fold. The initial value of the fold state @s@ is returned by
321−-- @initial@. The @step@ function consumes an input and either returns the
322−-- final result @b@ if the fold is done or the next intermediate state (see
323−-- 'Step'). At any point the fold driver can extract the result from the
324−-- intermediate state using the @extract@ function.
357+-- internal to the specific fold definition. The initial value of the fold
358+-- state @s@ is returned by @initial@. The @step@ function consumes an input
359+-- and either returns the final result @b@ if the fold is done or the next
360+-- intermediate state (see 'Step'). At any point the fold driver can extract
361+-- the result from the intermediate state using the @extract@ function.
325362--
326−-- NOTE: If you think you need the constructor of this type please consider
327−-- using the smart constructors in "Streamly.Internal.Data.Fold' instead.
363+-- NOTE: The constructor is not yet exposed via exposed modules, smart
364+-- constructors are provided to create folds. If you think you need the
365+-- constructor of this type please consider using the smart constructors in
366+-- "Streamly.Internal.Data.Fold' instead.
367+--
368+-- /since 0.8.0 (type changed)/
328369--
329370-- @since 0.7.0
330371
@@ -332,6 +373,10 @@ data Fold m a b =
332373 -- | @Fold @ @ step @ @ initial @ @ extract@
333374 forall s. Fold (s -> a -> m (Step s b)) (m (Step s b)) (s -> m b)
334375
376+------------------------------------------------------------------------------
377+-- Mapping on the output
378+------------------------------------------------------------------------------
379+
335380-- | Map a monadic function on the output of a fold.
336381--
337382-- @since 0.8.0
@@ -361,7 +406,9 @@ rmapM f (Fold step initial extract) = Fold step1 initial1 (extract >=> f)
361406-- mkfoldlx step initial extract = fmap extract (mkFoldl step initial)
362407-- @
363408--
364−-- /Pre-release/
409+-- See also: "Streamly.Prelude.foldl'"
410+--
411+-- @since 0.8.0
365412--
366413{-# INLINE mkFoldl #-}
367414mkFoldl :: Monad m => (b -> a -> b) -> b -> Fold m a b
@@ -377,42 +424,65 @@ mkFoldl step initial =
377424-- A fold with an extract function can be expressed using rmapM:
378425--
379426-- @
380−-- mkAccumM :: Functor m => (s -> a -> m s) -> m s -> (s -> m b) -> Fold m a b
381−-- mkAccumM step initial extract = rmapM extract (mkFoldlM step initial)
427+-- mkFoldlxM :: Functor m => (s -> a -> m s) -> m s -> (s -> m b) -> Fold m a b
428+-- mkFoldlxM step initial extract = rmapM extract (mkFoldlM step initial)
382429-- @
383430--
384−-- /Pre-release/
431+-- See also: "Streamly.Prelude.foldlM'"
432+--
433+-- @since 0.8.0
385434--
386435{-# INLINE mkFoldlM #-}
387436mkFoldlM :: Monad m => (b -> a -> m b) -> m b -> Fold m a b
388437mkFoldlM step initial =
389438 Fold (\s a -> Partial <$> step s a) (Partial <$> initial) return
390439
440+-- | Make a strict left fold, for non-empty streams, using first element as the
441+-- starting value. Returns Nothing if the stream is empty.
442+--
443+-- See also: "Streamly.Prelude.foldl1'"
444+--
445+-- /Pre-release/
446+{-# INLINE mkFoldl1 #-}
447+mkFoldl1 :: Monad m => (a -> a -> a) -> Fold m a (Maybe a)
448+mkFoldl1 step = fmap toMaybe $ mkFoldl step1 Nothing'
449+
450+ where
451+
452+ step1 Nothing' a = Just' a
453+ step1 (Just' x) a = Just' $ step x a
454+
391455------------------------------------------------------------------------------
392456-- Right fold constructors
393457------------------------------------------------------------------------------
394458
395459-- | Make a fold using a right fold style step function and a terminal value.
396−-- It performs a right fold via a left fold using function composition.
397−-- This can be useful for constructing structures. For reductions this may be
398−-- very inefficient compared to using a direct fold implementation using
399−-- 'mkFold'.
460+-- It performs a strict right fold via a left fold using function composition.
461+-- Note that this is strict fold, it can only be useful for constructing strict
462+-- structures in memory. For reductions this will be very inefficient.
400463--
401464-- For example,
402465--
403466-- > toList = mkFoldr (:) []
404467--
405−-- /Pre-release/
468+-- See also: "Streamly.Prelude.foldr"
469+--
470+-- @since 0.8.0
406471{-# INLINE mkFoldr #-}
407472mkFoldr :: Monad m => (a -> b -> b) -> b -> Fold m a b
408473mkFoldr g z = fmap ($ z) $ mkFoldl (\f x -> f . g x) id
409474
475+-- XXX we have not seen any use of this yet, not releasing until we have a use
476+-- case.
477+--
410478-- | Like 'mkFoldr' but with a monadic step function.
411479--
412480-- For example,
413481--
414482-- > toList = mkFoldrM (\a xs -> return $ a : xs) (return [])
415483--
484+-- See also: "Streamly.Prelude.foldrM"
485+--
416486-- /Pre-release/
417487{-# INLINE mkFoldrM #-}
418488mkFoldrM :: Monad m => (a -> b -> m b) -> m b -> Fold m a b
@@ -423,6 +493,17 @@ mkFoldrM g z =
423493-- General fold constructors
424494------------------------------------------------------------------------------
425495
496+-- XXX If the Step yield gives the result each time along with the state then
497+-- we can make the type of this as
498+--
499+-- mkFold :: Monad m => (s -> a -> Step s b) -> Step s b -> Fold m a b
500+--
501+-- Then similar to mkFoldl and mkFoldr we can just fmap extract on it to extend
502+-- it to the version where an 'extract' function is required. Or do we even
503+-- need that?
504+--
505+-- Until we investigate this we are not releasing these.
506+
426507-- | Make a terminating fold using a pure step function, a pure initial state
427508-- and a pure state extraction function.
428509--
@@ -513,6 +594,8 @@ drain = mkFoldl (\_ _ -> ()) ()
513594-- very inefficient, consider using "Streamly.Data.Array.Foreign"
514595-- instead.
515596--
597+-- > toList = mkFoldr (:) []
598+--
516599-- @since 0.7.0
517600{-# INLINABLE toList #-}
518601toList :: Monad m => Fold m a [a]
@@ -558,16 +641,26 @@ yieldM b = Fold undefined (Done <$> b) pure
558641data SeqFoldState sl f sr = SeqFoldL !sl | SeqFoldR !f !sr
559642
560643-- | Sequential fold application. Apply two folds sequentially to an input
561−-- stream. The input is provided to the first fold, when it is done the
644+-- stream. The input is provided to the first fold, when it is done - the
562645-- remaining input is provided to the second fold. When the second fold is done
563646-- or if the input stream is over, the outputs of the two folds are combined
564647-- using the supplied function.
565648--
566−-- Note: This is a folding dual of appending streams using
567−-- 'Streamly.Prelude.serial', it splits the streams using two folds and zips
568−-- the results. This has the same caveats as ParseD's @serialWith@
649+-- >>> f = Fold.serialWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)
650+-- >>> Stream.fold f $ Stream.fromList "header: hello\n"
651+-- ("header: ","hello\n")
569652--
570−-- /Pre-release/
653+-- Note: This is dual to appending streams using 'Streamly.Prelude.serial'.
654+--
655+-- Note: this implementation allows for stream fusion but has quadratic time
656+-- complexity, because each composition adds a new branch that each subsequent
657+-- fold's input element has to traverse, therefore, it cannot scale to a large
658+-- number of compositions. After around 100 compositions the performance starts
659+-- dipping rapidly compared to a CPS style implementation.
660+--
661+-- /Time: O(n^2) where n is the number of compositions./
662+--
663+-- @since 0.8.0
571664--
572665{-# INLINE serialWith #-}
573666serialWith :: Monad m => (a -> b -> c) -> Fold m x a -> Fold m x b -> Fold m x c
@@ -627,6 +720,17 @@ data GenericRunner sL sR bL bR
627720-- | @teeWith k f1 f2@ distributes its input to both @f1@ and @f2@ until both
628721-- of them terminate and combines their output using @k@.
629722--
723+-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
724+-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]
725+-- 50.5
726+--
727+-- > teeWith k f1 f2 = fmap (uncurry k) ((Fold.tee f1 f2)
728+--
729+-- For applicative composition using this combinator see
730+-- "Streamly.Internal.Data.Fold.Tee".
731+--
732+-- See also: "Streamly.Internal.Data.Fold.Tee"
733+--
630734-- @since 0.8.0
631735--
632736{-# INLINE teeWith #-}
@@ -687,7 +791,7 @@ teeWith f (Fold stepL beginL doneL) (Fold stepR beginR doneR) =
687791 bR <- doneR sR
688792 return $ f bL bR
689793
690−-- | Like 'teeWith' but terminates when the first fold terminates.
794+-- | Like 'teeWith' but terminates as soon as the first fold terminates.
691795--
692796-- /Unimplemented/
693797--
@@ -695,7 +799,8 @@ teeWith f (Fold stepL beginL doneL) (Fold stepR beginR doneR) =
695799teeWithFst :: (b -> c -> d) -> Fold m a b -> Fold m a c -> Fold m a d
696800teeWithFst = undefined
697801
698−-- | Like 'teeWith' but terminates when any fold terminates.
802+-- | Like 'teeWith' but terminates as soon as any one of the two folds
803+-- terminates.
699804--
700805-- /Unimplemented/
701806--
@@ -729,14 +834,26 @@ data ConcatMapState m sa a c
729834 = B !sa
730835 | forall s. C (s -> a -> m (Step s c)) !s (s -> m c)
731836
837+-- Compare with foldIterate.
838+--
732839-- | Map a 'Fold' returning function on the result of a 'Fold' and run the
733−-- returned fold.
840+-- returned fold. This operation can be used to express data dependencies
841+-- between fold operations.
842+--
843+-- Let's say the first element in the stream is a count of the following
844+-- elements that we have to add, then:
734845--
735846-- >>> import Data.Maybe (fromJust)
736−-- >>> Stream.fold (Fold.concatMap (flip Fold.take Fold.sum) (Fold.rmapM (return . fromJust) Fold.head)) $ Stream.fromList [10,9..1]
847+-- >>> count = fmap fromJust Fold.head
848+-- >>> total n = Fold.take n Fold.sum
849+-- >>> Stream.fold (Fold.concatMap total count) $ Stream.fromList [10,9..1]
737850-- 45
738851--
739−-- /Pre-release/
852+-- /Time: O(n^2) where @n@ is the number of compositions./
853+--
854+-- See also: "Streamly.Internal.Data.Stream.IsStream.foldIterateM"
855+--
856+-- @since 0.8.0
740857--
741858{-# INLINE concatMap #-}
742859concatMap :: Monad m => (b -> Fold m a c) -> Fold m a b -> Fold m a c
@@ -781,12 +898,14 @@ concatMap f (Fold stepa initiala extracta) = Fold stepc initialc extractc
781898-- Mapping on input
782899------------------------------------------------------------------------------
783900
784−-- | @(lmap f fold)@ maps the function @f@ on the input of the fold.
901+-- | @lmap f fold@ maps the function @f@ on the input of the fold.
785902--
786903-- >>> Stream.fold (Fold.lmap (\x -> x * x) Fold.sum) (Stream.enumerateFromTo 1 100)
787904-- 338350
788905--
789−-- /Pre-release/
906+-- > lmap = Fold.lmapM return
907+--
908+-- @since 0.8.0
790909{-# INLINABLE lmap #-}
791910lmap :: (a -> b) -> Fold m b r -> Fold m a r
792911lmap f (Fold step begin done) = Fold step' begin done
@@ -800,9 +919,9 @@ lmap f (Fold step begin done) = Fold step' begin done
800919map :: (a -> b) -> Fold m b r -> Fold m a r
801920map = lmap
802921
803−-- | @(lmapM f fold)@ maps the monadic function @f@ on the input of the fold.
922+-- | @lmapM f fold@ maps the monadic function @f@ on the input of the fold.
804923--
805−-- /Pre-release/
924+-- @since 0.8.0
806925{-# INLINABLE lmapM #-}
807926lmapM :: Monad m => (a -> m b) -> Fold m b r -> Fold m a r
808927lmapM f (Fold step begin done) = Fold step' begin done
@@ -818,10 +937,9 @@ lmapM f (Fold step begin done) = Fold step' begin done
818937-- >>> Stream.fold (Fold.filter (> 5) Fold.sum) $ Stream.fromList [1..10]
819938-- 40
820939--
821−-- >>> Stream.fold (Fold.filter (< 5) Fold.sum) $ Stream.fromList [1..10]
822−-- 10
940+-- > filter f = Fold.filterM (return . f)
823941--
824−-- @since 0.7.0
942+-- @since 0.8.0
825943{-# INLINABLE filter #-}
826944filter :: Monad m => (a -> Bool) -> Fold m a r -> Fold m a r
827945filter f (Fold step begin done) = Fold step' begin done
@@ -830,7 +948,7 @@ filter f (Fold step begin done) = Fold step' begin done
830948
831949-- | Like 'filter' but with a monadic predicate.
832950--
833−-- @since 0.7.0
951+-- @since 0.8.0
834952{-# INLINABLE filterM #-}
835953filterM :: Monad m => (a -> m Bool) -> Fold m a r -> Fold m a r
836954filterM f (Fold step begin done) = Fold step' begin done
@@ -842,7 +960,7 @@ filterM f (Fold step begin done) = Fold step' begin done
842960-- | Modify a fold to receive a 'Maybe' input, the 'Just' values are unwrapped
843961-- and sent to the original fold, 'Nothing' values are discarded.
844962--
845−-- /Pre-release/
963+-- @since 0.8.0
846964{-# INLINE catMaybes #-}
847965catMaybes :: Monad m => Fold m a b -> Fold m (Maybe a) b
848966catMaybes = filter isJust . map fromJust
@@ -851,15 +969,13 @@ catMaybes = filter isJust . map fromJust
851969-- Parsing
852970------------------------------------------------------------------------------
853971
854−-- | Take at most @n@ input elements and fold them using the supplied fold.
972+-- | Take at most @n@ input elements and fold them using the supplied fold. A
973+-- negative count is treated as 0.
855974--
856−-- >>> Stream.fold (Fold.take 1 Fold.toList) $ Stream.fromList [1]
857−-- [1]
975+-- >>> Stream.fold (Fold.take 2 Fold.toList) $ Stream.fromList [1..10]
976+-- [1,2]
858977--
859−-- >>> Stream.fold (Fold.take (-1) Fold.toList) $ Stream.fromList [1]
860−-- []
861−--
862−-- /Pre-release/
978+-- @since 0.8.0
863979{-# INLINE take #-}
864980take :: Monad m => Int -> Fold m a b -> Fold m a b
865981take n (Fold fstep finitial fextract) = Fold step initial extract
@@ -898,11 +1014,13 @@ take n (Fold fstep finitial fextract) = Fold step initial extract
8981014-- i.e. it uses the last accumulator value as the initial value of the
8991015-- accumulator. Thus we can resume the fold later and feed it more input.
9001016--
901−-- >> do
902−-- > more <- S.fold (FL.duplicate FL.sum) (S.enumerateFromTo 1 10)
903−-- > evenMore <- S.fold (FL.duplicate more) (S.enumerateFromTo 11 20)
904−-- > S.fold evenMore (S.enumerateFromTo 21 30)
905−-- > 465
1017+-- >>> :{
1018+-- do
1019+-- more <- Stream.fold (Fold.duplicate Fold.sum) (Stream.enumerateFromTo 1 10)
1020+-- evenMore <- Stream.fold (Fold.duplicate more) (Stream.enumerateFromTo 11 20)
1021+-- Stream.fold evenMore (Stream.enumerateFromTo 21 30)
1022+-- :}
1023+-- 465
9061024--
9071025-- /Pre-release/
9081026{-# INLINABLE duplicate #-}
@@ -959,11 +1077,16 @@ data ManyState s1 s2
9591077-- the @split@ fold repeatedly on the input stream and accumulates zero or more
9601078-- fold results using @collect@.
9611079--
1080+-- >>> two = Fold.take 2 Fold.toList
1081+-- >>> twos = Fold.many two Fold.toList
1082+-- >>> Stream.fold twos $ Stream.fromList [1..10]
1083+-- [[1,2],[3,4],[5,6],[7,8],[9,10]]
1084+--
9621085-- Stops when @collect@ stops.
9631086--
964−-- /Pre-release/
1087+-- See also: "Streamly.Prelude.concatMap", "Streamly.Prelude.foldMany"
9651088--
966−-- /See also: Streamly.Prelude.concatMap, Streamly.Prelude.foldMany/
1089+-- @since 0.8.0
9671090--
9681091{-# INLINE many #-}
9691092many :: Monad m => Fold m a b -> Fold m b c -> Fold m a c
@@ -1077,11 +1200,15 @@ manyPost (Fold sstep sinitial sextract) (Fold cstep cinitial cextract) =
10771200-- of @n@ items in the input stream and supplies the result to the @collect@
10781201-- fold.
10791202--
1203+-- >>> twos = Fold.chunksOf 2 Fold.toList Fold.toList
1204+-- >>> Stream.fold twos $ Stream.fromList [1..10]
1205+-- [[1,2],[3,4],[5,6],[7,8],[9,10]]
1206+--
10801207-- > chunksOf n split = many (take n split)
10811208--
10821209-- Stops when @collect@ stops.
10831210--
1084−-- /Pre-release/
1211+-- @since 0.8.0
10851212--
10861213{-# INLINE chunksOf #-}
10871214chunksOf :: Monad m => Int -> Fold m a b -> Fold m b c -> Fold m a c
@@ -1119,10 +1246,15 @@ chunksOf2 n (Fold step1 initial1 extract1) (Fold2 step2 inject2 extract2) =
11191246-- XXX We can use asyncClock here. A parser can be used to return an input that
11201247-- arrives after the timeout.
11211248-- XXX If n is 0 return immediately in initial.
1249+-- XXX we should probably discard the input received after the timeout like
1250+-- takeEndBy_.
11221251--
11231252-- | @takeInterval n fold@ uses @fold@ to fold the input items arriving within
11241253-- a window of first @n@ seconds.
11251254--
1255+-- >>> Stream.fold (Fold.takeInterval 1.0 Fold.toList) $ Stream.delay 0.1 $ Stream.fromList [1..]
1256+-- [1,2,3,4,5,6,7,8,9,10,11]
1257+--
11261258-- Stops when @fold@ stops or when the timeout occurs. Note that the fold needs
11271259-- an input after the timeout to stop. For example, if no input is pushed to
11281260-- the fold until one hour after the timeout had occurred, then the fold will
@@ -1177,18 +1309,16 @@ takeInterval n (Fold step initial done) = Fold step' initial' done'
11771309 handleChildException :: MVar Bool -> SomeException -> IO ()
11781310 handleChildException mv _ = void $ swapMVar mv True
11791311
1180−-- | Group the input stream into windows of n second each and then fold each
1181−-- group using the provided fold function.
1182−--
11831312-- For example, we can copy and distribute a stream to multiple folds where
11841313-- each fold can group the input differently e.g. by one second, one minute and
11851314-- one hour windows respectively and fold each resulting stream of folds.
1315+
1316+-- | Group the input stream into windows of n second each using the first fold
1317+-- and then fold the resulting groups using the second fold.
11861318--
1187−-- @
1188−--
1189−-- -----Fold m a b----|-Fold n a c-|-Fold n a c-|-...-|----Fold m a c
1190−--
1191−-- @
1319+-- >>> intervals = Fold.intervalsOf 0.5 Fold.toList Fold.toList
1320+-- >>> Stream.fold intervals $ Stream.delay 0.2 $ Stream.fromList [1..10]
1321+-- [[1,2,3,4],[5,6,7],[8,9,10]]
11921322--
11931323-- > intervalsOf n split = many (takeInterval n split)
11941324--
src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs+6−3
@@ -332,14 +332,17 @@ runSink = fold . toFold
332332-- Running a Parser
333333------------------------------------------------------------------------------
334334
335−-- | Parse a stream using the supplied 'Parser'.
335+-- | Parse a stream using the supplied ParserD 'PRD.Parser'.
336336--
337−-- /Pre-release/
337+-- /Internal/
338338--
339339{-# INLINE_NORMAL parseD #-}
340340parseD :: MonadThrow m => PRD.Parser m a b -> SerialT m a -> m b
341341parseD p = D.parse p . toStreamD
342342
343+-- | Parse a stream using the supplied ParserK 'PRK.Parser'.
344+--
345+-- /Internal/
343346{-# INLINE parseK #-}
344347parseK :: MonadThrow m => PRK.Parser m a b -> SerialT m a -> m b
345348parseK = parse
@@ -877,7 +880,7 @@ isPrefixOf m1 m2 = D.isPrefixOf (toStreamD m1) (toStreamD m2)
877880--
878881-- /Pre-release/
879882--
880−-- /Requires 'Storable' constraint/ - Help wanted.
883+-- /Requires 'Storable' constraint/
881884--
882885{-# INLINE isInfixOf #-}
883886isInfixOf :: (MonadIO m, Eq a, Enum a, Storable a)
src/Streamly/Internal/Data/Stream/IsStream/Nesting.hs+1−1
@@ -980,7 +980,7 @@ foldManyPost f m = D.fromStreamD $ D.foldManyPost f (D.toStreamD m)
980980-- Note @foldMany (take 0)@ would result in an infinite loop in a non-empty
981981-- stream.
982982--
983−-- /Pre-release/
983+-- @since 0.8.0
984984--
985985{-# INLINE foldMany #-}
986986foldMany
src/Streamly/Internal/Data/Stream/IsStream/Transform.hs+1−1
@@ -1192,7 +1192,7 @@ interspersePrefix_ m = mapM (\x -> void m >> return x)
11921192-- (AbsTime (TimeSpec {sec = ..., nsec = ...}),2)
11931193-- (AbsTime (TimeSpec {sec = ..., nsec = ...}),3)
11941194--
1195−-- /Pre-release/
1195+-- @since 0.8.0
11961196--
11971197{-# INLINE delay #-}
11981198delay :: (IsStream t, MonadIO m) => Double -> t m a -> t m a
src/Streamly/Internal/Network/Socket.hs−1
@@ -98,7 +98,6 @@ import Streamly.Data.Fold (Fold)
9898-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
9999
100100import qualified Streamly.Data.Fold as FL
101−import qualified Streamly.Internal.Data.Fold.Type as FL
102101import qualified Streamly.Internal.Data.Unfold as UF
103102import qualified Streamly.Internal.Data.Array.Foreign as IA
104103import qualified Streamly.Data.Array.Foreign as A
src/Streamly/Prelude.hs+2
@@ -247,6 +247,7 @@ module Streamly.Prelude
247247 -- $runningfolds
248248
249249 , fold
250+ , foldMany
250251
251252 -- ** Full Folds
252253 -- | Folds that are guaranteed to evaluate the whole stream.
@@ -343,6 +344,7 @@ module Streamly.Prelude
343344 , mapM_
344345 , trace
345346 , tap
347+ , delay
346348
347349 -- ** Scanning
348350 --
test/lib/Streamly/Test/Prelude/Common.hs−1
@@ -134,7 +134,6 @@ import Streamly.Prelude (avgRate, rate, maxBuffer, maxThreads)
134134import qualified Streamly.Prelude as S
135135import qualified Streamly.Data.Fold as FL
136136import qualified Streamly.Internal.Data.Stream.IsStream as S
137−import qualified Streamly.Internal.Data.Fold as FL
138137import qualified Streamly.Internal.Data.Unfold as UF
139138
140139import Streamly.Test.Common