git.iko.soy/streamly
on master help sign in
Update Fold module docs and expose some APIs
3b1a26cHarendra Kumar
Changelog.md+7−5
compare with master
unifiedsplit
@@ -2,9 +2,11 @@
2
2
3### Behavioral changes
3### Behavioral changes
4
4
5−* `Streamly.Prelude.fold` can now terminate early without consuming the entire
5+* `Streamly.Prelude.fold` may now terminate early without consuming the entire
6 stream. For example, `fold Fold.head stream` would now terminate immediately
6 stream. For example, `fold Fold.head stream` would now terminate immediately
7− after consuming the head element from `stream`.
7+ after consuming the head element from `stream`. This may result in change of
8+ behavior in existing programs if the program relies on the evaluation of the
9+ full stream.
8* Change the associativity of combinators `serial`, `wSerial`,
10* Change the associativity of combinators `serial`, `wSerial`,
9 `ahead`, `async`, `wAsync`, `parallel` to be the same as `<>`.
11 `ahead`, `async`, `wAsync`, `parallel` to be the same as `<>`.
10* `encodeUtf8`, `decodeUtf8` now replace any invalid character encountered
12* `encodeUtf8`, `decodeUtf8` now replace any invalid character encountered
@@ -26,14 +28,14 @@
26 require an additional `MonadAsync` constraint. Several other
28 require an additional `MonadAsync` constraint. Several other
27 functions that used these functions also now require the additional
29 functions that used these functions also now require the additional
28 constraint.
30 constraint.
29−* Remove `Applicative` instance of folds. Please use `teeWith` or the `Tee` type
31+* Remove `Applicative` instance of folds. Please use `teeWith` or the `Tee`
30− as an alternative to Fold applicative.
32+ type as an alternative to Fold applicative.
31
33
32### Enhancements
34### Enhancements
33
35
34* New encoding/decoding routines, `encodeUtf8'`, `encodeLatin1'`, `decodeUtf8'`,
36* New encoding/decoding routines, `encodeUtf8'`, `encodeLatin1'`, `decodeUtf8'`,
35 are added, these routines fail when they encounter any invalid characters.
37 are added, these routines fail when they encounter any invalid characters.
36−* `teeWith` is released as a part of `Streamly.Data.Fold`
38+* Several new functions added to `Streamly.Data.Fold`
37
39
38### Bug Fixes
40### Bug Fixes
39
41
benchmark/Streamly/Benchmark/Data/Fold.hs−1
@@ -19,7 +19,6 @@ import Streamly.Internal.Data.Fold (Fold(..))
19
19
20import qualified Data.Map.Strict as Map
20import qualified Data.Map.Strict as Map
21import qualified Streamly.Internal.Data.Fold as FL
21import qualified Streamly.Internal.Data.Fold as FL
22−import qualified Streamly.Internal.Data.Fold.Type as FL
23import qualified Streamly.Internal.Data.Pipe as Pipe
22import qualified Streamly.Internal.Data.Pipe as Pipe
24import qualified Streamly.Internal.Data.Sink as Sink
23import qualified Streamly.Internal.Data.Sink as Sink
25import qualified Streamly.Internal.Data.Stream.IsStream as IP
24import qualified Streamly.Internal.Data.Stream.IsStream as IP
benchmark/Streamly/Benchmark/Prelude/Concurrent.hs+20−17
@@ -8,9 +8,12 @@
8
8
9import Control.Concurrent
9import Control.Concurrent
10import Control.Monad (when, replicateM)
10import Control.Monad (when, replicateM)
11+import Streamly.Prelude
12+ ( IsStream, SerialT, serial, async, asyncly, ahead, aheadly, wAsync
13+ , wAsyncly, parallel, parallely
14+ )
11
15
12import Gauge
16import Gauge
13−import Streamly.Prelude hiding (mapM_, replicateM)
14import qualified Streamly.Prelude as S
17import qualified Streamly.Prelude as S
15
18
16-------------------------------------------------------------------------------
19-------------------------------------------------------------------------------
@@ -28,8 +31,8 @@ append buflen tcount d t =
28 let work = (\i -> when (d /= 0) (threadDelay d) >> return i)
31 let work = (\i -> when (d /= 0) (threadDelay d) >> return i)
29 in S.drain
32 in S.drain
30 $ t
33 $ t
31− $ maxBuffer buflen
34+ $ S.maxBuffer buflen
32− $ maxThreads (-1)
35+ $ S.maxThreads (-1)
33 $ S.fromFoldableM $ fmap work [1..tcount]
36 $ S.fromFoldableM $ fmap work [1..tcount]
34
37
35-- | Run @threads@ concurrently, each producing streams of @elems@ elements
38-- | Run @threads@ concurrently, each producing streams of @elems@ elements
@@ -48,28 +51,28 @@ concated
48concated buflen threads d elems t =
51concated buflen threads d elems t =
49 let work = \i -> S.replicateM i (when (d /= 0) (threadDelay d) >> return i)
52 let work = \i -> S.replicateM i (when (d /= 0) (threadDelay d) >> return i)
50 in S.drain
53 in S.drain
51− $ adapt
54+ $ S.adapt
52− $ maxThreads (-1)
55+ $ S.maxThreads (-1)
53− $ maxBuffer buflen
56+ $ S.maxBuffer buflen
54 $ S.concatMapWith t work
57 $ S.concatMapWith t work
55 $ S.replicate threads elems
58 $ S.replicate threads elems
56
59
57appendGroup :: Int -> Int -> Int -> [Benchmark]
60appendGroup :: Int -> Int -> Int -> [Benchmark]
58−appendGroup buflen threads delay =
61+appendGroup buflen threads usec =
59 [ -- bench "serial" $ nfIO $ append buflen threads delay serially
62 [ -- bench "serial" $ nfIO $ append buflen threads delay serially
60− bench "ahead" $ nfIO $ append buflen threads delay aheadly
63+ bench "ahead" $ nfIO $ append buflen threads usec aheadly
61− , bench "async" $ nfIO $ append buflen threads delay asyncly
64+ , bench "async" $ nfIO $ append buflen threads usec asyncly
62− , bench "wAsync" $ nfIO $ append buflen threads delay wAsyncly
65+ , bench "wAsync" $ nfIO $ append buflen threads usec wAsyncly
63− , bench "parallel" $ nfIO $ append buflen threads delay parallely
66+ , bench "parallel" $ nfIO $ append buflen threads usec parallely
64 ]
67 ]
65
68
66concatGroup :: Int -> Int -> Int -> Int -> [Benchmark]
69concatGroup :: Int -> Int -> Int -> Int -> [Benchmark]
67−concatGroup buflen threads delay n =
70+concatGroup buflen threads usec n =
68− [ bench "serial" $ nfIO $ concated buflen threads delay n serial
71+ [ bench "serial" $ nfIO $ concated buflen threads usec n serial
69− , bench "ahead" $ nfIO $ concated buflen threads delay n ahead
72+ , bench "ahead" $ nfIO $ concated buflen threads usec n ahead
70− , bench "async" $ nfIO $ concated buflen threads delay n async
73+ , bench "async" $ nfIO $ concated buflen threads usec n async
71− , bench "wAsync" $ nfIO $ concated buflen threads delay n wAsync
74+ , bench "wAsync" $ nfIO $ concated buflen threads usec n wAsync
72− , bench "parallel" $ nfIO $ concated buflen threads delay n parallel
75+ , bench "parallel" $ nfIO $ concated buflen threads usec n parallel
73 ]
76 ]
74
77
75main :: IO ()
78main :: IO ()
src/Streamly/Data/Fold.hs+168−179
@@ -1,79 +1,142 @@
1-- |
1-- |
2-- Module : Streamly.Data.Fold
2-- Module : Streamly.Data.Fold
3-- Copyright : (c) 2019 Composewell Technologies
3-- Copyright : (c) 2019 Composewell Technologies
4−-- License : BSD3
4+-- License : BSD-3-Clause
5-- Maintainer : streamly@composewell.com
5-- Maintainer : streamly@composewell.com
6-- Stability : experimental
6-- Stability : experimental
7-- Portability : GHC
7-- Portability : GHC
8--
8--
9+-- A 'Fold' is a sink or a consumer of a stream of values. The 'Fold' type
9−-- The 'Fold' type represents an effectful action that absorbs a value into an
10+-- consists of an accumulator and an effectful action that absorbs a value into
10−-- accumulator. For example, a 'sum' 'Fold' represents adding the input to the
11+-- the accumulator.
11−-- accumulated sum. A fold driver e.g. 'Streamly.Prelude.fold' pushes values
12−-- from a stream to the 'Fold' one at a time, reducing the stream to a single
13−-- value. Therefore, a fold can also be thought of as a sink or a stream
14−-- consumer.
15--
12--
16−-- A 'Fold' is in fact a data representation of a standard left fold ('foldl').
13+-- >>> import qualified Streamly.Data.Fold as Fold
17−-- Unlike the standard left folds, 'Fold's can terminate at any point e.g. the
14+-- >>> import qualified Streamly.Prelude as Stream
18−-- 'head' fold would terminate immediately after returning the head element.
15+--
16+-- For example, a 'sum' Fold represents adding the input to the accumulated
17+-- sum. A fold driver e.g. 'Streamly.Prelude.fold' pushes values from a stream
18+-- to the 'Fold' one at a time, reducing the stream to a single value.
19+--
20+-- >>> Stream.fold Fold.sum $ Stream.fromList [1..100]
21+-- 5050
22+--
23+-- Conceptually, a 'Fold' is a data type that can mimic a strict left fold
24+-- ('Data.List.foldl') as well as lazy right fold ('Prelude.foldr'). The above
25+-- example is similar to a left fold using @(+)@ as the step and @0@ as the
26+-- initial value of the accumulator:
27+--
28+-- >>> Data.List.foldl' (+) 0 [1..100]
29+-- 5050
30+--
31+-- 'Fold's have an early termination capability e.g. the 'head' fold would
32+-- terminate on an infinite stream:
33+--
34+-- >>> Stream.fold Fold.head $ Stream.fromList [1..]
35+-- Just 1
36+--
37+-- The above example is similar to the following right fold:
38+--
39+-- >>> Prelude.foldr (\x _ -> Just x) Nothing [1..]
40+-- Just 1
41+--
42+-- 'Fold's can be combined together using combinators. For example, to create a
43+-- fold that sums first two elements in a stream:
44+--
45+-- >>> sumTwo = Fold.take 2 Fold.sum
46+-- >>> Stream.fold sumTwo $ Stream.fromList [1..100]
47+-- 3
48+--
49+-- Folds can be combined to run in parallel on the same input. For example, to
50+-- compute the average of numbers in a stream without going through the stream
51+-- twice:
52+--
53+-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
54+-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]
55+-- 50.5
56+--
57+-- Folds can be combined so as to partition the input stream over multiple
58+-- folds. For example, to count even and odd numbers in a stream:
19--
59--
20−-- 'Fold's can be combined efficiently using combinators provided in this
60+-- >>> split n = if even n then Left n else Right n
21−-- module; a stream supplied to the combined fold is provided to the
61+-- >>> stream = Stream.map split $ Stream.fromList [1..100]
22−-- constituent folds according to the behavior of the combinator. For example,
62+-- >>> countEven = fmap (("Even " ++) . show) Fold.length
23−-- the 'tee' combinator distributes the input stream to two folds and then
63+-- >>> countOdd = fmap (("Odd " ++) . show) Fold.length
24−-- combines the resulting fold outputs. Similarly, a 'partition' combinator
64+-- >>> f = Fold.partition countEven countOdd
25−-- divides the input among constituent folds.
65+-- >>> Stream.fold f stream
66+-- ("Even 50","Odd 50")
26--
67--
27−-- = Accumulators and Terminating Folds
68+-- Terminating folds can be combined to parse the stream serially such that the
69+-- first fold consumes the input until it terminates and the second fold
70+-- consumes the rest of the input until it terminates:
28--
71--
29−-- Folds in this module can be classified in two categories viz. accumulators
72+-- >>> f = Fold.serialWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)
30−-- and terminating folds. Accumulators do not have a terminating condition,
73+-- >>> Stream.fold f $ Stream.fromList "header: hello\n"
31−-- they run forever and consume the entire stream, for example the 'length'
74+-- ("header: ","hello\n")
32−-- fold. Terminating folds have a terminating condition and can terminate
33−-- without consuming the entire stream, for example, the 'head' fold.
34--
75--
35−-- = Monoids
76+-- A 'Fold' can be applied repeatedly on a stream to transform it to a stream
77+-- of fold results. To split a stream on newlines:
36--
78--
37−-- Monoids allow generalized, modular folding. The accumulators in this module
79+-- >>> f = Fold.takeEndBy (== '\n') Fold.toList
38−-- can be expressed using 'mconcat' and a suitable 'Monoid'. Instead of
80+-- >>> Stream.toList $ Stream.foldMany f $ Stream.fromList "Hello there!\nHow are you\n"
39−-- writing folds we can write Monoids and turn them into folds.
81+-- ["Hello there!\n","How are you\n"]
40--
82--
41−-- = Performance Notes
83+-- Similarly, we can split the input of a fold too:
42--
84--
43−-- 'Streamly.Prelude' module provides fold functions to directly fold streams
85+-- >>> Stream.fold (Fold.many f Fold.toList) $ Stream.fromList "Hello there!\nHow are you\n"
44−-- e.g. Streamly.Prelude/'Streamly.Prelude.sum' serves the same purpose as
86+-- ["Hello there!\n","How are you\n"]
45−-- Fold/'sum'. However, the functions in Streamly.Prelude cannot be
46−-- efficiently combined together e.g. we cannot drive the input stream through
47−-- @sum@ and @length@ fold functions simultaneously. Using the 'Fold' type we
48−-- can efficiently split the stream across mutliple folds because it allows the
49−-- compiler to perform stream fusion optimizations.
50--
87--
51−-- = Programmer Notes
88+-- Please see "Streamly.Internal.Data.Fold" for additional @Pre-release@
89+-- functions.
52--
90--
53−-- > import qualified Streamly.Data.Fold as Fold
91+-- = Folds vs. Streams
54--
92--
55−-- More, not yet exposed, fold combinators can be found in
93+-- We can often use streams or folds to achieve the same goal. However, streams
56−-- "Streamly.Internal.Data.Fold".
94+-- allow efficient composition of producers (e.g. 'Streamly.Prelude.serial' or
95+-- 'Streamly.Prelude.mergeBy') whereas folds allow efficient composition of
96+-- consumers (e.g. 'serialWith', 'partition' or 'teeWith').
97+--
98+-- Streams are producers, transformations on streams happen on the output side:
99+--
100+-- >>> f = Stream.sum . Stream.map (+1) . Stream.filter odd
101+-- >>> f $ Stream.fromList [1..100]
102+-- 2550
103+--
104+-- Folds are stream consumers with an input stream and an output value, stream
105+-- transformations on folds happen on the input side:
106+--
107+-- >>> f = Fold.filter odd $ Fold.lmap (+1) $ Fold.sum
108+-- >>> Stream.fold f $ Stream.fromList [1..100]
109+-- 2550
110+--
111+-- Notice the composition by @.@ vs @$@ and the order of operations in the
112+-- above examples, the difference is due to output vs input side
113+-- transformations.
57
114
58module Streamly.Data.Fold
115module Streamly.Data.Fold
59 (
116 (
60 -- * Fold Type
117 -- * Fold Type
61− -- |
62− -- A 'Fold' can be run over a stream using the 'Streamly.Prelude.fold'
63− -- combinator:
64− --
65− -- >>> Stream.fold Fold.sum (Stream.enumerateFromTo 1 100)
66− -- 5050
67
118
68 Fold -- (..)
119 Fold -- (..)
69
120
121+ -- * Constructors
122+ , mkFoldl
123+ , mkFoldlM
124+ , mkFoldr
125+
126+ -- * Folds
70− -- * Accumulators
127+ -- ** Accumulators
71− -- ** Monoids
128+ -- | Folds that never terminate, these folds are much like strict left
129+ -- folds. 'mconcat' is the fundamental accumulator. All other accumulators
130+ -- can be expressed in terms of 'mconcat' using a suitable Monoid. Instead
131+ -- of writing folds we could write Monoids and turn them into folds.
132+
133+ -- Monoids
134+ , sconcat
72 , mconcat
135 , mconcat
73 , foldMap
136 , foldMap
74 , foldMapM
137 , foldMapM
75
138
76− -- ** Reducers
139+ -- Reducers
77 , drain
140 , drain
78 , drainBy
141 , drainBy
79 , last
142 , last
@@ -88,22 +151,16 @@ module Streamly.Data.Fold
88 , variance
151 , variance
89 , stdDev
152 , stdDev
90
153
91− -- ** Collectors
92− -- | Avoid using these folds in scalable or performance critical
93− -- applications, they buffer all the input in GC memory which can be
94− -- detrimental to performance if the input is large.
95
154
155+ -- Collectors
96 , toList
156 , toList
157+ , toListRev
158+
159+ -- ** Terminating Folds
160+ -- | These are much like lazy right folds.
97
161
98− -- * Terminating Folds
99− -- , drainN
100− -- , drainWhile
101− -- , lastN
102− -- , (!!)
103− -- , genericIndex
104 , index
162 , index
105 , head
163 , head
106− -- , findM
107 , find
164 , find
108 , lookup
165 , lookup
109 , findIndex
166 , findIndex
@@ -116,143 +173,82 @@ module Streamly.Data.Fold
116 , and
173 , and
117 , or
174 , or
118
175
119− -- * Output Transformations
176+ -- * Combinators
120− -- | Unlike stream producer types (e.g. @SerialT m a@) which have only
177+ -- | Combinators are modifiers of folds. In the type @Fold m a b@, @a@ is
121− -- output side, folds have an input side as well as an output side. In the
122− -- type @Fold m a b@, the input type is @a@ and the output type is @b@.
178+ -- the input type and @b@ is the output type. Transformations can be
123− -- Transformations can be applied either on the input side or on the output
179+ -- applied either on the input side or on the output side. Therefore,
180+ -- combinators are of one of the following general shapes:
181+ --
182+ -- * @... -> Fold m a b -> Fold m c b@ (input transformation)
183+ -- * @... -> Fold m a b -> Fold m a c@ (output transformation)
184+ --
185+ -- Output transformations are also known as covariant transformations, and
186+ -- input transformations are also known as contravariant transformations.
187+ -- The input side transformations are more interesting for folds. Most of
188+ -- the following sections describe the input transformation operations on a
189+ -- fold. The names and signatures of the operations are consistent with
190+ -- corresponding operations in "Streamly.Prelude". When an operation makes
191+ -- sense on both input and output side we use the prefix @l@ (for left) for
192+ -- input side operations and the prefix @r@ (for right) for output side
193+ -- operations.
194+
195+ -- ** Mapping on output
124− -- side. The 'Functor' instance of a fold maps on the output of the fold:
196+ -- | The 'Functor' instance of a fold maps on the output of the fold:
125 --
197 --
126 -- >>> Stream.fold (fmap show Fold.sum) (Stream.enumerateFromTo 1 100)
198 -- >>> Stream.fold (fmap show Fold.sum) (Stream.enumerateFromTo 1 100)
127 -- "5050"
199 -- "5050"
128 --
200 --
129− -- Note: Output transformations are also known as covariant
130− -- transformations.
131− , sequence
132 , rmapM
201 , rmapM
133− , mapM
134−
135− {-
136− -- * Input Transformations
137− -- The input side transformations are more interesting for folds. The
138− -- following sections describe the input transformation operations on a
139− -- fold. The names and signatures of the operations are consistent with
140− -- corresponding operations in "Streamly.Prelude".
141− --
142− -- Note: Input transformations are also known as contravariant
143− -- transformations.
144−
145− -- ** Mapping
146− --, transform
147− -- , lmap
148− --, lsequence
149− -- , lmapM
150−
151− -- -- ** Filtering
152− -- , filter
153− -- , filterM
154− -- , ldeleteBy
155− -- , luniq
156−
157− -- ** Mapping Filters
158− , lmapMaybe
159− , lmapMaybeM
160−
161− -- ** Scanning Filters
162− , lfindIndices
163− , lelemIndices
164
202
165− -- ** Insertion
203+ -- ** Mapping on Input
166− -- | Insertion adds more elements to the stream.
204+ , lmap
205+ , lmapM
167
206
168− , linsertBy
207+ -- ** Filtering
169− , lintersperseM
208+ , filter
209+ , filterM
170
210
171− -- ** Reordering
211+ -- -- ** Mapping Filters
172− , lreverse
212+ , catMaybes
173− -}
213+ , mapMaybe
174
214
175− {-
176 -- ** Trimming
215 -- ** Trimming
177 , take
216 , take
178− -- takeByTime
217+ -- , takeInterval
179− , ldrop
218+ , takeEndBy_
180− , ldropWhile
219+ , takeEndBy
181− , ldropWhileM
182− -}
183
220
184− -- * Distributing
221+ -- ** Serial Append
185− -- |
222+ , serialWith
186− -- The 'Applicative' instance of a distributing 'Fold' distributes one copy
223+
187− -- of the stream to each fold and combines the results using a function.
224+ -- ** Parallel Distribution
188− --
225+ -- | For applicative composition using distribution see
189− -- @
226+ -- "Streamly.Internal.Data.Fold.Tee".
190− --
191− -- |-------Fold m a b--------|
192− -- ---stream m a---| |---m (b,c,...)
193− -- |-------Fold m a c--------|
194− -- | |
195− -- ...
196− -- @
197− --
198− -- To compute the average of numbers in a stream without going through the
199− -- stream twice:
200− --
201− -- >>> let avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
202− -- >>> Stream.fold avg (Stream.enumerateFromTo 1.0 100.0)
203− -- 50.5
204− --
205− -- The 'Semigroup' and 'Monoid' instances of a distributing fold distribute
206− -- the input to both the folds and combines the outputs using Monoid or
207− -- Semigroup instances of the output types:
208− --
209− -- >>> import Data.Monoid (Sum(..))
210− -- >>> Stream.fold (Fold.teeWith (<>) Fold.head Fold.last) (fmap Sum $ Stream.enumerateFromTo 1.0 100.0)
211− -- Just (Sum {getSum = 101.0})
212− --
213− -- The 'Num', 'Floating', and 'Fractional' instances work in the same way.
214
227
215− , tee
216 , teeWith
228 , teeWith
229+ , tee
217 , distribute
230 , distribute
218
231
219− -- * Partitioning
232+ -- ** Partitioning
220− -- |
221− -- Direct items in the input stream to different folds using a binary
233+ -- | Direct items in the input stream to different folds using a binary
222 -- fold selector.
234 -- fold selector.
223
235
224− -- , partitionByM
225− -- , partitionBy
226 , partition
236 , partition
227
237
228− {-
238+ -- ** Unzipping
229− -- * Demultiplexing
239+ , unzip
230− -- | Direct values in the input stream to different folds using an n-ary
231− -- fold selector.
232−
233− , demux
234− -- , demuxWith
235− , demux_
236− -- , demuxWith_
237−
238− -- * Classifying
239− -- | In an input stream of key value pairs fold values for different keys
240− -- in individual output buckets using the given fold.
241
240
242− , classify
241+ -- ** Splitting
243− -- , classifyWith
242+ , many
244− -}
243+ , chunksOf
244+ -- , intervalsOf
245
245
246− -- * Unzipping
246+ -- ** Nesting
247− , unzip
247+ , concatMap
248− -- These can be expressed using lmap/lmapM and unzip
249− -- , unzipWith
250− -- , unzipWithM
251
248
252− -- -- * Nesting
249+ -- * Deprecated
253− -- , concatMap
250+ , sequence
254− -- , chunksOf
251+ , mapM
255− -- , duplicate -- experimental
256 )
252 )
257where
253where
258
254
@@ -265,10 +261,3 @@ import Prelude
265 span, splitAt, break, mapM)
261 span, splitAt, break, mapM)
266
262
267import Streamly.Internal.Data.Fold
263import Streamly.Internal.Data.Fold
268−
269−--
270−-- $setup
271−-- >>> :m
272−-- >>> import Prelude hiding (head, sum, last, length)
273−-- >>> import qualified Streamly.Prelude as Stream
274−-- >>> import qualified Streamly.Data.Fold as Fold
src/Streamly/Internal/Data/Fold.hs+171−176
@@ -7,7 +7,8 @@
7-- Stability : experimental
7-- Stability : experimental
8-- Portability : GHC
8-- Portability : GHC
9--
9--
10−-- See "Streamly.Data.Fold" for an overview.
10+-- See "Streamly.Data.Fold" for an overview and
11+-- "Streamly.Internal.Data.Fold.Types" for design notes.
11--
12--
12-- IMPORTANT: keep the signatures consistent with the folds in Streamly.Prelude
13-- IMPORTANT: keep the signatures consistent with the folds in Streamly.Prelude
13
14
@@ -17,9 +18,10 @@ module Streamly.Internal.Data.Fold
17 Step (..)
18 Step (..)
18 , Fold (..)
19 , Fold (..)
19
20
20− -- * Creating
21+ -- * Constructors
21 , mkFoldl
22 , mkFoldl
22 , mkFoldlM
23 , mkFoldlM
24+ , mkFoldl1
23 , mkFoldr
25 , mkFoldr
24 , mkFoldrM
26 , mkFoldrM
25 , mkFold
27 , mkFold
@@ -27,21 +29,21 @@ module Streamly.Internal.Data.Fold
27 , mkFoldM
29 , mkFoldM
28 , mkFoldM_
30 , mkFoldM_
29
31
30− -- * Generators
32+ -- * Folds
33+ -- ** Identity
31 , yield
34 , yield
32 , yieldM
35 , yieldM
33
36
34− -- * Accumulators
37+ -- ** Accumulators
35− -- ** Semigroups and Monoids
38+ -- *** Semigroups and Monoids
36 , sconcat
39 , sconcat
37 , mconcat
40 , mconcat
38 , foldMap
41 , foldMap
39 , foldMapM
42 , foldMapM
40
43
41− -- ** Reducers
44+ -- *** Reducers
42 , drain
45 , drain
43 , drainBy
46 , drainBy
44− , drainBy2
45 , last
47 , last
46 , length
48 , length
47 , mean
49 , mean
@@ -52,7 +54,7 @@ module Streamly.Internal.Data.Fold
52 , rollingHashFirstN
54 , rollingHashFirstN
53 -- , rollingHashLastN
55 -- , rollingHashLastN
54
56
55− -- ** Saturating Reducers
57+ -- *** Saturating Reducers
56 -- | 'product' terminates if it becomes 0. Other folds can theoretically
58 -- | 'product' terminates if it becomes 0. Other folds can theoretically
57 -- saturate on bounded types, and therefore terminate, however, they will
59 -- saturate on bounded types, and therefore terminate, however, they will
58 -- run forever on unbounded types like Integer/Double.
60 -- run forever on unbounded types like Integer/Double.
@@ -63,14 +65,17 @@ module Streamly.Internal.Data.Fold
63 , minimumBy
65 , minimumBy
64 , minimum
66 , minimum
65
67
66− -- ** Collectors
68+ -- *** Collectors
69+ -- | Avoid using these folds in scalable or performance critical
70+ -- applications, they buffer all the input in GC memory which can be
71+ -- detrimental to performance if the input is large.
67 , toList
72 , toList
68 , toListRev
73 , toListRev
69 -- $toListRev
74 -- $toListRev
70 , toStream
75 , toStream
71 , toStreamRev
76 , toStreamRev
72
77
73− -- * Terminating Folds
78+ -- ** Terminating Folds
74 , drainN
79 , drainN
75 -- , lastN
80 -- , lastN
76 -- , (!!)
81 -- , (!!)
@@ -91,26 +96,21 @@ module Streamly.Internal.Data.Fold
91 , or
96 , or
92 -- , the
97 -- , the
93
98
94− -- * Adapting
99+ -- * Combinators
100+ -- ** Utilities
101+ , with
102+
103+ -- ** Transforming the Monad
95 , hoist
104 , hoist
96 , generally
105 , generally
97
106
98− -- * Running Incrementally
107+ -- ** Mapping on output
99− , initialize
100− , runStep
101−
102− -- * Output Transformations
103− , sequence
104 , rmapM
108 , rmapM
105− , mapM
106
109
107− -- * Input Transformations
108−
109− -- ** Mapping
110+ -- ** Mapping on Input
110 , transform
111 , transform
111 , map
112 , map
112 , lmap
113 , lmap
113−
114 --, lsequence
114 --, lsequence
115 , lmapM
115 , lmapM
116 , indexed
116 , indexed
@@ -125,63 +125,67 @@ module Streamly.Internal.Data.Fold
125 -- ** Mapping Filters
125 -- ** Mapping Filters
126 , catMaybes
126 , catMaybes
127 , mapMaybe
127 , mapMaybe
128+ -- , mapMaybeM
128
129
129 {-
130 {-
130 -- ** Scanning Filters
131 -- ** Scanning Filters
131− , lfindIndices
132+ , findIndices
132− , lelemIndices
133+ , elemIndices
133
134
134 -- ** Insertion
135 -- ** Insertion
135 -- | Insertion adds more elements to the stream.
136 -- | Insertion adds more elements to the stream.
136
137
137− , linsertBy
138+ , insertBy
138− , lintersperseM
139+ , intersperseM
139
140
140 -- ** Reordering
141 -- ** Reordering
141− , lreverse
142+ , reverse
142 -}
143 -}
143
144
144 -- ** Trimming
145 -- ** Trimming
145 , take
146 , take
146 , takeInterval
147 , takeInterval
148+
147 -- By elements
149 -- By elements
148− , takeEndBy_
149 , takeEndBy
150 , takeEndBy
151+ , takeEndBy_
150 -- , takeEndBySeq
152 -- , takeEndBySeq
151 {-
153 {-
152− , ldrop
154+ , drop
153− , ldropWhile
155+ , dropWhile
154− , ldropWhileM
156+ , dropWhileM
155 -}
157 -}
156
158
157− -- * Splitting
159+ -- ** Serial Append
158−
160+ , serialWith
159− -- Binary
160 -- , tail
161 -- , tail
161 -- , init
162 -- , init
162 , splitAt -- spanN
163 , splitAt -- spanN
163 -- , splitIn -- sessionN
164 -- , splitIn -- sessionN
164
165
165− -- * Distributing
166+ -- ** Parallel Distribution
166−
167− , tee
168 , teeWith
167 , teeWith
168+ , tee
169 , teeWithFst
169 , teeWithFst
170 , teeWithMin
170 , teeWithMin
171 , distribute
171 , distribute
172 -- , distributeFst
172 -- , distributeFst
173 -- , distributeMin
173 -- , distributeMin
174
174
175− -- * Partitioning
175+ -- ** Parallel Alternative
176+ , shortest
177+ , longest
176
178
179+ -- ** Partitioning
177 , partitionByM
180 , partitionByM
178 , partitionByFstM
181 , partitionByFstM
179 , partitionByMinM
182 , partitionByMinM
180 , partitionBy
183 , partitionBy
181 , partition
184 , partition
182
185
183− -- * Demultiplexing
186+ -- ** Demultiplexing
184−
187+ -- | Direct values in the input stream to different folds using an n-ary
188+ -- fold selector.
185 , demux -- XXX rename this to demux_
189 , demux -- XXX rename this to demux_
186 , demuxWith
190 , demuxWith
187 , demuxDefault -- XXX rename this to demux
191 , demuxDefault -- XXX rename this to demux
@@ -189,31 +193,47 @@ module Streamly.Internal.Data.Fold
189 -- , demuxWithSel
193 -- , demuxWithSel
190 -- , demuxWithMin
194 -- , demuxWithMin
191
195
192− -- * Classifying
196+ -- ** Classifying
193−
197+ -- | In an input stream of key value pairs fold values for different keys
198+ -- in individual output buckets using the given fold.
194 , classify
199 , classify
195 , classifyWith
200 , classifyWith
196 -- , classifyWithSel
201 -- , classifyWithSel
197 -- , classifyWithMin
202 -- , classifyWithMin
198
203
199− -- * Unzipping
204+ -- ** Unzipping
200 , unzip
205 , unzip
206+ -- These two can be expressed using lmap/lmapM and unzip
201 , unzipWith
207 , unzipWith
202 , unzipWithM
208 , unzipWithM
203 , unzipWithFstM
209 , unzipWithFstM
204 , unzipWithMinM
210 , unzipWithMinM
205
211
206− -- * Nesting
212+ -- ** Zipping
213+ , zipWithM
214+ , zip
215+
216+ -- ** Splitting
207 , many
217 , many
208 , intervalsOf
218 , intervalsOf
209 , chunksOf
219 , chunksOf
210 , chunksBetween
220 , chunksBetween
211− , zipWithM
212− , zip
213
221
222+ -- ** Nesting
214 , concatSequence
223 , concatSequence
215 , concatMap
224 , concatMap
225+
226+ -- * Running Partially
227+ , initialize
228+ , runStep
216 , duplicate
229 , duplicate
230+
231+ -- * Fold2
232+ , drainBy2
233+
234+ -- * Deprecated
235+ , sequence
236+ , mapM
217 )
237 )
218where
238where
219
239
@@ -228,7 +248,6 @@ import Data.Semigroup (Semigroup((<>)))
228#endif
248#endif
229import Streamly.Internal.Data.Either.Strict
249import Streamly.Internal.Data.Either.Strict
230 (Either'(..), fromLeft', fromRight', isLeft', isRight')
250 (Either'(..), fromLeft', fromRight', isLeft', isRight')
231−import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)
232import Streamly.Internal.Data.Pipe.Type (Pipe (..), PipeState(..))
251import Streamly.Internal.Data.Pipe.Type (Pipe (..), PipeState(..))
233import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
252import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
234import Streamly.Internal.Data.Stream.Serial (SerialT)
253import Streamly.Internal.Data.Stream.Serial (SerialT)
@@ -270,7 +289,7 @@ hoist f (Fold step initial extract) =
270
289
271-- | Adapt a pure fold to any monad
290-- | Adapt a pure fold to any monad
272--
291--
273−-- > generally = hoist (return . runIdentity)
292+-- > generally = Fold.hoist (return . runIdentity)
274--
293--
275-- /Pre-release/
294-- /Pre-release/
276generally :: Monad m => Fold Identity a b -> Fold m a b
295generally :: Monad m => Fold Identity a b -> Fold m a b
@@ -305,7 +324,7 @@ mapM = rmapM
305-- >>> Stream.fold fld (Stream.enumerateFromTo 1 10)
324-- >>> Stream.fold fld (Stream.enumerateFromTo 1 10)
306-- [2,4,6,8,10]
325-- [2,4,6,8,10]
307--
326--
308−-- /Pre-release/
327+-- @since 0.8.0
309{-# INLINE mapMaybe #-}
328{-# INLINE mapMaybe #-}
310mapMaybe :: (Monad m) => (a -> Maybe b) -> Fold m b r -> Fold m a r
329mapMaybe :: (Monad m) => (a -> Maybe b) -> Fold m b r -> Fold m a r
311mapMaybe f = map f . filter isJust . map fromJust
330mapMaybe f = map f . filter isJust . map fromJust
@@ -318,7 +337,7 @@ mapMaybe f = map f . filter isJust . map fromJust
318--
337--
319-- | Apply a transformation on a 'Fold' using a 'Pipe'.
338-- | Apply a transformation on a 'Fold' using a 'Pipe'.
320--
339--
321−-- @since 0.7.0
340+-- /Pre-release/
322{-# INLINE transform #-}
341{-# INLINE transform #-}
323transform :: Monad m => Pipe m a b -> Fold m b c -> Fold m a c
342transform :: Monad m => Pipe m a b -> Fold m b c -> Fold m a c
324transform (Pipe pstep1 pstep2 pinitial) (Fold fstep finitial fextract) =
343transform (Pipe pstep1 pstep2 pinitial) (Fold fstep finitial fextract) =
@@ -355,23 +374,6 @@ transform (Pipe pstep1 pstep2 pinitial) (Fold fstep finitial fextract) =
355
374
356 extract (Tuple' _ fs) = fextract fs
375 extract (Tuple' _ fs) = fextract fs
357
376
358−------------------------------------------------------------------------------
359−-- Utilities
360−------------------------------------------------------------------------------
361−
362−-- | @_Fold1 step@ returns a new 'Fold' using just a step function that has the
363−-- same type for the accumulator and the element. The result type is the
364−-- accumulator type wrapped in 'Maybe'. The initial accumulator is retrieved
365−-- from the 'Foldable', the result is 'None' for empty containers.
366−{-# INLINABLE _Fold1 #-}
367−_Fold1 :: Monad m => (a -> a -> a) -> Fold m a (Maybe a)
368−_Fold1 step = fmap toMaybe $ mkFoldl step_ Nothing'
369−
370− where
371−
372− step_ Nothing' a = Just' a
373− step_ (Just' x) a = Just' $ step x a
374−
375------------------------------------------------------------------------------
377------------------------------------------------------------------------------
376-- Left folds
378-- Left folds
377------------------------------------------------------------------------------
379------------------------------------------------------------------------------
@@ -382,11 +384,13 @@ _Fold1 step = fmap toMaybe $ mkFoldl step_ Nothing'
382
384
383-- |
385-- |
384-- > drainBy f = lmapM f drain
386-- > drainBy f = lmapM f drain
385−-- > drainBy = FL.foldMapM (void . f)
387+-- > drainBy = Fold.foldMapM (void . f)
386--
388--
387-- Drain all input after passing it through a monadic function. This is the
389-- Drain all input after passing it through a monadic function. This is the
388-- dual of mapM_ on stream producers.
390-- dual of mapM_ on stream producers.
389--
391--
392+-- See also: "Streamly.Prelude.mapM_"
393+--
390-- @since 0.7.0
394-- @since 0.7.0
391{-# INLINABLE drainBy #-}
395{-# INLINABLE drainBy #-}
392drainBy :: Monad m => (a -> m b) -> Fold m a ()
396drainBy :: Monad m => (a -> m b) -> Fold m a ()
@@ -401,12 +405,12 @@ drainBy2 f = Fold2 (const (void . f)) (\_ -> return ()) return
401
405
402-- | Extract the last element of the input stream, if any.
406-- | Extract the last element of the input stream, if any.
403--
407--
404−-- > last = fmap getLast $ FL.foldMap (Last . Just)
408+-- > last = fmap getLast $ Fold.foldMap (Last . Just)
405--
409--
406-- @since 0.7.0
410-- @since 0.7.0
407{-# INLINABLE last #-}
411{-# INLINABLE last #-}
408last :: Monad m => Fold m a (Maybe a)
412last :: Monad m => Fold m a (Maybe a)
409−last = _Fold1 (\_ x -> x)
413+last = mkFoldl1 (\_ x -> x)
410
414
411------------------------------------------------------------------------------
415------------------------------------------------------------------------------
412-- To Summary
416-- To Summary
@@ -416,14 +420,14 @@ last = _Fold1 (\_ x -> x)
416--
420--
417-- > genericLength = fmap getSum $ foldMap (Sum . const 1)
421-- > genericLength = fmap getSum $ foldMap (Sum . const 1)
418--
422--
419−-- @since 0.7.0
423+-- /Pre-release/
420{-# INLINE genericLength #-}
424{-# INLINE genericLength #-}
421genericLength :: (Monad m, Num b) => Fold m a b
425genericLength :: (Monad m, Num b) => Fold m a b
422genericLength = mkFoldl (\n _ -> n + 1) 0
426genericLength = mkFoldl (\n _ -> n + 1) 0
423
427
424-- | Determine the length of the input stream.
428-- | Determine the length of the input stream.
425--
429--
426−-- > length = fmap getSum $ foldMap (Sum . const 1)
430+-- > length = fmap getSum $ Fold.foldMap (Sum . const 1)
427--
431--
428-- @since 0.7.0
432-- @since 0.7.0
429{-# INLINE length #-}
433{-# INLINE length #-}
@@ -434,7 +438,7 @@ length = genericLength
434-- identity (@0@) when the stream is empty. Note that this is not numerically
438-- identity (@0@) when the stream is empty. Note that this is not numerically
435-- stable for floating point numbers.
439-- stable for floating point numbers.
436--
440--
437−-- > sum = fmap getSum $ FL.foldMap Sum
441+-- > sum = fmap getSum $ Fold.foldMap Sum
438--
442--
439-- @since 0.7.0
443-- @since 0.7.0
440{-# INLINE sum #-}
444{-# INLINE sum #-}
@@ -445,7 +449,7 @@ sum = mkFoldl (+) 0
445-- multiplicative identity (@1@) when the stream is empty. The fold terminates
449-- multiplicative identity (@1@) when the stream is empty. The fold terminates
446-- when it encounters (@0@) in its input.
450-- when it encounters (@0@) in its input.
447--
451--
448−-- > product = fmap getProduct $ FL.foldMap Product
452+-- Compare with @Fold.foldMap Product@.
449--
453--
450-- @since 0.7.0
454-- @since 0.7.0
451-- /Since 0.8.0 (Added 'Eq' constraint)/
455-- /Since 0.8.0 (Added 'Eq' constraint)/
@@ -470,7 +474,7 @@ product = mkFold_ step (Partial 1)
470-- @since 0.7.0
474-- @since 0.7.0
471{-# INLINE maximumBy #-}
475{-# INLINE maximumBy #-}
472maximumBy :: Monad m => (a -> a -> Ordering) -> Fold m a (Maybe a)
476maximumBy :: Monad m => (a -> a -> Ordering) -> Fold m a (Maybe a)
473−maximumBy cmp = _Fold1 max'
477+maximumBy cmp = mkFoldl1 max'
474
478
475 where
479 where
476
480
@@ -481,24 +485,24 @@ maximumBy cmp = _Fold1 max'
481
485
482-- |
486-- |
483-- @
487-- @
484−-- maximum = 'maximumBy' compare
488+-- maximum = Fold.maximumBy compare
485-- @
489-- @
486--
490--
487-- Determine the maximum element in a stream.
491-- Determine the maximum element in a stream.
488--
492--
489−-- Compare with @FL.foldMap Max@.
493+-- Compare with @Fold.foldMap Max@.
490--
494--
491-- @since 0.7.0
495-- @since 0.7.0
492{-# INLINE maximum #-}
496{-# INLINE maximum #-}
493maximum :: (Monad m, Ord a) => Fold m a (Maybe a)
497maximum :: (Monad m, Ord a) => Fold m a (Maybe a)
494−maximum = _Fold1 max
498+maximum = mkFoldl1 max
495
499
496-- | Computes the minimum element with respect to the given comparison function
500-- | Computes the minimum element with respect to the given comparison function
497--
501--
498-- @since 0.7.0
502-- @since 0.7.0
499{-# INLINE minimumBy #-}
503{-# INLINE minimumBy #-}
500minimumBy :: Monad m => (a -> a -> Ordering) -> Fold m a (Maybe a)
504minimumBy :: Monad m => (a -> a -> Ordering) -> Fold m a (Maybe a)
501−minimumBy cmp = _Fold1 min'
505+minimumBy cmp = mkFoldl1 min'
502
506
503 where
507 where
504
508
@@ -514,12 +518,12 @@ minimumBy cmp = _Fold1 min'
514-- minimum = 'minimumBy' compare
518-- minimum = 'minimumBy' compare
515-- @
519-- @
516--
520--
517−-- Compare with @FL.foldMap Min@.
521+-- Compare with @Fold.foldMap Min@.
518--
522--
519-- @since 0.7.0
523-- @since 0.7.0
520{-# INLINE minimum #-}
524{-# INLINE minimum #-}
521minimum :: (Monad m, Ord a) => Fold m a (Maybe a)
525minimum :: (Monad m, Ord a) => Fold m a (Maybe a)
522−minimum = _Fold1 min
526+minimum = mkFoldl1 min
523
527
524------------------------------------------------------------------------------
528------------------------------------------------------------------------------
525-- To Summary (Statistical)
529-- To Summary (Statistical)
@@ -603,7 +607,7 @@ defaultSalt = -2578643520546668380
603
607
604-- | Compute an 'Int' sized polynomial rolling hash of a stream.
608-- | Compute an 'Int' sized polynomial rolling hash of a stream.
605--
609--
606−-- > rollingHash = rollingHashWithSalt defaultSalt
610+-- > rollingHash = Fold.rollingHashWithSalt defaultSalt
607--
611--
608-- /Pre-release/
612-- /Pre-release/
609{-# INLINABLE rollingHash #-}
613{-# INLINABLE rollingHash #-}
@@ -613,7 +617,7 @@ rollingHash = rollingHashWithSalt defaultSalt
613-- | Compute an 'Int' sized polynomial rolling hash of the first n elements of
617-- | Compute an 'Int' sized polynomial rolling hash of the first n elements of
614-- a stream.
618-- a stream.
615--
619--
616−-- > rollingHashFirstN = take n rollingHash
620+-- > rollingHashFirstN = Fold.take n Fold.rollingHash
617--
621--
618-- /Pre-release/
622-- /Pre-release/
619{-# INLINABLE rollingHashFirstN #-}
623{-# INLINABLE rollingHashFirstN #-}
@@ -626,10 +630,14 @@ rollingHashFirstN n = take n rollingHash
626
630
627-- | Append the elements of an input stream to a provided starting value.
631-- | Append the elements of an input stream to a provided starting value.
628--
632--
629−-- > S.fold (FL.sconcat 10) (S.map Sum $ S.enumerateFromTo 1 10)
633+-- >>> Stream.fold (Fold.sconcat 10) (Stream.map Data.Monoid.Sum $ Stream.enumerateFromTo 1 10)
634+-- Sum {getSum = 65}
630--
635--
631−-- /Pre-release/
636+-- @
637+-- sconcat = Fold.mkFoldl (<>)
638+-- @
632--
639--
640+-- @since 0.8.0
633{-# INLINE sconcat #-}
641{-# INLINE sconcat #-}
634sconcat :: (Monad m, Semigroup a) => a -> Fold m a a
642sconcat :: (Monad m, Semigroup a) => a -> Fold m a a
635sconcat = mkFoldl (<>)
643sconcat = mkFoldl (<>)
@@ -637,7 +645,10 @@ sconcat = mkFoldl (<>)
637-- | Fold an input stream consisting of monoidal elements using 'mappend'
645-- | Fold an input stream consisting of monoidal elements using 'mappend'
638-- and 'mempty'.
646-- and 'mempty'.
639--
647--
640−-- > S.fold FL.mconcat (S.map Sum $ S.enumerateFromTo 1 10)
648+-- >>> Stream.fold Fold.mconcat (Stream.map Data.Monoid.Sum $ Stream.enumerateFromTo 1 10)
649+-- Sum {getSum = 55}
650+--
651+-- > mconcat = Fold.sconcat mempty
641--
652--
642-- @since 0.7.0
653-- @since 0.7.0
643{-# INLINE mconcat #-}
654{-# INLINE mconcat #-}
@@ -650,12 +661,13 @@ mconcat ::
650mconcat = sconcat mempty
661mconcat = sconcat mempty
651
662
652-- |
663-- |
653−-- > foldMap f = map f mconcat
664+-- > foldMap f = Fold.lmap f Fold.mconcat
654--
665--
655-- Make a fold from a pure function that folds the output of the function
666-- Make a fold from a pure function that folds the output of the function
656-- using 'mappend' and 'mempty'.
667-- using 'mappend' and 'mempty'.
657--
668--
658−-- > S.fold (FL.foldMap Sum) $ S.enumerateFromTo 1 10
669+-- >>> Stream.fold (Fold.foldMap Data.Monoid.Sum) $ Stream.enumerateFromTo 1 10
670+-- Sum {getSum = 55}
659--
671--
660-- @since 0.7.0
672-- @since 0.7.0
661{-# INLINABLE foldMap #-}
673{-# INLINABLE foldMap #-}
@@ -664,15 +676,16 @@ foldMap :: (Monad m, Monoid b
664 , Semigroup b
676 , Semigroup b
665#endif
677#endif
666 ) => (a -> b) -> Fold m a b
678 ) => (a -> b) -> Fold m a b
667−foldMap f = map f mconcat
679+foldMap f = lmap f mconcat
668
680
669-- |
681-- |
670−-- > foldMapM f = lmapM f mconcat
682+-- > foldMapM f = Fold.lmapM f Fold.mconcat
671--
683--
672-- Make a fold from a monadic function that folds the output of the function
684-- Make a fold from a monadic function that folds the output of the function
673-- using 'mappend' and 'mempty'.
685-- using 'mappend' and 'mempty'.
674--
686--
675−-- > S.fold (FL.foldMapM (return . Sum)) $ S.enumerateFromTo 1 10
687+-- >>> Stream.fold (Fold.foldMapM (return . Data.Monoid.Sum)) $ Stream.enumerateFromTo 1 10
688+-- Sum {getSum = 55}
676--
689--
677-- @since 0.7.0
690-- @since 0.7.0
678{-# INLINABLE foldMapM #-}
691{-# INLINABLE foldMapM #-}
@@ -695,10 +708,12 @@ foldMapM act = mkFoldlM step (pure mempty)
695
708
696-- | Buffers the input stream to a list in the reverse order of the input.
709-- | Buffers the input stream to a list in the reverse order of the input.
697--
710--
711+-- > toListRev = Fold.mkFoldl (flip (:)) []
712+--
698-- /Warning!/ working on large lists accumulated as buffers in memory could be
713-- /Warning!/ working on large lists accumulated as buffers in memory could be
699-- very inefficient, consider using "Streamly.Array" instead.
714-- very inefficient, consider using "Streamly.Array" instead.
700--
715--
701−-- @since 0.7.0
716+-- @since 0.8.0
702
717
703-- xn : ... : x2 : x1 : []
718-- xn : ... : x2 : x1 : []
704{-# INLINABLE toListRev #-}
719{-# INLINABLE toListRev #-}
@@ -711,6 +726,10 @@ toListRev = mkFoldl (flip (:)) []
711
726
712-- | A fold that drains the first n elements of its input, running the effects
727-- | A fold that drains the first n elements of its input, running the effects
713-- and discarding the results.
728-- and discarding the results.
729+--
730+-- > drainN n = Fold.take n Fold.drain
731+--
732+-- /Pre-release/
714{-# INLINABLE drainN #-}
733{-# INLINABLE drainN #-}
715drainN :: Monad m => Int -> Fold m a ()
734drainN :: Monad m => Int -> Fold m a ()
716drainN n = take n drain
735drainN n = take n drain
@@ -735,6 +754,8 @@ genericIndex i = mkFold step (Partial 0) (const Nothing)
735
754
736-- | Lookup the element at the given index.
755-- | Lookup the element at the given index.
737--
756--
757+-- See also: "Streamly.Prelude.!!"
758+--
738-- @since 0.7.0
759-- @since 0.7.0
739{-# INLINABLE index #-}
760{-# INLINABLE index #-}
740index :: Monad m => Int -> Fold m a (Maybe a)
761index :: Monad m => Int -> Fold m a (Maybe a)
@@ -742,8 +763,6 @@ index = genericIndex
742
763
743-- | Extract the first element of the stream, if any.
764-- | Extract the first element of the stream, if any.
744--
765--
745−-- > head = fmap getFirst $ FL.foldMap (First . Just)
746−--
747-- @since 0.7.0
766-- @since 0.7.0
748{-# INLINABLE head #-}
767{-# INLINABLE head #-}
749head :: Monad m => Fold m a (Maybe a)
768head :: Monad m => Fold m a (Maybe a)
@@ -751,11 +770,6 @@ head = mkFold_ (const (Done . Just)) (Partial Nothing)
751
770
752-- | Returns the first element that satisfies the given predicate.
771-- | Returns the first element that satisfies the given predicate.
753--
772--
754−-- @
755−-- find p = fmap getFirst $
756−-- FL.foldMap (\x -> First (if p x then Just x else Nothing))
757−-- @
758−--
759-- @since 0.7.0
773-- @since 0.7.0
760{-# INLINABLE find #-}
774{-# INLINABLE find #-}
761find :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
775find :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
@@ -771,7 +785,7 @@ find predicate = mkFold step (Partial ()) (const Nothing)
771-- | In a stream of (key-value) pairs @(a, b)@, return the value @b@ of the
785-- | In a stream of (key-value) pairs @(a, b)@, return the value @b@ of the
772-- first pair where the key equals the given value @a@.
786-- first pair where the key equals the given value @a@.
773--
787--
774−-- > lookup = snd <$> find ((==) . fst)
788+-- > lookup = snd <$> Fold.find ((==) . fst)
775--
789--
776-- @since 0.7.0
790-- @since 0.7.0
777{-# INLINABLE lookup #-}
791{-# INLINABLE lookup #-}
@@ -801,7 +815,7 @@ findIndex predicate = mkFold step (Partial 0) (const Nothing)
801
815
802-- | Returns the first index where a given value is found in the stream.
816-- | Returns the first index where a given value is found in the stream.
803--
817--
804−-- > elemIndex a = findIndex (== a)
818+-- > elemIndex a = Fold.findIndex (== a)
805--
819--
806-- @since 0.7.0
820-- @since 0.7.0
807{-# INLINABLE elemIndex #-}
821{-# INLINABLE elemIndex #-}
@@ -814,22 +828,20 @@ elemIndex a = findIndex (a ==)
814
828
815-- | Return 'True' if the input stream is empty.
829-- | Return 'True' if the input stream is empty.
816--
830--
817−-- > null = fmap isJust head
831+-- > null = fmap isJust Fold.head
818--
832--
819-- @since 0.7.0
833-- @since 0.7.0
820{-# INLINABLE null #-}
834{-# INLINABLE null #-}
821null :: Monad m => Fold m a Bool
835null :: Monad m => Fold m a Bool
822null = mkFold (\() _ -> Done False) (Partial ()) (const True)
836null = mkFold (\() _ -> Done False) (Partial ()) (const True)
823
837
824−--
825−-- > any p = map p or
826−-- > any p = fmap getAny . FL.foldMap (Any . p)
827−--
828-- | Returns 'True' if any of the elements of a stream satisfies a predicate.
838-- | Returns 'True' if any of the elements of a stream satisfies a predicate.
829--
839--
830-- >>> Stream.fold (Fold.any (== 0)) $ Stream.fromList [1,0,1]
840-- >>> Stream.fold (Fold.any (== 0)) $ Stream.fromList [1,0,1]
831-- True
841-- True
832--
842--
843+-- > any p = Fold.lmap p Fold.or
844+--
833-- @since 0.7.0
845-- @since 0.7.0
834{-# INLINE any #-}
846{-# INLINE any #-}
835any :: Monad m => (a -> Bool) -> Fold m a Bool
847any :: Monad m => (a -> Bool) -> Fold m a Bool
@@ -846,22 +858,20 @@ any predicate = mkFold_ step initial
846
858
847-- | Return 'True' if the given element is present in the stream.
859-- | Return 'True' if the given element is present in the stream.
848--
860--
849−-- > elem a = any (== a)
861+-- > elem a = Fold.any (== a)
850--
862--
851-- @since 0.7.0
863-- @since 0.7.0
852{-# INLINABLE elem #-}
864{-# INLINABLE elem #-}
853elem :: (Eq a, Monad m) => a -> Fold m a Bool
865elem :: (Eq a, Monad m) => a -> Fold m a Bool
854elem a = any (a ==)
866elem a = any (a ==)
855
867
856−--
857−-- > all p = map p and
858−-- > all p = fmap getAll . FL.foldMap (All . p)
859−--
860-- | Returns 'True' if all elements of a stream satisfy a predicate.
868-- | Returns 'True' if all elements of a stream satisfy a predicate.
861--
869--
862-- >>> Stream.fold (Fold.all (== 0)) $ Stream.fromList [1,0,1]
870-- >>> Stream.fold (Fold.all (== 0)) $ Stream.fromList [1,0,1]
863-- False
871-- False
864--
872--
873+-- > all p = Fold.lmap p Fold.and
874+--
865-- @since 0.7.0
875-- @since 0.7.0
866{-# INLINABLE all #-}
876{-# INLINABLE all #-}
867all :: Monad m => (a -> Bool) -> Fold m a Bool
877all :: Monad m => (a -> Bool) -> Fold m a Bool
@@ -878,7 +888,7 @@ all predicate = mkFold_ step initial
878
888
879-- | Returns 'True' if the given element is not present in the stream.
889-- | Returns 'True' if the given element is not present in the stream.
880--
890--
881−-- > notElem a = all (/= a)
891+-- > notElem a = Fold.all (/= a)
882--
892--
883-- @since 0.7.0
893-- @since 0.7.0
884{-# INLINABLE notElem #-}
894{-# INLINABLE notElem #-}
@@ -887,8 +897,7 @@ notElem a = all (a /=)
887
897
888-- | Returns 'True' if all elements are 'True', 'False' otherwise
898-- | Returns 'True' if all elements are 'True', 'False' otherwise
889--
899--
890−-- > and = all (== True)
900+-- > and = Fold.all (== True)
891−-- > and = fmap getAll . FL.foldMap All
892--
901--
893-- @since 0.7.0
902-- @since 0.7.0
894{-# INLINE and #-}
903{-# INLINE and #-}
@@ -897,8 +906,7 @@ and = all (== True)
897
906
898-- | Returns 'True' if any element is 'True', 'False' otherwise
907-- | Returns 'True' if any element is 'True', 'False' otherwise
899--
908--
900−-- > or = any (== True)
909+-- > or = Fold.any (== True)
901−-- > or = fmap getAny . FL.foldMap Any
902--
910--
903-- @since 0.7.0
911-- @since 0.7.0
904{-# INLINE or #-}
912{-# INLINE or #-}
@@ -917,8 +925,6 @@ or = any (== True)
917-- Binary APIs
925-- Binary APIs
918------------------------------------------------------------------------------
926------------------------------------------------------------------------------
919
927
920−-- XXX These would just be applicative compositions of terminating folds.
921−
922-- | @splitAt n f1 f2@ composes folds @f1@ and @f2@ such that first @n@
928-- | @splitAt n f1 f2@ composes folds @f1@ and @f2@ such that first @n@
923-- elements of its input are consumed by fold @f1@ and the rest of the stream
929-- elements of its input are consumed by fold @f1@ and the rest of the stream
924-- is consumed by fold @f2@.
930-- is consumed by fold @f2@.
@@ -943,7 +949,7 @@ or = any (== True)
943-- >>> splitAt_ 4 [1,2,3]
949-- >>> splitAt_ 4 [1,2,3]
944-- ([1,2,3],[])
950-- ([1,2,3],[])
945--
951--
946−-- > splitAt n f1 f2 = serialWith (,) (take n f1) f2
952+-- > splitAt n f1 f2 = Fold.serialWith (,) (Fold.take n f1) f2
947--
953--
948-- /Internal/
954-- /Internal/
949
955
@@ -967,53 +973,22 @@ splitAt n fld = serialWith (,) (take n fld)
967-- Note: Keep this consistent with S.splitOn. In fact we should eliminate
973-- Note: Keep this consistent with S.splitOn. In fact we should eliminate
968-- S.splitOn in favor of the fold.
974-- S.splitOn in favor of the fold.
969--
975--
970−-- | Consume the input until it encounters an infixed separator element (i.e.
976+-- XXX Use Fold.many instead once it is fixed.
971−-- when the supplied predicate succeeds), dropping the separator.
972−--
973−-- Repeated applications of 'takeEndBy_' splits the stream on separator
974−-- elements determined by the supplied predicate, separator is considered as
975−-- infixed between two segments, if one side of the separator is missing then
976−-- it is parsed as an empty stream. The supplied 'Fold' is applied on the
977−-- split segments. With '-' representing non-separator elements and '.' as
978−-- separator, repeated 'takeEndBy_' splits the stream as follows:
979−--
980−-- @
981−-- "--.--" => "--" "--"
982−-- "--." => "--" ""
983−-- ".--" => "" "--"
984−-- @
985−--
986−-- Repeated applications of @Fold.takeEndBy_ (== x)@ on the input stream gives us
987−-- an inverse of @Stream.intercalate (Stream.yield x)@
988−--
989−-- > Stream.splitOn pred f = Stream.foldMany (Fold.takeEndBy_ pred f)
990−--
991−-- Let's use the following definition for illustration:
992−--
993−-- >>> splitOn p = Stream.foldMany (Fold.takeEndBy_ pred Fold.toList)
994−-- >>> splitOn' p = Stream.toList . splitOn p . Stream.fromList
995−
996−-- >>> splitOn' (== '.') ""
997−-- [""]
998−--
999−-- >>> splitOn' (== '.') "."
1000−-- ["",""]
1001--
977--
1002−-- >>> splitOn' (== '.') ".a"
978+-- | Like 'takeEndBy' but drops the element on which the predicate succeeds.
1003−-- ["","a"]
1004--
979--
1005−-- >>> splitOn' (== '.') "a."
980+-- >>> Stream.fold (Fold.takeEndBy_ (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1006−-- ["a",""]
981+-- "hello"
1007--
982--
1008−-- >>> splitOn' (== '.') "a.b"
983+-- >>> Stream.toList $ Stream.foldMany (Fold.takeEndBy_ (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1009−-- ["a","b"]
984+-- ["hello","there"]
1010--
985--
1011−-- >>> splitOn' (== '.') "a..b"
986+-- > Stream.splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)
1012−-- ["a","","b"]
1013--
987--
1014−-- Stops - when the predicate succeeds.
988+-- See 'Streamly.Prelude.splitOnSuffix' for more details on splitting a
989+-- stream using 'takeEndBy_'.
1015--
990--
1016−-- /Pre-release/
991+-- @since 0.8.0
1017{-# INLINE takeEndBy_ #-}
992{-# INLINE takeEndBy_ #-}
1018takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
993takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
1019takeEndBy_ predicate (Fold fstep finitial fextract) =
994takeEndBy_ predicate (Fold fstep finitial fextract) =
@@ -1026,16 +1001,21 @@ takeEndBy_ predicate (Fold fstep finitial fextract) =
1026 then fstep s a
1001 then fstep s a
1027 else Done <$> fextract s
1002 else Done <$> fextract s
1028
1003
1029−-- | Collect stream elements until an element succeeds the predicate. Also take
1004+-- | Take the input, stop when the predicate succeeds taking the succeeding
1030−-- the element on which the predicate succeeded. The succeeding element is
1005+-- element as well.
1031−-- treated as a suffix separator which is kept in the output segement.
1032--
1006--
1033−-- * Stops - when the predicate succeeds.
1007+-- >>> Stream.fold (Fold.takeEndBy (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1008+-- "hello\n"
1034--
1009--
1035−-- > Stream.splitWithSuffix pred f = Stream.foldMany (Fold.takeEndBy pred f)
1010+-- >>> Stream.toList $ Stream.foldMany (Fold.takeEndBy (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1011+-- ["hello\n","there\n"]
1036--
1012--
1037−-- /Pre-release/
1013+-- > Stream.splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)
1038--
1014--
1015+-- See 'Streamly.Prelude.splitWithSuffix' for more details on splitting a
1016+-- stream using 'takeEndBy'.
1017+--
1018+-- @since 0.8.0
1039{-# INLINE takeEndBy #-}
1019{-# INLINE takeEndBy #-}
1040takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
1020takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
1041takeEndBy predicate (Fold fstep finitial fextract) =
1021takeEndBy predicate (Fold fstep finitial fextract) =
@@ -1086,6 +1066,8 @@ breakOn pat f m = undefined
1086-- >>> Stream.fold (Fold.tee Fold.sum Fold.length) (Stream.enumerateFromTo 1.0 100.0)
1066-- >>> Stream.fold (Fold.tee Fold.sum Fold.length) (Stream.enumerateFromTo 1.0 100.0)
1087-- (5050.0,100)
1067-- (5050.0,100)
1088--
1068--
1069+-- > tee = teeWith (,)
1070+--
1089-- @since 0.7.0
1071-- @since 0.7.0
1090{-# INLINE tee #-}
1072{-# INLINE tee #-}
1091tee :: Monad m => Fold m a b -> Fold m a c -> Fold m a (b,c)
1073tee :: Monad m => Fold m a b -> Fold m a c -> Fold m a (b,c)
@@ -1110,6 +1092,8 @@ tee = teeWith (,)
1110-- >>> Stream.fold (Fold.distribute [Fold.sum, Fold.length]) (Stream.enumerateFromTo 1 5)
1092-- >>> Stream.fold (Fold.distribute [Fold.sum, Fold.length]) (Stream.enumerateFromTo 1 5)
1111-- [15,5]
1093-- [15,5]
1112--
1094--
1095+-- > distribute = Prelude.foldr (Fold.teeWith (:)) (Fold.yield [])
1096+--
1113-- This is the consumer side dual of the producer side 'sequence' operation.
1097-- This is the consumer side dual of the producer side 'sequence' operation.
1114--
1098--
1115-- Stops when all the folds stop.
1099-- Stops when all the folds stop.
@@ -1171,7 +1155,7 @@ distribute = foldr (teeWith (:)) (yield [])
1171--
1155--
1172-- /See also: 'partitionByFstM' and 'partitionByMinM'./
1156-- /See also: 'partitionByFstM' and 'partitionByMinM'./
1173--
1157--
1174−-- @since 0.7.0
1158+-- /Pre-release/
1175{-# INLINE partitionByM #-}
1159{-# INLINE partitionByM #-}
1176partitionByM :: Monad m
1160partitionByM :: Monad m
1177 => (a -> m (Either b c)) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
1161 => (a -> m (Either b c)) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -1270,7 +1254,7 @@ partitionByMinM = undefined
1270-- :}
1254-- :}
1271-- ("Even 50","Odd 50")
1255-- ("Even 50","Odd 50")
1272--
1256--
1273−-- @since 0.7.0
1257+-- /Pre-release/
1274{-# INLINE partitionBy #-}
1258{-# INLINE partitionBy #-}
1275partitionBy :: Monad m
1259partitionBy :: Monad m
1276 => (a -> Either b c) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
1260 => (a -> Either b c) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -1346,7 +1330,9 @@ demuxWith f kv = fmap fst $ demuxDefaultWith f kv drain
1346-- :}
1330-- :}
1347-- fromList [("PRODUCT",8),("SUM",4)]
1331-- fromList [("PRODUCT",8),("SUM",4)]
1348--
1332--
1349−-- @since 0.7.0
1333+-- > demux = demuxWith id
1334+--
1335+-- /Pre-release/
1350{-# INLINE demux #-}
1336{-# INLINE demux #-}
1351demux :: (Monad m, Ord k)
1337demux :: (Monad m, Ord k)
1352 => Map k (Fold m a b) -> Fold m (k, a) (Map k b)
1338 => Map k (Fold m a b) -> Fold m (k, a) (Map k b)
@@ -1476,6 +1462,10 @@ demuxDefaultWith f kv (Fold dstep dinitial dextract) =
1476 b <- dextract dacc
1462 b <- dextract dacc
1477 return (doneMap, b)
1463 return (doneMap, b)
1478
1464
1465+-- |
1466+-- > demuxDefault = demuxDefaultWith id
1467+--
1468+-- /Pre-release/
1479{-# INLINE demuxDefault #-}
1469{-# INLINE demuxDefault #-}
1480demuxDefault :: (Monad m, Ord k)
1470demuxDefault :: (Monad m, Ord k)
1481 => Map k (Fold m a b) -> Fold m (k, a) b -> Fold m (k, a) (Map k b, b)
1471 => Map k (Fold m a b) -> Fold m (k, a) b -> Fold m (k, a) (Map k b, b)
@@ -1555,12 +1545,11 @@ classifyWith f (Fold step1 initial1 extract1) =
1555-- :}
1545-- :}
1556-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
1546-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
1557--
1547--
1558−-- @since 0.7.0
1559−--
1560-- Same as:
1548-- Same as:
1561--
1549--
1562−-- > classify fld = classifyWith fst (map snd fld)
1550+-- > classify fld = Fold.classifyWith fst (map snd fld)
1563--
1551--
1552+-- /Pre-release/
1564{-# INLINE classify #-}
1553{-# INLINE classify #-}
1565classify :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)
1554classify :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)
1566classify fld = classifyWith fst (map snd fld)
1555classify fld = classifyWith fst (map snd fld)
@@ -1571,9 +1560,9 @@ classify fld = classifyWith fst (map snd fld)
1571
1560
1572-- | Like 'unzipWith' but with a monadic splitter function.
1561-- | Like 'unzipWith' but with a monadic splitter function.
1573--
1562--
1574−-- -- @unzipWithM k f1 f2 = lmapM k (unzip f1 f2)@
1563+-- @unzipWithM k f1 f2 = lmapM k (unzip f1 f2)@
1575--
1564--
1576−-- @since 0.7.0
1565+-- /Pre-release/
1577{-# INLINE unzipWithM #-}
1566{-# INLINE unzipWithM #-}
1578unzipWithM :: Monad m
1567unzipWithM :: Monad m
1579 => (a -> m (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
1568 => (a -> m (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
@@ -1653,11 +1642,11 @@ unzipWithMinM = undefined
1653-- | Split elements in the input stream into two parts using a pure splitter
1642-- | Split elements in the input stream into two parts using a pure splitter
1654-- function, direct each part to a different fold and zip the results.
1643-- function, direct each part to a different fold and zip the results.
1655--
1644--
1656−-- @unzipWith f fld1 fld2 = map f (unzip fld1 fld2)@
1645+-- @unzipWith f fld1 fld2 = Fold.lmap f (Fold.unzip fld1 fld2)@
1657--
1646--
1658-- This fold terminates when both the input folds terminate.
1647-- This fold terminates when both the input folds terminate.
1659--
1648--
1660−-- @since 0.7.0
1649+-- /Pre-release/
1661{-# INLINE unzipWith #-}
1650{-# INLINE unzipWith #-}
1662unzipWith :: Monad m
1651unzipWith :: Monad m
1663 => (a -> (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
1652 => (a -> (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
@@ -1674,6 +1663,8 @@ unzipWith f = unzipWithM (return . f)
1674--
1663--
1675-- @
1664-- @
1676--
1665--
1666+-- > unzip = Fold.unzipWith id
1667+--
1677-- This is the consumer side dual of the producer side 'zip' operation.
1668-- This is the consumer side dual of the producer side 'zip' operation.
1678--
1669--
1679-- @since 0.7.0
1670-- @since 0.7.0
@@ -1773,6 +1764,8 @@ chunksBetween _low _high _f1 _f2 = undefined
1773-- /Warning!/ working on large streams accumulated as buffers in memory could
1764-- /Warning!/ working on large streams accumulated as buffers in memory could
1774-- be very inefficient, consider using "Streamly.Data.Array" instead.
1765-- be very inefficient, consider using "Streamly.Data.Array" instead.
1775--
1766--
1767+-- > toStream = mkFoldr K.cons K.nil
1768+--
1776-- /Pre-release/
1769-- /Pre-release/
1777{-# INLINE toStream #-}
1770{-# INLINE toStream #-}
1778toStream :: Monad m => Fold m a (SerialT Identity a)
1771toStream :: Monad m => Fold m a (SerialT Identity a)
@@ -1784,6 +1777,8 @@ toStream = mkFoldr K.cons K.nil
1784-- | Buffers the input stream to a pure stream in the reverse order of the
1777-- | Buffers the input stream to a pure stream in the reverse order of the
1785-- input.
1778-- input.
1786--
1779--
1780+-- > toStreamRev = mkFoldl (flip K.cons) K.nil
1781+--
1787-- /Warning!/ working on large streams accumulated as buffers in memory could
1782-- /Warning!/ working on large streams accumulated as buffers in memory could
1788-- be very inefficient, consider using "Streamly.Data.Array" instead.
1783-- be very inefficient, consider using "Streamly.Data.Array" instead.
1789--
1784--
src/Streamly/Internal/Data/Fold/Tee.hs+57−25
@@ -6,10 +6,35 @@
6-- Stability : experimental
6-- Stability : experimental
7-- Portability : GHC
7-- Portability : GHC
8--
8--
9+-- The @Tee@ type is a newtype wrapper over the 'Fold' type providing
10+-- distributing 'Applicative', 'Semigroup', 'Monoid', 'Num', 'Floating' and
11+-- 'Fractional' instances. The input received by the composed 'Tee' is
12+-- replicated and distributed to both the constituent Tee's.
13+--
14+-- For example, to compute the average of numbers in a stream without going
15+-- through the stream twice:
16+--
17+-- >>> import Streamly.Internal.Data.Fold.Tee (mkT, toFold)
18+-- >>> import Streamly.Internal.Data.Fold as Fold
19+--
20+-- >>> avg = (/) <$> (mkT Fold.sum) <*> (mkT $ fmap fromIntegral Fold.length)
21+-- >>> Stream.fold (toFold avg) $ Stream.fromList [1.0..100.0]
22+-- 50.5
23+--
24+-- Similarly, the 'Semigroup' and 'Monoid' instances of 'Tee' distribute the
25+-- input to both the folds and combines the outputs using Monoid or Semigroup
26+-- instances of the output types:
27+--
28+-- >>> import Data.Monoid (Sum(..))
29+-- >>> t = mkT Fold.head <> mkT Fold.last
30+-- >>> Stream.fold (toFold t) (fmap Sum $ Stream.enumerateFromTo 1.0 100.0)
31+-- Just (Sum {getSum = 101.0})
32+--
33+-- The 'Num', 'Floating', and 'Fractional' instances work in the same way.
34+--
9module Streamly.Internal.Data.Fold.Tee
35module Streamly.Internal.Data.Fold.Tee
10 ( Tee(..)
36 ( Tee(..)
11− , fromFold
37+ , mkT
12− , toFold
13 )
38 )
14where
39where
15
40
@@ -22,44 +47,44 @@ import Streamly.Internal.Data.Fold.Type (Fold)
22
47
23import qualified Streamly.Internal.Data.Fold.Type as Fold
48import qualified Streamly.Internal.Data.Fold.Type as Fold
24
49
25−-- | The type @Tee m a b@ represents a left fold over an input stream of values
50+-- | @Tee@ is a newtype wrapper over the 'Fold' type providing distributing
26−-- of type @a@ to a single value of type @b@ in 'Monad' @m@.
51+-- 'Applicative', 'Semigroup', 'Monoid', 'Num', 'Floating' and 'Fractional'
27−--
52+-- instances.
28−-- @Tee@ is a wrapper over 'Fold' that uses 'teeWith' to define the applicative
29−-- instance.
30--
53--
31-- /Pre-release/
54-- /Pre-release/
32newtype Tee m a b =
55newtype Tee m a b =
33− Tee { runTee :: Fold m a b }
56+ Tee { toFold :: Fold m a b }
34 deriving (Functor)
57 deriving (Functor)
35
58
36−-- | Convert a 'Tee' to 'Fold'.
59+-- | Make a 'Tee' from a 'Fold'.
60+--
61+-- /Pre-release/
37−{-# INLINE toFold #-}
62+{-# INLINE mkT #-}
38−toFold :: Tee m a b -> Fold m a b
63+mkT :: Fold m a b -> Tee m a b
39−toFold = coerce
64+mkT = coerce
40
65
41−-- | Convert a 'Fold' to 'Tee'.
42−{-# INLINE fromFold #-}
43−fromFold :: Fold m a b -> Tee m a b
44−fromFold = coerce
45
66
46−-- | The 'Tee' resulting from '<*>' distributes its input to both the argument
67+-- | '<*>' distributes the input to both the argument 'Tee's and combines their
47−-- 'Tee's and combines their output using function application.
68+-- outputs using function application.
48--
69--
49instance Monad m => Applicative (Tee m a) where
70instance Monad m => Applicative (Tee m a) where
50
71
51 {-# INLINE pure #-}
72 {-# INLINE pure #-}
52− pure a = fromFold (Fold.yield a)
73+ pure a = mkT (Fold.yield a)
53
74
54 {-# INLINE (<*>) #-}
75 {-# INLINE (<*>) #-}
55− (<*>) a b = fromFold (Fold.teeWith ($) (toFold a) (toFold b))
76+ (<*>) a b = mkT (Fold.teeWith ($) (toFold a) (toFold b))
56
77
78+-- | '<>' distributes the input to both the argument 'Tee's and combines their
57−-- | Combines the outputs (the type @b@) using their 'Semigroup' instances.
79+-- outputs using the 'Semigroup' instance of the output type.
80+--
58instance (Semigroup b, Monad m) => Semigroup (Tee m a b) where
81instance (Semigroup b, Monad m) => Semigroup (Tee m a b) where
59 {-# INLINE (<>) #-}
82 {-# INLINE (<>) #-}
60 (<>) = liftA2 (<>)
83 (<>) = liftA2 (<>)
61
84
62−-- | Combines the outputs (the type @b@) using their 'Monoid' instances.
85+-- | '<>' distributes the input to both the argument 'Tee's and combines their
86+-- outputs using the 'Monoid' instance of the output type.
87+--
63instance (Semigroup b, Monoid b, Monad m) => Monoid (Tee m a b) where
88instance (Semigroup b, Monoid b, Monad m) => Monoid (Tee m a b) where
64 {-# INLINE mempty #-}
89 {-# INLINE mempty #-}
65 mempty = pure mempty
90 mempty = pure mempty
@@ -67,7 +92,9 @@ instance (Semigroup b, Monoid b, Monad m) => Monoid (Tee m a b) where
67 {-# INLINE mappend #-}
92 {-# INLINE mappend #-}
68 mappend = (<>)
93 mappend = (<>)
69
94
70−-- | Combines the outputs (type @b@) using their 'Num' instances.
95+-- | Binary 'Num' operations distribute the input to both the argument 'Tee's
96+-- and combine their outputs using the 'Num' instance of the output type.
97+--
71instance (Monad m, Num b) => Num (Tee m a b) where
98instance (Monad m, Num b) => Num (Tee m a b) where
72 {-# INLINE fromInteger #-}
99 {-# INLINE fromInteger #-}
73 fromInteger = pure . fromInteger
100 fromInteger = pure . fromInteger
@@ -90,7 +117,10 @@ instance (Monad m, Num b) => Num (Tee m a b) where
90 {-# INLINE (-) #-}
117 {-# INLINE (-) #-}
91 (-) = liftA2 (-)
118 (-) = liftA2 (-)
92
119
120+-- | Binary 'Fractional' operations distribute the input to both the argument
93−-- | Combines the outputs (type @b@) using their 'Fractional' instances.
121+-- 'Tee's and combine their outputs using the 'Fractional' instance of the
122+-- output type.
123+--
94instance (Monad m, Fractional b) => Fractional (Tee m a b) where
124instance (Monad m, Fractional b) => Fractional (Tee m a b) where
95 {-# INLINE fromRational #-}
125 {-# INLINE fromRational #-}
96 fromRational = pure . fromRational
126 fromRational = pure . fromRational
@@ -101,7 +131,9 @@ instance (Monad m, Fractional b) => Fractional (Tee m a b) where
101 {-# INLINE (/) #-}
131 {-# INLINE (/) #-}
102 (/) = liftA2 (/)
132 (/) = liftA2 (/)
103
133
104−-- | Combines the outputs using their 'Floating' instances.
134+-- | Binary 'Floating' operations distribute the input to both the argument
135+-- 'Tee's and combine their outputs using the 'Floating' instance of the output
136+-- type.
105instance (Monad m, Floating b) => Floating (Tee m a b) where
137instance (Monad m, Floating b) => Floating (Tee m a b) where
106 {-# INLINE pi #-}
138 {-# INLINE pi #-}
107 pi = pure pi
139 pi = pure pi
src/Streamly/Internal/Data/Fold/Type.hs+225−95
@@ -164,16 +164,41 @@
164-- can be used for the scan use case, instead of using extract. Extract would
164-- can be used for the scan use case, instead of using extract. Extract would
165-- then be used only for the case when the stream stops before the fold
165-- then be used only for the case when the stream stops before the fold
166-- completes.
166-- completes.
167−
167+--
168+-- = Accumulators and Terminating Folds
169+--
170+-- Folds in this module can be classified in two categories viz. accumulators
171+-- and terminating folds. Accumulators do not have a terminating condition,
172+-- they run forever and consume the entire stream, for example the 'length'
173+-- fold. Terminating folds have a terminating condition and can terminate
174+-- without consuming the entire stream, for example, the 'head' fold.
175+--
176+-- = Monoids
177+--
178+-- Monoids allow generalized, modular folding. The accumulators in this module
179+-- can be expressed using 'mconcat' and a suitable 'Monoid'. Instead of
180+-- writing folds we can write Monoids and turn them into folds.
181+--
182+-- = Performance Notes
183+--
184+-- 'Streamly.Prelude' module provides fold functions to directly fold streams
185+-- e.g. Streamly.Prelude/'Streamly.Prelude.sum' serves the same purpose as
186+-- Fold/'sum'. However, the functions in Streamly.Prelude cannot be
187+-- efficiently combined together e.g. we cannot drive the input stream through
188+-- @sum@ and @length@ fold functions simultaneously. Using the 'Fold' type we
189+-- can efficiently split the stream across multiple folds because it allows the
190+-- compiler to perform stream fusion optimizations.
191+--
168module Streamly.Internal.Data.Fold.Type
192module Streamly.Internal.Data.Fold.Type
169 (
193 (
170 -- * Types
194 -- * Types
171 Step (..)
195 Step (..)
172 , Fold (..)
196 , Fold (..)
173
197
174− -- * Fold constructors
198+ -- * Constructors
175 , mkFoldl
199 , mkFoldl
176 , mkFoldlM
200 , mkFoldlM
201+ , mkFoldl1
177 , mkFoldr
202 , mkFoldr
178 , mkFoldrM
203 , mkFoldrM
179 , mkFold
204 , mkFold
@@ -181,55 +206,64 @@ module Streamly.Internal.Data.Fold.Type
181 , mkFoldM
206 , mkFoldM
182 , mkFoldM_
207 , mkFoldM_
183
208
184− -- * Fold2
185− , Fold2 (..)
186− , simplify
187−
188− -- * Basic Folds
209+ -- * Folds
210+ , yield
211+ , yieldM
189 , drain
212 , drain
190 , toList
213 , toList
191
214
192− -- * Generators
215+ -- * Combinators
193− , yield
194− , yieldM
195
216
196− -- * Transformations
217+ -- ** Mapping output
197 , rmapM
218 , rmapM
219+
220+ -- ** Mapping Input
198 , map
221 , map
199 , lmap
222 , lmap
200 , lmapM
223 , lmapM
224+
225+ -- ** Filtering
201 , filter
226 , filter
202 , filterM
227 , filterM
203 , catMaybes
228 , catMaybes
229+
230+ -- ** Trimming
204 , take
231 , take
205 , takeInterval
232 , takeInterval
206
233
207− -- * Distributing
234+ -- ** Serial Append
235+ , serialWith
236+ , serial_
237+
238+ -- ** Parallel Distribution
239+ , GenericRunner(..)
208 , teeWith
240 , teeWith
209 , teeWithFst
241 , teeWithFst
210 , teeWithMin
242 , teeWithMin
243+
244+ -- ** Parallel Alternative
211 , shortest
245 , shortest
212 , longest
246 , longest
213
247
214− -- * Serial Application
248+ -- ** Splitting
215− , serialWith
216− , serial_
217−
218− -- * Nested Application
219− , concatMap
220 , ManyState
249 , ManyState
221 , many
250 , many
222 , manyPost
251 , manyPost
223− , intervalsOf
224 , chunksOf
252 , chunksOf
225− , chunksOf2
253+ , intervalsOf
254+
255+ -- ** Nesting
256+ , concatMap
226
257
258+ -- * Running Partially
227 , duplicate
259 , duplicate
228 , initialize
260 , initialize
229 , runStep
261 , runStep
230
262
231− -- * Misc
263+ -- * Fold2
232− , GenericRunner(..) -- Is used in multiple step functions
264+ , Fold2 (..)
265+ , simplify
266+ , chunksOf2
233 )
267 )
234where
268where
235
269
@@ -242,6 +276,7 @@ import Control.Monad.Trans.Control (control)
242import Data.Bifunctor (Bifunctor(..))
276import Data.Bifunctor (Bifunctor(..))
243import Data.Maybe (isJust, fromJust)
277import Data.Maybe (isJust, fromJust)
244import Fusion.Plugin.Types (Fuse(..))
278import Fusion.Plugin.Types (Fuse(..))
279+import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)
245import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
280import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
246import Streamly.Internal.Data.SVar (MonadAsync)
281import Streamly.Internal.Data.SVar (MonadAsync)
247
282
@@ -255,9 +290,14 @@ import Prelude hiding (concatMap, filter, map, take)
255-- >>> import qualified Streamly.Internal.Data.Fold as Fold
290-- >>> import qualified Streamly.Internal.Data.Fold as Fold
256
291
257------------------------------------------------------------------------------
292------------------------------------------------------------------------------
258−-- Monadic left folds
293+-- Step of a fold
259------------------------------------------------------------------------------
294------------------------------------------------------------------------------
260
295
296+-- The Step functor around b allows expressing early termination like a right
297+-- fold. Traditional list right folds use function composition and laziness to
298+-- terminate early whereas we use data constructors. It allows stream fusion in
299+-- contrast to the foldr/build fusion when composing with functions.
300+
261-- | Represents the result of the @step@ of a 'Fold'. 'Partial' returns an
301-- | Represents the result of the @step@ of a 'Fold'. 'Partial' returns an
262-- intermediate state of the fold, the fold step can be called again with the
302-- intermediate state of the fold, the fold step can be called again with the
263-- state or the driver can use @extract@ on the state to get the result out.
303-- state or the driver can use @extract@ on the state to get the result out.
@@ -270,10 +310,7 @@ data Step s b
270 = Partial !s
310 = Partial !s
271 | Done !b
311 | Done !b
272
312
273−-- | A bifunctor instance on 'Step'. @first@ maps on the value held by 'Partial'
274−-- and @second@ maps on the result held by 'Done'.
313+-- | 'first' maps over 'Partial' and 'second' maps over 'Done'.
275−--
276−-- /Pre-release/
277--
314--
278instance Bifunctor Step where
315instance Bifunctor Step where
279 {-# INLINE bimap #-}
316 {-# INLINE bimap #-}
@@ -288,17 +325,19 @@ instance Bifunctor Step where
288 second _ (Partial x) = Partial x
325 second _ (Partial x) = Partial x
289 second f (Done a) = Done (f a)
326 second f (Done a) = Done (f a)
290
327
291−-- | Maps the function over the result held by 'Done'.
328+-- | 'fmap' maps over 'Done'.
329+--
292-- @
330-- @
293-- fmap = 'second'
331-- fmap = 'second'
294-- @
332-- @
295--
333--
296−-- /Pre-release/
297−--
298instance Functor (Step s) where
334instance Functor (Step s) where
299 {-# INLINE fmap #-}
335 {-# INLINE fmap #-}
300 fmap = second
336 fmap = second
301
337
338+-- | Map a monadic function over the result @b@ in @Step s b@.
339+--
340+-- /Internal/
302{-# INLINE mapMStep #-}
341{-# INLINE mapMStep #-}
303mapMStep :: Applicative m => (a -> m b) -> Step s a -> m (Step s b)
342mapMStep :: Applicative m => (a -> m b) -> Step s a -> m (Step s b)
304mapMStep f res =
343mapMStep f res =
@@ -306,25 +345,27 @@ mapMStep f res =
306 Partial s -> pure $ Partial s
345 Partial s -> pure $ Partial s
307 Done b -> Done <$> f b
346 Done b -> Done <$> f b
308
347
309−-- The Step functor around b allows expressing early termination like a right
348+------------------------------------------------------------------------------
310−-- fold. Traditional list right folds use function composition and laziness to
349+-- The Fold type
311−-- terminate early whereas we use data constructors. It allows stream fusion in
350+------------------------------------------------------------------------------
312−-- contrast to the foldr/build fusion when composing with functions.
313
351
314-- | The type @Fold m a b@ having constructor @Fold step initial extract@
352-- | The type @Fold m a b@ having constructor @Fold step initial extract@
315−-- represents a left fold over an input stream of values of type @a@ to a
353+-- represents a fold over an input stream of values of type @a@ to a final
316−-- single value of type @b@ in 'Monad' @m@. The constructor is not exposed via
354+-- value of type @b@ in 'Monad' @m@.
317−-- exposed modules, smart constructors are provided to create folds.
318--
355--
319-- The fold uses an intermediate state @s@ as accumulator, the type @s@ is
356-- The fold uses an intermediate state @s@ as accumulator, the type @s@ is
320−-- specific to the fold. The initial value of the fold state @s@ is returned by
357+-- internal to the specific fold definition. The initial value of the fold
321−-- @initial@. The @step@ function consumes an input and either returns the
358+-- state @s@ is returned by @initial@. The @step@ function consumes an input
322−-- final result @b@ if the fold is done or the next intermediate state (see
359+-- and either returns the final result @b@ if the fold is done or the next
323−-- 'Step'). At any point the fold driver can extract the result from the
360+-- intermediate state (see 'Step'). At any point the fold driver can extract
324−-- intermediate state using the @extract@ function.
361+-- the result from the intermediate state using the @extract@ function.
325--
362--
363+-- NOTE: The constructor is not yet exposed via exposed modules, smart
364+-- constructors are provided to create folds. If you think you need the
326−-- NOTE: If you think you need the constructor of this type please consider
365+-- constructor of this type please consider using the smart constructors in
327−-- using the smart constructors in "Streamly.Internal.Data.Fold' instead.
366+-- "Streamly.Internal.Data.Fold' instead.
367+--
368+-- /since 0.8.0 (type changed)/
328--
369--
329-- @since 0.7.0
370-- @since 0.7.0
330
371
@@ -332,6 +373,10 @@ data Fold m a b =
332 -- | @Fold @ @ step @ @ initial @ @ extract@
373 -- | @Fold @ @ step @ @ initial @ @ extract@
333 forall s. Fold (s -> a -> m (Step s b)) (m (Step s b)) (s -> m b)
374 forall s. Fold (s -> a -> m (Step s b)) (m (Step s b)) (s -> m b)
334
375
376+------------------------------------------------------------------------------
377+-- Mapping on the output
378+------------------------------------------------------------------------------
379+
335-- | Map a monadic function on the output of a fold.
380-- | Map a monadic function on the output of a fold.
336--
381--
337-- @since 0.8.0
382-- @since 0.8.0
@@ -361,7 +406,9 @@ rmapM f (Fold step initial extract) = Fold step1 initial1 (extract >=> f)
361-- mkfoldlx step initial extract = fmap extract (mkFoldl step initial)
406-- mkfoldlx step initial extract = fmap extract (mkFoldl step initial)
362-- @
407-- @
363--
408--
364−-- /Pre-release/
409+-- See also: "Streamly.Prelude.foldl'"
410+--
411+-- @since 0.8.0
365--
412--
366{-# INLINE mkFoldl #-}
413{-# INLINE mkFoldl #-}
367mkFoldl :: Monad m => (b -> a -> b) -> b -> Fold m a b
414mkFoldl :: Monad m => (b -> a -> b) -> b -> Fold m a b
@@ -377,42 +424,65 @@ mkFoldl step initial =
377-- A fold with an extract function can be expressed using rmapM:
424-- A fold with an extract function can be expressed using rmapM:
378--
425--
379-- @
426-- @
380−-- mkAccumM :: Functor m => (s -> a -> m s) -> m s -> (s -> m b) -> Fold m a b
427+-- mkFoldlxM :: Functor m => (s -> a -> m s) -> m s -> (s -> m b) -> Fold m a b
381−-- mkAccumM step initial extract = rmapM extract (mkFoldlM step initial)
428+-- mkFoldlxM step initial extract = rmapM extract (mkFoldlM step initial)
382-- @
429-- @
383--
430--
384−-- /Pre-release/
431+-- See also: "Streamly.Prelude.foldlM'"
432+--
433+-- @since 0.8.0
385--
434--
386{-# INLINE mkFoldlM #-}
435{-# INLINE mkFoldlM #-}
387mkFoldlM :: Monad m => (b -> a -> m b) -> m b -> Fold m a b
436mkFoldlM :: Monad m => (b -> a -> m b) -> m b -> Fold m a b
388mkFoldlM step initial =
437mkFoldlM step initial =
389 Fold (\s a -> Partial <$> step s a) (Partial <$> initial) return
438 Fold (\s a -> Partial <$> step s a) (Partial <$> initial) return
390
439
440+-- | Make a strict left fold, for non-empty streams, using first element as the
441+-- starting value. Returns Nothing if the stream is empty.
442+--
443+-- See also: "Streamly.Prelude.foldl1'"
444+--
445+-- /Pre-release/
446+{-# INLINE mkFoldl1 #-}
447+mkFoldl1 :: Monad m => (a -> a -> a) -> Fold m a (Maybe a)
448+mkFoldl1 step = fmap toMaybe $ mkFoldl step1 Nothing'
449+
450+ where
451+
452+ step1 Nothing' a = Just' a
453+ step1 (Just' x) a = Just' $ step x a
454+
391------------------------------------------------------------------------------
455------------------------------------------------------------------------------
392-- Right fold constructors
456-- Right fold constructors
393------------------------------------------------------------------------------
457------------------------------------------------------------------------------
394
458
395-- | Make a fold using a right fold style step function and a terminal value.
459-- | Make a fold using a right fold style step function and a terminal value.
396−-- It performs a right fold via a left fold using function composition.
460+-- It performs a strict right fold via a left fold using function composition.
461+-- Note that this is strict fold, it can only be useful for constructing strict
397−-- This can be useful for constructing structures. For reductions this may be
462+-- structures in memory. For reductions this will be very inefficient.
398−-- very inefficient compared to using a direct fold implementation using
399−-- 'mkFold'.
400--
463--
401-- For example,
464-- For example,
402--
465--
403-- > toList = mkFoldr (:) []
466-- > toList = mkFoldr (:) []
404--
467--
405−-- /Pre-release/
468+-- See also: "Streamly.Prelude.foldr"
469+--
470+-- @since 0.8.0
406{-# INLINE mkFoldr #-}
471{-# INLINE mkFoldr #-}
407mkFoldr :: Monad m => (a -> b -> b) -> b -> Fold m a b
472mkFoldr :: Monad m => (a -> b -> b) -> b -> Fold m a b
408mkFoldr g z = fmap ($ z) $ mkFoldl (\f x -> f . g x) id
473mkFoldr g z = fmap ($ z) $ mkFoldl (\f x -> f . g x) id
409
474
475+-- XXX we have not seen any use of this yet, not releasing until we have a use
476+-- case.
477+--
410-- | Like 'mkFoldr' but with a monadic step function.
478-- | Like 'mkFoldr' but with a monadic step function.
411--
479--
412-- For example,
480-- For example,
413--
481--
414-- > toList = mkFoldrM (\a xs -> return $ a : xs) (return [])
482-- > toList = mkFoldrM (\a xs -> return $ a : xs) (return [])
415--
483--
484+-- See also: "Streamly.Prelude.foldrM"
485+--
416-- /Pre-release/
486-- /Pre-release/
417{-# INLINE mkFoldrM #-}
487{-# INLINE mkFoldrM #-}
418mkFoldrM :: Monad m => (a -> b -> m b) -> m b -> Fold m a b
488mkFoldrM :: Monad m => (a -> b -> m b) -> m b -> Fold m a b
@@ -423,6 +493,17 @@ mkFoldrM g z =
423-- General fold constructors
493-- General fold constructors
424------------------------------------------------------------------------------
494------------------------------------------------------------------------------
425
495
496+-- XXX If the Step yield gives the result each time along with the state then
497+-- we can make the type of this as
498+--
499+-- mkFold :: Monad m => (s -> a -> Step s b) -> Step s b -> Fold m a b
500+--
501+-- Then similar to mkFoldl and mkFoldr we can just fmap extract on it to extend
502+-- it to the version where an 'extract' function is required. Or do we even
503+-- need that?
504+--
505+-- Until we investigate this we are not releasing these.
506+
426-- | Make a terminating fold using a pure step function, a pure initial state
507-- | Make a terminating fold using a pure step function, a pure initial state
427-- and a pure state extraction function.
508-- and a pure state extraction function.
428--
509--
@@ -513,6 +594,8 @@ drain = mkFoldl (\_ _ -> ()) ()
513-- very inefficient, consider using "Streamly.Data.Array.Foreign"
594-- very inefficient, consider using "Streamly.Data.Array.Foreign"
514-- instead.
595-- instead.
515--
596--
597+-- > toList = mkFoldr (:) []
598+--
516-- @since 0.7.0
599-- @since 0.7.0
517{-# INLINABLE toList #-}
600{-# INLINABLE toList #-}
518toList :: Monad m => Fold m a [a]
601toList :: Monad m => Fold m a [a]
@@ -558,16 +641,26 @@ yieldM b = Fold undefined (Done <$> b) pure
558data SeqFoldState sl f sr = SeqFoldL !sl | SeqFoldR !f !sr
641data SeqFoldState sl f sr = SeqFoldL !sl | SeqFoldR !f !sr
559
642
560-- | Sequential fold application. Apply two folds sequentially to an input
643-- | Sequential fold application. Apply two folds sequentially to an input
561−-- stream. The input is provided to the first fold, when it is done the
644+-- stream. The input is provided to the first fold, when it is done - the
562-- remaining input is provided to the second fold. When the second fold is done
645-- remaining input is provided to the second fold. When the second fold is done
563-- or if the input stream is over, the outputs of the two folds are combined
646-- or if the input stream is over, the outputs of the two folds are combined
564-- using the supplied function.
647-- using the supplied function.
565--
648--
566−-- Note: This is a folding dual of appending streams using
649+-- >>> f = Fold.serialWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)
567−-- 'Streamly.Prelude.serial', it splits the streams using two folds and zips
650+-- >>> Stream.fold f $ Stream.fromList "header: hello\n"
568−-- the results. This has the same caveats as ParseD's @serialWith@
651+-- ("header: ","hello\n")
569--
652--
570−-- /Pre-release/
653+-- Note: This is dual to appending streams using 'Streamly.Prelude.serial'.
654+--
655+-- Note: this implementation allows for stream fusion but has quadratic time
656+-- complexity, because each composition adds a new branch that each subsequent
657+-- fold's input element has to traverse, therefore, it cannot scale to a large
658+-- number of compositions. After around 100 compositions the performance starts
659+-- dipping rapidly compared to a CPS style implementation.
660+--
661+-- /Time: O(n^2) where n is the number of compositions./
662+--
663+-- @since 0.8.0
571--
664--
572{-# INLINE serialWith #-}
665{-# INLINE serialWith #-}
573serialWith :: Monad m => (a -> b -> c) -> Fold m x a -> Fold m x b -> Fold m x c
666serialWith :: Monad m => (a -> b -> c) -> Fold m x a -> Fold m x b -> Fold m x c
@@ -627,6 +720,17 @@ data GenericRunner sL sR bL bR
627-- | @teeWith k f1 f2@ distributes its input to both @f1@ and @f2@ until both
720-- | @teeWith k f1 f2@ distributes its input to both @f1@ and @f2@ until both
628-- of them terminate and combines their output using @k@.
721-- of them terminate and combines their output using @k@.
629--
722--
723+-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
724+-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]
725+-- 50.5
726+--
727+-- > teeWith k f1 f2 = fmap (uncurry k) ((Fold.tee f1 f2)
728+--
729+-- For applicative composition using this combinator see
730+-- "Streamly.Internal.Data.Fold.Tee".
731+--
732+-- See also: "Streamly.Internal.Data.Fold.Tee"
733+--
630-- @since 0.8.0
734-- @since 0.8.0
631--
735--
632{-# INLINE teeWith #-}
736{-# INLINE teeWith #-}
@@ -687,7 +791,7 @@ teeWith f (Fold stepL beginL doneL) (Fold stepR beginR doneR) =
687 bR <- doneR sR
791 bR <- doneR sR
688 return $ f bL bR
792 return $ f bL bR
689
793
690−-- | Like 'teeWith' but terminates when the first fold terminates.
794+-- | Like 'teeWith' but terminates as soon as the first fold terminates.
691--
795--
692-- /Unimplemented/
796-- /Unimplemented/
693--
797--
@@ -695,7 +799,8 @@ teeWith f (Fold stepL beginL doneL) (Fold stepR beginR doneR) =
695teeWithFst :: (b -> c -> d) -> Fold m a b -> Fold m a c -> Fold m a d
799teeWithFst :: (b -> c -> d) -> Fold m a b -> Fold m a c -> Fold m a d
696teeWithFst = undefined
800teeWithFst = undefined
697
801
698−-- | Like 'teeWith' but terminates when any fold terminates.
802+-- | Like 'teeWith' but terminates as soon as any one of the two folds
803+-- terminates.
699--
804--
700-- /Unimplemented/
805-- /Unimplemented/
701--
806--
@@ -729,14 +834,26 @@ data ConcatMapState m sa a c
729 = B !sa
834 = B !sa
730 | forall s. C (s -> a -> m (Step s c)) !s (s -> m c)
835 | forall s. C (s -> a -> m (Step s c)) !s (s -> m c)
731
836
837+-- Compare with foldIterate.
838+--
732-- | Map a 'Fold' returning function on the result of a 'Fold' and run the
839-- | Map a 'Fold' returning function on the result of a 'Fold' and run the
733−-- returned fold.
840+-- returned fold. This operation can be used to express data dependencies
841+-- between fold operations.
842+--
843+-- Let's say the first element in the stream is a count of the following
844+-- elements that we have to add, then:
734--
845--
735-- >>> import Data.Maybe (fromJust)
846-- >>> import Data.Maybe (fromJust)
847+-- >>> count = fmap fromJust Fold.head
848+-- >>> total n = Fold.take n Fold.sum
736−-- >>> Stream.fold (Fold.concatMap (flip Fold.take Fold.sum) (Fold.rmapM (return . fromJust) Fold.head)) $ Stream.fromList [10,9..1]
849+-- >>> Stream.fold (Fold.concatMap total count) $ Stream.fromList [10,9..1]
737-- 45
850-- 45
738--
851--
739−-- /Pre-release/
852+-- /Time: O(n^2) where @n@ is the number of compositions./
853+--
854+-- See also: "Streamly.Internal.Data.Stream.IsStream.foldIterateM"
855+--
856+-- @since 0.8.0
740--
857--
741{-# INLINE concatMap #-}
858{-# INLINE concatMap #-}
742concatMap :: Monad m => (b -> Fold m a c) -> Fold m a b -> Fold m a c
859concatMap :: Monad m => (b -> Fold m a c) -> Fold m a b -> Fold m a c
@@ -781,12 +898,14 @@ concatMap f (Fold stepa initiala extracta) = Fold stepc initialc extractc
781-- Mapping on input
898-- Mapping on input
782------------------------------------------------------------------------------
899------------------------------------------------------------------------------
783
900
784−-- | @(lmap f fold)@ maps the function @f@ on the input of the fold.
901+-- | @lmap f fold@ maps the function @f@ on the input of the fold.
785--
902--
786-- >>> Stream.fold (Fold.lmap (\x -> x * x) Fold.sum) (Stream.enumerateFromTo 1 100)
903-- >>> Stream.fold (Fold.lmap (\x -> x * x) Fold.sum) (Stream.enumerateFromTo 1 100)
787-- 338350
904-- 338350
788--
905--
789−-- /Pre-release/
906+-- > lmap = Fold.lmapM return
907+--
908+-- @since 0.8.0
790{-# INLINABLE lmap #-}
909{-# INLINABLE lmap #-}
791lmap :: (a -> b) -> Fold m b r -> Fold m a r
910lmap :: (a -> b) -> Fold m b r -> Fold m a r
792lmap f (Fold step begin done) = Fold step' begin done
911lmap f (Fold step begin done) = Fold step' begin done
@@ -800,9 +919,9 @@ lmap f (Fold step begin done) = Fold step' begin done
800map :: (a -> b) -> Fold m b r -> Fold m a r
919map :: (a -> b) -> Fold m b r -> Fold m a r
801map = lmap
920map = lmap
802
921
803−-- | @(lmapM f fold)@ maps the monadic function @f@ on the input of the fold.
922+-- | @lmapM f fold@ maps the monadic function @f@ on the input of the fold.
804--
923--
805−-- /Pre-release/
924+-- @since 0.8.0
806{-# INLINABLE lmapM #-}
925{-# INLINABLE lmapM #-}
807lmapM :: Monad m => (a -> m b) -> Fold m b r -> Fold m a r
926lmapM :: Monad m => (a -> m b) -> Fold m b r -> Fold m a r
808lmapM f (Fold step begin done) = Fold step' begin done
927lmapM f (Fold step begin done) = Fold step' begin done
@@ -818,10 +937,9 @@ lmapM f (Fold step begin done) = Fold step' begin done
818-- >>> Stream.fold (Fold.filter (> 5) Fold.sum) $ Stream.fromList [1..10]
937-- >>> Stream.fold (Fold.filter (> 5) Fold.sum) $ Stream.fromList [1..10]
819-- 40
938-- 40
820--
939--
821−-- >>> Stream.fold (Fold.filter (< 5) Fold.sum) $ Stream.fromList [1..10]
940+-- > filter f = Fold.filterM (return . f)
822−-- 10
823--
941--
824−-- @since 0.7.0
942+-- @since 0.8.0
825{-# INLINABLE filter #-}
943{-# INLINABLE filter #-}
826filter :: Monad m => (a -> Bool) -> Fold m a r -> Fold m a r
944filter :: Monad m => (a -> Bool) -> Fold m a r -> Fold m a r
827filter f (Fold step begin done) = Fold step' begin done
945filter f (Fold step begin done) = Fold step' begin done
@@ -830,7 +948,7 @@ filter f (Fold step begin done) = Fold step' begin done
830
948
831-- | Like 'filter' but with a monadic predicate.
949-- | Like 'filter' but with a monadic predicate.
832--
950--
833−-- @since 0.7.0
951+-- @since 0.8.0
834{-# INLINABLE filterM #-}
952{-# INLINABLE filterM #-}
835filterM :: Monad m => (a -> m Bool) -> Fold m a r -> Fold m a r
953filterM :: Monad m => (a -> m Bool) -> Fold m a r -> Fold m a r
836filterM f (Fold step begin done) = Fold step' begin done
954filterM f (Fold step begin done) = Fold step' begin done
@@ -842,7 +960,7 @@ filterM f (Fold step begin done) = Fold step' begin done
842-- | Modify a fold to receive a 'Maybe' input, the 'Just' values are unwrapped
960-- | Modify a fold to receive a 'Maybe' input, the 'Just' values are unwrapped
843-- and sent to the original fold, 'Nothing' values are discarded.
961-- and sent to the original fold, 'Nothing' values are discarded.
844--
962--
845−-- /Pre-release/
963+-- @since 0.8.0
846{-# INLINE catMaybes #-}
964{-# INLINE catMaybes #-}
847catMaybes :: Monad m => Fold m a b -> Fold m (Maybe a) b
965catMaybes :: Monad m => Fold m a b -> Fold m (Maybe a) b
848catMaybes = filter isJust . map fromJust
966catMaybes = filter isJust . map fromJust
@@ -851,15 +969,13 @@ catMaybes = filter isJust . map fromJust
851-- Parsing
969-- Parsing
852------------------------------------------------------------------------------
970------------------------------------------------------------------------------
853
971
854−-- | Take at most @n@ input elements and fold them using the supplied fold.
972+-- | Take at most @n@ input elements and fold them using the supplied fold. A
973+-- negative count is treated as 0.
855--
974--
856−-- >>> Stream.fold (Fold.take 1 Fold.toList) $ Stream.fromList [1]
975+-- >>> Stream.fold (Fold.take 2 Fold.toList) $ Stream.fromList [1..10]
857−-- [1]
976+-- [1,2]
858--
977--
859−-- >>> Stream.fold (Fold.take (-1) Fold.toList) $ Stream.fromList [1]
978+-- @since 0.8.0
860−-- []
861−--
862−-- /Pre-release/
863{-# INLINE take #-}
979{-# INLINE take #-}
864take :: Monad m => Int -> Fold m a b -> Fold m a b
980take :: Monad m => Int -> Fold m a b -> Fold m a b
865take n (Fold fstep finitial fextract) = Fold step initial extract
981take n (Fold fstep finitial fextract) = Fold step initial extract
@@ -898,11 +1014,13 @@ take n (Fold fstep finitial fextract) = Fold step initial extract
898-- i.e. it uses the last accumulator value as the initial value of the
1014-- i.e. it uses the last accumulator value as the initial value of the
899-- accumulator. Thus we can resume the fold later and feed it more input.
1015-- accumulator. Thus we can resume the fold later and feed it more input.
900--
1016--
1017+-- >>> :{
901−-- >> do
1018+-- do
902−-- > more <- S.fold (FL.duplicate FL.sum) (S.enumerateFromTo 1 10)
1019+-- more <- Stream.fold (Fold.duplicate Fold.sum) (Stream.enumerateFromTo 1 10)
903−-- > evenMore <- S.fold (FL.duplicate more) (S.enumerateFromTo 11 20)
1020+-- evenMore <- Stream.fold (Fold.duplicate more) (Stream.enumerateFromTo 11 20)
904−-- > S.fold evenMore (S.enumerateFromTo 21 30)
1021+-- Stream.fold evenMore (Stream.enumerateFromTo 21 30)
1022+-- :}
905−-- > 465
1023+-- 465
906--
1024--
907-- /Pre-release/
1025-- /Pre-release/
908{-# INLINABLE duplicate #-}
1026{-# INLINABLE duplicate #-}
@@ -959,11 +1077,16 @@ data ManyState s1 s2
959-- the @split@ fold repeatedly on the input stream and accumulates zero or more
1077-- the @split@ fold repeatedly on the input stream and accumulates zero or more
960-- fold results using @collect@.
1078-- fold results using @collect@.
961--
1079--
1080+-- >>> two = Fold.take 2 Fold.toList
1081+-- >>> twos = Fold.many two Fold.toList
1082+-- >>> Stream.fold twos $ Stream.fromList [1..10]
1083+-- [[1,2],[3,4],[5,6],[7,8],[9,10]]
1084+--
962-- Stops when @collect@ stops.
1085-- Stops when @collect@ stops.
963--
1086--
964−-- /Pre-release/
1087+-- See also: "Streamly.Prelude.concatMap", "Streamly.Prelude.foldMany"
965--
1088--
966−-- /See also: Streamly.Prelude.concatMap, Streamly.Prelude.foldMany/
1089+-- @since 0.8.0
967--
1090--
968{-# INLINE many #-}
1091{-# INLINE many #-}
969many :: Monad m => Fold m a b -> Fold m b c -> Fold m a c
1092many :: Monad m => Fold m a b -> Fold m b c -> Fold m a c
@@ -1077,11 +1200,15 @@ manyPost (Fold sstep sinitial sextract) (Fold cstep cinitial cextract) =
1077-- of @n@ items in the input stream and supplies the result to the @collect@
1200-- of @n@ items in the input stream and supplies the result to the @collect@
1078-- fold.
1201-- fold.
1079--
1202--
1203+-- >>> twos = Fold.chunksOf 2 Fold.toList Fold.toList
1204+-- >>> Stream.fold twos $ Stream.fromList [1..10]
1205+-- [[1,2],[3,4],[5,6],[7,8],[9,10]]
1206+--
1080-- > chunksOf n split = many (take n split)
1207-- > chunksOf n split = many (take n split)
1081--
1208--
1082-- Stops when @collect@ stops.
1209-- Stops when @collect@ stops.
1083--
1210--
1084−-- /Pre-release/
1211+-- @since 0.8.0
1085--
1212--
1086{-# INLINE chunksOf #-}
1213{-# INLINE chunksOf #-}
1087chunksOf :: Monad m => Int -> Fold m a b -> Fold m b c -> Fold m a c
1214chunksOf :: Monad m => Int -> Fold m a b -> Fold m b c -> Fold m a c
@@ -1119,10 +1246,15 @@ chunksOf2 n (Fold step1 initial1 extract1) (Fold2 step2 inject2 extract2) =
1119-- XXX We can use asyncClock here. A parser can be used to return an input that
1246-- XXX We can use asyncClock here. A parser can be used to return an input that
1120-- arrives after the timeout.
1247-- arrives after the timeout.
1121-- XXX If n is 0 return immediately in initial.
1248-- XXX If n is 0 return immediately in initial.
1249+-- XXX we should probably discard the input received after the timeout like
1250+-- takeEndBy_.
1122--
1251--
1123-- | @takeInterval n fold@ uses @fold@ to fold the input items arriving within
1252-- | @takeInterval n fold@ uses @fold@ to fold the input items arriving within
1124-- a window of first @n@ seconds.
1253-- a window of first @n@ seconds.
1125--
1254--
1255+-- >>> Stream.fold (Fold.takeInterval 1.0 Fold.toList) $ Stream.delay 0.1 $ Stream.fromList [1..]
1256+-- [1,2,3,4,5,6,7,8,9,10,11]
1257+--
1126-- Stops when @fold@ stops or when the timeout occurs. Note that the fold needs
1258-- Stops when @fold@ stops or when the timeout occurs. Note that the fold needs
1127-- an input after the timeout to stop. For example, if no input is pushed to
1259-- an input after the timeout to stop. For example, if no input is pushed to
1128-- the fold until one hour after the timeout had occurred, then the fold will
1260-- the fold until one hour after the timeout had occurred, then the fold will
@@ -1177,18 +1309,16 @@ takeInterval n (Fold step initial done) = Fold step' initial' done'
1177 handleChildException :: MVar Bool -> SomeException -> IO ()
1309 handleChildException :: MVar Bool -> SomeException -> IO ()
1178 handleChildException mv _ = void $ swapMVar mv True
1310 handleChildException mv _ = void $ swapMVar mv True
1179
1311
1180−-- | Group the input stream into windows of n second each and then fold each
1181−-- group using the provided fold function.
1182−--
1183-- For example, we can copy and distribute a stream to multiple folds where
1312-- For example, we can copy and distribute a stream to multiple folds where
1184-- each fold can group the input differently e.g. by one second, one minute and
1313-- each fold can group the input differently e.g. by one second, one minute and
1185-- one hour windows respectively and fold each resulting stream of folds.
1314-- one hour windows respectively and fold each resulting stream of folds.
1315+
1316+-- | Group the input stream into windows of n second each using the first fold
1317+-- and then fold the resulting groups using the second fold.
1186--
1318--
1187−-- @
1319+-- >>> intervals = Fold.intervalsOf 0.5 Fold.toList Fold.toList
1188−--
1320+-- >>> Stream.fold intervals $ Stream.delay 0.2 $ Stream.fromList [1..10]
1189−-- -----Fold m a b----|-Fold n a c-|-Fold n a c-|-...-|----Fold m a c
1321+-- [[1,2,3,4],[5,6,7],[8,9,10]]
1190−--
1191−-- @
1192--
1322--
1193-- > intervalsOf n split = many (takeInterval n split)
1323-- > intervalsOf n split = many (takeInterval n split)
1194--
1324--
src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs+6−3
@@ -332,14 +332,17 @@ runSink = fold . toFold
332-- Running a Parser
332-- Running a Parser
333------------------------------------------------------------------------------
333------------------------------------------------------------------------------
334
334
335−-- | Parse a stream using the supplied 'Parser'.
335+-- | Parse a stream using the supplied ParserD 'PRD.Parser'.
336--
336--
337−-- /Pre-release/
337+-- /Internal/
338--
338--
339{-# INLINE_NORMAL parseD #-}
339{-# INLINE_NORMAL parseD #-}
340parseD :: MonadThrow m => PRD.Parser m a b -> SerialT m a -> m b
340parseD :: MonadThrow m => PRD.Parser m a b -> SerialT m a -> m b
341parseD p = D.parse p . toStreamD
341parseD p = D.parse p . toStreamD
342
342
343+-- | Parse a stream using the supplied ParserK 'PRK.Parser'.
344+--
345+-- /Internal/
343{-# INLINE parseK #-}
346{-# INLINE parseK #-}
344parseK :: MonadThrow m => PRK.Parser m a b -> SerialT m a -> m b
347parseK :: MonadThrow m => PRK.Parser m a b -> SerialT m a -> m b
345parseK = parse
348parseK = parse
@@ -877,7 +880,7 @@ isPrefixOf m1 m2 = D.isPrefixOf (toStreamD m1) (toStreamD m2)
877--
880--
878-- /Pre-release/
881-- /Pre-release/
879--
882--
880−-- /Requires 'Storable' constraint/ - Help wanted.
883+-- /Requires 'Storable' constraint/
881--
884--
882{-# INLINE isInfixOf #-}
885{-# INLINE isInfixOf #-}
883isInfixOf :: (MonadIO m, Eq a, Enum a, Storable a)
886isInfixOf :: (MonadIO m, Eq a, Enum a, Storable a)
src/Streamly/Internal/Data/Stream/IsStream/Nesting.hs+1−1
@@ -980,7 +980,7 @@ foldManyPost f m = D.fromStreamD $ D.foldManyPost f (D.toStreamD m)
980-- Note @foldMany (take 0)@ would result in an infinite loop in a non-empty
980-- Note @foldMany (take 0)@ would result in an infinite loop in a non-empty
981-- stream.
981-- stream.
982--
982--
983−-- /Pre-release/
983+-- @since 0.8.0
984--
984--
985{-# INLINE foldMany #-}
985{-# INLINE foldMany #-}
986foldMany
986foldMany
src/Streamly/Internal/Data/Stream/IsStream/Transform.hs+1−1
@@ -1192,7 +1192,7 @@ interspersePrefix_ m = mapM (\x -> void m >> return x)
1192-- (AbsTime (TimeSpec {sec = ..., nsec = ...}),2)
1192-- (AbsTime (TimeSpec {sec = ..., nsec = ...}),2)
1193-- (AbsTime (TimeSpec {sec = ..., nsec = ...}),3)
1193-- (AbsTime (TimeSpec {sec = ..., nsec = ...}),3)
1194--
1194--
1195−-- /Pre-release/
1195+-- @since 0.8.0
1196--
1196--
1197{-# INLINE delay #-}
1197{-# INLINE delay #-}
1198delay :: (IsStream t, MonadIO m) => Double -> t m a -> t m a
1198delay :: (IsStream t, MonadIO m) => Double -> t m a -> t m a
src/Streamly/Internal/Network/Socket.hs−1
@@ -98,7 +98,6 @@ import Streamly.Data.Fold (Fold)
98-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
98-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
99
99
100import qualified Streamly.Data.Fold as FL
100import qualified Streamly.Data.Fold as FL
101−import qualified Streamly.Internal.Data.Fold.Type as FL
102import qualified Streamly.Internal.Data.Unfold as UF
101import qualified Streamly.Internal.Data.Unfold as UF
103import qualified Streamly.Internal.Data.Array.Foreign as IA
102import qualified Streamly.Internal.Data.Array.Foreign as IA
104import qualified Streamly.Data.Array.Foreign as A
103import qualified Streamly.Data.Array.Foreign as A
src/Streamly/Prelude.hs+2
@@ -247,6 +247,7 @@ module Streamly.Prelude
247 -- $runningfolds
247 -- $runningfolds
248
248
249 , fold
249 , fold
250+ , foldMany
250
251
251 -- ** Full Folds
252 -- ** Full Folds
252 -- | Folds that are guaranteed to evaluate the whole stream.
253 -- | Folds that are guaranteed to evaluate the whole stream.
@@ -343,6 +344,7 @@ module Streamly.Prelude
343 , mapM_
344 , mapM_
344 , trace
345 , trace
345 , tap
346 , tap
347+ , delay
346
348
347 -- ** Scanning
349 -- ** Scanning
348 --
350 --
test/lib/Streamly/Test/Prelude/Common.hs−1
@@ -134,7 +134,6 @@ import Streamly.Prelude (avgRate, rate, maxBuffer, maxThreads)
134import qualified Streamly.Prelude as S
134import qualified Streamly.Prelude as S
135import qualified Streamly.Data.Fold as FL
135import qualified Streamly.Data.Fold as FL
136import qualified Streamly.Internal.Data.Stream.IsStream as S
136import qualified Streamly.Internal.Data.Stream.IsStream as S
137−import qualified Streamly.Internal.Data.Fold as FL
138import qualified Streamly.Internal.Data.Unfold as UF
137import qualified Streamly.Internal.Data.Unfold as UF
139
138
140import Streamly.Test.Common
139import Streamly.Test.Common