@@ -8,12 +8,13 @@
88-- Portability : GHC
99--
1010-- See "Streamly.Data.Fold" for an overview and
11−-- "Streamly.Internal.Data.Fold.Types" for design notes.
12−--
13−-- IMPORTANT: keep the signatures consistent with the folds in Streamly.Prelude
11+-- "Streamly.Internal.Data.Fold.Type" for design notes.
1412
1513module Streamly.Internal.Data.Fold
1614 (
15+ -- * Imports
16+ -- $setup
17+
1718 -- * Fold Type
1819 Step (..)
1920 , Fold (..)
@@ -22,19 +23,17 @@ module Streamly.Internal.Data.Fold
2223 -- | Which constructor to use?
2324 --
2425 -- * @foldl*@: If the fold never terminates i.e. does not use the 'Done'
25− -- constructor otherwise use the @mkFold@ variants.
26+ -- constructor otherwise use the @foldt*@ variants.
2627 -- * @*M@: Use the @M@ suffix variants if any of the step, initial, or
2728 -- extract function is monadic, otherwise use the pure variants.
28− -- * @*_@: Use the @_@ suffix variants if the extract function is 'id' or
29− -- 'return'.
3029 --
3130 , foldl'
3231 , foldlM'
3332 , foldl1'
3433 , foldt'
3534 , foldtM'
36− , foldr
37− , foldrM
35+ , foldr'
36+ , foldrM'
3837
3938 -- * Mappers
4039 -- | Monadic functions useful with mapM/lmapM on folds or streams.
@@ -42,9 +41,6 @@ module Streamly.Internal.Data.Fold
4241 , trace
4342
4443 -- * Folds
45− -- ** Identity
46− , fromPure
47− , fromEffect
4844
4945 -- ** Accumulators
5046 -- *** Semigroups and Monoids
@@ -55,15 +51,13 @@ module Streamly.Internal.Data.Fold
5551
5652 -- *** Reducers
5753 , drain
58− , drainBy
54+ , drainMapM
5955 , the
6056 , length
61− , genericLength
62− , countDistinct
63− , countDistinctInt
64− , frequency
57+ , lengthGeneric
6558 , mean
6659 , rollingHash
60+ , defaultSalt
6761 , rollingHashWithSalt
6862 , rollingHashFirstN
6963 -- , rollingHashLastN
@@ -88,8 +82,7 @@ module Streamly.Internal.Data.Fold
8882 -- $toListRev
8983 , toStream
9084 , toStreamRev
91− , toMap
92− , topBy
85+ , bottomBy
9386 , top
9487 , bottom
9588
@@ -98,7 +91,7 @@ module Streamly.Internal.Data.Fold
9891 -- the 'scanMaybe' combinator. For scanners the result of the fold is
9992 -- usually a transformation of the current element rather than an
10093 -- aggregation of all elements till now.
101− , last -- XXX prev
94+ , latest
10295 -- , nthLast -- using Ring array
10396 , indexingWith
10497 , indexing
@@ -109,8 +102,6 @@ module Streamly.Internal.Data.Fold
109102 -- | Useful in combination with the 'scanMaybe' combinator.
110103 , filtering
111104 , deleteBy
112− , nub
113− , nubInt
114105 , uniqBy
115106 , uniq
116107 , repeated
@@ -118,18 +109,25 @@ module Streamly.Internal.Data.Fold
118109 , elemIndices
119110
120111 -- ** Terminating Folds
121− -- Element folds. Terminate after inspecting one element. All these can be
122− -- implemented in terms of the 'maybe' fold.
112+ -- *** Empty folds
113+ -- | Folds that return a result without consuming any input.
114+ , fromPure
115+ , fromEffect
116+
117+ -- *** Singleton folds
118+ -- | Folds that terminate after consuming exactly one input element. All
119+ -- these can be implemented in terms of the 'maybe' fold.
123120 , one
124121 , null
125122 , satisfy
126123 , maybe
127124
128− -- Sequence folds. Terminate after inspecting a sequence of elements.
125+ -- *** Multi folds
126+ -- | Terminate after consuming one or more elements.
129127 , drainN
130128 -- , lastN
131129 -- , (!!)
132− , genericIndex
130+ , indexGeneric
133131 , index
134132 , findM
135133 , find
@@ -155,6 +153,20 @@ module Streamly.Internal.Data.Fold
155153 , droppingWhile
156154 , prune
157155
156+ -- * Running A Fold
157+ , drive
158+ , driveBreak
159+
160+ -- * Building Incrementally
161+ , extractM
162+ , reduce
163+ , isClosed
164+ , snoc
165+ , snocl
166+ , snocM
167+ , snoclM
168+ , augment
169+
158170 -- * Combinators
159171 -- ** Utilities
160172 , with
@@ -182,8 +194,8 @@ module Streamly.Internal.Data.Fold
182194 , indexed
183195
184196 -- ** Zipping Input
185− , zipWithM
186− , zip
197+ , zipStreamWithM
198+ , zipStream
187199
188200 -- ** Filtering Input
189201 , catMaybes
@@ -260,41 +272,6 @@ module Streamly.Internal.Data.Fold
260272 , partitionBy
261273 , partition
262274
263− -- ** Demultiplexing
264− -- | Direct values in the input stream to different folds using an n-ary
265− -- fold selector. 'demux' is a generalization of 'classify' (and
266− -- 'partition') where each key of the classifier can use a different fold.
267− , demux
268− , demuxWith
269− , demuxScanWith
270− , demuxScanMutWith
271− , demuxMutWith
272−
273− -- TODO: These can be implemented using the above operations
274− -- , demuxWithSel -- Stop when the fold for the specified key stops
275− -- , demuxWithMin -- Stop when any of the folds stop
276− -- , demuxWithAll -- Stop when all the folds stop (run once)
277−
278− -- ** Classifying
279− -- | In an input stream of key value pairs fold values for different keys
280− -- in individual output buckets using the given fold. 'classify' is a
281− -- special case of 'demux' where all the branches of the demultiplexer use
282− -- the same fold.
283− --
284− -- Different types of maps can be used with these combinators via the IsMap
285− -- type class. Hashmap performs better when there are more collisions, trie
286− -- Map performs better otherwise. Trie has an advantage of sorting the keys
287− -- at the same time. For example if we want to store a dictionary of words
288− -- and their meanings then trie Map would be better if we also want to
289− -- display them in sorted order.
290−
291− , classify
292− , classifyWith
293− , classifyMutWith
294− , classifyScanWith
295− -- , classifyWithSel
296− -- , classifyWithMin
297−
298275 -- ** Splitting
299276 , many
300277 , chunksOf
@@ -304,20 +281,12 @@ module Streamly.Internal.Data.Fold
304281 , unfoldMany
305282 , concatSequence
306283 , concatMap
307−
308− -- * Running A Fold
309− -- | Normally you would run a fold to completion by supplying it a stream,
310− -- e.g. using 'Stream.fold'. However, you could also run a fold partially
311− -- by using 'duplicate' on it and then running it with a stream.
312− -- Alternatively, 'initialize', 'snoc' and 'finish' can be used to run a
313− -- fold incrementally, however, that may not be the most efficient way to
314− -- run a fold.
315− , initialize
316− , snoc
317284 , duplicate
318− , finish
319285
320286 -- * Deprecated
287+ , foldr
288+ , drainBy
289+ , last
321290 , head
322291 , sequence
323292 , mapM
@@ -336,11 +305,8 @@ import Data.Bits (shiftL, shiftR, (.|.), (.&.))
336305import Data.Either (isLeft, isRight, fromLeft, fromRight)
337306import Data.Functor.Identity (Identity(..))
338307import Data.Int (Int64)
339−import Data.IORef (newIORef, readIORef, writeIORef)
340−import Data.Map.Strict (Map)
341308import Data.Word (Word32)
342309import Foreign.Storable (peek, sizeOf)
343−import Streamly.Internal.Data.IsMap (IsMap(..))
344310import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)
345311import Streamly.Internal.Data.Pipe.Type (Pipe (..), PipeState(..))
346312import Streamly.Internal.Data.Unboxed (Unbox)
@@ -348,16 +314,14 @@ import Streamly.Internal.Data.Unfold.Type (Unfold(..))
348314import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
349315import Streamly.Internal.Data.Stream.Type (Stream)
350316
351−import qualified Data.IntSet as IntSet
352−import qualified Data.Set as Set
353317import qualified Prelude
354318import qualified Streamly.Internal.Data.Array.Unboxed.Mut.Type as MA
355319import qualified Streamly.Internal.Data.Array.Unboxed.Type as Array
356320import qualified Streamly.Internal.Data.Fold.Window as FoldW
357−import qualified Streamly.Internal.Data.IsMap as IsMap
358321import qualified Streamly.Internal.Data.Pipe.Type as Pipe
359322import qualified Streamly.Internal.Data.Ring.Unboxed as Ring
360323import qualified Streamly.Internal.Data.Stream.StreamD.Type as StreamD
324+import qualified Streamly.Internal.Data.Stream.StreamK as K
361325import qualified Streamly.Internal.Data.Stream.Type as Stream
362326
363327import Prelude hiding
@@ -371,7 +335,13 @@ import Streamly.Internal.Data.Fold.Type
371335
372336-- $setup
373337-- >>> :m
374−-- >>> import Prelude hiding (break, map, span, splitAt)
338+-- >>> :set -package random
339+-- >>> import Control.Monad (void)
340+-- >>> import Data.Functor.Identity (runIdentity)
341+-- >>> import Data.Maybe (isJust)
342+-- >>> import Data.Monoid (Last(..), Sum(..))
343+-- >>> import Streamly.Data.Stream (Stream)
344+-- >>> import Data.IORef (newIORef, readIORef, writeIORef)
375345-- >>> import qualified Streamly.Data.Array.Unboxed as Array
376346-- >>> import qualified Streamly.Data.Fold as Fold
377347-- >>> import qualified Streamly.Data.Stream as Stream
@@ -382,8 +352,60 @@ import Streamly.Internal.Data.Fold.Type
382352-- >>> import qualified Streamly.Internal.Data.Parser as Parser
383353-- >>> import qualified Streamly.Internal.Data.Stream.Type as Stream
384354-- >>> import qualified Streamly.Internal.Data.Unfold as Unfold
385−-- >>> import Streamly.Internal.Data.Stream.Type (Stream)
386−-- >>> import Data.IORef (newIORef, readIORef, writeIORef)
355+-- >>> import Prelude hiding (break, map, span, splitAt)
356+
357+------------------------------------------------------------------------------
358+-- Running
359+------------------------------------------------------------------------------
360+
361+-- | Drive a fold using the supplied 'Stream', reducing the resulting
362+-- expression strictly at each step.
363+--
364+-- Definition:
365+--
366+-- >>> drive = flip Stream.fold
367+--
368+-- Example:
369+--
370+-- >>> Fold.drive (Stream.enumerateFromTo 1 100) Fold.sum
371+-- 5050
372+--
373+{-# INLINE drive #-}
374+drive :: Monad m => Stream m a -> Fold m a b -> m b
375+drive strm fl = StreamD.fold fl $ StreamD.fromStreamK $ Stream.toStreamK strm
376+
377+-- | Like 'drive' but also returns the remaining stream. The resulting stream
378+-- would be 'Stream.nil' if the stream finished before the fold.
379+--
380+-- Definition:
381+--
382+-- >>> driveBreak = flip Stream.foldBreak
383+--
384+-- /Not fused/
385+--
386+{-# INLINE driveBreak #-}
387+driveBreak :: Monad m => Stream m a -> Fold m a b -> m (b, Stream m a)
388+driveBreak strm fl = fmap f $ K.foldBreak fl (Stream.toStreamK strm)
389+
390+ where
391+
392+ f (b, str) = (b, Stream.fromStreamK str)
393+
394+-- | Augment a fold strictly by adding a stream to it.
395+--
396+-- Definitions:
397+--
398+-- >>> augment stream = Fold.drive stream . Fold.duplicate
399+--
400+-- >>> :{
401+-- Fold.augment (Stream.enumerateFromTo 1 10) Fold.sum
402+-- >>= Fold.augment (Stream.enumerateFromTo 11 20)
403+-- >>= Fold.drive (Stream.enumerateFromTo 21 30)
404+-- :}
405+-- 465
406+--
407+augment :: Monad m => Stream m a -> Fold m a b -> m (Fold m a b)
408+augment stream = drive stream . duplicate
387409
388410------------------------------------------------------------------------------
389411-- hoist
@@ -398,7 +420,7 @@ hoist f (Fold step initial extract) =
398420
399421-- | Adapt a pure fold to any monad
400422--
401−-- > generally = Fold.hoist (return . runIdentity)
423+-- >>> generally = Fold.hoist (return . runIdentity)
402424--
403425-- /Pre-release/
404426generally :: Monad m => Fold Identity a b -> Fold m a b
@@ -575,10 +597,12 @@ scanMany = scanWith True
575597-- Filters
576598------------------------------------------------------------------------------
577599
578−-- | A filtering scan that deletes the first occurrence of the element in the
579−-- stream that satisfies the given equality predicate.
600+-- | Returns the latest element omitting the first occurrence that satisfies
601+-- the given equality predicate.
602+--
603+-- Example:
580604--
581−-- >>> input = Stream.unfold Unfold.fromList [1,3,3,5]
605+-- >>> input = Stream.fromList [1,3,3,5]
582606-- >>> Stream.fold Fold.toList $ Stream.scanMaybe (Fold.deleteBy (==) 3) input
583607-- [1,3,5]
584608--
@@ -614,20 +638,19 @@ slide2 (Fold step1 initial1 extract1) = Fold step initial extract
614638
615639 extract (Tuple' _ s) = extract1 s
616640
617−-- | Drop repeated elements that are adjacent to each other using the supplied
618−-- comparison function.
641+-- | Return the latest unique element using the supplied comparison function.
642+-- Returns 'Nothing' if the current element is same as the last element
643+-- otherwise returns 'Just'.
619644--
620−-- To strip duplicate path separators:
645+-- Example, strip duplicate path separators:
621646--
622−-- >>> input = Stream.unfold Unfold.fromList "//a//b"
647+-- >>> input = Stream.fromList "//a//b"
623648-- >>> f x y = x == '/' && y == '/'
624649-- >>> Stream.fold Fold.toList $ Stream.scanMaybe (Fold.uniqBy f) input
625650-- "/a/b"
626651--
627652-- Space: @O(1)@
628653--
629−-- See also: 'nubBy'.
630−--
631654-- /Pre-release/
632655--
633656{-# INLINE uniqBy #-}
@@ -641,7 +664,9 @@ uniqBy eq = rollingMap f
641664 Nothing -> Just curr
642665 Just x -> if x `eq` curr then Nothing else Just curr
643666
644−-- | Drop repeated elements that are adjacent to each other.
667+-- | See 'uniqBy'.
668+--
669+-- Definition:
645670--
646671-- >>> uniq = Fold.uniqBy (==)
647672--
@@ -676,43 +701,6 @@ repeated :: -- (Monad m, Eq a) =>
676701 Fold m a (Maybe a)
677702repeated = error "Not implemented yet!"
678703
679−-- | Used as a scan. Returns 'Just' for the first occurrence of an element,
680−-- returns 'Nothing' for any other occurrences.
681−--
682−-- >>> stream = Stream.fromList [1::Int,1,2,3,4,4,5,1,5,7]
683−-- >>> Stream.fold Fold.toList $ Stream.mapMaybe id $ Stream.postscan Fold.nub stream
684−-- [1,2,3,4,5,7]
685−--
686−-- /Pre-release/
687−{-# INLINE nub #-}
688−nub :: (Monad m, Ord a) => Fold m a (Maybe a)
689−nub = fmap (\(Tuple' _ x) -> x) $ foldl' step initial
690−
691− where
692−
693− initial = Tuple' Set.empty Nothing
694−
695− step (Tuple' set _) x =
696− if Set.member x set
697− then Tuple' set Nothing
698− else Tuple' (Set.insert x set) (Just x)
699−
700−-- | Like 'nub' but specialized to a stream of 'Int', for better performance.
701−--
702−-- /Pre-release/
703−{-# INLINE nubInt #-}
704−nubInt :: Monad m => Fold m Int (Maybe Int)
705−nubInt = fmap (\(Tuple' _ x) -> x) $ foldl' step initial
706−
707− where
708−
709− initial = Tuple' IntSet.empty Nothing
710−
711− step (Tuple' set _) x =
712− if IntSet.member x set
713− then Tuple' set Nothing
714− else Tuple' (IntSet.insert x set) (Just x)
715−
716704------------------------------------------------------------------------------
717705-- Left folds
718706------------------------------------------------------------------------------
@@ -722,28 +710,40 @@ nubInt = fmap (\(Tuple' _ x) -> x) $ foldl' step initial
722710------------------------------------------------------------------------------
723711
724712-- |
725−-- > drainBy f = lmapM f drain
726−-- > drainBy = Fold.foldMapM (void . f)
713+-- Definitions:
714+--
715+-- >>> drainMapM f = Fold.lmapM f Fold.drain
716+-- >>> drainMapM f = Fold.foldMapM (void . f)
727717--
728718-- Drain all input after passing it through a monadic function. This is the
729719-- dual of mapM_ on stream producers.
730720--
731−-- See also: 'Streamly.Prelude.mapM_'
732−--
721+{-# INLINE drainMapM #-}
722+drainMapM :: Monad m => (a -> m b) -> Fold m a ()
723+drainMapM f = lmapM f drain
724+
725+{-# DEPRECATED drainBy "Please use 'drainMapM' instead." #-}
733726{-# INLINE drainBy #-}
734727drainBy :: Monad m => (a -> m b) -> Fold m a ()
735−drainBy f = lmapM f drain
728+drainBy = drainMapM
736729
737−-- | Extract the last element of the input stream, if any.
730+-- | Returns the latest element of the input stream, if any.
738731--
739−-- > last = fmap getLast $ Fold.foldMap (Last . Just)
732+-- >>> latest = Fold.foldl1' (\_ x -> x)
733+-- >>> latest = fmap getLast $ Fold.foldMap (Last . Just)
740734--
735+{-# INLINE latest #-}
736+latest :: Monad m => Fold m a (Maybe a)
737+latest = foldl1' (\_ x -> x)
738+
739+{-# DEPRECATED last "Please use 'latest' instead." #-}
741740{-# INLINE last #-}
742741last :: Monad m => Fold m a (Maybe a)
743−last = foldl1' (\_ x -> x)
742+last = latest
744743
745−-- | If the stream consists of one or more occurences of the same element
746−-- then returns that element else terminates and returns 'Nothing'.
744+-- | Terminates with 'Nothing' as soon as it finds an element different than
745+-- the previous one, returns 'the' element if the entire input consists of the
746+-- same element.
747747--
748748{-# INLINE the #-}
749749the :: (Monad m, Eq a) => Fold m a (Maybe a)
@@ -765,82 +765,27 @@ the = foldt' step initial id
765765
766766-- | Like 'length', except with a more general 'Num' return value
767767--
768−-- > genericLength = fmap getSum $ foldMap (Sum . const 1)
768+-- Definition:
769+--
770+-- >>> lengthGeneric = fmap getSum $ Fold.foldMap (Sum . const 1)
771+-- >>> lengthGeneric = Fold.foldl' (\n _ -> n + 1) 0
769772--
770773-- /Pre-release/
771−{-# INLINE genericLength #-}
772−genericLength :: (Monad m, Num b) => Fold m a b
773−genericLength = foldl' (\n _ -> n + 1) 0
774+{-# INLINE lengthGeneric #-}
775+lengthGeneric :: (Monad m, Num b) => Fold m a b
776+lengthGeneric = foldl' (\n _ -> n + 1) 0
774777
775778-- | Determine the length of the input stream.
776779--
777−-- > length = fmap getSum $ Fold.foldMap (Sum . const 1)
780+-- Definition:
781+--
782+-- >>> length = Fold.lengthGeneric
783+-- >>> length = fmap getSum $ Fold.foldMap (Sum . const 1)
778784--
779785{-# INLINE length #-}
780786length :: Monad m => Fold m a Int
781−length = genericLength
782−
783−-- XXX Try Hash set
784−-- XXX Add a countDistinct window fold
785−-- XXX Add a bloom filter fold
786−
787−-- | Count non-duplicate elements in the stream.
788−--
789−-- Equivalent to using 'nub' followed by 'length' on a stream.
790−--
791−-- >>> countDistinct = Fold.postscan Fold.nub $ Fold.catMaybes $ Fold.length
792−--
793−-- The memory used is proportional to the number of distinct elements in the
794−-- stream, to guard against using too much memory use it as a scan and
795−-- terminate if the count reaches more than a threshold.
796−--
797−-- /Space/: \(\mathcal{O}(n)\)
798−--
799−-- /Pre-release/
800−--
801−{-# INLINE countDistinct #-}
802−countDistinct :: (Monad m, Ord a) => Fold m a Int
803−countDistinct = postscan nub $ catMaybes length
804−{-
805−countDistinct = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
806−
807− where
787+length = lengthGeneric
808788
809− initial = Tuple' Set.empty 0
810−
811− step (Tuple' set n) x = do
812− if Set.member x set
813− then
814− Tuple' set n
815− else
816− let cnt = n + 1
817− in Tuple' (Set.insert x set) cnt
818−-}
819−
820−-- | Like 'countDistinct' but specialized to a stream of 'Int', for better
821−-- performance.
822−--
823−-- >>> countDistinctInt = Fold.postscan Fold.nubInt $ Fold.catMaybes $ Fold.length
824−--
825−-- /Pre-release/
826−{-# INLINE countDistinctInt #-}
827−countDistinctInt :: Monad m => Fold m Int Int
828−countDistinctInt = postscan nubInt $ catMaybes length
829−{-
830−countDistinctInt = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
831−
832− where
833−
834− initial = Tuple' IntSet.empty 0
835−
836− step (Tuple' set n) x = do
837− if IntSet.member x set
838− then
839− Tuple' set n
840− else
841− let cnt = n + 1
842− in Tuple' (IntSet.insert x set) cnt
843− -}
844789
845790-- | Determine the sum of all elements of a stream of numbers. Returns additive
846791-- identity (@0@) when the stream is empty. Note that this is not numerically
@@ -861,7 +806,9 @@ sum = FoldW.cumulative FoldW.sum
861806-- multiplicative identity (@1@) when the stream is empty. The fold terminates
862807-- when it encounters (@0@) in its input.
863808--
864−-- Compare with @Fold.foldMap Product@.
809+-- Same as the following but terminates on multiplication by @0@:
810+--
811+-- >>> product = fmap Data.Monoid.getProduct $ Fold.foldMap Data.Monoid.Product
865812--
866813{-# INLINE product #-}
867814product :: (Monad m, Num a, Eq a) => Fold m a a
@@ -892,14 +839,17 @@ maximumBy cmp = foldl1' max'
892839 GT -> x
893840 _ -> y
894841
895−-- |
896−-- @
897−-- maximum = Fold.maximumBy compare
898−-- @
842+-- | Determine the maximum element in a stream.
843+--
844+-- Definitions:
899845--
900−-- Determine the maximum element in a stream.
846+-- >>> maximum = Fold.maximumBy compare
847+-- >>> maximum = Fold.foldl1' max
901848--
902−-- Compare with @Fold.foldMap Max@.
849+-- Same as the following but without a default maximum. The 'Max' Monoid uses
850+-- the 'minBound' as the default maximum:
851+--
852+-- >>> maximum = fmap Data.Semigroup.getMax $ Fold.foldMap Data.Semigroup.Max
903853--
904854{-# INLINE maximum #-}
905855maximum :: (Monad m, Ord a) => Fold m a (Maybe a)
@@ -921,11 +871,15 @@ minimumBy cmp = foldl1' min'
921871-- | Determine the minimum element in a stream using the supplied comparison
922872-- function.
923873--
924−-- @
925−-- minimum = 'minimumBy' compare
926−-- @
874+-- Definitions:
875+--
876+-- >>> minimum = Fold.minimumBy compare
877+-- >>> minimum = Fold.foldl1' min
878+--
879+-- Same as the following but without a default minimum. The 'Min' Monoid uses the
880+-- 'maxBound' as the default maximum:
927881--
928−-- Compare with @Fold.foldMap Min@.
882+-- >>> maximum = fmap Data.Semigroup.getMin $ Fold.foldMap Data.Semigroup.Min
929883--
930884{-# INLINE minimum #-}
931885minimum :: (Monad m, Ord a) => Fold m a (Maybe a)
@@ -1011,7 +965,7 @@ defaultSalt = -2578643520546668380
1011965
1012966-- | Compute an 'Int' sized polynomial rolling hash of a stream.
1013967--
1014−-- > rollingHash = Fold.rollingHashWithSalt defaultSalt
968+-- >>> rollingHash = Fold.rollingHashWithSalt Fold.defaultSalt
1015969--
1016970{-# INLINE rollingHash #-}
1017971rollingHash :: (Monad m, Enum a) => Fold m a Int64
@@ -1020,7 +974,7 @@ rollingHash = rollingHashWithSalt defaultSalt
1020974-- | Compute an 'Int' sized polynomial rolling hash of the first n elements of
1021975-- a stream.
1022976--
1023−-- > rollingHashFirstN = Fold.take n Fold.rollingHash
977+-- >>> rollingHashFirstN n = Fold.take n Fold.rollingHash
1024978--
1025979-- /Pre-release/
1026980{-# INLINE rollingHashFirstN #-}
@@ -1053,6 +1007,9 @@ rollingMapM f = Fold step initial extract
10531007
10541008 extract = return . snd
10551009
1010+-- |
1011+-- >>> rollingMap f = Fold.rollingMapM (\x y -> return $ f x y)
1012+--
10561013{-# INLINE rollingMap #-}
10571014rollingMap :: Monad m => (Maybe a -> a -> b) -> Fold m a b
10581015rollingMap f = rollingMapM (\x y -> return $ f x y)
@@ -1061,26 +1018,31 @@ rollingMap f = rollingMapM (\x y -> return $ f x y)
10611018-- Monoidal left folds
10621019------------------------------------------------------------------------------
10631020
1064−-- | Append the elements of an input stream to a provided starting value.
1021+-- | Semigroup concat. Append the elements of an input stream to a provided
1022+-- starting value.
10651023--
1066−-- >>> Stream.fold (Fold.sconcat 10) (fmap Data.Monoid.Sum $ Stream.enumerateFromTo 1 10)
1067−-- Sum {getSum = 65}
1024+-- Definition:
10681025--
1069−-- @
1070−-- sconcat = Fold.foldl' (<>)
1071−-- @
1026+-- >>> sconcat = Fold.foldl' (<>)
1027+--
1028+-- >>> semigroups = fmap Data.Monoid.Sum $ Stream.enumerateFromTo 1 10
1029+-- >>> Stream.fold (Fold.sconcat 10) semigroups
1030+-- Sum {getSum = 65}
10721031--
10731032{-# INLINE sconcat #-}
10741033sconcat :: (Monad m, Semigroup a) => a -> Fold m a a
10751034sconcat = foldl' (<>)
10761035
1077−-- | Fold an input stream consisting of monoidal elements using 'mappend'
1078−-- and 'mempty'.
1036+-- | Monoid concat. Fold an input stream consisting of monoidal elements using
1037+-- 'mappend' and 'mempty'.
10791038--
1080−-- >>> Stream.fold Fold.mconcat (fmap Data.Monoid.Sum $ Stream.enumerateFromTo 1 10)
1081−-- Sum {getSum = 55}
1039+-- Definition:
10821040--
1083−-- > mconcat = Fold.sconcat mempty
1041+-- >>> mconcat = Fold.sconcat mempty
1042+--
1043+-- >>> monoids = fmap Data.Monoid.Sum $ Stream.enumerateFromTo 1 10
1044+-- >>> Stream.fold Fold.mconcat monoids
1045+-- Sum {getSum = 55}
10841046--
10851047{-# INLINE mconcat #-}
10861048mconcat ::
@@ -1089,26 +1051,31 @@ mconcat ::
10891051mconcat = sconcat mempty
10901052
10911053-- |
1092−-- > foldMap f = Fold.lmap f Fold.mconcat
1054+-- Definition:
1055+--
1056+-- >>> foldMap f = Fold.lmap f Fold.mconcat
10931057--
10941058-- Make a fold from a pure function that folds the output of the function
10951059-- using 'mappend' and 'mempty'.
10961060--
1097−-- >>> Stream.fold (Fold.foldMap Data.Monoid.Sum) $ Stream.enumerateFromTo 1 10
1061+-- >>> sum = Fold.foldMap Data.Monoid.Sum
1062+-- >>> Stream.fold sum $ Stream.enumerateFromTo 1 10
10981063-- Sum {getSum = 55}
10991064--
11001065{-# INLINE foldMap #-}
1101−foldMap :: (Monad m, Monoid b
1102− ) => (a -> b) -> Fold m a b
1066+foldMap :: (Monad m, Monoid b) => (a -> b) -> Fold m a b
11031067foldMap f = lmap f mconcat
11041068
11051069-- |
1106−-- > foldMapM f = Fold.lmapM f Fold.mconcat
1070+-- Definition:
1071+--
1072+-- >>> foldMapM f = Fold.lmapM f Fold.mconcat
11071073--
11081074-- Make a fold from a monadic function that folds the output of the function
11091075-- using 'mappend' and 'mempty'.
11101076--
1111−-- >>> Stream.fold (Fold.foldMapM (return . Data.Monoid.Sum)) $ Stream.enumerateFromTo 1 10
1077+-- >>> sum = Fold.foldMapM (return . Data.Monoid.Sum)
1078+-- >>> Stream.fold sum $ Stream.enumerateFromTo 1 10
11121079-- Sum {getSum = 55}
11131080--
11141081{-# INLINE foldMapM #-}
@@ -1131,7 +1098,9 @@ foldMapM act = foldlM' step (pure mempty)
11311098
11321099-- | Buffers the input stream to a list in the reverse order of the input.
11331100--
1134−-- > toListRev = Fold.foldl' (flip (:)) []
1101+-- Definition:
1102+--
1103+-- >>> toListRev = Fold.foldl' (flip (:)) []
11351104--
11361105-- /Warning!/ working on large lists accumulated as buffers in memory could be
11371106-- very inefficient, consider using "Streamly.Array" instead.
@@ -1149,7 +1118,9 @@ toListRev = foldl' (flip (:)) []
11491118-- | A fold that drains the first n elements of its input, running the effects
11501119-- and discarding the results.
11511120--
1152−-- > drainN n = Fold.take n Fold.drain
1121+-- Definition:
1122+--
1123+-- >>> drainN n = Fold.take n Fold.drain
11531124--
11541125-- /Pre-release/
11551126{-# INLINE drainN #-}
@@ -1163,9 +1134,9 @@ drainN n = take n drain
11631134-- | Like 'index', except with a more general 'Integral' argument
11641135--
11651136-- /Pre-release/
1166−{-# INLINE genericIndex #-}
1167−genericIndex :: (Integral i, Monad m) => i -> Fold m a (Maybe a)
1168−genericIndex i = foldt' step (Partial 0) (const Nothing)
1137+{-# INLINE indexGeneric #-}
1138+indexGeneric :: (Integral i, Monad m) => i -> Fold m a (Maybe a)
1139+indexGeneric i = foldt' step (Partial 0) (const Nothing)
11691140
11701141 where
11711142
@@ -1174,15 +1145,18 @@ genericIndex i = foldt' step (Partial 0) (const Nothing)
11741145 then Done $ Just a
11751146 else Partial (j + 1)
11761147
1177−-- | Lookup the element at the given index.
1148+-- | Return the element at the given index.
11781149--
1179−-- See also: 'Streamly.Prelude.!!'
1150+-- Definition:
1151+--
1152+-- >>> index = Fold.indexGeneric
11801153--
11811154{-# INLINE index #-}
11821155index :: Monad m => Int -> Fold m a (Maybe a)
1183−index = genericIndex
1156+index = indexGeneric
11841157
1185−-- | Map a 'Maybe' returning function on the next element in the stream.
1158+-- | Consume a single input and transform it using the supplied 'Maybe'
1159+-- returning function.
11861160--
11871161-- /Pre-release/
11881162--
@@ -1190,16 +1164,24 @@ index = genericIndex
11901164maybe :: Monad m => (a -> Maybe b) -> Fold m a (Maybe b)
11911165maybe f = foldt' (const (Done . f)) (Partial Nothing) id
11921166
1193−-- | Test if the next element satisfies the supplied predicate.
1167+-- | Consume a single element and return it if it passes the predicate else
1168+-- return 'Nothing'.
1169+--
1170+-- Definition:
1171+--
1172+-- >>> satisfy f = Fold.maybe (\a -> if f a then Just a else Nothing)
11941173--
11951174-- /Pre-release/
11961175{-# INLINE satisfy #-}
11971176satisfy :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
1177+satisfy f = maybe (\a -> if f a then Just a else Nothing)
1178+{-
11981179satisfy f = Fold step (return $ Partial ()) (const (return Nothing))
11991180
12001181 where
12011182
12021183 step () a = return $ Done $ if f a then Just a else Nothing
1184+-}
12031185
12041186-- Naming notes:
12051187--
@@ -1228,6 +1210,12 @@ satisfy f = Fold step (return $ Partial ()) (const (return Nothing))
12281210
12291211-- | Take one element from the stream and stop.
12301212--
1213+-- Definition:
1214+--
1215+-- >>> one = Fold.maybe Just
1216+--
1217+-- This is similar to the stream 'Stream.uncons' operation.
1218+--
12311219{-# INLINE one #-}
12321220one :: Monad m => Fold m a (Maybe a)
12331221one = maybe Just
@@ -1266,7 +1254,9 @@ find p = findM (return . p)
12661254-- | In a stream of (key-value) pairs @(a, b)@, return the value @b@ of the
12671255-- first pair where the key equals the given value @a@.
12681256--
1269−-- > lookup = snd <$> Fold.find ((==) . fst)
1257+-- Definition:
1258+--
1259+-- >>> lookup x = fmap snd <$> Fold.find ((== x) . fst)
12701260--
12711261{-# INLINE lookup #-}
12721262lookup :: (Eq a, Monad m) => a -> Fold m (a,b) (Maybe b)
@@ -1292,10 +1282,8 @@ findIndex predicate = foldt' step (Partial 0) (const Nothing)
12921282 then Done $ Just i
12931283 else Partial (i + 1)
12941284
1295−-- XXX If we have a Continue contructor we won't need a Maybe type for the
1296−-- scan. The fold will have the ability to skip producing output.
1297−--
1298−-- | Returns all indices that satisfy the given predicate.
1285+-- | Returns the index of the latest element if the element satisfies the given
1286+-- predicate.
12991287--
13001288{-# INLINE findIndices #-}
13011289findIndices :: Monad m => (a -> Bool) -> Fold m a (Maybe Int)
@@ -1310,8 +1298,10 @@ findIndices predicate =
13101298 then Right (either id id i + 1)
13111299 else Left (either id id i + 1)
13121300
1313−-- | Find all the indices where the value of the element in the stream is equal
1314−-- to the given value.
1301+-- | Returns the index of the latest element if the element matches the given
1302+-- value.
1303+--
1304+-- Definition:
13151305--
13161306-- >>> elemIndices a = Fold.findIndices (== a)
13171307--
@@ -1321,31 +1311,46 @@ elemIndices a = findIndices (== a)
13211311
13221312-- | Returns the first index where a given value is found in the stream.
13231313--
1324−-- > elemIndex a = Fold.findIndex (== a)
1314+-- Definition:
1315+--
1316+-- >>> elemIndex a = Fold.findIndex (== a)
13251317--
13261318{-# INLINE elemIndex #-}
13271319elemIndex :: (Eq a, Monad m) => a -> Fold m a (Maybe Int)
1328−elemIndex a = findIndex (a ==)
1320+elemIndex a = findIndex (== a)
13291321
13301322------------------------------------------------------------------------------
13311323-- To Boolean
13321324------------------------------------------------------------------------------
13331325
1334−-- | Return 'True' if the input stream is empty.
1326+-- Similar to 'eof' parser, but the fold consumes and discards an input element
1327+-- when not at eof. XXX Remove or Rename to "eof"?
1328+
1329+-- | Consume one element, return 'True' if successful else return 'False'. In
1330+-- other words, test if the input is empty or not.
13351331--
1336−-- > null = fmap isJust Fold.one
1332+-- WARNING! It consumes one element if the stream is not empty. If that is not
1333+-- what you want please use the eof parser instead.
1334+--
1335+-- Definition:
1336+--
1337+-- >>> null = fmap isJust Fold.one
13371338--
13381339{-# INLINE null #-}
13391340null :: Monad m => Fold m a Bool
13401341null = foldt' (\() _ -> Done False) (Partial ()) (const True)
13411342
1342−-- | Returns 'True' if any of the elements of a stream satisfies a predicate.
1343+-- | Returns 'True' if any element of the input satisfies the predicate.
1344+--
1345+-- Definition:
1346+--
1347+-- >>> any p = Fold.lmap p Fold.or
1348+--
1349+-- Example:
13431350--
13441351-- >>> Stream.fold (Fold.any (== 0)) $ Stream.fromList [1,0,1]
13451352-- True
13461353--
1347−-- > any p = Fold.lmap p Fold.or
1348−--
13491354{-# INLINE any #-}
13501355any :: Monad m => (a -> Bool) -> Fold m a Bool
13511356any predicate = foldt' step initial id
@@ -1361,19 +1366,25 @@ any predicate = foldt' step initial id
13611366
13621367-- | Return 'True' if the given element is present in the stream.
13631368--
1364−-- > elem a = Fold.any (== a)
1369+-- Definition:
1370+--
1371+-- >>> elem a = Fold.any (== a)
13651372--
13661373{-# INLINE elem #-}
13671374elem :: (Eq a, Monad m) => a -> Fold m a Bool
1368−elem a = any (a ==)
1375+elem a = any (== a)
13691376
1370−-- | Returns 'True' if all elements of a stream satisfy a predicate.
1377+-- | Returns 'True' if all elements of the input satisfy the predicate.
1378+--
1379+-- Definition:
1380+--
1381+-- >>> all p = Fold.lmap p Fold.and
1382+--
1383+-- Example:
13711384--
13721385-- >>> Stream.fold (Fold.all (== 0)) $ Stream.fromList [1,0,1]
13731386-- False
13741387--
1375−-- > all p = Fold.lmap p Fold.and
1376−--
13771388{-# INLINE all #-}
13781389all :: Monad m => (a -> Bool) -> Fold m a Bool
13791390all predicate = foldt' step initial id
@@ -1389,15 +1400,19 @@ all predicate = foldt' step initial id
13891400
13901401-- | Returns 'True' if the given element is not present in the stream.
13911402--
1392−-- > notElem a = Fold.all (/= a)
1403+-- Definition:
1404+--
1405+-- >>> notElem a = Fold.all (/= a)
13931406--
13941407{-# INLINE notElem #-}
13951408notElem :: (Eq a, Monad m) => a -> Fold m a Bool
1396−notElem a = all (a /=)
1409+notElem a = all (/= a)
13971410
13981411-- | Returns 'True' if all elements are 'True', 'False' otherwise
13991412--
1400−-- > and = Fold.all (== True)
1413+-- Definition:
1414+--
1415+-- >>> and = Fold.all (== True)
14011416--
14021417{-# INLINE and #-}
14031418and :: Monad m => Fold m Bool Bool
@@ -1405,7 +1420,9 @@ and = all (== True)
14051420
14061421-- | Returns 'True' if any element is 'True', 'False' otherwise
14071422--
1408−-- > or = Fold.any (== True)
1423+-- Definition:
1424+--
1425+-- >>> or = Fold.any (== True)
14091426--
14101427{-# INLINE or #-}
14111428or :: Monad m => Fold m Bool Bool
@@ -1531,6 +1548,9 @@ droppingWhileM p = Fold step initial (return . toMaybe)
15311548 else Just' a
15321549 step _ a = return $ Partial $ Just' a
15331550
1551+-- |
1552+-- >>> droppingWhile p = Fold.droppingWhileM (return . p)
1553+--
15341554{-# INLINE droppingWhile #-}
15351555droppingWhile :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
15361556droppingWhile p = droppingWhileM (return . p)
@@ -1539,20 +1559,20 @@ droppingWhile p = droppingWhileM (return . p)
15391559-- S.splitOn in favor of the fold.
15401560--
15411561-- XXX Use Fold.many instead once it is fixed.
1542−--
1562+-- > Stream.splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)
1563+
15431564-- | Like 'takeEndBy' but drops the element on which the predicate succeeds.
15441565--
1545−-- >>> Stream.fold (Fold.takeEndBy_ (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1566+-- Example:
1567+--
1568+-- >>> input = Stream.fromList "hello\nthere\n"
1569+-- >>> line = Fold.takeEndBy_ (== '\n') Fold.toList
1570+-- >>> Stream.fold line input
15461571-- "hello"
15471572--
1548−-- >>> Stream.fold Fold.toList $ Stream.foldMany (Fold.takeEndBy_ (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1573+-- >>> Stream.fold Fold.toList $ Stream.foldMany line input
15491574-- ["hello","there"]
15501575--
1551−-- > Stream.splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)
1552−--
1553−-- See 'Streamly.Prelude.splitOnSuffix' for more details on splitting a
1554−-- stream using 'takeEndBy_'.
1555−--
15561576{-# INLINE takeEndBy_ #-}
15571577takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
15581578-- takeEndBy_ predicate = scanMaybe (takingEndBy_ predicate)
@@ -1566,20 +1586,22 @@ takeEndBy_ predicate (Fold fstep finitial fextract) =
15661586 then fstep s a
15671587 else Done <$> fextract s
15681588
1589+-- Note:
1590+-- > Stream.splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)
1591+
15691592-- | Take the input, stop when the predicate succeeds taking the succeeding
15701593-- element as well.
15711594--
1572−-- >>> Stream.fold (Fold.takeEndBy (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1595+-- Example:
1596+--
1597+-- >>> input = Stream.fromList "hello\nthere\n"
1598+-- >>> line = Fold.takeEndBy (== '\n') Fold.toList
1599+-- >>> Stream.fold line input
15731600-- "hello\n"
15741601--
1575−-- >>> Stream.fold Fold.toList $ Stream.foldMany (Fold.takeEndBy (== '\n') Fold.toList) $ Stream.fromList "hello\nthere\n"
1602+-- >>> Stream.fold Fold.toList $ Stream.foldMany line input
15761603-- ["hello\n","there\n"]
15771604--
1578−-- > Stream.splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)
1579−--
1580−-- See 'Streamly.Prelude.splitWithSuffix' for more details on splitting a
1581−-- stream using 'takeEndBy'.
1582−--
15831605{-# INLINE takeEndBy #-}
15841606takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
15851607-- takeEndBy predicate = scanMaybe (takingEndBy predicate)
@@ -1924,10 +1946,16 @@ takeEndBySeq_ patArr (Fold fstep finitial fextract) =
19241946-- ---stream m a---| |---m (b,c)
19251947-- |-------Fold m a c--------|
19261948-- @
1927−-- >>> Stream.fold (Fold.tee Fold.sum Fold.length) (Stream.enumerateFromTo 1.0 100.0)
1928−-- (5050.0,100)
19291949--
1930−-- > tee = teeWith (,)
1950+-- Definition:
1951+--
1952+-- >>> tee = Fold.teeWith (,)
1953+--
1954+-- Example:
1955+--
1956+-- >>> t = Fold.tee Fold.sum Fold.length
1957+-- >>> Stream.fold t (Stream.enumerateFromTo 1.0 100.0)
1958+-- (5050.0,100)
19311959--
19321960{-# INLINE tee #-}
19331961tee :: Monad m => Fold m a b -> Fold m a c -> Fold m a (b,c)
@@ -1992,16 +2020,15 @@ partitionByMUsing t f fld1 fld2 =
19922020-- |-------Fold c y--------|
19932021-- @
19942022--
1995−-- Send input to either fold randomly:
2023+-- Example, send input to either fold randomly:
19962024--
1997−-- @
1998−-- > import System.Random (randomIO)
1999−-- > randomly a = randomIO >>= \\x -> return $ if x then Left a else Right a
2000−-- > Stream.fold (Fold.partitionByM randomly Fold.length Fold.length) (Stream.enumerateFromTo 1 100)
2001−-- (59,41)
2002−-- @
2025+-- >>> import System.Random (randomIO)
2026+-- >>> randomly a = randomIO >>= \x -> return $ if x then Left a else Right a
2027+-- >>> f = Fold.partitionByM randomly Fold.length Fold.length
2028+-- >>> Stream.fold f (Stream.enumerateFromTo 1 100)
2029+-- ...
20032030--
2004−-- Send input to the two folds in a proportion of 2:1:
2031+-- Example, send input to the two folds in a proportion of 2:1:
20052032--
20062033-- >>> :{
20072034-- proportionately m n = do
@@ -2014,8 +2041,9 @@ partitionByMUsing t f fld1 fld2 =
20142041--
20152042-- >>> :{
20162043-- main = do
2017−-- f <- proportionately 2 1
2018−-- r <- Stream.fold (Fold.partitionByM f Fold.length Fold.length) (Stream.enumerateFromTo (1 :: Int) 100)
2044+-- g <- proportionately 2 1
2045+-- let f = Fold.partitionByM g Fold.length Fold.length
2046+-- r <- Stream.fold f (Stream.enumerateFromTo (1 :: Int) 100)
20192047-- print r
20202048-- :}
20212049--
@@ -2040,7 +2068,6 @@ partitionByM = partitionByMUsing teeWith
20402068
20412069-- | Similar to 'partitionByM' but terminates when the first fold terminates.
20422070--
2043−--
20442071{-# INLINE partitionByFstM #-}
20452072partitionByFstM :: Monad m
20462073 => (a -> m (Either b c)) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -2048,7 +2075,6 @@ partitionByFstM = partitionByMUsing teeWithFst
20482075
20492076-- | Similar to 'partitionByM' but terminates when any fold terminates.
20502077--
2051−--
20522078{-# INLINE partitionByMinM #-}
20532079partitionByMinM :: Monad m =>
20542080 (a -> m (Either b c)) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -2057,9 +2083,10 @@ partitionByMinM = partitionByMUsing teeWithMin
20572083-- Note: we could use (a -> Bool) instead of (a -> Either b c), but the latter
20582084-- makes the signature clearer as to which case belongs to which fold.
20592085-- XXX need to check the performance in both cases.
2086+
20602087-- | Same as 'partitionByM' but with a pure partition function.
20612088--
2062−-- Count even and odd numbers in a stream:
2089+-- Example, count even and odd numbers in a stream:
20632090--
20642091-- >>> :{
20652092-- let f = Fold.partitionBy (\n -> if even n then Left n else Right n)
@@ -2079,7 +2106,9 @@ partitionBy f = partitionByM (return . f)
20792106-- and routes the 'Left' values to the first fold and 'Right' values to the
20802107-- second fold.
20812108--
2082−-- > partition = partitionBy id
2109+-- Definition:
2110+--
2111+-- >>> partition = Fold.partitionBy id
20832112--
20842113{-# INLINE partition #-}
20852114partition :: Monad m
@@ -2094,412 +2123,6 @@ partition = partitionBy id
20942123-- partitionN fs = Fold step begin done
20952124-}
20962125
2097−------------------------------------------------------------------------------
2098−-- demux: in a key value stream fold each key sub-stream with a different fold
2099−------------------------------------------------------------------------------
2100−
2101−-- TODO Demultiplex an input element into a number of typed variants. We want
2102−-- to statically restrict the target values within a set of predefined types,
2103−-- an enumeration of a GADT.
2104−--
2105−-- This is the consumer side dual of the producer side 'mux' operation (XXX to
2106−-- be implemented).
2107−--
2108−-- XXX If we use Refold in it, it can perhaps fuse/be more efficient. For
2109−-- example we can store just the result rather than storing the whole fold in
2110−-- the Map.
2111−--
2112−-- Note: There are separate functions to determine Key and Fold from the input
2113−-- because key is to be determined on each input whereas fold is to be
2114−-- determined only once for a key.
2115−
2116−-- | In a key value stream, fold values corresponding to each key with a key
2117−-- specific fold. The fold returns the fold result as the second component of
2118−-- the output tuple whenever a fold terminates. The first component of the
2119−-- tuple is a container of in-progress folds. If a fold terminates, another
2120−-- instance of the fold is started upon receiving an input with that key.
2121−--
2122−-- This can be used to scan a stream and collect the results from the scan
2123−-- output.
2124−--
2125−-- /Pre-release/
2126−--
2127−{-# INLINE demuxScanWith #-}
2128−demuxScanWith :: (Monad m, IsMap f, Traversable f) =>
2129− (a -> Key f)
2130− -> (a -> m (Fold m a b))
2131− -> Fold m a (m (f b), Maybe (Key f, b))
2132−demuxScanWith getKey getFold = fmap extract $ foldlM' step initial
2133−
2134− where
2135−
2136− initial = return $ Tuple' IsMap.mapEmpty Nothing
2137−
2138− {-# INLINE runFold #-}
2139− runFold kv (Fold step1 initial1 extract1) (k, a) = do
2140− res <- initial1
2141− case res of
2142− Partial s -> do
2143− res1 <- step1 s a
2144− return
2145− $ case res1 of
2146− Partial _ ->
2147− let fld = Fold step1 (return res1) extract1
2148− in Tuple' (IsMap.mapInsert k fld kv) Nothing
2149− Done b -> Tuple' (IsMap.mapDelete k kv) (Just (k, b))
2150− Done b -> return $ Tuple' kv (Just (k, b))
2151−
2152− step (Tuple' kv _) a = do
2153− let k = getKey a
2154− case IsMap.mapLookup k kv of
2155− Nothing -> do
2156− fld <- getFold a
2157− runFold kv fld (k, a)
2158− Just f -> runFold kv f (k, a)
2159−
2160− extract (Tuple' kv x) = (Prelude.mapM f kv, x)
2161−
2162− where
2163−
2164− f (Fold _ i e) = do
2165− r <- i
2166− case r of
2167− Partial s -> e s
2168− Done b -> return b
2169−
2170−-- | This is specialized version of 'demuxScanWith' that uses mutable cells as
2171−-- fold accumulators for better performance.
2172−--
2173−-- >>> demuxScanMutWith = Fold.demuxScanWith
2174−--
2175−{-# INLINE demuxScanMutWith #-}
2176−demuxScanMutWith :: (MonadIO m, IsMap f, Traversable f) =>
2177− (a -> Key f)
2178− -> (a -> m (Fold m a b))
2179− -> Fold m a (m (f b), Maybe (Key f, b))
2180−demuxScanMutWith getKey getFold = fmap extract $ foldlM' step initial
2181−
2182− where
2183−
2184− initial = return $ Tuple' IsMap.mapEmpty Nothing
2185−
2186− {-# INLINE initFold #-}
2187− initFold kv (Fold step1 initial1 extract1) (k, a) = do
2188− res <- initial1
2189− case res of
2190− Partial s -> do
2191− res1 <- step1 s a
2192− case res1 of
2193− Partial _ -> do
2194− let fld = Fold step1 (return res1) extract1
2195− ref <- liftIO $ newIORef fld
2196− return $ Tuple' (IsMap.mapInsert k ref kv) Nothing
2197− Done b -> return $ Tuple' kv (Just (k, b))
2198− Done b -> return $ Tuple' kv (Just (k, b))
2199−
2200− {-# INLINE runFold #-}
2201− runFold kv ref (Fold step1 initial1 extract1) (k, a) = do
2202− res <- initial1
2203− case res of
2204− Partial s -> do
2205− res1 <- step1 s a
2206− case res1 of
2207− Partial _ -> do
2208− let fld = Fold step1 (return res1) extract1
2209− liftIO $ writeIORef ref fld
2210− return $ Tuple' kv Nothing
2211− Done b ->
2212− let kv1 = IsMap.mapDelete k kv
2213− in return $ Tuple' kv1 (Just (k, b))
2214− Done _ -> error "demuxScanMutWith: unreachable"
2215−
2216− step (Tuple' kv _) a = do
2217− let k = getKey a
2218− case IsMap.mapLookup k kv of
2219− Nothing -> do
2220− f <- getFold a
2221− initFold kv f (k, a)
2222− Just ref -> do
2223− f <- liftIO $ readIORef ref
2224− runFold kv ref f (k, a)
2225−
2226− extract (Tuple' kv x) = (Prelude.mapM f kv, x)
2227−
2228− where
2229−
2230− f ref = do
2231− (Fold _ i e) <- liftIO $ readIORef ref
2232− r <- i
2233− case r of
2234− Partial s -> e s
2235− Done b -> return b
2236−
2237−-- | This collects all the results of 'demuxScanWith' in a container.
2238−--
2239−{-# INLINE demuxWith #-}
2240−demuxWith :: (Monad m, IsMap f, Traversable f) =>
2241− (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)
2242−demuxWith getKey getFold =
2243− let
2244− classifier = demuxScanWith getKey getFold
2245− getMap Nothing = pure IsMap.mapEmpty
2246− getMap (Just action) = action
2247− aggregator =
2248− teeWith IsMap.mapUnion
2249− (rmapM getMap $ lmap fst last)
2250− (lmap snd $ catMaybes toMap)
2251− in postscan classifier aggregator
2252−
2253−-- | Same as 'demuxWith' but uses 'demuxScanMutWith' for better performance.
2254−--
2255−-- >>> demuxMutWith = Fold.demuxWith
2256−--
2257−{-# INLINE demuxMutWith #-}
2258−demuxMutWith :: (MonadIO m, IsMap f, Traversable f) =>
2259− (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)
2260−demuxMutWith getKey getFold =
2261− let
2262− classifier = demuxScanMutWith getKey getFold
2263− getMap Nothing = pure IsMap.mapEmpty
2264− getMap (Just action) = action
2265− aggregator =
2266− teeWith IsMap.mapUnion
2267− (rmapM getMap $ lmap fst last)
2268− (lmap snd $ catMaybes toMap)
2269− in postscan classifier aggregator
2270−
2271−-- | Fold a stream of key value pairs using a function that maps keys to folds.
2272−--
2273−-- >>> import Data.Map (Map)
2274−-- >>> :{
2275−-- let f "SUM" = return Fold.sum
2276−-- f _ = return Fold.product
2277−-- input = Stream.fromList [("SUM",1),("PRODUCT",2),("SUM",3),("PRODUCT",4)]
2278−-- in Stream.fold (Fold.demux f) input :: IO (Map String Int)
2279−-- :}
2280−-- fromList [("PRODUCT",8),("SUM",4)]
2281−--
2282−-- >>> demux f = Fold.demuxWith fst (Fold.lmap snd . f)
2283−--
2284−-- /Pre-release/
2285−{-# INLINE demux #-}
2286−demux :: (Monad m, IsMap f, Traversable f) =>
2287− (Key f -> m (Fold m a b)) -> Fold m (Key f, a) (f b)
2288−demux f = demuxWith fst (\(k, _) -> fmap (lmap snd) (f k))
2289−
2290−------------------------------------------------------------------------------
2291−-- Classify: Like demux but uses the same fold for all keys.
2292−------------------------------------------------------------------------------
2293−
2294−-- XXX Change these to make the behavior similar to demux* variants. We can
2295−-- implement this using classifyScanManyWith. Maintain a set of done folds in
2296−-- the underlying monad, and when initial is called look it up, if the fold is
2297−-- done then initial would set a flag in the state to ignore the input or
2298−-- return an error.
2299−
2300−-- | Folds the values for each key using the supplied fold. When scanning, as
2301−-- soon as the fold is complete, its result is available in the second
2302−-- component of the tuple. The first component of the tuple is a snapshot of
2303−-- the in-progress folds.
2304−--
2305−-- Once the fold for a key is done, any future values of the key are ignored.
2306−--
2307−-- >>> classifyScanWith f fld = Fold.demuxScanWith f (const fld)
2308−--
2309−{-# INLINE classifyScanWith #-}
2310−classifyScanWith :: (Monad m, IsMap f, Traversable f, Ord (Key f)) =>
2311− -- Note: we need to return the Map itself to display the in-progress values
2312− -- e.g. to implement top. We could possibly create a separate abstraction
2313− -- for that use case. We return an action because we want it to be lazy so
2314− -- that the downstream consumers can choose to process or discard it.
2315− (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))
2316−classifyScanWith f (Fold step1 initial1 extract1) =
2317− fmap extract $ foldlM' step initial
2318−
2319− where
2320−
2321− initial = return $ Tuple3' IsMap.mapEmpty Set.empty Nothing
2322−
2323− {-# INLINE initFold #-}
2324− initFold kv set k a = do
2325− x <- initial1
2326− case x of
2327− Partial s -> do
2328− r <- step1 s a
2329− return
2330− $ case r of
2331− Partial s1 ->
2332− Tuple3' (IsMap.mapInsert k s1 kv) set Nothing
2333− Done b ->
2334− Tuple3' kv set (Just (k, b))
2335− Done b -> return (Tuple3' kv (Set.insert k set) (Just (k, b)))
2336−
2337− step (Tuple3' kv set _) a = do
2338− let k = f a
2339− case IsMap.mapLookup k kv of
2340− Nothing -> do
2341− if Set.member k set
2342− then return (Tuple3' kv set Nothing)
2343− else initFold kv set k a
2344− Just s -> do
2345− r <- step1 s a
2346− return
2347− $ case r of
2348− Partial s1 ->
2349− Tuple3' (IsMap.mapInsert k s1 kv) set Nothing
2350− Done b ->
2351− let kv1 = IsMap.mapDelete k kv
2352− in Tuple3' kv1 (Set.insert k set) (Just (k, b))
2353−
2354− extract (Tuple3' kv _ x) = (Prelude.mapM extract1 kv, x)
2355−
2356−-- XXX we can use a Prim IORef if we can constrain the state "s" to be Prim
2357−--
2358−-- The code is almost the same as classifyScanWith except the IORef operations.
2359−--
2360−-- | Same as classifyScanWith except that it uses mutable IORef cells in the
2361−-- Map providing better performance. Be aware that if this is used as a scan,
2362−-- the values in the intermediate Maps would be mutable.
2363−--
2364−-- >>> classifyScanMutWith = classifyScanWith
2365−-- >>> classifyScanMutWith f fld = demuxScanMutWith f (const fld)
2366−--
2367−{-# INLINE classifyScanMutWith #-}
2368−classifyScanMutWith :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) =>
2369− (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))
2370−classifyScanMutWith f (Fold step1 initial1 extract1) =
2371− fmap extract $ foldlM' step initial
2372−
2373− where
2374−
2375− initial = return $ Tuple3' IsMap.mapEmpty Set.empty Nothing
2376−
2377− {-# INLINE initFold #-}
2378− initFold kv set k a = do
2379− x <- initial1
2380− case x of
2381− Partial s -> do
2382− r <- step1 s a
2383− case r of
2384− Partial s1 -> do
2385− ref <- liftIO $ newIORef s1
2386− return $ Tuple3' (IsMap.mapInsert k ref kv) set Nothing
2387− Done b ->
2388− return $ Tuple3' kv set (Just (k, b))
2389− Done b -> return (Tuple3' kv (Set.insert k set) (Just (k, b)))
2390−
2391− step (Tuple3' kv set _) a = do
2392− let k = f a
2393− case IsMap.mapLookup k kv of
2394− Nothing -> do
2395− if Set.member k set
2396− then return (Tuple3' kv set Nothing)
2397− else initFold kv set k a
2398− Just ref -> do
2399− s <- liftIO $ readIORef ref
2400− r <- step1 s a
2401− case r of
2402− Partial s1 -> do
2403− liftIO $ writeIORef ref s1
2404− return $ Tuple3' kv set Nothing
2405− Done b ->
2406− let kv1 = IsMap.mapDelete k kv
2407− in return
2408− $ Tuple3' kv1 (Set.insert k set) (Just (k, b))
2409−
2410− extract (Tuple3' kv _ x) =
2411− (Prelude.mapM (\ref -> liftIO (readIORef ref) >>= extract1) kv, x)
2412−
2413−-- | Fold a key value stream to a key-value container. If the same key appears
2414−-- multiple times, only the last value is retained.
2415−{-# INLINE toMap #-}
2416−toMap :: (Monad m, IsMap f) => Fold m (Key f, a) (f a)
2417−toMap = foldl' (\kv (k, v) -> IsMap.mapInsert k v kv) IsMap.mapEmpty
2418−
2419−-- | Split the input stream based on a key field and fold each split using the
2420−-- given fold. Useful for map/reduce, bucketizing the input in different bins
2421−-- or for generating histograms.
2422−--
2423−-- >>> import Data.Map.Strict (Map)
2424−-- >>> :{
2425−-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]
2426−-- classify = Fold.classifyWith fst (Fold.lmap snd Fold.toList)
2427−-- in Stream.fold classify input :: IO (Map String [Double])
2428−-- :}
2429−-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
2430−--
2431−-- Once the classifier fold terminates for a particular key any further inputs
2432−-- in that bucket are ignored.
2433−--
2434−-- Space used is proportional to the number of keys seen till now and
2435−-- monotonically increases because it stores whether a key has been seen or
2436−-- not.
2437−--
2438−-- /Stops: never/
2439−--
2440−-- /Pre-release/
2441−--
2442−{-# INLINE classifyWith #-}
2443−classifyWith :: (Monad m, IsMap f, Traversable f, Ord (Key f)) =>
2444− (a -> Key f) -> Fold m a b -> Fold m a (f b)
2445−classifyWith f fld =
2446− let
2447− classifier = classifyScanWith f fld
2448− getMap Nothing = pure IsMap.mapEmpty
2449− getMap (Just action) = action
2450− aggregator =
2451− teeWith IsMap.mapUnion
2452− (rmapM getMap $ lmap fst last)
2453− (lmap snd $ catMaybes toMap)
2454− in postscan classifier aggregator
2455−
2456−-- | Same as 'classifyWith' but maybe faster because it uses mutable cells as
2457−-- fold accumulators in the Map.
2458−--
2459−-- >>> classifyMutWith = Fold.classifyWith
2460−--
2461−{-# INLINE classifyMutWith #-}
2462−classifyMutWith :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) =>
2463− (a -> Key f) -> Fold m a b -> Fold m a (f b)
2464−classifyMutWith f fld =
2465− let
2466− classifier = classifyScanMutWith f fld
2467− getMap Nothing = pure IsMap.mapEmpty
2468− getMap (Just action) = action
2469− aggregator =
2470− teeWith IsMap.mapUnion
2471− (rmapM getMap $ lmap fst last)
2472− (lmap snd $ catMaybes toMap)
2473− in postscan classifier aggregator
2474−
2475−-- | Given an input stream of key value pairs and a fold for values, fold all
2476−-- the values belonging to each key. Useful for map/reduce, bucketizing the
2477−-- input in different bins or for generating histograms.
2478−--
2479−-- >>> :{
2480−-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]
2481−-- in Stream.fold (Fold.classify Fold.toList) input
2482−-- :}
2483−-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
2484−--
2485−-- Same as:
2486−--
2487−-- >>> classify = Fold.classifyWith fst . Fold.lmap snd
2488−--
2489−-- /Pre-release/
2490−{-# INLINE classify #-}
2491−classify :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)
2492−classify = classifyWith fst . lmap snd
2493−
2494−-- | Determine the frequency of each element in the stream.
2495−--
2496−-- You can just collect the keys of the resulting map to get the unique
2497−-- elements in the stream.
2498−--
2499−{-# INLINE frequency #-}
2500−frequency :: (Monad m, Ord a) => Fold m a (Map a Int)
2501−frequency = classifyWith id length
2502−
25032126------------------------------------------------------------------------------
25042127-- Unzipping
25052128------------------------------------------------------------------------------
@@ -2522,7 +2145,9 @@ unzipWithMUsing t f fld1 fld2 =
25222145
25232146-- | Like 'unzipWith' but with a monadic splitter function.
25242147--
2525−-- @unzipWithM k f1 f2 = lmapM k (unzip f1 f2)@
2148+-- Definition:
2149+--
2150+-- >>> unzipWithM k f1 f2 = Fold.lmapM k (Fold.unzip f1 f2)
25262151--
25272152-- /Pre-release/
25282153{-# INLINE unzipWithM #-}
@@ -2532,7 +2157,6 @@ unzipWithM = unzipWithMUsing teeWith
25322157
25332158-- | Similar to 'unzipWithM' but terminates when the first fold terminates.
25342159--
2535−--
25362160{-# INLINE unzipWithFstM #-}
25372161unzipWithFstM :: Monad m =>
25382162 (a -> m (b, c)) -> Fold m b x -> Fold m c y -> Fold m a (x, y)
@@ -2540,7 +2164,6 @@ unzipWithFstM = unzipWithMUsing teeWithFst
25402164
25412165-- | Similar to 'unzipWithM' but terminates when any fold terminates.
25422166--
2543−--
25442167{-# INLINE unzipWithMinM #-}
25452168unzipWithMinM :: Monad m =>
25462169 (a -> m (b,c)) -> Fold m b x -> Fold m c y -> Fold m a (x,y)
@@ -2549,7 +2172,10 @@ unzipWithMinM = unzipWithMUsing teeWithMin
25492172-- | Split elements in the input stream into two parts using a pure splitter
25502173-- function, direct each part to a different fold and zip the results.
25512174--
2552−-- @unzipWith f fld1 fld2 = Fold.lmap f (Fold.unzip fld1 fld2)@
2175+-- Definitions:
2176+--
2177+-- >>> unzipWith f = Fold.unzipWithM (return . f)
2178+-- >>> unzipWith f fld1 fld2 = Fold.lmap f (Fold.unzip fld1 fld2)
25532179--
25542180-- This fold terminates when both the input folds terminate.
25552181--
@@ -2570,7 +2196,9 @@ unzipWith f = unzipWithM (return . f)
25702196--
25712197-- @
25722198--
2573−-- > unzip = Fold.unzipWith id
2199+-- Definition:
2200+--
2201+-- >>> unzip = Fold.unzipWith id
25742202--
25752203-- This is the consumer side dual of the producer side 'zip' operation.
25762204--
@@ -2588,22 +2216,27 @@ unzip = unzipWith id
25882216-- handled in parsers using the Continue facility. See zipWithM in the Parser
25892217-- module.
25902218--
2219+-- cmpBy, eqBy, isPrefixOf, isSubsequenceOf etc can be implemented using
2220+-- zipStream.
2221+
25912222-- | Zip a stream with the input of a fold using the supplied function.
25922223--
25932224-- /Unimplemented/
25942225--
2595−{-# INLINE zipWithM #-}
2596−zipWithM :: -- Monad m =>
2597− (a -> b -> m c) -> t m a -> Fold m c x -> Fold m b x
2598−zipWithM = undefined
2226+{-# INLINE zipStreamWithM #-}
2227+zipStreamWithM :: -- Monad m =>
2228+ (a -> b -> m c) -> Stream m a -> Fold m c x -> Fold m b x
2229+zipStreamWithM = undefined
25992230
26002231-- | Zip a stream with the input of a fold.
26012232--
2233+-- >>> zip = Fold.zipStreamWithM (curry return)
2234+--
26022235-- /Unimplemented/
26032236--
2604−{-# INLINE zip #-}
2605−zip :: Monad m => t m a -> Fold m (a, b) x -> Fold m b x
2606−zip = zipWithM (curry return)
2237+{-# INLINE zipStream #-}
2238+zipStream :: Monad m => Stream m a -> Fold m (a, b) x -> Fold m b x
2239+zipStream = zipStreamWithM (curry return)
26072240
26082241-- | Pair each element of a fold input with its index, starting from index 0.
26092242--
@@ -2618,16 +2251,24 @@ indexingWith i f = fmap toMaybe $ foldl' step initial
26182251 step Nothing' a = Just' (i, a)
26192252 step (Just' (n, _)) a = Just' (f n, a)
26202253
2254+-- |
2255+-- >>> indexing = Fold.indexingWith 0 (+ 1)
2256+--
26212257{-# INLINE indexing #-}
26222258indexing :: Monad m => Fold m a (Maybe (Int, a))
26232259indexing = indexingWith 0 (+ 1)
26242260
2261+-- |
2262+-- >>> indexingRev n = Fold.indexingWith n (subtract 1)
2263+--
26252264{-# INLINE indexingRev #-}
26262265indexingRev :: Monad m => Int -> Fold m a (Maybe (Int, a))
26272266indexingRev n = indexingWith n (subtract 1)
26282267
26292268-- | Pair each element of a fold input with its index, starting from index 0.
26302269--
2270+-- >>> indexed = Fold.scanMaybe Fold.indexing
2271+--
26312272{-# INLINE indexed #-}
26322273indexed :: Monad m => Fold m (Int, a) b -> Fold m a b
26332274indexed = scanMaybe indexing
@@ -2636,8 +2277,9 @@ indexed = scanMaybe indexing
26362277-- additional state input @(s, a) -> b@. Convenient to filter with an
26372278-- addiitonal index or time input.
26382279--
2280+-- >>> filterWithIndex = Fold.with Fold.indexed Fold.filter
2281+--
26392282-- @
2640−-- filterWithIndex = with indexed filter
26412283-- filterWithAbsTime = with timestamped filter
26422284-- filterWithRelTime = with timeIndexed filter
26432285-- @
@@ -2756,12 +2398,14 @@ unfoldMany (Unfold ustep inject) (Fold fstep initial extract) =
27562398 {-# INLINE_LATE consume #-}
27572399 consume s a = inject a >>= produce s
27582400
2759−{-# INLINE topBy #-}
2760−topBy :: (MonadIO m, Unbox a) =>
2401+-- | Get the bottom most @n@ elements using the supplied comparison function.
2402+--
2403+{-# INLINE bottomBy #-}
2404+bottomBy :: (MonadIO m, Unbox a) =>
27612405 (a -> a -> Ordering)
27622406 -> Int
27632407 -> Fold m a (MA.Array a)
2764−topBy cmp n = Fold step initial extract
2408+bottomBy cmp n = Fold step initial extract
27652409
27662410 where
27672411
@@ -2790,6 +2434,10 @@ topBy cmp n = Fold step initial extract
27902434
27912435-- | Fold the input stream to top n elements.
27922436--
2437+-- Definition:
2438+--
2439+-- >>> top = Fold.bottomBy $ flip compare
2440+--
27932441-- >>> stream = Stream.fromList [2::Int,7,9,3,1,5,6,11,17]
27942442-- >>> Stream.fold (Fold.top 3) stream >>= MA.toList
27952443-- [17,11,9]
@@ -2797,10 +2445,14 @@ topBy cmp n = Fold step initial extract
27972445-- /Pre-release/
27982446{-# INLINE top #-}
27992447top :: (MonadIO m, Unbox a, Ord a) => Int -> Fold m a (MA.Array a)
2800−top = topBy $ flip compare
2448+top = bottomBy $ flip compare
28012449
28022450-- | Fold the input stream to bottom n elements.
28032451--
2452+-- Definition:
2453+--
2454+-- >>> bottom = Fold.bottomBy compare
2455+--
28042456-- >>> stream = Stream.fromList [2::Int,7,9,3,1,5,6,11,17]
28052457-- >>> Stream.fold (Fold.bottom 3) stream >>= MA.toList
28062458-- [1,2,3]
@@ -2808,4 +2460,4 @@ top = topBy $ flip compare
28082460-- /Pre-release/
28092461{-# INLINE bottom #-}
28102462bottom :: (MonadIO m, Unbox a, Ord a) => Int -> Fold m a (MA.Array a)
2811−bottom = topBy compare
2463+bottom = bottomBy compare