@@ -12,27 +12,38 @@ module Streamly.Internal.Data.Fold.Container
12 -- * Imports
12 -- * Imports
15− countDistinct
18+ , countDistinct
16 , countDistinctInt
19 , countDistinctInt
22 -- ** Demultiplexing
26 -- ** Demultiplexing
23 -- | Direct values in the input stream to different folds using an n-ary
27 -- | Direct values in the input stream to different folds using an n-ary
24 -- fold selector. 'demux' is a generalization of 'classify' (and
28 -- fold selector. 'demux' is a generalization of 'classify' (and
25 -- 'partition') where each key of the classifier can use a different fold.
29 -- 'partition') where each key of the classifier can use a different fold.
27− , demuxWith
40+ , demuxGenericIO
28− , demuxScanWith
41+ , demuxIO
32 -- TODO: These can be implemented using the above operations
43 -- TODO: These can be implemented using the above operations
33− -- , demuxWithSel -- Stop when the fold for the specified key stops
44+ -- , demuxSel -- Stop when the fold for the specified key stops
34− -- , demuxWithMin -- Stop when any of the folds stop
45+ -- , demuxMin -- Stop when any of the folds stop
35− -- , demuxWithAll -- Stop when all the folds stop (run once)
46+ -- , demuxAll -- Stop when all the folds stop (run once)
37 -- ** Classifying
48 -- ** Classifying
38 -- | In an input stream of key value pairs fold values for different keys
49 -- | In an input stream of key value pairs fold values for different keys
@@ -47,12 +58,19 @@ module Streamly.Internal.Data.Fold.Container
47 -- and their meanings then trie Map would be better if we also want to
58 -- and their meanings then trie Map would be better if we also want to
48 -- display them in sorted order.
59 -- display them in sorted order.
50 , classify
69 , classify
51− , classifyWith
70+ , classifyGenericIO
52− , classifyMutWith
71+ , classifyIO
53− , classifyScanWith
72+ -- , toContainerSel
54− -- , classifyWithSel
73+ -- , toContainerMin
@@ -62,29 +80,48 @@ where
62import Control.Monad.IO.Class (MonadIO(..))
80import Control.Monad.IO.Class (MonadIO(..))
63import Data.IORef (newIORef, readIORef, writeIORef)
81import Data.IORef (newIORef, readIORef, writeIORef)
64import Data.Map.Strict (Map)
82import Data.Map.Strict (Map)
83+import Data.IntSet (IntSet)
65import Streamly.Internal.Data.IsMap (IsMap(..))
85import Streamly.Internal.Data.IsMap (IsMap(..))
66import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
86import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..))
68import qualified Data.IntSet as IntSet
88import qualified Data.IntSet as IntSet
69import qualified Data.Set as Set
89import qualified Data.Set as Set
70−import qualified Prelude
71import qualified Streamly.Internal.Data.IsMap as IsMap
90import qualified Streamly.Internal.Data.IsMap as IsMap
73−import Prelude hiding
92+import Prelude hiding (length)
74− ( filter, foldl1, drop, dropWhile, take, takeWhile, zipWith
75− , foldl, foldr, map, mapM_, sequence, all, any, sum, product, elem
76− , notElem, maximum, minimum, head, last, tail, length, null
77− , reverse, iterate, init, and, or, lookup, (!!)
78− , scanl, scanl1, replicate, concatMap, mconcat, foldMap, unzip
79− , span, splitAt, break, mapM, zip, maybe)
80import Streamly.Internal.Data.Fold
93import Streamly.Internal.Data.Fold
84-- >>> :set -XFlexibleContexts
97-- >>> :set -XFlexibleContexts
98+-- >>> import qualified Data.Map as Map
99+-- >>> import qualified Data.Set as Set
100+-- >>> import qualified Data.IntSet as IntSet
85-- >>> import qualified Streamly.Data.Fold as Fold
101-- >>> import qualified Streamly.Data.Fold as Fold
86-- >>> import qualified Streamly.Data.Stream as Stream
102-- >>> import qualified Streamly.Data.Stream as Stream
87−-- >>> import qualified Streamly.Internal.Data.Fold.Extra as Fold
103+-- >>> import qualified Streamly.Internal.Data.Fold.Container as Fold
105+-- | Fold the input to a set.
109+-- >>> toSet = Fold.foldl' (flip Set.insert) Set.empty
112+toSet :: (Monad m, Ord a) => Fold m a (Set a)
113+toSet = foldl' (flip Set.insert) Set.empty
115+-- | Fold the input to an int set. For integer inputs this performs better than
120+-- >>> toIntSet = Fold.foldl' (flip IntSet.insert) IntSet.empty
122+{-# INLINE toIntSet #-}
123+toIntSet :: Monad m => Fold m Int IntSet
124+toIntSet = foldl' (flip IntSet.insert) IntSet.empty
89-- XXX Name as nubOrd? Or write a nubGeneric
126-- XXX Name as nubOrd? Or write a nubGeneric
@@ -113,10 +150,6 @@ nub = fmap (\(Tuple' _ x) -> x) $ foldl' step initial
114-- | Like 'nub' but specialized to a stream of 'Int', for better performance.
151-- | Like 'nub' but specialized to a stream of 'Int', for better performance.
118−-- >>> nubInt = Fold.nub
120-- /Pre-release/
153-- /Pre-release/
121{-# INLINE nubInt #-}
154{-# INLINE nubInt #-}
122nubInt :: Monad m => Fold m Int (Maybe Int)
155nubInt :: Monad m => Fold m Int (Maybe Int)
@@ -139,6 +172,7 @@ nubInt = fmap (\(Tuple' _ x) -> x) $ foldl' step initial
140-- Definition:
173-- Definition:
175+-- >>> countDistinct = fmap Set.size Fold.toSet
142-- >>> countDistinct = Fold.postscan Fold.nub $ Fold.catMaybes $ Fold.length
176-- >>> countDistinct = Fold.postscan Fold.nub $ Fold.catMaybes $ Fold.length
144-- The memory used is proportional to the number of distinct elements in the
178-- The memory used is proportional to the number of distinct elements in the
@@ -151,7 +185,8 @@ nubInt = fmap (\(Tuple' _ x) -> x) $ foldl' step initial
152{-# INLINE countDistinct #-}
186{-# INLINE countDistinct #-}
153countDistinct :: (Monad m, Ord a) => Fold m a Int
187countDistinct :: (Monad m, Ord a) => Fold m a Int
154−countDistinct = postscan nub $ catMaybes length
188+-- countDistinct = postscan nub $ catMaybes length
189+countDistinct = fmap Set.size toSet
156countDistinct = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
191countDistinct = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
@@ -173,12 +208,14 @@ countDistinct = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
174-- Definition:
209-- Definition:
211+-- >>> countDistinctInt = fmap IntSet.size Fold.toIntSet
176-- >>> countDistinctInt = Fold.postscan Fold.nubInt $ Fold.catMaybes $ Fold.length
212-- >>> countDistinctInt = Fold.postscan Fold.nubInt $ Fold.catMaybes $ Fold.length
178-- /Pre-release/
214-- /Pre-release/
179{-# INLINE countDistinctInt #-}
215{-# INLINE countDistinctInt #-}
180countDistinctInt :: Monad m => Fold m Int Int
216countDistinctInt :: Monad m => Fold m Int Int
181−countDistinctInt = postscan nubInt $ catMaybes length
217+-- countDistinctInt = postscan nubInt $ catMaybes length
218+countDistinctInt = fmap IntSet.size toIntSet
183countDistinctInt = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
220countDistinctInt = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
@@ -214,23 +251,12 @@ countDistinctInt = fmap (\(Tuple' _ n) -> n) $ foldl' step initial
214-- because key is to be determined on each input whereas fold is to be
251-- because key is to be determined on each input whereas fold is to be
215-- determined only once for a key.
252-- determined only once for a key.
217−-- | In a key value stream, fold values corresponding to each key with a key
254+{-# INLINE demuxGeneric #-}
218−-- specific fold. The fold returns the fold result as the second component of
219−-- the output tuple whenever a fold terminates. The first component of the
220−-- tuple is a container of in-progress folds. If a fold terminates, another
221−-- instance of the fold is started upon receiving an input with that key.
223−-- This can be used to scan a stream and collect the results from the scan
228−{-# INLINE demuxScanWith #-}
229−demuxScanWith :: (Monad m, IsMap f, Traversable f) =>
255+demuxGeneric :: (Monad m, IsMap f, Traversable f) =>
230 (a -> Key f)
256 (a -> Key f)
231 -> (a -> m (Fold m a b))
257 -> (a -> m (Fold m a b))
232 -> Fold m a (m (f b), Maybe (Key f, b))
258 -> Fold m a (m (f b), Maybe (Key f, b))
233−demuxScanWith getKey getFold = fmap extract $ foldlM' step initial
259+demuxGeneric getKey getFold = fmap extract $ foldlM' step initial
@@ -268,19 +294,30 @@ demuxScanWith getKey getFold = fmap extract $ foldlM' step initial
268 Partial s -> e s
294 Partial s -> e s
269 Done b -> return b
295 Done b -> return b
271−-- | This is specialized version of 'demuxScanWith' that uses mutable cells as
297+-- | In a key value stream, fold values corresponding to each key with a key
272−-- fold accumulators for better performance.
298+-- specific fold. The fold returns the fold result as the second component of
299+-- the output tuple whenever a fold terminates. The first component of the
300+-- tuple is a Map of in-progress folds. If a fold terminates, another
301+-- instance of the fold is started upon receiving an input with that key.
274−-- Definition:
303+-- This can be used to scan a stream and collect the results from the scan
276−-- >>> demuxScanMutWith = Fold.demuxScanWith
306+-- /Pre-release/
278−{-# INLINE demuxScanMutWith #-}
308+{-# INLINE demux #-}
309+demux :: (Monad m, Ord k) =>
311+ -> (a -> m (Fold m a b))
312+ -> Fold m a (m (Map k b), Maybe (k, b))
315+{-# INLINE demuxGenericIO #-}
279−demuxScanMutWith :: (MonadIO m, IsMap f, Traversable f) =>
316+demuxGenericIO :: (MonadIO m, IsMap f, Traversable f) =>
280 (a -> Key f)
317 (a -> Key f)
281 -> (a -> m (Fold m a b))
318 -> (a -> m (Fold m a b))
282 -> Fold m a (m (f b), Maybe (Key f, b))
319 -> Fold m a (m (f b), Maybe (Key f, b))
283−demuxScanMutWith getKey getFold = fmap extract $ foldlM' step initial
320+demuxGenericIO getKey getFold = fmap extract $ foldlM' step initial
@@ -314,7 +351,7 @@ demuxScanMutWith getKey getFold = fmap extract $ foldlM' step initial
314 Done b ->
351 Done b ->
315 let kv1 = IsMap.mapDelete k kv
352 let kv1 = IsMap.mapDelete k kv
316 in return $ Tuple' kv1 (Just (k, b))
353 in return $ Tuple' kv1 (Just (k, b))
317− Done _ -> error "demuxScanMutWith: unreachable"
354+ Done _ -> error "demuxGenericIO: unreachable"
319 step (Tuple' kv _) a = do
356 step (Tuple' kv _) a = do
320 let k = getKey a
357 let k = getKey a
@@ -337,47 +374,68 @@ demuxScanMutWith getKey getFold = fmap extract $ foldlM' step initial
337 Partial s -> e s
374 Partial s -> e s
338 Done b -> return b
375 Done b -> return b
340−-- | This collects all the results of 'demuxScanWith' in a container.
377+-- | This is specialized version of 'demux' that uses mutable IO cells as
378+-- fold accumulators for better performance.
342−{-# INLINE demuxWith #-}
380+{-# INLINE demuxIO #-}
381+demuxIO :: (MonadIO m, Ord k) =>
383+ -> (a -> m (Fold m a b))
384+ -> Fold m a (m (Map k b), Maybe (k, b))
385+demuxIO = demuxGenericIO
387+{-# INLINE demuxToContainer #-}
343−demuxWith :: (Monad m, IsMap f, Traversable f) =>
388+demuxToContainer :: (Monad m, IsMap f, Traversable f) =>
344 (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)
389 (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)
345−demuxWith getKey getFold =
390+demuxToContainer getKey getFold =
347− classifier = demuxScanWith getKey getFold
392+ classifier = demuxGeneric getKey getFold
348 getMap Nothing = pure IsMap.mapEmpty
393 getMap Nothing = pure IsMap.mapEmpty
349 getMap (Just action) = action
394 getMap (Just action) = action
350 aggregator =
395 aggregator =
351 teeWith IsMap.mapUnion
396 teeWith IsMap.mapUnion
352 (rmapM getMap $ lmap fst latest)
397 (rmapM getMap $ lmap fst latest)
353− (lmap snd $ catMaybes toMap)
398+ (lmap snd $ catMaybes kvToMapOverwriteGeneric)
354 in postscan classifier aggregator
399 in postscan classifier aggregator
356−-- | Same as 'demuxWith' but uses 'demuxScanMutWith' for better performance.
401+-- | This collects all the results of 'demux' in a Map.
358−-- Definition:
403+{-# INLINE demuxToMap #-}
359−--
404+demuxToMap :: (Monad m, Ord k) =>
360−-- >>> demuxMutWith = Fold.demuxWith
405+ (a -> k) -> (a -> m (Fold m a b)) -> Fold m a (Map k b)
361−--
406+demuxToMap = demuxToContainer
362−{-# INLINE demuxMutWith #-}
407+
408+{-# INLINE demuxToContainerIO #-}
363−demuxMutWith :: (MonadIO m, IsMap f, Traversable f) =>
409+demuxToContainerIO :: (MonadIO m, IsMap f, Traversable f) =>
364 (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)
410 (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)
365−demuxMutWith getKey getFold =
411+demuxToContainerIO getKey getFold =
367− classifier = demuxScanMutWith getKey getFold
413+ classifier = demuxGenericIO getKey getFold
368 getMap Nothing = pure IsMap.mapEmpty
414 getMap Nothing = pure IsMap.mapEmpty
369 getMap (Just action) = action
415 getMap (Just action) = action
370 aggregator =
416 aggregator =
371 teeWith IsMap.mapUnion
417 teeWith IsMap.mapUnion
372 (rmapM getMap $ lmap fst latest)
418 (rmapM getMap $ lmap fst latest)
373− (lmap snd $ catMaybes toMap)
419+ (lmap snd $ catMaybes kvToMapOverwriteGeneric)
374 in postscan classifier aggregator
420 in postscan classifier aggregator
422+-- | Same as 'demuxToMap' but uses 'demuxIO' for better performance.
424+{-# INLINE demuxToMapIO #-}
425+demuxToMapIO :: (MonadIO m, Ord k) =>
426+ (a -> k) -> (a -> m (Fold m a b)) -> Fold m a (Map k b)
427+demuxToMapIO = demuxToContainerIO
429+{-# INLINE demuxKvToContainer #-}
430+demuxKvToContainer :: (Monad m, IsMap f, Traversable f) =>
431+ (Key f -> m (Fold m a b)) -> Fold m (Key f, a) (f b)
432+demuxKvToContainer f = demuxToContainer fst (\(k, _) -> fmap (lmap snd) (f k))
376-- | Fold a stream of key value pairs using a function that maps keys to folds.
434-- | Fold a stream of key value pairs using a function that maps keys to folds.
378-- Definition:
436-- Definition:
380−-- >>> demux f = Fold.demuxWith fst (Fold.lmap snd . f)
438+-- >>> demuxKvToMap f = Fold.demuxToContainer fst (Fold.lmap snd . f)
382-- Example:
440-- Example:
@@ -386,15 +444,15 @@ demuxMutWith getKey getFold =
386-- let f "SUM" = return Fold.sum
444-- let f "SUM" = return Fold.sum
387-- f _ = return Fold.product
445-- f _ = return Fold.product
388-- input = Stream.fromList [("SUM",1),("PRODUCT",2),("SUM",3),("PRODUCT",4)]
446-- input = Stream.fromList [("SUM",1),("PRODUCT",2),("SUM",3),("PRODUCT",4)]
389−-- in Stream.fold (Fold.demux f) input :: IO (Map String Int)
447+-- in Stream.fold (Fold.demuxKvToMap f) input :: IO (Map String Int)
391-- fromList [("PRODUCT",8),("SUM",4)]
449-- fromList [("PRODUCT",8),("SUM",4)]
393-- /Pre-release/
451-- /Pre-release/
394−{-# INLINE demux #-}
452+{-# INLINE demuxKvToMap #-}
395−demux :: (Monad m, IsMap f, Traversable f) =>
453+demuxKvToMap :: (Monad m, Ord k) =>
396− (Key f -> m (Fold m a b)) -> Fold m (Key f, a) (f b)
454+ (k -> m (Fold m a b)) -> Fold m (k, a) (Map k b)
397−demux f = demuxWith fst (\(k, _) -> fmap (lmap snd) (f k))
455+demuxKvToMap = demuxKvToContainer
399------------------------------------------------------------------------------
457------------------------------------------------------------------------------
400-- Classify: Like demux but uses the same fold for all keys.
458-- Classify: Like demux but uses the same fold for all keys.
@@ -406,25 +464,14 @@ demux f = demuxWith fst (\(k, _) -> fmap (lmap snd) (f k))
406-- done then initial would set a flag in the state to ignore the input or
464-- done then initial would set a flag in the state to ignore the input or
407-- return an error.
465-- return an error.
409−-- | Folds the values for each key using the supplied fold. When scanning, as
467+{-# INLINE classifyGeneric #-}
410−-- soon as the fold is complete, its result is available in the second
411−-- component of the tuple. The first component of the tuple is a snapshot of
412−-- the in-progress folds.
414−-- Once the fold for a key is done, any future values of the key are ignored.
418−-- >>> classifyScanWith f fld = Fold.demuxScanWith f (const fld)
420−{-# INLINE classifyScanWith #-}
421−classifyScanWith :: (Monad m, IsMap f, Traversable f, Ord (Key f)) =>
468+classifyGeneric :: (Monad m, IsMap f, Traversable f, Ord (Key f)) =>
422 -- Note: we need to return the Map itself to display the in-progress values
469 -- Note: we need to return the Map itself to display the in-progress values
423 -- e.g. to implement top. We could possibly create a separate abstraction
470 -- e.g. to implement top. We could possibly create a separate abstraction
424 -- for that use case. We return an action because we want it to be lazy so
471 -- for that use case. We return an action because we want it to be lazy so
425 -- that the downstream consumers can choose to process or discard it.
472 -- that the downstream consumers can choose to process or discard it.
426 (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))
473 (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))
427−classifyScanWith f (Fold step1 initial1 extract1) =
474+classifyGeneric f (Fold step1 initial1 extract1) =
428 fmap extract $ foldlM' step initial
475 fmap extract $ foldlM' step initial
@@ -464,23 +511,30 @@ classifyScanWith f (Fold step1 initial1 extract1) =
465 extract (Tuple3' kv _ x) = (Prelude.mapM extract1 kv, x)
512 extract (Tuple3' kv _ x) = (Prelude.mapM extract1 kv, x)
467−-- XXX we can use a Prim IORef if we can constrain the state "s" to be Prim
514+-- | Folds the values for each key using the supplied fold. When scanning, as
515+-- soon as the fold is complete, its result is available in the second
516+-- component of the tuple. The first component of the tuple is a snapshot of
517+-- the in-progress folds.
469−-- The code is almost the same as classifyScanWith except the IORef operations.
519+-- Once the fold for a key is done, any future values of the key are ignored.
471−-- | Same as classifyScanWith except that it uses mutable IORef cells in the
472−-- Map providing better performance. Be aware that if this is used as a scan,
473−-- the values in the intermediate Maps would be mutable.
475−-- Definitions:
521+-- Definition:
523+-- >>> classify f fld = Fold.demux f (const fld)
477−-- >>> classifyScanMutWith = Fold.classifyScanWith
525+{-# INLINE classify #-}
478−-- >>> classifyScanMutWith f fld = Fold.demuxScanMutWith f (const fld)
526+classify :: (Monad m, Ord k) =>
527+ (a -> k) -> Fold m a b -> Fold m a (m (Map k b), Maybe (k, b))
528+classify = classifyGeneric
530+-- XXX we can use a Prim IORef if we can constrain the state "s" to be Prim
480−{-# INLINE classifyScanMutWith #-}
532+-- The code is almost the same as classifyGeneric except the IORef operations.
534+{-# INLINE classifyGenericIO #-}
481−classifyScanMutWith :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) =>
535+classifyGenericIO :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) =>
482 (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))
536 (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))
483−classifyScanMutWith f (Fold step1 initial1 extract1) =
537+classifyGenericIO f (Fold step1 initial1 extract1) =
484 fmap extract $ foldlM' step initial
538 fmap extract $ foldlM' step initial
@@ -523,11 +577,39 @@ classifyScanMutWith f (Fold step1 initial1 extract1) =
523 extract (Tuple3' kv _ x) =
577 extract (Tuple3' kv _ x) =
524 (Prelude.mapM (\ref -> liftIO (readIORef ref) >>= extract1) kv, x)
578 (Prelude.mapM (\ref -> liftIO (readIORef ref) >>= extract1) kv, x)
580+-- | Same as classify except that it uses mutable IORef cells in the
581+-- Map providing better performance. Be aware that if this is used as a scan,
582+-- the values in the intermediate Maps would be mutable.
586+-- >>> classifyIO f fld = Fold.demuxIO f (const fld)
588+{-# INLINE classifyIO #-}
589+classifyIO :: (MonadIO m, Ord k) =>
590+ (a -> k) -> Fold m a b -> Fold m a (m (Map k b), Maybe (k, b))
591+classifyIO = classifyGenericIO
526−-- | Fold a key value stream to a key-value container. If the same key appears
593+-- | Fold a key value stream to a key-value Map. If the same key appears
527-- multiple times, only the last value is retained.
594-- multiple times, only the last value is retained.
528−{-# INLINE toMap #-}
595+{-# INLINE kvToMapOverwriteGeneric #-}
529−toMap :: (Monad m, IsMap f) => Fold m (Key f, a) (f a)
596+kvToMapOverwriteGeneric :: (Monad m, IsMap f) => Fold m (Key f, a) (f a)
597+kvToMapOverwriteGeneric =
530−toMap = foldl' (\kv (k, v) -> IsMap.mapInsert k v kv) IsMap.mapEmpty
598+ foldl' (\kv (k, v) -> IsMap.mapInsert k v kv) IsMap.mapEmpty
600+{-# INLINE toContainer #-}
601+toContainer :: (Monad m, IsMap f, Traversable f, Ord (Key f)) =>
602+ (a -> Key f) -> Fold m a b -> Fold m a (f b)
605+ classifier = classifyGeneric f fld
606+ getMap Nothing = pure IsMap.mapEmpty
607+ getMap (Just action) = action
609+ teeWith IsMap.mapUnion
610+ (rmapM getMap $ lmap fst latest)
611+ (lmap snd $ catMaybes kvToMapOverwriteGeneric)
612+ in postscan classifier aggregator
532-- | Split the input stream based on a key field and fold each split using the
614-- | Split the input stream based on a key field and fold each split using the
533-- given fold. Useful for map/reduce, bucketizing the input in different bins
615-- given fold. Useful for map/reduce, bucketizing the input in different bins
@@ -538,7 +620,7 @@ toMap = foldl' (\kv (k, v) -> IsMap.mapInsert k v kv) IsMap.mapEmpty
538-- >>> import Data.Map.Strict (Map)
620-- >>> import Data.Map.Strict (Map)
540-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]
622-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]
541−-- classify = Fold.classifyWith fst (Fold.lmap snd Fold.toList)
623+-- classify = Fold.toMap fst (Fold.lmap snd Fold.toList)
542-- in Stream.fold classify input :: IO (Map String [Double])
624-- in Stream.fold classify input :: IO (Map String [Double])
544-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
626-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
@@ -550,44 +632,42 @@ toMap = foldl' (\kv (k, v) -> IsMap.mapInsert k v kv) IsMap.mapEmpty
550-- monotonically increases because it stores whether a key has been seen or
632-- monotonically increases because it stores whether a key has been seen or
635+-- See 'demuxToMap' for a more powerful version where you can use a different
636+-- fold for each key. A simpler version of 'toMap' retaining only the last
637+-- value for a key can be written as:
639+-- >>> toMap = Fold.foldl' (\kv (k, v) -> Map.insert k v kv) Map.empty
553-- /Stops: never/
641-- /Stops: never/
555-- /Pre-release/
643-- /Pre-release/
557−{-# INLINE classifyWith #-}
645+{-# INLINE toMap #-}
646+toMap :: (Monad m, Ord k) =>
647+ (a -> k) -> Fold m a b -> Fold m a (Map k b)
650+{-# INLINE toContainerIO #-}
558−classifyWith :: (Monad m, IsMap f, Traversable f, Ord (Key f)) =>
651+toContainerIO :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) =>
559 (a -> Key f) -> Fold m a b -> Fold m a (f b)
652 (a -> Key f) -> Fold m a b -> Fold m a (f b)
560−classifyWith f fld =
653+toContainerIO f fld =
562− classifier = classifyScanWith f fld
655+ classifier = classifyGenericIO f fld
563 getMap Nothing = pure IsMap.mapEmpty
656 getMap Nothing = pure IsMap.mapEmpty
564 getMap (Just action) = action
657 getMap (Just action) = action
565 aggregator =
658 aggregator =
566 teeWith IsMap.mapUnion
659 teeWith IsMap.mapUnion
567 (rmapM getMap $ lmap fst latest)
660 (rmapM getMap $ lmap fst latest)
568− (lmap snd $ catMaybes toMap)
661+ (lmap snd $ catMaybes kvToMapOverwriteGeneric)
569 in postscan classifier aggregator
662 in postscan classifier aggregator
571−-- | Same as 'classifyWith' but maybe faster because it uses mutable cells as
664+-- | Same as 'toMap' but maybe faster because it uses mutable cells as
572-- fold accumulators in the Map.
665-- fold accumulators in the Map.
574−-- Definition:
667+{-# INLINE toMapIO #-}
575−--
668+toMapIO :: (MonadIO m, Ord k) =>
576−-- >>> classifyMutWith = Fold.classifyWith
578−{-# INLINE classifyMutWith #-}
579−classifyMutWith :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) =>
580− (a -> Key f) -> Fold m a b -> Fold m a (f b)
669+ (a -> k) -> Fold m a b -> Fold m a (Map k b)
581−classifyMutWith f fld =
670+toMapIO = toContainerIO
583− classifier = classifyScanMutWith f fld
584− getMap Nothing = pure IsMap.mapEmpty
585− getMap (Just action) = action
587− teeWith IsMap.mapUnion
588− (rmapM getMap $ lmap fst latest)
589− (lmap snd $ catMaybes toMap)
590− in postscan classifier aggregator
592-- | Given an input stream of key value pairs and a fold for values, fold all
672-- | Given an input stream of key value pairs and a fold for values, fold all
593-- the values belonging to each key. Useful for map/reduce, bucketizing the
673-- the values belonging to each key. Useful for map/reduce, bucketizing the
@@ -595,20 +675,20 @@ classifyMutWith f fld =
596-- Definition:
676-- Definition:
598−-- >>> classify = Fold.classifyWith fst . Fold.lmap snd
678+-- >>> kvToMap = Fold.toMap fst . Fold.lmap snd
600-- Example:
680-- Example:
603-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]
683-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]
604−-- in Stream.fold (Fold.classify Fold.toList) input
684+-- in Stream.fold (Fold.kvToMap Fold.toList) input
606-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
686-- fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]
608-- /Pre-release/
688-- /Pre-release/
609−{-# INLINE classify #-}
689+{-# INLINE kvToMap #-}
610−classify :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)
690+kvToMap :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)
611−classify = classifyWith fst . lmap snd
691+kvToMap = toMap fst . lmap snd
613-- | Determine the frequency of each element in the stream.
693-- | Determine the frequency of each element in the stream.
@@ -617,8 +697,8 @@ classify = classifyWith fst . lmap snd
618-- Definition:
698-- Definition:
620−-- >>> frequency = Fold.classifyWith id Fold.length
700+-- >>> frequency = Fold.toMap id Fold.length
622{-# INLINE frequency #-}
702{-# INLINE frequency #-}
623frequency :: (Monad m, Ord a) => Fold m a (Map a Int)
703frequency :: (Monad m, Ord a) => Fold m a (Map a Int)
624−frequency = classifyWith id length
704+frequency = toMap id length