@@ -36,6 +36,11 @@ module Streamly.Internal.Data.Fold
3636 , foldr
3737 , foldrM
3838
39+ -- * Mappers
40+ -- | Monadic functions useful with mapM/lmapM on folds or streams.
41+ , tracing
42+ , trace
43+
3944 -- * Folds
4045 -- ** Identity
4146 , fromPure
@@ -51,7 +56,6 @@ module Streamly.Internal.Data.Fold
5156 -- *** Reducers
5257 , drain
5358 , drainBy
54− , last
5559 , the
5660 , length
5761 , genericLength
@@ -63,7 +67,6 @@ module Streamly.Internal.Data.Fold
6367 , rollingHashWithSalt
6468 , rollingHashFirstN
6569 -- , rollingHashLastN
66− , rollingMapM
6770
6871 -- *** Saturating Reducers
6972 -- | 'product' terminates if it becomes 0. Other folds can theoretically
@@ -86,15 +89,41 @@ module Streamly.Internal.Data.Fold
8689 , toStream
8790 , toStreamRev
8891 , toMap
92+ , topBy
8993 , top
9094 , bottom
9195
96+ -- *** Scanners
97+ -- | Stateful transformation of the elements. Useful in combination with
98+ -- the 'scanMaybe' combinator. For scanners the result of the fold is
99+ -- usually a transformation of the current element rather than an
100+ -- aggregation of all elements till now.
101+ , last -- XXX prev
102+ -- , nthLast -- using Ring array
103+ , indexingWith
104+ , indexing
105+ , indexingRev
106+ , rollingMapM
107+
108+ -- *** Filters
109+ -- | Useful in combination with the 'scanMaybe' combinator.
110+ , filtering
111+ , deleteBy
112+ , nub
113+ , nubInt
114+ , uniqBy
115+ , uniq
116+ , repeated
117+ , findIndices
118+ , elemIndices
119+
92120 -- ** Terminating Folds
93121 -- Element folds. Terminate after inspecting one element. All these can be
94122 -- implemented in terms of the 'maybe' fold.
95123 , head
96124 , one
97125 , null
126+ , satisfy
98127 , maybe
99128
100129 -- Sequence folds. Terminate after inspecting a sequence of elements.
@@ -115,6 +144,18 @@ module Streamly.Internal.Data.Fold
115144 , and
116145 , or
117146
147+ -- ** Trimmers
148+ -- | Useful in combination with the 'scanMaybe' combinator.
149+ , taking
150+ , dropping
151+ , takingEndByM
152+ , takingEndBy
153+ , takingEndByM_
154+ , takingEndBy_
155+ , droppingWhileM
156+ , droppingWhile
157+ , prune
158+
118159 -- * Combinators
119160 -- ** Utilities
120161 , with
@@ -131,42 +172,35 @@ module Streamly.Internal.Data.Fold
131172 , lmap
132173 --, lsequence
133174 , lmapM
175+
176+ -- ** Sliding Window
177+ , slide2
178+
179+ -- ** Scanning Input
134180 , scan
135181 , scanMany
136182 , postscan
137183 , indexed
138− , trace
139184
140185 -- ** Zipping Input
141186 , zipWithM
142187 , zip
143188
144− -- ** Filtering
189+ -- ** Filtering Input
190+ , catMaybes
191+ , mapMaybeM
192+ , mapMaybe
193+ , scanMaybe
145194 , filter
146195 , filterM
147− , foldFilter
148− , satisfy
149196 , sampleFromthen
150− -- , ldeleteBy
151− -- , luniq
152− , nub
153− , nubInt
154−
155− -- ** Mapping Filters
156− , catMaybes
157− , mapMaybe
158− -- , mapMaybeM
159197
160198 -- Either streams
161199 , lefts
162200 , rights
163201 , both
164202
165− -- ** Scanning Filters
166− , findIndices
167203 {-
168− , elemIndices
169−
170204 -- ** Insertion
171205 -- | Insertion adds more elements to the stream.
172206
@@ -304,10 +338,10 @@ import Data.Functor.Identity (Identity(..))
304338import Data.Int (Int64)
305339import Data.IORef (newIORef, readIORef, writeIORef)
306340import Data.Map.Strict (Map)
307−import Data.Maybe (isJust, fromJust)
308341import Data.Word (Word32)
309342import Foreign.Storable (peek, sizeOf)
310343import Streamly.Internal.Data.IsMap (IsMap(..))
344+import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)
311345import Streamly.Internal.Data.Pipe.Type (Pipe (..), PipeState(..))
312346import Streamly.Internal.Data.Unboxed (Unboxed)
313347import Streamly.Internal.Data.Unfold.Type (Unfold(..))
@@ -339,17 +373,18 @@ import Streamly.Internal.Data.Fold.Type
339373-- >>> :m
340374-- >>> :set -package streamly
341375-- >>> import Prelude hiding (break, map, span, splitAt)
342−-- >>> import qualified Streamly.Data.Stream as Stream
376+-- >>> import qualified Streamly.Data.Array.Unboxed as Array
343377-- >>> import qualified Streamly.Data.Fold as Fold
378+-- >>> import qualified Streamly.Data.Stream as Stream
379+-- >>> import qualified Streamly.Internal.Data.Array.Unboxed.Mut.Type as MA
344380-- >>> import qualified Streamly.Internal.Data.Fold as Fold
345381-- >>> import qualified Streamly.Internal.Data.Fold.Type as Fold
346382-- >>> import qualified Streamly.Internal.Data.Fold.Window as FoldW
347−-- >>> import qualified Streamly.Data.Array.Unboxed as Array
348383-- >>> import qualified Streamly.Internal.Data.Parser as Parser
349384-- >>> import qualified Streamly.Internal.Data.Stream.Type as Stream
385+-- >>> import qualified Streamly.Internal.Data.Unfold as Unfold
350386-- >>> import Streamly.Internal.Data.Stream.Type (Stream)
351387-- >>> import Data.IORef (newIORef, readIORef, writeIORef)
352−-- >>> import qualified Streamly.Internal.Data.Array.Unboxed.Mut.Type as MA
353388
354389------------------------------------------------------------------------------
355390-- hoist
@@ -390,10 +425,20 @@ sequence = rmapM id
390425mapM :: Monad m => (b -> m c) -> Fold m a b -> Fold m a c
391426mapM = rmapM
392427
428+-- |
429+-- >>> mapMaybeM f = Fold.lmapM f . Fold.catMaybes
430+--
431+{-# INLINE mapMaybeM #-}
432+mapMaybeM :: Monad m => (a -> m (Maybe b)) -> Fold m b r -> Fold m a r
433+mapMaybeM f = lmapM f . catMaybes
434+
393435-- | @mapMaybe f fold@ maps a 'Maybe' returning function @f@ on the input of
394436-- the fold, filters out 'Nothing' elements, and return the values extracted
395437-- from 'Just'.
396438--
439+-- >>> mapMaybe f = Fold.lmap f . Fold.catMaybes
440+-- >>> mapMaybe f = Fold.mapMaybeM (return . f)
441+--
397442-- >>> f x = if even x then Just x else Nothing
398443-- >>> fld = Fold.mapMaybe f Fold.toList
399444-- >>> Stream.fold fld (Stream.enumerateFromTo 1 10)
@@ -401,13 +446,25 @@ mapM = rmapM
401446--
402447-- @since 0.8.0
403448{-# INLINE mapMaybe #-}
404−mapMaybe :: (Monad m) => (a -> Maybe b) -> Fold m b r -> Fold m a r
405−mapMaybe f = lmap f . filter isJust . lmap fromJust
449+mapMaybe :: Monad m => (a -> Maybe b) -> Fold m b r -> Fold m a r
450+mapMaybe f = lmap f . catMaybes
406451
407452------------------------------------------------------------------------------
408453-- Transformations on fold inputs
409454------------------------------------------------------------------------------
410455
456+-- | Apply a monadic function on the input and return the input.
457+--
458+-- >>> Stream.fold (Fold.lmapM (Fold.tracing print) Fold.drain) $ (Stream.enumerateFromTo (1 :: Int) 2)
459+-- 1
460+-- 2
461+--
462+-- /Pre-release/
463+--
464+{-# INLINE tracing #-}
465+tracing :: Monad m => (a -> m b) -> (a -> m a)
466+tracing f x = void (f x) >> return x
467+
411468-- | Apply a monadic function to each element flowing through and discard the
412469-- results.
413470--
@@ -415,10 +472,12 @@ mapMaybe f = lmap f . filter isJust . lmap fromJust
415472-- 1
416473-- 2
417474--
475+-- >>> trace f = Fold.lmapM (Fold.tracing f)
476+--
418477-- /Pre-release/
419478{-# INLINE trace #-}
420479trace :: Monad m => (a -> m b) -> Fold m a r -> Fold m a r
421−trace f = lmapM (\x -> void (f x) >> return x)
480+trace f = lmapM (tracing f)
422481
423482-- rename to lpipe?
424483--
@@ -516,82 +575,110 @@ scan = scanWith False
516575scanMany :: Monad m => Fold m a b -> Fold m b c -> Fold m a c
517576scanMany = scanWith True
518577
519−-- | Postscan the input of a 'Fold' to change it in a stateful manner using
520−-- another 'Fold'.
521−-- /Pre-release/
522−{-# INLINE postscan #-}
523−postscan :: Monad m => Fold m a b -> Fold m b c -> Fold m a c
524−postscan (Fold stepL initialL extractL) (Fold stepR initialR extractR) =
525− Fold step initial extract
578+------------------------------------------------------------------------------
579+-- Filters
580+------------------------------------------------------------------------------
581+
582+-- | A filtering scan that deletes the first occurrence of the element in the
583+-- stream that satisfies the given equality predicate.
584+--
585+-- >>> input = Stream.unfold Unfold.fromList [1,3,3,5]
586+-- >>> Stream.fold Fold.toList $ Stream.scanMaybe (Fold.deleteBy (==) 3) input
587+-- [1,3,5]
588+--
589+{-# INLINE_NORMAL deleteBy #-}
590+deleteBy :: Monad m => (a -> a -> Bool) -> a -> Fold m a (Maybe a)
591+deleteBy eq x0 = fmap extract $ foldl' step (Tuple' False Nothing)
526592
527593 where
528594
529− {-# INLINE runStep #-}
530− runStep actionL sR = do
531− rL <- actionL
532− case rL of
533− Done bL -> do
534− rR <- stepR sR bL
535− case rR of
536− Partial sR1 -> Done <$> extractR sR1
537− Done bR -> return $ Done bR
538− Partial sL -> do
539− !b <- extractL sL
540− rR <- stepR sR b
541− return
542− $ case rR of
543− Partial sR1 -> Partial (sL, sR1)
544− Done bR -> Done bR
595+ step (Tuple' False _) x =
596+ if eq x x0
597+ then Tuple' True Nothing
598+ else Tuple' False (Just x)
599+ step (Tuple' True _) x = Tuple' True (Just x)
545600
546− initial = do
547− r <- initialR
548− rL <- initialL
549− case r of
550− Partial sR ->
551− case rL of
552− Done _ -> Done <$> extractR sR
553− Partial sL -> return $ Partial (sL, sR)
554− Done b -> return $ Done b
601+ extract (Tuple' _ x) = x
555602
556− step (sL, sR) x = runStep (stepL sL x) sR
603+-- | Provide a sliding window of length 2 elements.
604+--
605+-- See "Streamly.Internal.Data.Fold.Window".
606+--
607+{-# INLINE slide2 #-}
608+slide2 :: Monad m => Fold m (a, Maybe a) b -> Fold m a b
609+slide2 (Fold step1 initial1 extract1) = Fold step initial extract
557610
558− extract = extractR . snd
611+ where
559612
560−------------------------------------------------------------------------------
561−-- Filters
562−------------------------------------------------------------------------------
613+ initial =
614+ first (Tuple' Nothing) <$> initial1
615+
616+ step (Tuple' prev s) cur =
617+ first (Tuple' (Just cur)) <$> step1 s (cur, prev)
618+
619+ extract (Tuple' _ s) = extract1 s
563620
564−-- | Convert a predicate into a filtering fold.
621+-- | Drop repeated elements that are adjacent to each other using the supplied
622+-- comparison function.
565623--
566−-- >>> f = Fold.foldFilter (Fold.satisfy (> 5)) Fold.sum
567−-- >>> Stream.fold f $ Stream.fromList [1..10]
568−-- 40
624+-- To strip duplicate path separators:
625+--
626+-- >>> input = Stream.unfold Unfold.fromList "//a//b"
627+-- >>> f x y = x == '/' && y == '/'
628+-- >>> Stream.fold Fold.toList $ Stream.scanMaybe (Fold.uniqBy f) input
629+-- "/a/b"
630+--
631+-- Space: @O(1)@
632+--
633+-- See also: 'nubBy'.
569634--
570635-- /Pre-release/
571−{-# INLINE satisfy #-}
572−satisfy :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
573−satisfy f = Fold step (return $ Partial ()) (const (return Nothing))
636+--
637+{-# INLINE uniqBy #-}
638+uniqBy :: Monad m => (a -> a -> Bool) -> Fold m a (Maybe a)
639+uniqBy eq = rollingMap f
574640
575641 where
576642
577− step () a = return $ Done $ if f a then Just a else Nothing
643+ f pre curr =
644+ case pre of
645+ Nothing -> Just curr
646+ Just x -> if x `eq` curr then Nothing else Just curr
578647
579−-- | Use a 'Maybe' returning fold as a filtering scan.
648+-- | Drop repeated elements that are adjacent to each other.
580649--
581−-- >>> f = Fold.foldFilter (Fold.satisfy (> 5)) Fold.sum
582−-- >>> Stream.fold f $ Stream.fromList [1..10]
583−-- 40
650+-- >>> uniq = Fold.uniqBy (==)
584651--
585−-- The above snippet is equivalent to:
652+{-# INLINE uniq #-}
653+uniq :: (Monad m, Eq a) => Fold m a (Maybe a)
654+uniq = uniqBy (==)
655+
656+-- | Strip all leading and trailing occurrences of an element passing a
657+-- predicate and make all other consecutive occurrences uniq.
586658--
587−-- >>> f = Fold.filter (> 5) Fold.sum
588−-- >>> Stream.fold f $ Stream.fromList [1..10]
589−-- 40
659+-- >> prune p = Stream.dropWhileAround p $ Stream.uniqBy (x y -> p x && p y)
590660--
591−-- /Pre-release/
592−{-# INLINE foldFilter #-}
593−foldFilter :: Monad m => Fold m a (Maybe b) -> Fold m b c -> Fold m a c
594−foldFilter f1 f2 = many f1 (catMaybes f2)
661+-- @
662+-- > Stream.prune isSpace (Stream.fromList " hello world! ")
663+-- "hello world!"
664+--
665+-- @
666+--
667+-- Space: @O(1)@
668+--
669+-- /Unimplemented/
670+{-# INLINE prune #-}
671+prune ::
672+ -- (Monad m, Eq a) =>
673+ (a -> Bool) -> Fold m a (Maybe a)
674+prune = error "Not implemented yet!"
675+
676+-- | Emit only repeated elements, once.
677+--
678+-- /Unimplemented/
679+repeated :: -- (Monad m, Eq a) =>
680+ Fold m a (Maybe a)
681+repeated = error "Not implemented yet!"
595682
596683-- | Used as a scan. Returns 'Just' for the first occurrence of an element,
597684-- returns 'Nothing' for any other occurrences.
@@ -960,6 +1047,9 @@ rollingHash = rollingHashWithSalt defaultSalt
9601047rollingHashFirstN :: (Monad m, Enum a) => Int -> Fold m a Int64
9611048rollingHashFirstN n = take n rollingHash
9621049
1050+-- XXX Compare this with the implementation in Fold.Window, preferrably use the
1051+-- latter if performance is good.
1052+
9631053-- | Apply a function on every two successive elements of a stream. The first
9641054-- argument of the map function is the previous element and the second argument
9651055-- is the current element. When processing the very first element in the
@@ -983,6 +1073,10 @@ rollingMapM f = Fold step initial extract
9831073
9841074 extract = return . snd
9851075
1076+{-# INLINE rollingMap #-}
1077+rollingMap :: Monad m => (Maybe a -> a -> b) -> Fold m a b
1078+rollingMap f = rollingMapM (\x y -> return $ f x y)
1079+
9861080------------------------------------------------------------------------------
9871081-- Monoidal left folds
9881082------------------------------------------------------------------------------
@@ -1122,6 +1216,17 @@ index = genericIndex
11221216maybe :: Monad m => (a -> Maybe b) -> Fold m a (Maybe b)
11231217maybe f = foldt' (const (Done . f)) (Partial Nothing) id
11241218
1219+-- | Test if the next element satisfies the supplied predicate.
1220+--
1221+-- /Pre-release/
1222+{-# INLINE satisfy #-}
1223+satisfy :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
1224+satisfy f = Fold step (return $ Partial ()) (const (return Nothing))
1225+
1226+ where
1227+
1228+ step () a = return $ Done $ if f a then Just a else Nothing
1229+
11251230-- Naming notes:
11261231--
11271232-- "head" and "next" are two alternative names for the same API. head sounds
@@ -1227,6 +1332,7 @@ findIndex predicate = foldt' step (Partial 0) (const Nothing)
12271332{-# INLINE findIndices #-}
12281333findIndices :: Monad m => (a -> Bool) -> Fold m a (Maybe Int)
12291334findIndices predicate =
1335+ -- XXX implement by combining indexing and filtering scans
12301336 fmap (either (const Nothing) Just) $ foldl' step (Left (-1))
12311337
12321338 where
@@ -1236,6 +1342,15 @@ findIndices predicate =
12361342 then Right (either id id i + 1)
12371343 else Left (either id id i + 1)
12381344
1345+-- | Find all the indices where the value of the element in the stream is equal
1346+-- to the given value.
1347+--
1348+-- >>> elemIndices a = Fold.findIndices (== a)
1349+--
1350+{-# INLINE elemIndices #-}
1351+elemIndices :: (Monad m, Eq a) => a -> Fold m a (Maybe Int)
1352+elemIndices a = findIndices (== a)
1353+
12391354-- | Returns the first index where a given value is found in the stream.
12401355--
12411356-- > elemIndex a = Fold.findIndex (== a)
@@ -1393,6 +1508,73 @@ splitAt n fld = serialWith (,) (take n fld)
13931508-- Binary APIs
13941509------------------------------------------------------------------------------
13951510
1511+{-# INLINE takingEndByM #-}
1512+takingEndByM :: Monad m => (a -> m Bool) -> Fold m a (Maybe a)
1513+takingEndByM p = Fold step initial (return . toMaybe)
1514+
1515+ where
1516+
1517+ initial = return $ Partial Nothing'
1518+
1519+ step _ a = do
1520+ r <- p a
1521+ return
1522+ $ if r
1523+ then Done $ Just a
1524+ else Partial $ Just' a
1525+
1526+-- |
1527+--
1528+-- >>> takingEndBy p = Fold.takingEndByM (return . p)
1529+--
1530+{-# INLINE takingEndBy #-}
1531+takingEndBy :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
1532+takingEndBy p = takingEndByM (return . p)
1533+
1534+{-# INLINE takingEndByM_ #-}
1535+takingEndByM_ :: Monad m => (a -> m Bool) -> Fold m a (Maybe a)
1536+takingEndByM_ p = Fold step initial (return . toMaybe)
1537+
1538+ where
1539+
1540+ initial = return $ Partial Nothing'
1541+
1542+ step _ a = do
1543+ r <- p a
1544+ return
1545+ $ if r
1546+ then Done Nothing
1547+ else Partial $ Just' a
1548+
1549+-- |
1550+--
1551+-- >>> takingEndBy_ p = Fold.takingEndByM_ (return . p)
1552+--
1553+{-# INLINE takingEndBy_ #-}
1554+takingEndBy_ :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
1555+takingEndBy_ p = takingEndByM_ (return . p)
1556+
1557+{-# INLINE droppingWhileM #-}
1558+droppingWhileM :: Monad m => (a -> m Bool) -> Fold m a (Maybe a)
1559+droppingWhileM p = Fold step initial (return . toMaybe)
1560+
1561+ where
1562+
1563+ initial = return $ Partial Nothing'
1564+
1565+ step Nothing' a = do
1566+ r <- p a
1567+ return
1568+ $ Partial
1569+ $ if r
1570+ then Nothing'
1571+ else Just' a
1572+ step _ a = return $ Partial $ Just' a
1573+
1574+{-# INLINE droppingWhile #-}
1575+droppingWhile :: Monad m => (a -> Bool) -> Fold m a (Maybe a)
1576+droppingWhile p = droppingWhileM (return . p)
1577+
13961578-- Note: Keep this consistent with S.splitOn. In fact we should eliminate
13971579-- S.splitOn in favor of the fold.
13981580--
@@ -1414,6 +1596,7 @@ splitAt n fld = serialWith (,) (take n fld)
14141596-- @since 0.8.0
14151597{-# INLINE takeEndBy_ #-}
14161598takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
1599+-- takeEndBy_ predicate = scanMaybe (takingEndBy_ predicate)
14171600takeEndBy_ predicate (Fold fstep finitial fextract) =
14181601 Fold step finitial fextract
14191602
@@ -1441,6 +1624,7 @@ takeEndBy_ predicate (Fold fstep finitial fextract) =
14411624-- @since 0.8.0
14421625{-# INLINE takeEndBy #-}
14431626takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b
1627+-- takeEndBy predicate = scanMaybe (takingEndBy predicate)
14441628takeEndBy predicate (Fold fstep finitial fextract) =
14451629 Fold step finitial fextract
14461630
@@ -2469,11 +2653,30 @@ zip = zipWithM (curry return)
24692653
24702654-- | Pair each element of a fold input with its index, starting from index 0.
24712655--
2472−-- /Unimplemented/
2656+{-# INLINE indexingWith #-}
2657+indexingWith :: Monad m => Int -> (Int -> Int) -> Fold m a (Maybe (Int, a))
2658+indexingWith i f = fmap toMaybe $ foldl' step initial
2659+
2660+ where
2661+
2662+ initial = Nothing'
2663+
2664+ step Nothing' a = Just' (i, a)
2665+ step (Just' (n, _)) a = Just' (f n, a)
2666+
2667+{-# INLINE indexing #-}
2668+indexing :: Monad m => Fold m a (Maybe (Int, a))
2669+indexing = indexingWith 0 (+ 1)
2670+
2671+{-# INLINE indexingRev #-}
2672+indexingRev :: Monad m => Int -> Fold m a (Maybe (Int, a))
2673+indexingRev n = indexingWith n (subtract 1)
2674+
2675+-- | Pair each element of a fold input with its index, starting from index 0.
2676+--
24732677{-# INLINE indexed #-}
2474−indexed :: -- forall m a b. Monad m =>
2475− Fold m (Int, a) b -> Fold m a b
2476−indexed = undefined -- zip (Stream.enumerateFrom 0 :: Stream m Int)
2678+indexed :: Monad m => Fold m (Int, a) b -> Fold m a b
2679+indexed = scanMaybe indexing
24772680
24782681-- | Change the predicate function of a Fold from @a -> b@ to accept an
24792682-- additional state input @(s, a) -> b@. Convenient to filter with an
@@ -2493,10 +2696,12 @@ with ::
24932696 -> (((s, a) -> c) -> Fold m a b -> Fold m a b)
24942697with f comb g = f . comb g . lmap snd
24952698
2699+-- XXX Implement as a filter
2700+-- sampleFromthen :: Monad m => Int -> Int -> Fold m a (Maybe a)
2701+
24962702-- | @sampleFromthen offset stride@ samples the element at @offset@ index and
24972703-- then every element at strides of @stride@.
24982704--
2499−-- /Unimplemented/
25002705{-# INLINE sampleFromthen #-}
25012706sampleFromthen :: Monad m => Int -> Int -> Fold m a b -> Fold m a b
25022707sampleFromthen offset size =