on f9a81d4 help sign in
core/src/DocTestDataStream.hs−2
since splitsnapshots
unifiedsplit
@@ -6,7 +6,6 @@
6>>> import Control.Monad.IO.Class (MonadIO (liftIO))
6>>> import Control.Monad.IO.Class (MonadIO (liftIO))
7>>> import Control.Monad.Trans.Class (lift)
7>>> import Control.Monad.Trans.Class (lift)
8>>> import Control.Monad.Trans.Identity (runIdentityT)
8>>> import Control.Monad.Trans.Identity (runIdentityT)
9−>>> import Data.Char (isSpace)
10>>> import Data.Either (fromLeft, fromRight, isLeft, isRight, either)
9>>> import Data.Either (fromLeft, fromRight, isLeft, isRight, either)
11>>> import Data.Maybe (fromJust, isJust)
10>>> import Data.Maybe (fromJust, isJust)
12>>> import Data.Function (fix, (&))
11>>> import Data.Function (fix, (&))
@@ -33,7 +32,6 @@ For APIs that have not been released yet.
33>>> import qualified Streamly.Internal.Data.Fold as Fold
32>>> import qualified Streamly.Internal.Data.Fold as Fold
34>>> import qualified Streamly.Internal.Data.Parser as Parser
33>>> import qualified Streamly.Internal.Data.Parser as Parser
35>>> import qualified Streamly.Internal.Data.Stream as Stream
34>>> import qualified Streamly.Internal.Data.Stream as Stream
36−>>> import qualified Streamly.Internal.Data.StreamK as StreamK
37>>> import qualified Streamly.Internal.Data.Unfold as Unfold
35>>> import qualified Streamly.Internal.Data.Unfold as Unfold
38>>> import qualified Streamly.Internal.FileSystem.Dir as Dir
36>>> import qualified Streamly.Internal.FileSystem.Dir as Dir
39-}
37-}
core/src/Streamly/Data/Stream.hs+20−129
@@ -70,15 +70,6 @@ module Streamly.Data.Stream
70
70
71 -- ** From Values
71 -- ** From Values
72 -- | Generate a monadic stream from a seed value or values.
72 -- | Generate a monadic stream from a seed value or values.
73− --
74− -- Non-empty pure streams can be cycled using repeat and unfold:
75− --
76− -- >>> cycle = Stream.unfoldMany Unfold.fromList . Stream.repeat
77− --
78− -- Non-empty StreamK can be cycled using append:
79− --
80− -- >>> cycle xs = let ys = xs `StreamK.append` ys in ys
81− --
82 , fromPure
73 , fromPure
83 , fromEffect
74 , fromEffect
84 , repeat
75 , repeat
@@ -221,57 +212,21 @@ module Streamly.Data.Stream
221 -- operations like mapping a function over the stream.
212 -- operations like mapping a function over the stream.
222 , foldrM
213 , foldrM
223 , foldr
214 , foldr
224− -- foldr1
225
215
226 -- ** Specific Folds
216 -- ** Specific Folds
227− -- | Streams are folded using folds in "Streamly.Data.Fold". Here are some
217+ -- | Usually you can use the folds in "Streamly.Data.Fold". However, some
228− -- idioms and equivalents of Data.List APIs using folds:
218+ -- folds that may be commonly used or may have an edge in performance in
219+ -- some cases are provided here.
220+ --
221+ -- Useful idioms:
229 --
222 --
230 -- >>> foldlM' f a = Stream.fold (Fold.foldlM' f a)
223 -- >>> foldlM' f a = Stream.fold (Fold.foldlM' f a)
231− -- >>> foldl1' f = Stream.fold (Fold.foldl1' f)
224+ -- >>> foldl1 f = Stream.fold (Fold.foldl1' f)
232 -- >>> foldl' f a = Stream.fold (Fold.foldl' f a)
225 -- >>> foldl' f a = Stream.fold (Fold.foldl' f a)
233 -- >>> drain = Stream.fold Fold.drain
226 -- >>> drain = Stream.fold Fold.drain
234 -- >>> mapM_ f = Stream.fold (Fold.drainMapM f)
227 -- >>> mapM_ f = Stream.fold (Fold.drainMapM f)
235 -- >>> length = Stream.fold Fold.length
228 -- >>> length = Stream.fold Fold.length
236− -- >>> genericLength = Stream.fold Fold.lengthGeneric
237 -- >>> head = Stream.fold Fold.one
229 -- >>> head = Stream.fold Fold.one
238− -- >>> last = Stream.fold Fold.latest
239− -- >>> null = Stream.fold Fold.null
240− -- >>> and = Stream.fold Fold.and
241− -- >>> or = Stream.fold Fold.or
242− -- >>> any p = Stream.fold (Fold.any p)
243− -- >>> all p = Stream.fold (Fold.all p)
244− -- >>> sum = Stream.fold Fold.sum
245− -- >>> product = Stream.fold Fold.product
246− -- >>> maximum = Stream.fold Fold.maximum
247− -- >>> maximumBy cmp = Stream.fold (Fold.maximumBy cmp)
248− -- >>> minimum = Stream.fold Fold.minimum
249− -- >>> minimumBy cmp = Stream.fold (Fold.minimumBy cmp)
250− -- >>> elem x = Stream.fold (Fold.elem x)
251− -- >>> notElem x = Stream.fold (Fold.notElem x)
252− -- >>> lookup x = Stream.fold (Fold.lookup x)
253− -- >>> find p = Stream.fold (Fold.find p)
254− -- >>> (!?) i = Stream.fold (Fold.index i)
255− -- >>> genericIndex i = Stream.fold (Fold.indexGeneric i)
256− -- >>> elemIndex x = Stream.fold (Fold.elemIndex x)
257− -- >>> findIndex p = Stream.fold (Fold.findIndex p)
258− --
259− -- Some equivalents of Data.List APIs from the Stream module:
260− --
261− -- >>> head = fmap (fmap fst) . Stream.uncons
262− -- >>> tail = fmap (fmap snd) . Stream.uncons
263− -- >>> tail = Stream.tail -- unreleased API
264− -- >>> init = Stream.init -- unreleased API
265− --
266− -- A Stream based toList fold implementation is provided below because it
267− -- has a better performance compared to the fold.
268−
269− -- Functions in Data.List, missing here:
270− -- unsnoc = Stream.parseBreak (Parser.init Fold.toList)
271− -- genericTake
272− -- genericDrop
273− -- genericSplitAt
274− -- genericReplicate
275 , toList
230 , toList
276
231
277 -- * Mapping
232 -- * Mapping
@@ -325,15 +280,6 @@ module Streamly.Data.Stream
325 , scan
280 , scan
326 , postscan
281 , postscan
327 -- XXX postscan1 can be implemented using Monoids or Refolds.
282 -- XXX postscan1 can be implemented using Monoids or Refolds.
328− -- The following scans from Data.List are not provided.
329− -- XXX scanl
330− -- XXX scanl1
331− -- XXX scanr
332− -- XXX scanr1
333− -- XXX mapAccumL
334− -- XXX mapAccumR
335− -- XXX inits
336− -- XXX tails
337
283
338 -- ** Specific scans
284 -- ** Specific scans
339 -- Indexing can be considered as a special type of zipping where we zip a
285 -- Indexing can be considered as a special type of zipping where we zip a
@@ -342,8 +288,6 @@ module Streamly.Data.Stream
342
288
343 -- * Insertion
289 -- * Insertion
344 -- | Add elements to the stream.
290 -- | Add elements to the stream.
345− --
346− -- >>> insert = Stream.insertBy compare
347
291
348 -- Inserting elements is a special case of interleaving/merging streams.
292 -- Inserting elements is a special case of interleaving/merging streams.
349 , insertBy
293 , insertBy
@@ -379,15 +323,12 @@ module Streamly.Data.Stream
379 -- "Streamly.Internal.Data.Fold" can be used along with 'scanMaybe' to
323 -- "Streamly.Internal.Data.Fold" can be used along with 'scanMaybe' to
380 -- perform stateful filtering operations in general.
324 -- perform stateful filtering operations in general.
381 --
325 --
382− -- Idioms and equivalents of Data.List APIs:
326+ -- Useful idioms:
383 --
327 --
384 -- >>> deleteBy cmp x = Stream.scanMaybe (Fold.deleteBy cmp x)
328 -- >>> deleteBy cmp x = Stream.scanMaybe (Fold.deleteBy cmp x)
385− -- >>> deleteBy = Stream.deleteBy -- unreleased API
386− -- >>> delete = deleteBy (==)
387 -- >>> findIndices p = Stream.scanMaybe (Fold.findIndices p)
329 -- >>> findIndices p = Stream.scanMaybe (Fold.findIndices p)
388 -- >>> elemIndices a = findIndices (== a)
330 -- >>> elemIndices a = findIndices (== a)
389 -- >>> uniq = Stream.scanMaybe (Fold.uniqBy (==))
331 -- >>> uniq = Stream.scanMaybe (Fold.uniqBy (==))
390− -- >>> partition p = Stream.fold (Fold.partition Fold.toList Fold.toList) . fmap (if p then Left else Right)
391 , scanMaybe
332 , scanMaybe
392 , take
333 , take
393 , takeWhile
334 , takeWhile
@@ -395,8 +336,6 @@ module Streamly.Data.Stream
395 , drop
336 , drop
396 , dropWhile
337 , dropWhile
397 , dropWhileM
338 , dropWhileM
398− -- XXX write to an array in reverse and then read in reverse
399− -- > dropWhileEnd = reverse . dropWhile p . reverse
400
339
401 -- XXX These are available as scans in folds. We need to check the
340 -- XXX These are available as scans in folds. We need to check the
402 -- performance though. If these are common and we need convenient stream
341 -- performance though. If these are common and we need convenient stream
@@ -433,9 +372,6 @@ module Streamly.Data.Stream
433 -- >>> s = s1 `Stream.append` s2
372 -- >>> s = s1 `Stream.append` s2
434
373
435 -- ** Appending
374 -- ** Appending
436− -- | Equivalent of Data.List append:
437− --
438− -- >>> (++) = Stream.append
439 , append
375 , append
440
376
441 -- ** Interleaving
377 -- ** Interleaving
@@ -446,14 +382,8 @@ module Streamly.Data.Stream
446 , mergeByM
382 , mergeByM
447
383
448 -- ** Zipping
384 -- ** Zipping
449− -- | Idioms and equivalents of Data.List APIs:
450− --
451− -- >>> zip = Stream.zipWith (,)
452− -- >>> unzip = Stream.fold (Fold.unzip Fold.toList Fold.toList)
453 , zipWith
385 , zipWith
454 , zipWithM
386 , zipWithM
455− -- XXX zipWith3,4,5,6,7
456− -- XXX unzip3,4,5,6,7
457 -- , ZipStream (..)
387 -- , ZipStream (..)
458
388
459 -- ** Cross Product
389 -- ** Cross Product
@@ -486,89 +416,48 @@ module Streamly.Data.Stream
486 -- One dimension loops are just a special case of nested loops. For
416 -- One dimension loops are just a special case of nested loops. For
487 -- example, 'concatMap' can degenerate to a simple map operation:
417 -- example, 'concatMap' can degenerate to a simple map operation:
488 --
418 --
489− -- >>> map f m = Stream.concatMap (\x -> Stream.fromPure (f x)) m
419+ -- > map f m = S.concatMap (\x -> S.fromPure (f x)) m
490 --
420 --
491 -- Similarly, 'concatMap' can perform filtering by mapping an element to a
421 -- Similarly, 'concatMap' can perform filtering by mapping an element to a
492 -- 'nil' stream:
422 -- 'nil' stream:
493 --
423 --
494− -- >>> filter p m = Stream.concatMap (\x -> if p x then Stream.fromPure x else Stream.nil) m
424+ -- > filter p m = S.concatMap (\x -> if p x then S.fromPure x else S.nil) m
495− --
496− -- Idioms and equivalents of Data.List APIs:
497 --
425 --
498− -- >>> concat = Stream.concatMap id
499− -- >>> unlines = Stream.interposeSuffix '\n'
500− -- >>> unlines = Stream.intercalateSuffix Unfold.fromList "\n"
501− -- >>> unwords = Stream.interpose ' '
502− -- >>> unwords = Stream.intercalate Unfold.fromList " "
503
426
504 , concatEffect
427 , concatEffect
505 , concatMap
428 , concatMap
506 , concatMapM
429 , concatMapM
507
430
508 -- * Repeated Fold
431 -- * Repeated Fold
509− -- | Idioms and equivalents of Data.List APIs:
432+ -- | Useful idioms:
510 --
433 --
434+ -- >>> splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)
435+ -- >>> splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)
436+ -- >>> groupsBy eq f = Stream.parseMany (Parser.groupBy eq f)
437+ -- >>> groupsByRolling eq f = Stream.parseMany (Parser.groupByRolling eq f)
511− -- >>> groupsOf n = Stream.foldMany (Fold.take n Fold.toList)
438+ -- >>> groupsOf n f = Stream.foldMany (Fold.take n f)
512− -- >>> groupBy eq = Stream.groupsWhile eq Fold.toList
513− -- >>> groupBy eq = Stream.parseMany (Parser.groupBy eq Fold.toList)
514− -- >>> groupsByRolling eq = Stream.parseMany (Parser.groupByRolling eq Fold.toList)
515− -- >>> groups = groupBy (==)
516 , foldMany -- XXX Rename to foldRepeat
439 , foldMany -- XXX Rename to foldRepeat
517 , groupsOf
440 , groupsOf
518 , parseMany
441 , parseMany
519
442
520 -- * Splitting
443 -- * Splitting
521− -- | Idioms and equivalents of Data.List APIs:
522− --
523− -- >>> splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)
524− -- >>> splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)
525− -- >>> lines = splitOnSuffix (== '\n')
526− -- >>> words = Stream.wordsBy isSpace
527− -- >>> splitAt n = Stream.fold (Fold.splitAt n Fold.toList Fold.toList)
528− -- >>> span p = Parser.splitWith (,) (Parser.takeWhile p Fold.toList) (Parser.fromFold Fold.toList)
529− -- >>> break p = span (not . p)
530 , splitOn
444 , splitOn
531 , wordsBy
445 , wordsBy
532
446
533 -- * Buffered Operations
447 -- * Buffered Operations
534 -- | Operations that require buffering of the stream.
448 -- | Operations that require buffering of the stream.
535 -- Reverse is essentially a left fold followed by an unfold.
449 -- Reverse is essentially a left fold followed by an unfold.
536− --
537− -- Idioms and equivalents of Data.List APIs:
538− --
539− -- >>> nub = Stream.fold Fold.toList . Stream.scanMaybe Fold.nub
540− -- >>> nub = Stream.ordNub -- unreleased API
541− -- >>> sortBy = StreamK.sortBy
542− -- >>> sortOn f = StreamK.sortOn -- unreleased API
543− -- >>> deleteFirstsBy = Stream.deleteFirstsBy -- unreleased
544− -- >>> (\\) = Stream.deleteFirstsBy (==) -- unreleased
545− -- >>> intersectBy = Stream.intersectBy -- unreleased
546− -- >>> intersect = Stream.intersectBy (==) -- unreleased
547− -- >>> unionBy = Stream.unionBy -- unreleased
548− -- >>> union = Stream.unionBy (==) -- unreleased
549− --
550 , reverse
450 , reverse
551− -- XXX transpose: write the streams to arrays and then stream transposed.
552− -- XXX subsequences
553− -- XXX permutations
554− -- , nub
555− -- , ordNub
556− -- , nubBy
557
451
558 -- * Multi-Stream folds
452 -- * Multi-Stream folds
559 -- | Operations that consume multiple streams at the same time.
453 -- | Operations that consume multiple streams at the same time.
560 , eqBy
454 , eqBy
561 , cmpBy
455 , cmpBy
562 , isPrefixOf
456 , isPrefixOf
563− , isInfixOf
564− -- , isSuffixOf
565− -- , isSuffixOfUnbox
566 , isSubsequenceOf
457 , isSubsequenceOf
567
458
568 -- trimming sequences
459 -- trimming sequences
569− -- , stripPrefix
460+ , stripPrefix
570− -- , stripSuffix
571− -- , stripSuffixUnbox
572
461
573 -- Exceptions and resource management depend on the "exceptions" package
462 -- Exceptions and resource management depend on the "exceptions" package
574 -- XXX We can have IO Stream operations not depending on "exceptions"
463 -- XXX We can have IO Stream operations not depending on "exceptions"
@@ -632,8 +521,10 @@ where
632import Streamly.Internal.Data.Stream
521import Streamly.Internal.Data.Stream
633import Prelude
522import Prelude
634 hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr,
523 hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr,
524+ foldl, map, mapM, mapM_, sequence, all, any, sum, product, elem,
525+ notElem, maximum, minimum, head, last, tail, length, null,
635− mapM, sequence, reverse, iterate, foldr1, repeat, replicate,
526+ reverse, iterate, init, and, or, lookup, foldr1, (!!),
636− concatMap)
527+ scanl, scanl1, repeat, replicate, concatMap, span)
637
528
638import qualified Streamly.Internal.Data.Array.Type as Array
529import qualified Streamly.Internal.Data.Array.Type as Array
639
530
core/src/Streamly/Internal/Data/Stream/Container.hs+48−56
@@ -11,17 +11,16 @@
11
11
12module Streamly.Internal.Data.Stream.Container
12module Streamly.Internal.Data.Stream.Container
13 (
13 (
14− -- * Deduplication
14+ nub
15− ordNub
16
15
17− -- * Joins
16+ -- * Joins for unconstrained types
18− , leftJoin
17+ , joinLeftGeneric
19− , outerJoin
18+ , joinOuterGeneric
20
19
21− -- * Ord Joins
20+ -- * Joins with Ord constraint
22− , innerOrdJoin
21+ , joinInner
23− , leftOrdJoin
22+ , joinLeft
24− , outerOrdJoin
23+ , joinOuter
25 )
24 )
26where
25where
27
26
@@ -48,15 +47,12 @@ import qualified Streamly.Internal.Data.Stream.Transformer as Stream
48
47
49#include "DocTestDataStream.hs"
48#include "DocTestDataStream.hs"
50
49
51−-- | @nub@ specialized to 'Ord' types for better performance. Returns a
52−-- subsequence of the stream removing any duplicate elements.
53−--
54−-- The memory used is proportional to the number of unique elements in the
50+-- | The memory used is proportional to the number of unique elements in the
55−-- stream. One way to limit the memory is to use @take@ on the resulting
51+-- stream. If we want to limit the memory we can just use "take" to limit the
56−-- stream to limit the unique elements in the stream.
52+-- uniq elements in the stream.
57−{-# INLINE_NORMAL ordNub #-}
53+{-# INLINE_NORMAL nub #-}
58−ordNub :: (Monad m, Ord a) => Stream m a -> Stream m a
54+nub :: (Monad m, Ord a) => Stream m a -> Stream m a
59−ordNub (Stream step1 state1) = Stream step (Set.empty, state1)
55+nub (Stream step1 state1) = Stream step (Set.empty, state1)
60
56
61 where
57 where
62
58
@@ -82,8 +78,8 @@ toMap =
82--
78--
83-- XXX An IntMap may be faster when the keys are Int.
79-- XXX An IntMap may be faster when the keys are Int.
84-- XXX Use hashmap instead of map?
80-- XXX Use hashmap instead of map?
85−
81+--
86−-- | 'innerJoin' specialized to 'Ord' types for better performance.
82+-- | Like 'joinInner' but uses a 'Map' for efficiency.
87--
83--
88-- If the input streams have duplicate keys, the behavior is undefined.
84-- If the input streams have duplicate keys, the behavior is undefined.
89--
85--
@@ -94,10 +90,10 @@ toMap =
94-- Time: O(m + n)
90-- Time: O(m + n)
95--
91--
96-- /Pre-release/
92-- /Pre-release/
97−{-# INLINE innerOrdJoin #-}
93+{-# INLINE joinInner #-}
98−innerOrdJoin :: (Monad m, Ord k) =>
94+joinInner :: (Monad m, Ord k) =>
99 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, b)
95 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, b)
100−innerOrdJoin s1 s2 =
96+joinInner s1 s2 =
101 Stream.concatEffect $ do
97 Stream.concatEffect $ do
102 km <- toMap s2
98 km <- toMap s2
103 pure $ Stream.mapMaybe (joinAB km) s1
99 pure $ Stream.mapMaybe (joinAB km) s1
@@ -110,33 +106,32 @@ innerOrdJoin s1 s2 =
110 Nothing -> Nothing
106 Nothing -> Nothing
111
107
112-- XXX We can do this concurrently.
108-- XXX We can do this concurrently.
113−-- XXX Check performance of StreamD vs StreamK
114-- XXX If the second stream is sorted and passed as an Array or a seek capable
109-- XXX If the second stream is sorted and passed as an Array or a seek capable
115-- stream then we could use binary search if we have an Ord instance or
110-- stream then we could use binary search if we have an Ord instance or
116-- Ordering returning function. The time complexity would then become (m x log
111-- Ordering returning function. The time complexity would then become (m x log
117-- n).
112-- n).
118
113
119−-- | Like 'innerJoin' but emits @(a, Just b)@ whenever a and b are equal, for
114+-- XXX Check performance of StreamD vs StreamK
115+
116+-- | Like 'joinInner' but emit @(a, Just b)@, and additionally, for those @a@'s
120−-- those @a@'s that are not equal to any @b@ emits @(a, Nothing)@.
117+-- that are not equal to any @b@ emit @(a, Nothing)@.
121−--
122−-- This is a generalization of 'innerJoin' to include all elements from the
123−-- left stream and not just those which have an equal in the right stream. This
124−-- is not a commutative operation, the order of the stream arguments matters.
125--
118--
126−-- All the caveats mentioned in 'innerJoin' apply here as well. Right join is
119+-- The second stream is evaluated multiple times. If the stream is a
127−-- not provided because it is just a flipped left join:
120+-- consume-once stream then the caller should cache it in an 'Data.Array.Array'
121+-- before calling this function. Caching may also improve performance if the
122+-- stream is expensive to evaluate.
128--
123--
129−-- >>> rightJoin eq = flip (Stream.leftJoin eq)
124+-- >>> joinRightGeneric eq = flip (Stream.joinLeftGeneric eq)
130--
125--
131-- Space: O(n) assuming the second stream is cached in memory.
126-- Space: O(n) assuming the second stream is cached in memory.
132--
127--
133-- Time: O(m x n)
128-- Time: O(m x n)
134--
129--
135-- /Unimplemented/
130-- /Unimplemented/
136−{-# INLINE leftJoin #-}
131+{-# INLINE joinLeftGeneric #-}
137−leftJoin :: Monad m =>
132+joinLeftGeneric :: Monad m =>
138 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
133 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
139−leftJoin eq s1 s2 = Stream.evalStateT (return False) $ unCross $ do
134+joinLeftGeneric eq s1 s2 = Stream.evalStateT (return False) $ unCross $ do
140 a <- mkCross (Stream.liftInner s1)
135 a <- mkCross (Stream.liftInner s1)
141 -- XXX should we use StreamD monad here?
136 -- XXX should we use StreamD monad here?
142 -- XXX Is there a better way to perform some action at the end of a loop
137 -- XXX Is there a better way to perform some action at the end of a loop
@@ -157,17 +152,19 @@ leftJoin eq s1 s2 = Stream.evalStateT (return False) $ unCross $ do
157 else mkCross Stream.nil
152 else mkCross Stream.nil
158 Nothing -> return (a, Nothing)
153 Nothing -> return (a, Nothing)
159
154
160−-- | 'leftJoin' specialized to 'Ord' types for better performance.
155+-- XXX rename to joinLeftOrd?
156+
157+-- | A more efficient 'joinLeft' using a hashmap for efficiency.
161--
158--
162-- Space: O(n)
159-- Space: O(n)
163--
160--
164-- Time: O(m + n)
161-- Time: O(m + n)
165--
162--
166-- /Pre-release/
163-- /Pre-release/
167−{-# INLINE leftOrdJoin #-}
164+{-# INLINE joinLeft #-}
168−leftOrdJoin :: (Ord k, Monad m) =>
165+joinLeft :: (Ord k, Monad m) =>
169 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, Maybe b)
166 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, Maybe b)
170−leftOrdJoin s1 s2 =
167+joinLeft s1 s2 =
171 Stream.concatEffect $ do
168 Stream.concatEffect $ do
172 km <- toMap s2
169 km <- toMap s2
173 return $ fmap (joinAB km) s1
170 return $ fmap (joinAB km) s1
@@ -180,19 +177,14 @@ leftOrdJoin s1 s2 =
180 Nothing -> (k, a, Nothing)
177 Nothing -> (k, a, Nothing)
181
178
182-- XXX We can do this concurrently.
179-- XXX We can do this concurrently.
183−-- XXX Check performance of StreamD vs StreamK cross operation.
184
180
181+-- XXX Check performance of StreamD vs StreamK
182+
185−-- | Like 'leftJoin' but emits a @(Just a, Just b)@. Like 'leftJoin', for those
183+-- | Like 'joinLeft' but emits a @(Just a, Just b)@. Like 'joinLeft', for those
186-- @a@'s that are not equal to any @b@ emit @(Just a, Nothing)@, but
184-- @a@'s that are not equal to any @b@ emit @(Just a, Nothing)@, but
187-- additionally, for those @b@'s that are not equal to any @a@ emit @(Nothing,
185-- additionally, for those @b@'s that are not equal to any @a@ emit @(Nothing,
188-- Just b)@.
186-- Just b)@.
189--
187--
190−-- This is a generalization of left join to include all the elements from the
191−-- right stream as well, in other words it is a combination of left and right
192−-- joins. This is a commutative operation. The order of stream arguments can be
193−-- changed without affecting results, except for the ordering of elements in
194−-- the resulting tuple.
195−--
196-- For space efficiency use the smaller stream as the second stream.
188-- For space efficiency use the smaller stream as the second stream.
197--
189--
198-- Space: O(n)
190-- Space: O(n)
@@ -200,15 +192,15 @@ leftOrdJoin s1 s2 =
200-- Time: O(m x n)
192-- Time: O(m x n)
201--
193--
202-- /Pre-release/
194-- /Pre-release/
203−{-# INLINE outerJoin #-}
195+{-# INLINE joinOuterGeneric #-}
204−outerJoin :: MonadIO m =>
196+joinOuterGeneric :: MonadIO m =>
205 (a -> b -> Bool)
197 (a -> b -> Bool)
206 -> Stream m a
198 -> Stream m a
207 -> Stream m b
199 -> Stream m b
208 -> Stream m (Maybe a, Maybe b)
200 -> Stream m (Maybe a, Maybe b)
209−outerJoin eq s1 s2 =
201+joinOuterGeneric eq s1 s =
210 Stream.concatEffect $ do
202 Stream.concatEffect $ do
211− inputArr <- Array.fromStream s2
203+ inputArr <- Array.fromStream s
212 let len = Array.length inputArr
204 let len = Array.length inputArr
213 foundArr <-
205 foundArr <-
214 Stream.fold
206 Stream.fold
@@ -262,18 +254,18 @@ outerJoin eq s1 s2 =
262-- a flag. At the end go through @Stream m b@ and find those that are not in that
254-- a flag. At the end go through @Stream m b@ and find those that are not in that
263-- hash to return (Nothing, b).
255-- hash to return (Nothing, b).
264
256
265−-- | 'outerJoin' specialized to 'Ord' types for better performance.
257+-- | Like 'joinOuter' but uses a 'Map' for efficiency.
266--
258--
267-- Space: O(m + n)
259-- Space: O(m + n)
268--
260--
269-- Time: O(m + n)
261-- Time: O(m + n)
270--
262--
271-- /Pre-release/
263-- /Pre-release/
272−{-# INLINE outerOrdJoin #-}
264+{-# INLINE joinOuter #-}
273−outerOrdJoin ::
265+joinOuter ::
274 (Ord k, MonadIO m) =>
266 (Ord k, MonadIO m) =>
275 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, Maybe a, Maybe b)
267 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, Maybe a, Maybe b)
276−outerOrdJoin s1 s2 =
268+joinOuter s1 s2 =
277 Stream.concatEffect $ do
269 Stream.concatEffect $ do
278 km1 <- kvFold s1
270 km1 <- kvFold s1
279 km2 <- kvFold s2
271 km2 <- kvFold s2
core/src/Streamly/Internal/Data/Stream/Nesting.hs+46−2
@@ -141,8 +141,9 @@ module Streamly.Internal.Data.Stream.Nesting
141
141
142 -- * Transform (Nested Containers)
142 -- * Transform (Nested Containers)
143 -- | Opposite to compact in ArrayStream
143 -- | Opposite to compact in ArrayStream
144− , splitInnerBy -- XXX innerSplitOn
144+ , splitInnerBy
145− , splitInnerBySuffix -- XXX innerSplitOnSuffix
145+ , splitInnerBySuffix
146+ , intersectBySorted
146
147
147 -- * Reduce By Streams
148 -- * Reduce By Streams
148 , dropPrefix
149 , dropPrefix
@@ -592,6 +593,49 @@ mergeFstBy :: -- Monad m =>
592mergeFstBy _f _m1 _m2 = undefined
593mergeFstBy _f _m1 _m2 = undefined
593 -- fromStreamK $ D.mergeFstBy f (toStreamD m1) (toStreamD m2)
594 -- fromStreamK $ D.mergeFstBy f (toStreamD m1) (toStreamD m2)
594
595
596+-------------------------------------------------------------------------------
597+-- Intersection of sorted streams
598+-------------------------------------------------------------------------------
599+
600+-- Assuming the streams are sorted in ascending order
601+{-# INLINE_NORMAL intersectBySorted #-}
602+intersectBySorted :: Monad m
603+ => (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
604+intersectBySorted cmp (Stream stepa ta) (Stream stepb tb) =
605+ Stream step
606+ ( ta -- left stream state
607+ , tb -- right stream state
608+ , Nothing -- left value
609+ , Nothing -- right value
610+ )
611+
612+ where
613+
614+ {-# INLINE_LATE step #-}
615+ -- step 1, fetch the first value
616+ step gst (sa, sb, Nothing, b) = do
617+ r <- stepa gst sa
618+ return $ case r of
619+ Yield a sa' -> Skip (sa', sb, Just a, b) -- step 2/3
620+ Skip sa' -> Skip (sa', sb, Nothing, b)
621+ Stop -> Stop
622+
623+ -- step 2, fetch the second value
624+ step gst (sa, sb, a@(Just _), Nothing) = do
625+ r <- stepb gst sb
626+ return $ case r of
627+ Yield b sb' -> Skip (sa, sb', a, Just b) -- step 3
628+ Skip sb' -> Skip (sa, sb', a, Nothing)
629+ Stop -> Stop
630+
631+ -- step 3, compare the two values
632+ step _ (sa, sb, Just a, Just b) = do
633+ let res = cmp a b
634+ return $ case res of
635+ GT -> Skip (sa, sb, Just a, Nothing) -- step 2
636+ LT -> Skip (sa, sb, Nothing, Just b) -- step 1
637+ EQ -> Yield a (sa, sb, Nothing, Just b) -- step 1
638+
595------------------------------------------------------------------------------
639------------------------------------------------------------------------------
596-- Combine N Streams - unfoldMany
640-- Combine N Streams - unfoldMany
597------------------------------------------------------------------------------
641------------------------------------------------------------------------------
core/src/Streamly/Internal/Data/Stream/Top.hs+115−243
@@ -8,73 +8,38 @@
8-- Portability : GHC
8-- Portability : GHC
9--
9--
10-- Top level module that can depend on all other lower level Stream modules.
10-- Top level module that can depend on all other lower level Stream modules.
11−--
12−-- Design notes:
13−--
14−-- The order of arguments in the join operations should ideally be opposite. It
15−-- should be such that the infinite stream is the last one. The transformation
16−-- should be on the last argument, so if you curry the functions with all other
17−-- arguments we get a @Stream -> Stream@ function. The first stream argument
18−-- may be considered as a config or modifier for the operation.
19−--
20−-- Benefit of changing the order is that we get a more intuitive Stream ->
21−-- Stream transformation after currying all other arguments. The inner loop
22−-- streams become arguments for the transformation, more like local modifiers
23−-- for the global outer stream as the last argument. Thus we can continue using
24−-- transformations on the outer stream in a composed pipeline. Otherwise we can
25−-- use flip to flip the order.
26−--
27−-- The fact that the inner stream can be used in the loop multiple times also
28−-- tells that this is not the real effectful stream, it is more like a pure
29−-- stream or an array. In fact we may consider using an Identity streams as
30−-- inner streams in which case these functions will not look nice.
31−--
32−-- Downsides:
33−--
34−-- * Maybe less intuitive to think about, because we usually think the first
35−-- stream as the outer loop and second as the inner.
36−-- * Zip and merge operations will continue using the opposite order.
37−-- * Need to change the order of cross, crossWith operations as well
38−-- * It will be inconsistent with Data.List. The functions cannot be used as
39−-- intuitive operators.
40−--
41−-- The choice is similar to concatMap vs bind. concatMap is pipeline
42−-- composition friendly but bind is user intuition friendly. Another option is
43−-- to have other functions with a different argument order e.g. flippedCross
44−-- instead of cross.
45−--
46−-- If we change the order we have to make sure that we have a consistent
47−-- convention for set-like and the cross join operations.
48
11
49module Streamly.Internal.Data.Stream.Top
12module Streamly.Internal.Data.Stream.Top
50 (
13 (
14+ -- * Transformation
51− -- * Sampling
15+ -- ** Sampling
52 -- | Value agnostic filtering.
16 -- | Value agnostic filtering.
53 strideFromThen
17 strideFromThen
54
18
55− -- * Straight Joins
19+ -- * Nesting
20+ -- ** Set like operations
56− -- | These are set-like operations but not exactly set operations because
21+ -- | These are not exactly set operations because streams are not
57− -- streams are not necessarily sets, they may have duplicated elements.
22+ -- necessarily sets, they may have duplicated elements. These operations
58− -- These operations are generic i.e. they work on streams of unconstrained
23+ -- are generic i.e. they work on streams of unconstrained types, therefore,
59− -- types, therefore, they have quadratic performance characterstics. For
24+ -- they have quadratic performance characterstics. For better performance
60− -- better performance using Set or Map structures see the
61− -- Streamly.Internal.Data.Stream.Container module.
25+ -- using Set structures see the Streamly.Internal.Data.Stream.Container
62− , intersectBy
26+ -- module.
63− , deleteFirstsBy
27+ , filterInStreamGenericBy
64− , unionBy
28+ , deleteInStreamGenericBy
65−
29+ , unionWithStreamGenericBy
30+
66− -- Set like operations on sorted streams
31+ -- ** Set like operations on sorted streams
67− , sortedIntersectBy
32+ , filterInStreamAscBy
68− , sortedDeleteFirstsBy
33+ , deleteInStreamAscBy
69− , sortedUnionBy
34+ , unionWithStreamAscBy
70−
35+
71− -- * Cross Joins
36+ -- ** Join operations
72− , innerJoin
37+ , joinInnerGeneric
73−
38+
74− -- Joins on sorted stream
39+ -- * Joins on sorted stream
75− , innerSortedJoin
40+ , joinInnerAscBy
76− , leftSortedJoin
41+ , joinLeftAscBy
77− , outerSortedJoin
42+ , joinOuterAscBy
78 )
43 )
79where
44where
80
45
@@ -83,7 +48,7 @@ where
83import Control.Monad.IO.Class (MonadIO(..))
48import Control.Monad.IO.Class (MonadIO(..))
84import Data.IORef (newIORef, readIORef, modifyIORef')
49import Data.IORef (newIORef, readIORef, modifyIORef')
85import Streamly.Internal.Data.Fold.Type (Fold)
50import Streamly.Internal.Data.Fold.Type (Fold)
86−import Streamly.Internal.Data.Stream.Type (Stream(..), Step(..), cross)
51+import Streamly.Internal.Data.Stream.Type (Stream, cross)
87
52
88import qualified Data.List as List
53import qualified Data.List as List
89import qualified Streamly.Internal.Data.Fold as Fold
54import qualified Streamly.Internal.Data.Fold as Fold
@@ -145,40 +110,29 @@ strideFromThen offset stride =
145-- binary search if we have an Ord instance or Ordering returning function. The
110-- binary search if we have an Ord instance or Ordering returning function. The
146-- time complexity would then become (m x log n).
111-- time complexity would then become (m x log n).
147
112
148−-- | Like 'cross' but emits only those tuples where @a == b@ using the supplied
113+-- | Like 'cross' but emits only those tuples where @a == b@ using the
149−-- equality predicate. This is essentially a @cross intersection@ of two
114+-- supplied equality predicate.
150−-- streams.
151--
115--
152-- Definition:
116-- Definition:
153--
117--
154−-- >>> innerJoin eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ Stream.cross s1 s2
118+-- >>> joinInnerGeneric eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ Stream.cross s1 s2
155−--
156−-- The second (inner) stream must be finite. Moreover, it must be either pure
157−-- or capable of multiple evaluations. If not then the caller should cache it
158−-- in an 'Data.Array.Array', if the type does not have an 'Unbox' instance then
159−-- use the Generic 'Data.Array.Generic.Array'. Convert the array to stream
160−-- before calling this function. Caching may also improve performance if the
161−-- stream is expensive to evaluate.
162--
119--
163−-- If you care about performance this function should be your last choice among
120+-- You should almost always prefer @joinInnerOrd@ over 'joinInnerGeneric' if
164−-- all inner joins. 'Streamly.Internal.Data.Unfold.innerJoin' is a much faster
165−-- fused alternative. 'innerSortedJoin' is a faster alternative when streams
166−-- are sorted. 'innerOrdJoin' is an order of magnitude faster alternative when
121+-- possible. @joinInnerOrd@ is an order of magnitude faster but may take more
167−-- the type has an 'Ord' instance.
122+-- space for caching the second stream.
168--
123--
169−-- Note: Conceptually, this is a commutative operation. Result includes all the
124+-- See 'Streamly.Internal.Data.Unfold.joinInnerGeneric' for a much faster fused
170−-- elements from the left and the right stream. The order of streams can be
125+-- alternative.
171−-- changed without affecting results, except for the ordering within the tuple.
172--
126--
173-- Time: O(m x n)
127-- Time: O(m x n)
174--
128--
175-- /Pre-release/
129-- /Pre-release/
176−{-# INLINE innerJoin #-}
130+{-# INLINE joinInnerGeneric #-}
177−innerJoin :: Monad m =>
131+joinInnerGeneric :: Monad m =>
178 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, b)
132 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, b)
179−innerJoin eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ cross s1 s2
133+joinInnerGeneric eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ cross s1 s2
180{-
134{-
181−innerJoin eq s1 s2 = do
135+joinInnerGeneric eq s1 s2 = do
182 -- ConcatMap works faster than bind
136 -- ConcatMap works faster than bind
183 Stream.concatMap (\a ->
137 Stream.concatMap (\a ->
184 Stream.concatMap (\b ->
138 Stream.concatMap (\b ->
@@ -189,44 +143,44 @@ innerJoin eq s1 s2 = do
189 ) s1
143 ) s1
190-}
144-}
191
145
192−-- | A more efficient 'innerJoin' for sorted streams.
146+-- | A more efficient 'joinInner' for sorted streams.
193--
147--
194-- Space: O(1)
148-- Space: O(1)
195--
149--
196-- Time: O(m + n)
150-- Time: O(m + n)
197--
151--
198-- /Unimplemented/
152-- /Unimplemented/
199−{-# INLINE innerSortedJoin #-}
153+{-# INLINE joinInnerAscBy #-}
200−innerSortedJoin ::
154+joinInnerAscBy ::
201 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, b)
155 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, b)
202−innerSortedJoin = undefined
156+joinInnerAscBy = undefined
203
157
204−-- | A more efficient 'leftJoin' for sorted streams.
158+-- | A more efficient 'joinLeft' for sorted streams.
205--
159--
206-- Space: O(1)
160-- Space: O(1)
207--
161--
208-- Time: O(m + n)
162-- Time: O(m + n)
209--
163--
210-- /Unimplemented/
164-- /Unimplemented/
211−{-# INLINE leftSortedJoin #-}
165+{-# INLINE joinLeftAscBy #-}
212−leftSortedJoin :: -- Monad m =>
166+joinLeftAscBy :: -- Monad m =>
213 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
167 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
214−leftSortedJoin _eq _s1 _s2 = undefined
168+joinLeftAscBy _eq _s1 _s2 = undefined
215
169
216−-- | A more efficient 'outerJoin' for sorted streams.
170+-- | A more efficient 'joinOuter' for sorted streams.
217--
171--
218-- Space: O(1)
172-- Space: O(1)
219--
173--
220-- Time: O(m + n)
174-- Time: O(m + n)
221--
175--
222-- /Unimplemented/
176-- /Unimplemented/
223−{-# INLINE outerSortedJoin #-}
177+{-# INLINE joinOuterAscBy #-}
224−outerSortedJoin :: -- Monad m =>
178+joinOuterAscBy :: -- Monad m =>
225 (a -> b -> Ordering)
179 (a -> b -> Ordering)
226 -> Stream m a
180 -> Stream m a
227 -> Stream m b
181 -> Stream m b
228 -> Stream m (Maybe a, Maybe b)
182 -> Stream m (Maybe a, Maybe b)
229−outerSortedJoin _eq _s1 _s2 = undefined
183+joinOuterAscBy _eq _s1 _s2 = undefined
230
184
231------------------------------------------------------------------------------
185------------------------------------------------------------------------------
232-- Set operations (special joins)
186-- Set operations (special joins)
@@ -236,8 +190,8 @@ outerSortedJoin _eq _s1 _s2 = undefined
236-- the best would be to use an Array with linear search. If the second stream
190-- the best would be to use an Array with linear search. If the second stream
237-- is sorted we can also use a binary search, using Ord constraint.
191-- is sorted we can also use a binary search, using Ord constraint.
238
192
239−-- | Keep only those elements in the first stream that are present in the
193+-- | Keep only those elements in the second stream that are present in the
240−-- second stream too. The second stream is folded to a container using the
194+-- first stream too. The first stream is folded to a container using the
241-- supplied fold and then the elements in the container are looked up using the
195-- supplied fold and then the elements in the container are looked up using the
242-- supplied lookup function.
196-- supplied lookup function.
243--
197--
@@ -252,30 +206,23 @@ filterStreamWith :: Monad m =>
252filterStreamWith fld member s1 s2 =
206filterStreamWith fld member s1 s2 =
253 Stream.concatEffect
207 Stream.concatEffect
254 $ do
208 $ do
255− xs <- Stream.fold fld s2
209+ xs <- Stream.fold fld s1
256− return $ Stream.filter (`member` xs) s1
210+ return $ Stream.filter (`member` xs) s2
257−
258−-- XXX instead of folding the second stream to a list we could use it directly.
259−-- If the user wants they can generate the stream from an array and also call
260−-- uniq or nub on it. We can provide a convenience Stream -> Stream to cache
261−-- a finite stream in an array and serve it from the cache. The user can decide
262−-- what is best based on the context. They can also choose to use a boxed or
263−-- unboxed array for caching. To force caching we can make the second stream
264−-- monad type Identity. But that may be less flexible. One option is to use
265−-- cachedIntersectBy etc for automatic caching.
266
211
267−-- | 'intersectBy' returns a subsequence of the first stream which intersects
212+-- | 'filterInStreamGenericBy' retains only those elements in the second stream that
268−-- with the second stream. Note that this is not a commutative operation unlike
213+-- are present in the first stream.
269−-- a set intersection, because of duplicate elements in the stream the order of
270−-- the streams matters. This is similar to 'Data.List.intersectBy'. Note that
271−-- intersectBy is a special case of 'innerJoin'.
272--
214--
273−-- >>> f s1 s2 = Stream.fold Fold.toList $ Stream.intersectBy (==) (Stream.fromList s1) (Stream.fromList s2)
215+-- >>> Stream.fold Fold.toList $ Stream.filterInStreamGenericBy (==) (Stream.fromList [1,2,2,4]) (Stream.fromList [2,1,1,3])
274−-- >>> f [1,3,4,4,5]) [2,3,4,5,5]
216+-- [2,1,1]
275−-- [3,4,4,5]
276--
217--
218+-- >>> Stream.fold Fold.toList $ Stream.filterInStreamGenericBy (==) (Stream.fromList [2,1,1,3]) (Stream.fromList [1,2,2,4])
219+-- [1,2,2]
220+--
221+-- Similar to the list intersectBy operation but with the stream argument order
222+-- flipped.
223+--
277−-- First stream can be infinite, the second stream must be finite and must be
224+-- The first stream must be finite and must not block. Second stream is
278−-- capable of multiple evaluations.
225+-- processed only after the first stream is fully realized.
279--
226--
280-- Space: O(n) where @n@ is the number of elements in the second stream.
227-- Space: O(n) where @n@ is the number of elements in the second stream.
281--
228--
@@ -283,87 +230,46 @@ filterStreamWith fld member s1 s2 =
283-- @n@ is the number of elements in the second stream.
230-- @n@ is the number of elements in the second stream.
284--
231--
285-- /Pre-release/
232-- /Pre-release/
286−{-# INLINE intersectBy #-}
233+{-# INLINE filterInStreamGenericBy #-}
287−intersectBy :: Monad m =>
234+filterInStreamGenericBy :: Monad m =>
288 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
235 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
289−intersectBy eq =
236+filterInStreamGenericBy eq =
290 -- XXX Use an (unboxed) array instead.
237 -- XXX Use an (unboxed) array instead.
291 filterStreamWith
238 filterStreamWith
292 (Fold.scanMaybe (Fold.uniqBy eq) Fold.toListRev)
239 (Fold.scanMaybe (Fold.uniqBy eq) Fold.toListRev)
293 (List.any . eq)
240 (List.any . eq)
294
241
295−-------------------------------------------------------------------------------
296−-- Intersection of sorted streams
297−-------------------------------------------------------------------------------
298−
299−-- XXX The sort order is not important as long both the streams have the same
300−-- sort order. We need to move only in one direction in each stream.
301−-- XXX Fix the argument order to use the same behavior as intersectBy.
302−
303−-- | Like 'intersectBy' but assumes that the input streams are sorted in
242+-- | Like 'filterInStreamGenericBy' but assumes that the input streams are sorted in
304-- ascending order. To use it on streams sorted in descending order pass an
243-- ascending order. To use it on streams sorted in descending order pass an
305-- inverted comparison function returning GT for less than and LT for greater
244-- inverted comparison function returning GT for less than and LT for greater
306-- than.
245-- than.
307--
246--
308−-- Both streams can be infinite.
309−--
310-- Space: O(1)
247-- Space: O(1)
311--
248--
312-- Time: O(m+n)
249-- Time: O(m+n)
313--
250--
314-- /Pre-release/
251-- /Pre-release/
315−{-# INLINE_NORMAL sortedIntersectBy #-}
252+{-# INLINE filterInStreamAscBy #-}
316−sortedIntersectBy :: Monad m =>
253+filterInStreamAscBy :: Monad m =>
317 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
254 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
318−sortedIntersectBy cmp (Stream stepa ta) (Stream stepb tb) =
255+filterInStreamAscBy eq s1 s2 = Stream.intersectBySorted eq s2 s1
319− Stream step
320− ( ta -- left stream state
321− , tb -- right stream state
322− , Nothing -- left value
323− , Nothing -- right value
324− )
325−
326− where
327
256
328− {-# INLINE_LATE step #-}
257+-- | Delete all elements of the first stream from the seconds stream. If an
329− -- step 1, fetch the first value
258+-- element occurs multiple times in the first stream as many occurrences of it
330− step gst (sa, sb, Nothing, b) = do
259+-- are deleted from the second stream.
331− r <- stepa gst sa
332− return $ case r of
333− Yield a sa' -> Skip (sa', sb, Just a, b) -- step 2/3
334− Skip sa' -> Skip (sa', sb, Nothing, b)
335− Stop -> Stop
336−
337− -- step 2, fetch the second value
338− step gst (sa, sb, a@(Just _), Nothing) = do
339− r <- stepb gst sb
340− return $ case r of
341− Yield b sb' -> Skip (sa, sb', a, Just b) -- step 3
342− Skip sb' -> Skip (sa, sb', a, Nothing)
343− Stop -> Stop
344−
345− -- step 3, compare the two values
346− step _ (sa, sb, Just a, Just b) = do
347− let res = cmp a b
348− return $ case res of
349− GT -> Skip (sa, sb, Just a, Nothing) -- step 2
350− LT -> Skip (sa, sb, Nothing, Just b) -- step 1
351− EQ -> Yield a (sa, sb, Nothing, Just b) -- step 1
352−
353−-- | Returns a subsequence of the first stream, deleting first occurrences of
354−-- those elements that are present in the second stream. Note that this is not
355−-- a commutative operation. This is similar to the 'Data.List.deleteFirstsBy'.
356--
260--
357−-- >>> f xs ys = Stream.fold Fold.toList $ Stream.deleteFirstsBy (==) (Stream.fromList xs) (Stream.fromList ys)
261+-- >>> Stream.fold Fold.toList $ Stream.deleteInStreamGenericBy (==) (Stream.fromList [1,2,3]) (Stream.fromList [1,2,2])
358−-- >>> f [1,2,2,3,3,5] [1,2,2,3,4]
359−-- [2,3,5]
262+-- [2]
360--
263--
361−-- The following holds:
264+-- The following laws hold:
362--
265--
363−-- > deleteFirstsBy (==) (Stream.nub s2 `append` s1) s2 === s1
266+-- > deleteInStreamGenericBy (==) s1 (s1 `append` s2) === s2
364−-- > deleteFirstsBy (==) (Stream.nub s2 `interleave` s1) s2 === s1
267+-- > deleteInStreamGenericBy (==) s1 (s1 `interleave` s2) === s2
365--
268--
366−-- First stream can be infinite, second stream must be finite.
269+-- Same as the list 'Data.List.//' operation but with argument order flipped.
270+--
271+-- The first stream must be finite and must not block. Second stream is
272+-- processed only after the first stream is fully realized.
367--
273--
368-- Space: O(m) where @m@ is the number of elements in the first stream.
274-- Space: O(m) where @m@ is the number of elements in the first stream.
369--
275--
@@ -371,83 +277,58 @@ sortedIntersectBy cmp (Stream stepa ta) (Stream stepb tb) =
371-- @n@ is the number of elements in the second stream.
277-- @n@ is the number of elements in the second stream.
372--
278--
373-- /Pre-release/
279-- /Pre-release/
374−{-# INLINE deleteFirstsBy #-}
280+{-# INLINE deleteInStreamGenericBy #-}
375−deleteFirstsBy :: Monad m =>
281+deleteInStreamGenericBy :: Monad m =>
376 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
282 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
377−deleteFirstsBy eq s2 s1 =
283+deleteInStreamGenericBy eq s1 s2 =
378− -- XXX s2 can be a sorted mutable array and we can use binary
284+ Stream.concatEffect
379− -- search to find. Mark the element deleted, count the deletions
285+ $ do
380− -- and reconsolidate the array when a min number of elements is
286+ -- This may work well if s1 is small
381− -- deleted.
287+ -- If s1 is big we can go through s1, deleting elements from s2 and
382−
288+ -- not emitting an element if it was successfully deleted from s2.
383− -- XXX Use StreamK or list as second argument instead of Stream to avoid
289+ -- we will need a deleteBy that can return whether the element was
384− -- concatEffect?
290+ -- deleted or not.
385− Stream.concatEffect $ do
386− xs <- Stream.toList s1
291+ xs <- Stream.fold Fold.toList s2
387− -- It reverses the list but that is fine.
292+ let f = Fold.foldl' (flip (List.deleteBy eq)) xs
388− let del x =
293+ fmap Stream.fromList $ Stream.fold f s1
389− List.foldl' (\(ys,res) y ->
390− if x `eq` y
391− then (ys, True)
392− else (x:ys, res)) ([], False)
393− g (ys,_) x =
394− let (ys1, deleted) = del x ys
395− in if deleted
396− then (ys1, Nothing)
397− else (ys1, Just x)
398− in return
399− $ Stream.catMaybes
400− $ fmap snd
401− $ Stream.postscanl' g (xs, Nothing) s2
402
294
403−-- | A more efficient 'deleteFirstsBy' for streams sorted in ascending order.
295+-- | A more efficient 'deleteInStreamGenericBy' for streams sorted in ascending order.
404−--
405−-- Both streams can be infinite.
406--
296--
407-- Space: O(1)
297-- Space: O(1)
408--
298--
409-- /Unimplemented/
299-- /Unimplemented/
410−{-# INLINE sortedDeleteFirstsBy #-}
300+{-# INLINE deleteInStreamAscBy #-}
411−sortedDeleteFirstsBy :: -- (Monad m) =>
301+deleteInStreamAscBy :: -- (Monad m) =>
412 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
302 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
413−sortedDeleteFirstsBy _eq _s1 _s2 = undefined
303+deleteInStreamAscBy _eq _s1 _s2 = undefined
414
304
415-- XXX Remove the MonadIO constraint. We can just cache one stream and then
305-- XXX Remove the MonadIO constraint. We can just cache one stream and then
416-- implement using differenceEqBy.
306-- implement using differenceEqBy.
417
307
418−-- | Returns the first stream appended with those unique elements from the
308+-- | This essentially appends to the second stream all the occurrences of
419−-- second stream that are not already present in the first stream. Note that
309+-- elements in the first stream that are not already present in the second
420−-- this is not a commutative operation unlike a set union, argument order
310+-- stream.
421−-- matters. The behavior is similar to the 'Data.List.unionBy'.
422--
311--
423-- Equivalent to the following except that @s2@ is evaluated only once:
312-- Equivalent to the following except that @s2@ is evaluated only once:
424--
313--
425−-- >>> unionBy eq s1 s2 = s1 `Stream.append` Stream.deleteFirstsBy eq s1 (Stream.nub s2)
314+-- >>> unionWithStreamGenericBy eq s1 s2 = s2 `Stream.append` (Stream.deleteInStreamGenericBy eq s2 s1)
426--
315--
427-- Example:
316-- Example:
428--
317--
429−-- >>> f s1 s2 = Stream.fold Fold.toList $ Stream.unionBy (==) (Stream.fromList s1) (Stream.fromList s2)
318+-- >>> Stream.fold Fold.toList $ Stream.unionWithStreamGenericBy (==) (Stream.fromList [1,1,2,3]) (Stream.fromList [1,2,2,4])
430−-- >>> f [1,2,2,4] [1,1,2,3,3]
431-- [1,2,2,4,3]
319-- [1,2,2,4,3]
432--
320--
433−-- First stream can be infinite, but second stream must be finite. Note that if
434−-- the first stream is infinite the union means just the first stream. Thus
435−-- union is useful only when both streams are finite. See 'sortedUnionBy' where
436−-- union can work on infinite streams if they are sorted.
437−--
438-- Space: O(n)
321-- Space: O(n)
439--
322--
440-- Time: O(m x n)
323-- Time: O(m x n)
441--
324--
442-- /Pre-release/
325-- /Pre-release/
443−{-# INLINE unionBy #-}
326+{-# INLINE unionWithStreamGenericBy #-}
444−unionBy :: MonadIO m =>
327+unionWithStreamGenericBy :: MonadIO m =>
445 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
328 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
446−unionBy eq s2 s1 =
329+unionWithStreamGenericBy eq s1 s2 =
447 Stream.concatEffect
330 Stream.concatEffect
448 $ do
331 $ do
449− -- XXX use a rewrite rule such that if a list converted to stream
450− -- is passed to unionBy then this becomes an identity operation.
451 xs <- Stream.fold Fold.toList s1
332 xs <- Stream.fold Fold.toList s1
452 -- XXX we can use postscanlMAfter' instead of IORef
333 -- XXX we can use postscanlMAfter' instead of IORef
453 ref <- liftIO $ newIORef $! List.nubBy eq xs
334 ref <- liftIO $ newIORef $! List.nubBy eq xs
@@ -460,21 +341,12 @@ unionBy eq s2 s1 =
460 return $ Stream.fromList xs1
341 return $ Stream.fromList xs1
461 return $ Stream.mapM f s2 `Stream.append` s3
342 return $ Stream.mapM f s2 `Stream.append` s3
462
343
463−-- | A more efficient 'unionBy' for sorted streams.
344+-- | A more efficient 'unionWithStreamGenericBy' for sorted streams.
464−--
465−-- Note that the behavior is different from 'unionBy'. In 'unionBy' we append
466−-- the unique elements from second stream only after exhausting the first one
467−-- whereas in sorted streams we can determine unique elements early even when
468−-- we are going through the first stream. Thus the result is an interleaving of
469−-- the two streams, merging those elements from the second stream that are not
470−-- present in the first.
471--
345--
472-- Space: O(1)
346-- Space: O(1)
473--
347--
474−-- Both streams can be infinite.
475−--
476-- /Unimplemented/
348-- /Unimplemented/
477−{-# INLINE sortedUnionBy #-}
349+{-# INLINE unionWithStreamAscBy #-}
478−sortedUnionBy :: -- (Monad m) =>
350+unionWithStreamAscBy :: -- (Monad m) =>
479 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
351 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
480−sortedUnionBy _eq _s1 _s2 = undefined
352+unionWithStreamAscBy _eq _s1 _s2 = undefined
core/src/Streamly/Internal/Data/Stream/Type.hs−4
@@ -572,12 +572,8 @@ foldAdd f =
572-- >>> Stream.foldrM step (return False) s
572-- >>> Stream.foldrM step (return False) s
573-- True
573-- True
574--
574--
575−-- >>> import Control.Monad (join)
576−-- >>> foldrM f z = join $ Stream.foldr f z
577−--
578{-# INLINE_NORMAL foldrM #-}
575{-# INLINE_NORMAL foldrM #-}
579foldrM :: Monad m => (a -> m b -> m b) -> m b -> Stream m a -> m b
576foldrM :: Monad m => (a -> m b -> m b) -> m b -> Stream m a -> m b
580−-- foldrM f z = join . Streamly.Internal.Data.Stream.StreamD.Type.foldr f z
581foldrM f z (Stream step state) = go SPEC state
577foldrM f z (Stream step state) = go SPEC state
582 where
578 where
583 {-# INLINE_LATE go #-}
579 {-# INLINE_LATE go #-}
core/src/Streamly/Internal/Data/Unfold.hs+4−6
@@ -82,7 +82,7 @@ module Streamly.Internal.Data.Unfold
82 , dropWhileM
82 , dropWhileM
83
83
84 -- ** Cross product
84 -- ** Cross product
85− , innerJoin
85+ , joinInnerGeneric
86
86
87 -- ** Resource Management
87 -- ** Resource Management
88 -- | 'bracket' is the most general resource management operation, all other
88 -- | 'bracket' is the most general resource management operation, all other
@@ -677,12 +677,10 @@ dropWhileM f (Unfold step inject) = Unfold step' inject'
677dropWhile :: Monad m => (b -> Bool) -> Unfold m a b -> Unfold m a b
677dropWhile :: Monad m => (b -> Bool) -> Unfold m a b -> Unfold m a b
678dropWhile f = dropWhileM (return . f)
678dropWhile f = dropWhileM (return . f)
679
679
680−-- | Cross intersection of two unfolds. See
681−-- 'Streamly.Internal.Data.Stream.innerJoin' for more details.
682−{-# INLINE_NORMAL innerJoin #-}
680+{-# INLINE_NORMAL joinInnerGeneric #-}
683−innerJoin :: Monad m =>
681+joinInnerGeneric :: Monad m =>
684 (b -> c -> Bool) -> Unfold m a b -> Unfold m a c -> Unfold m a (b, c)
682 (b -> c -> Bool) -> Unfold m a b -> Unfold m a c -> Unfold m a (b, c)
685−innerJoin eq s1 s2 = filter (\(a, b) -> a `eq` b) $ cross s1 s2
683+joinInnerGeneric eq s1 s2 = filter (\(a, b) -> a `eq` b) $ cross s1 s2
686
684
687------------------------------------------------------------------------------
685------------------------------------------------------------------------------
688-- Exceptions
686-- Exceptions
core/src/Streamly/Internal/Unicode/Stream.hs+1−1
@@ -1057,7 +1057,7 @@ stripHead = Stream.dropWhile isSpace
1057-- >>> Stream.fold Fold.toList $ Unicode.lines Fold.toList (Stream.fromList "lines\nthis\nstring\n\n\n")
1057-- >>> Stream.fold Fold.toList $ Unicode.lines Fold.toList (Stream.fromList "lines\nthis\nstring\n\n\n")
1058-- ["lines","this","string","",""]
1058-- ["lines","this","string","",""]
1059--
1059--
1060−-- >>> lines f = Stream.foldMany (Fold.takeEndBy_ (== '\n') f)
1060+-- > lines = Stream.splitOnSuffix (== '\n')
1061--
1061--
1062-- /Pre-release/
1062-- /Pre-release/
1063{-# INLINE lines #-}
1063{-# INLINE lines #-}
docs/Developer/optimization-guidelines.md−12
@@ -191,18 +191,6 @@ the stages of a loop, it is often not useful to inline the whole loop
191itself, in fact we may have to occasionally use a `NOINLINE` so that the
191itself, in fact we may have to occasionally use a `NOINLINE` so that the
192compiler does not inline it.
192compiler does not inline it.
193
193
194−We need to be careful about NOINLINE on polymorphic functions. When
195−NOINLINE is used, the function cannot be specialized causing huge
196−performance degradation. To avoid that we can either make the
197−function monomorphic or sometimes using `NOINLINE [0]` can do the
198−trick. `NOINLINE [0]` blocks inlining in earlier phases but lets it
199−specialize in the last phase, inlining in the last phase is left to
200−the compiler. `INLINE [0]` may sound the same as `NOINLINE [0]`,
201−however, it behaves differently because we ask the compiler to INLINE
202−it compulsorily and it may not give us the desired results. The
203−Data.Fold.writeLastN benchmark is one such case where `NOINLINE [0]`
204−provides much better performance than `INLINE [0]`.
205−
206### NoSpecConstr
194### NoSpecConstr
207
195
208It is not always useful to specialize function calls for all constructors, some
196It is not always useful to specialize function calls for all constructors, some
src/Streamly/Internal/Data/Stream/IsStream/Top.hs+1−1
@@ -649,7 +649,7 @@ intersectBySorted :: (IsStream t, Monad m) =>
649 (a -> a -> Ordering) -> t m a -> t m a -> t m a
649 (a -> a -> Ordering) -> t m a -> t m a -> t m a
650intersectBySorted eq s1 =
650intersectBySorted eq s1 =
651 IsStream.fromStreamD
651 IsStream.fromStreamD
652− . StreamD.sortedIntersectBy eq (IsStream.toStreamD s1)
652+ . StreamD.intersectBySorted eq (IsStream.toStreamD s1)
653 . IsStream.toStreamD
653 . IsStream.toStreamD
654
654
655-- Roughly joinLeft s1 s2 = s1 `difference` s2 + s1 `intersection` s2
655-- Roughly joinLeft s1 s2 = s1 `difference` s2 + s1 `intersection` s2