on master help sign in
Refactor Stream.Top, move transformers, containers ops
a6e62aeHarendra Kumar
benchmark/Streamly/Benchmark/Data/Stream/Expand.hs+16−16
@@ -465,22 +465,22 @@ o_n_heap_buffering :: Int -> [Benchmark]
465o_n_heap_buffering value =
465o_n_heap_buffering value =
466 [ bgroup "buffered"
466 [ bgroup "buffered"
467 [
467 [
468− benchIOSrc1 "joinInner (sqrtVal)"
468+ benchIOSrc1 "joinInnerGeneric (sqrtVal)"
469− $ joinWith S.joinInner sqrtVal
469+ $ joinWith S.joinInnerGeneric sqrtVal
470− , benchIOSrc1 "joinInnerMap"
470+ , benchIOSrc1 "joinInner"
471− $ joinMapWith S.joinInnerMap halfVal
471+ $ joinMapWith S.joinInner halfVal
472− , benchIOSrc1 "joinLeft (sqrtVal)"
472+ , benchIOSrc1 "joinLeftGeneric (sqrtVal)"
473− $ joinWith S.joinLeft sqrtVal
473+ $ joinWith S.joinLeftGeneric sqrtVal
474− , benchIOSrc1 "joinLeftMap "
474+ , benchIOSrc1 "joinLeft "
475− $ joinMapWith S.joinLeftMap halfVal
475+ $ joinMapWith S.joinLeft halfVal
476− , benchIOSrc1 "joinOuter (sqrtVal)"
476+ , benchIOSrc1 "joinOuterGeneric (sqrtVal)"
477− $ joinWith S.joinOuter sqrtVal
477+ $ joinWith S.joinOuterGeneric sqrtVal
478− , benchIOSrc1 "joinOuterMap"
478+ , benchIOSrc1 "joinOuter"
479− $ joinMapWith S.joinOuterMap halfVal
479+ $ joinMapWith S.joinOuter halfVal
480− , benchIOSrc1 "intersectBy (sqrtVal)"
480+ , benchIOSrc1 "filterInStreamGenericBy (sqrtVal)"
481− $ joinWith S.intersectBy sqrtVal
481+ $ joinWith S.filterInStreamGenericBy sqrtVal
482− , benchIOSrc1 "intersectBySorted"
482+ , benchIOSrc1 "filterInStreamAscBy"
483− $ joinMapWith (S.intersectBySorted compare) halfVal
483+ $ joinMapWith (S.filterInStreamAscBy compare) halfVal
484 ]
484 ]
485 ]
485 ]
486
486
core/src/Streamly/Data/Stream.hs+11
@@ -388,6 +388,17 @@ module Streamly.Data.Stream
388 -- , zipWith2
388 -- , zipWith2
389 -- , zipWithM2
389 -- , zipWithM2
390
390
391+ -- ** Cross Product
392+ -- XXX The argument order in this operation is such that it seems we are
393+ -- transforming the first stream using the second stream because the second
394+ -- stream is evaluated many times or buffered and better be finite, first
395+ -- stream could potentially be infinite. In the tradition of using the
396+ -- transformed stream at the end we can have a flipped version called
397+ -- "crossMap" or "nestWith".
398+ , crossWith
399+ -- , cross
400+ -- , joinInner
401+
391 -- * Unfold Each
402 -- * Unfold Each
392 , unfoldMany
403 , unfoldMany
393 , intercalate
404 , intercalate
core/src/Streamly/Internal/Data/Stream/Top.hs+134−253
@@ -21,42 +21,42 @@ module Streamly.Internal.Data.Stream.Top
21 -- * Nesting
21 -- * Nesting
22 -- ** Set like operations
22 -- ** Set like operations
23 -- | These are not exactly set operations because streams are not
23 -- | These are not exactly set operations because streams are not
24− -- necessarily sets, they may have duplicated elements.
24+ -- necessarily sets, they may have duplicated elements. These operations
25− , intersectBy
25+ -- are generic i.e. they work on streams of unconstrained types, therefore,
26− , intersectBySorted
26+ -- they have quadratic performance characterstics. For better performance
27− , differenceBy
27+ -- using Set structures see the Streamly.Internal.Data.Stream.Container
28− , mergeDifferenceBy
28+ -- module.
29− , unionBy
29+ , filterInStreamGenericBy
30− , mergeUnionBy
30+ , deleteInStreamGenericBy
31+ , unionWithStreamGenericBy
32+
33+ -- ** Set like operations on sorted streams
34+ , filterInStreamAscBy
35+ , deleteInStreamAscBy
36+ , unionWithStreamAscBy
31
37
32 -- ** Join operations
38 -- ** Join operations
33− , crossJoin
39+ , joinInnerGeneric
34− , joinInner
40+
35− , joinInnerMerge
41+ -- * Joins on sorted stream
36− , joinLeft
42+ , joinInnerAscBy
37− , mergeLeftJoin
43+ , joinLeftAscBy
38− , joinOuter
44+ , joinOuterAscBy
39− , mergeOuterJoin
40 )
45 )
41where
46where
42
47
43#include "inline.hs"
48#include "inline.hs"
44
49
45import Control.Monad.IO.Class (MonadIO(..))
50import Control.Monad.IO.Class (MonadIO(..))
46−import Control.Monad.Trans.State.Strict (get, put)
47−import Data.Function ((&))
48import Data.IORef (newIORef, readIORef, modifyIORef')
51import Data.IORef (newIORef, readIORef, modifyIORef')
49−import Data.Maybe (isJust)
52+import Streamly.Internal.Data.Fold.Type (Fold)
50import Streamly.Internal.Data.Stream.Common ()
53import Streamly.Internal.Data.Stream.Common ()
51−import Streamly.Internal.Data.Stream.Cross (CrossStream (..))
54+import Streamly.Internal.Data.Stream.Type
52−import Streamly.Internal.Data.Stream.Type (Stream, fromStreamD, toStreamD)
55+ (Stream, fromStreamD, toStreamD, cross)
53
56
54import qualified Data.List as List
57import qualified Data.List as List
55−import qualified Streamly.Internal.Data.Array.Generic as Array
56−import qualified Streamly.Internal.Data.Array.Mut.Type as MA
57import qualified Streamly.Internal.Data.Fold as Fold
58import qualified Streamly.Internal.Data.Fold as Fold
58import qualified Streamly.Internal.Data.Parser as Parser
59import qualified Streamly.Internal.Data.Parser as Parser
59−import qualified Streamly.Internal.Data.Stream.Transformer as Stream
60import qualified Streamly.Internal.Data.Stream.Eliminate as Stream
60import qualified Streamly.Internal.Data.Stream.Eliminate as Stream
61import qualified Streamly.Internal.Data.Stream.Generate as Stream
61import qualified Streamly.Internal.Data.Stream.Generate as Stream
62import qualified Streamly.Internal.Data.Stream.Expand as Stream
62import qualified Streamly.Internal.Data.Stream.Expand as Stream
@@ -140,59 +140,37 @@ sortBy cmp =
140-- source changes during iterations. We can also use an Unfold instead of
140-- source changes during iterations. We can also use an Unfold instead of
141-- stream. We probably need a way to distinguish streams that can be read
141-- stream. We probably need a way to distinguish streams that can be read
142-- mutliple times without any interference (e.g. unfolding a stream using an
142-- mutliple times without any interference (e.g. unfolding a stream using an
143−-- immutable handle would work i.e. using pread/pwrite instead of maintianing
143+-- immutable handle would work i.e. using pread/pwrite instead of maintaining
144-- an offset in the handle).
144-- an offset in the handle).
145
145
146−-- XXX We can do this concurrently.
147−--
148−-- | This is the same as 'Streamly.Internal.Data.Unfold.outerProduct' but less
149−-- efficient.
150−--
151−-- The second stream is evaluated multiple times. If the second stream is
152−-- consume-once stream then it can be cached in an 'Data.Array.Array' before
153−-- calling this function. Caching may also improve performance if the stream is
154−-- expensive to evaluate.
155−--
156−-- Time: O(m x n)
157−--
158−-- /Pre-release/
159−{-# INLINE crossJoin #-}
160−crossJoin :: Monad m => Stream m a -> Stream m b -> Stream m (a, b)
161−crossJoin s1 s2 = unCrossStream $ do
162− -- XXX use concatMap instead?
163− a <- CrossStream s1
164− b <- CrossStream s2
165− return (a, b)
166−
167-- XXX We can do this concurrently.
146-- XXX We can do this concurrently.
168-- XXX If the second stream is sorted and passed as an Array we could use
147-- XXX If the second stream is sorted and passed as an Array we could use
169-- binary search if we have an Ord instance or Ordering returning function. The
148-- binary search if we have an Ord instance or Ordering returning function. The
170-- time complexity would then become (m x log n).
149-- time complexity would then become (m x log n).
150+
151+-- | Like 'cross' but emits only those tuples where @a == b@ using the
152+-- supplied equality predicate.
171--
153--
172−-- | For all elements in @Stream m a@, for all elements in @Stream m b@ if @a@ and @b@
154+-- Definition:
173−-- are equal by the given equality pedicate then return the tuple (a, b).
174−--
175−-- The second stream is evaluated multiple times. If the stream is a
176−-- consume-once stream then the caller should cache it (e.g. in a
177−-- 'Data.Array.Array') before calling this function. Caching may also improve
178−-- performance if the stream is expensive to evaluate.
179--
155--
180−-- For space efficiency use the smaller stream as the second stream.
156+-- >>> joinInnerGeneric eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ Stream.cross s1 s2
181--
157--
182−-- You should almost always use joinInnerMap instead of joinInner. joinInnerMap
158+-- You should almost always prefer @joinInnerOrd@ over 'joinInnerGeneric' if
183−-- is an order of magnitude faster. joinInner may be used when the second
159+-- possible. @joinInnerOrd@ is an order of magnitude faster but may take more
184−-- stream is generated from a seed, therefore, need not be stored in memory and
160+-- space for caching the second stream.
185−-- the amount of memory it takes is a concern.
186--
161--
187−-- Space: O(n) assuming the second stream is cached in memory.
162+-- See 'Streamly.Internal.Data.Unfold.joinInnerGeneric' for a much faster fused
163+-- alternative.
188--
164--
189-- Time: O(m x n)
165-- Time: O(m x n)
190--
166--
191-- /Pre-release/
167-- /Pre-release/
192−{-# INLINE joinInner #-}
168+{-# INLINE joinInnerGeneric #-}
193−joinInner :: Monad m =>
169+joinInnerGeneric :: Monad m =>
194 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, b)
170 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, b)
195−joinInner eq s1 s2 = do
171+joinInnerGeneric eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ cross s1 s2
172+{-
173+joinInnerGeneric eq s1 s2 = do
196 -- ConcatMap works faster than bind
174 -- ConcatMap works faster than bind
197 Stream.concatMap (\a ->
175 Stream.concatMap (\a ->
198 Stream.concatMap (\b ->
176 Stream.concatMap (\b ->
@@ -201,165 +179,46 @@ joinInner eq s1 s2 = do
201 else Stream.nil
179 else Stream.nil
202 ) s2
180 ) s2
203 ) s1
181 ) s1
182+-}
204
183
205−-- | Like 'joinInner' but works only on sorted streams.
184+-- | A more efficient 'joinInner' for sorted streams.
206--
185--
207-- Space: O(1)
186-- Space: O(1)
208--
187--
209-- Time: O(m + n)
188-- Time: O(m + n)
210--
189--
211-- /Unimplemented/
190-- /Unimplemented/
212−{-# INLINE joinInnerMerge #-}
191+{-# INLINE joinInnerAscBy #-}
213−joinInnerMerge ::
192+joinInnerAscBy ::
214 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, b)
193 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, b)
215−joinInnerMerge = undefined
194+joinInnerAscBy = undefined
216
195
217−-- XXX We can do this concurrently.
196+-- | A more efficient 'joinLeft' for sorted streams.
218−-- XXX If the second stream is sorted and passed as an Array or a seek capable
219−-- stream then we could use binary search if we have an Ord instance or
220−-- Ordering returning function. The time complexity would then become (m x log
221−-- n).
222−--
223−-- | For all elements in @Stream m a@, for all elements in @Stream m b@ if @a@ and @b@
224−-- are equal then return the tuple @(a, Just b)@. If @a@ is not present in @t
225−-- m b@ then return @(a, Nothing)@.
226−--
227−-- The second stream is evaluated multiple times. If the stream is a
228−-- consume-once stream then the caller should cache it in an 'Data.Array.Array'
229−-- before calling this function. Caching may also improve performance if the
230−-- stream is expensive to evaluate.
231−--
232−-- @
233−-- rightJoin = flip joinLeft
234−-- @
235−--
236−-- Space: O(n) assuming the second stream is cached in memory.
237−--
238−-- Time: O(m x n)
239−--
240−-- /Unimplemented/
241−{-# INLINE joinLeft #-}
242−joinLeft :: Monad m =>
243− (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
244−joinLeft eq s1 s2 = Stream.evalStateT (return False) $ unCrossStream $ do
245− a <- CrossStream (Stream.liftInner s1)
246− -- XXX should we use StreamD monad here?
247− -- XXX Is there a better way to perform some action at the end of a loop
248− -- iteration?
249− CrossStream (Stream.fromEffect $ put False)
250− let final = Stream.concatEffect $ do
251− r <- get
252− if r
253− then pure Stream.nil
254− else pure (Stream.fromPure Nothing)
255− b <- CrossStream (fmap Just (Stream.liftInner s2) <> final)
256− case b of
257− Just b1 ->
258− if a `eq` b1
259− then do
260− CrossStream (Stream.fromEffect $ put True)
261− return (a, Just b1)
262− else CrossStream Stream.nil
263− Nothing -> return (a, Nothing)
264−
265−-- | Like 'joinLeft' but works only on sorted streams.
266--
197--
267-- Space: O(1)
198-- Space: O(1)
268--
199--
269-- Time: O(m + n)
200-- Time: O(m + n)
270--
201--
271-- /Unimplemented/
202-- /Unimplemented/
272−{-# INLINE mergeLeftJoin #-}
203+{-# INLINE joinLeftAscBy #-}
273−mergeLeftJoin :: -- Monad m =>
204+joinLeftAscBy :: -- Monad m =>
274 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
205 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
275−mergeLeftJoin _eq _s1 _s2 = undefined
206+joinLeftAscBy _eq _s1 _s2 = undefined
276
207
277−-- XXX We can do this concurrently.
278−--
279−-- | For all elements in @t m a@, for all elements in @t m b@ if @a@ and @b@
280−-- are equal by the given equality pedicate then return the tuple (Just a, Just
281−-- b). If @a@ is not found in @t m b@ then return (a, Nothing), return
282−-- (Nothing, b) for vice-versa.
283−--
284−-- For space efficiency use the smaller stream as the second stream.
285−--
286−-- Space: O(n)
287−--
288−-- Time: O(m x n)
289−--
290−-- /Pre-release/
291−{-# INLINE joinOuter #-}
292−joinOuter :: MonadIO m =>
293− (a -> b -> Bool)
294− -> Stream m a
295− -> Stream m b
296− -> Stream m (Maybe a, Maybe b)
297−joinOuter eq s1 s =
298− Stream.concatEffect $ do
299− inputArr <- Array.fromStream s
300− let len = Array.length inputArr
301− foundArr <-
302− Stream.fold
303− (MA.writeN len)
304− (Stream.fromList (Prelude.replicate len False))
305− return $ go inputArr foundArr <> leftOver inputArr foundArr
306−
307− where
308−
309− leftOver inputArr foundArr =
310− let stream1 = Array.read inputArr
311− stream2 = Stream.unfold MA.reader foundArr
312− in Stream.filter
313− isJust
314− ( Stream.zipWith (\x y ->
315− if y
316− then Nothing
317− else Just (Nothing, Just x)
318− ) stream1 stream2
319− ) & Stream.catMaybes
320−
321− evalState = Stream.evalStateT (return False) . unCrossStream
322−
323− go inputArr foundArr = evalState $ do
324− a <- CrossStream (Stream.liftInner s1)
325− -- XXX should we use StreamD monad here?
326− -- XXX Is there a better way to perform some action at the end of a loop
327− -- iteration?
328− CrossStream (Stream.fromEffect $ put False)
329− let final = Stream.concatEffect $ do
330− r <- get
331− if r
332− then pure Stream.nil
333− else pure (Stream.fromPure Nothing)
334− (i, b) <-
335− let stream = Array.read inputArr
336− in CrossStream
337− (Stream.indexed $ fmap Just (Stream.liftInner stream) <> final)
338−
339− case b of
340− Just b1 ->
341− if a `eq` b1
342− then do
343− CrossStream (Stream.fromEffect $ put True)
344− MA.putIndex i True foundArr
345− return (Just a, Just b1)
346− else CrossStream Stream.nil
347− Nothing -> return (Just a, Nothing)
348−
349−-- | Like 'joinOuter' but works only on sorted streams.
208+-- | A more efficient 'joinOuter' for sorted streams.
350--
209--
351-- Space: O(1)
210-- Space: O(1)
352--
211--
353-- Time: O(m + n)
212-- Time: O(m + n)
354--
213--
355-- /Unimplemented/
214-- /Unimplemented/
356−{-# INLINE mergeOuterJoin #-}
215+{-# INLINE joinOuterAscBy #-}
357−mergeOuterJoin :: -- Monad m =>
216+joinOuterAscBy :: -- Monad m =>
358 (a -> b -> Ordering)
217 (a -> b -> Ordering)
359 -> Stream m a
218 -> Stream m a
360 -> Stream m b
219 -> Stream m b
361 -> Stream m (Maybe a, Maybe b)
220 -> Stream m (Maybe a, Maybe b)
362−mergeOuterJoin _eq _s1 _s2 = undefined
221+joinOuterAscBy _eq _s1 _s2 = undefined
363
222
364------------------------------------------------------------------------------
223------------------------------------------------------------------------------
365-- Set operations (special joins)
224-- Set operations (special joins)
@@ -369,19 +228,39 @@ mergeOuterJoin _eq _s1 _s2 = undefined
369-- the best would be to use an Array with linear search. If the second stream
228-- the best would be to use an Array with linear search. If the second stream
370-- is sorted we can also use a binary search, using Ord constraint.
229-- is sorted we can also use a binary search, using Ord constraint.
371
230
372−-- | 'intersectBy' is essentially a filtering operation that retains only those
373−-- elements in the first stream that are present in the second stream.
231+-- | Keep only those elements in the second stream that are present in the
232+-- first stream too. The first stream is folded to a container using the
233+-- supplied fold and then the elements in the container are looked up using the
234+-- supplied lookup function.
374--
235--
375−-- >>> Stream.fold Fold.toList $ Stream.intersectBy (==) (Stream.fromList [1,2,2,4]) (Stream.fromList [2,1,1,3])
236+-- The first stream must be finite and must not block.
376−-- [1,2,2]
237+{-# INLINE filterStreamWith #-}
238+filterStreamWith :: Monad m =>
239+ Fold m a (f a)
240+ -> (a -> f a -> Bool)
241+ -> Stream m a
242+ -> Stream m a
243+ -> Stream m a
244+filterStreamWith fld member s1 s2 =
245+ Stream.concatEffect
246+ $ do
247+ xs <- Stream.fold fld s1
248+ return $ Stream.filter (`member` xs) s2
249+
250+-- | 'filterInStreamGenericBy' retains only those elements in the second stream that
251+-- are present in the first stream.
377--
252--
378−-- >>> Stream.fold Fold.toList $ Stream.intersectBy (==) (Stream.fromList [2,1,1,3]) (Stream.fromList [1,2,2,4])
253+-- >>> Stream.fold Fold.toList $ Stream.filterInStreamGenericBy (==) (Stream.fromList [1,2,2,4]) (Stream.fromList [2,1,1,3])
379-- [2,1,1]
254-- [2,1,1]
380--
255--
381−-- 'intersectBy' is similar to but not the same as 'joinInner':
256+-- >>> Stream.fold Fold.toList $ Stream.filterInStreamGenericBy (==) (Stream.fromList [2,1,1,3]) (Stream.fromList [1,2,2,4])
257+-- [1,2,2]
258+--
259+-- Similar to the list intersectBy operation but with the stream argument order
260+-- flipped.
382--
261--
383−-- >>> Stream.fold Fold.toList $ fmap fst $ Stream.joinInner (==) (Stream.fromList [1,2,2,4]) (Stream.fromList [2,1,1,3])
262+-- The first stream must be finite and must not block. Second stream is
384−-- [1,1,2,2]
263+-- processed only after the first stream is fully realized.
385--
264--
386-- Space: O(n) where @n@ is the number of elements in the second stream.
265-- Space: O(n) where @n@ is the number of elements in the second stream.
387--
266--
@@ -389,48 +268,49 @@ mergeOuterJoin _eq _s1 _s2 = undefined
389-- @n@ is the number of elements in the second stream.
268-- @n@ is the number of elements in the second stream.
390--
269--
391-- /Pre-release/
270-- /Pre-release/
392−{-# INLINE intersectBy #-}
271+{-# INLINE filterInStreamGenericBy #-}
393−intersectBy :: Monad m =>
272+filterInStreamGenericBy :: Monad m =>
394 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
273 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
395−intersectBy eq s1 s2 =
274+filterInStreamGenericBy eq =
396− Stream.concatEffect
275+ -- XXX Use an (unboxed) array instead.
397− $ do
276+ filterStreamWith
398− -- This may work well when s2 is small
277+ (Fold.scanMaybe (Fold.uniqBy eq) Fold.toListRev)
399− xs <- Stream.fold Fold.toListRev $ Stream.uniqBy eq s2
278+ (List.any . eq)
400− return $ Stream.filter (\x -> List.any (eq x) xs) s1
401
279
402−-- | Like 'intersectBy' but works only on streams sorted in ascending order.
280+-- | Like 'filterInStreamGenericBy' but assumes that the input streams are sorted in
281+-- ascending order. To use it on streams sorted in descending order pass an
282+-- inverted comparison function returning GT for less than and LT for greater
283+-- than.
403--
284--
404-- Space: O(1)
285-- Space: O(1)
405--
286--
406-- Time: O(m+n)
287-- Time: O(m+n)
407--
288--
408-- /Pre-release/
289-- /Pre-release/
409−{-# INLINE intersectBySorted #-}
290+{-# INLINE filterInStreamAscBy #-}
410−intersectBySorted :: (Monad m) =>
291+filterInStreamAscBy :: Monad m =>
411 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
292 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
412−intersectBySorted eq s1 =
293+filterInStreamAscBy eq s1 s2 =
413 fromStreamD
294 fromStreamD
414− . StreamD.intersectBySorted eq (toStreamD s1)
295+ $ StreamD.intersectBySorted eq (toStreamD s2)
415− . toStreamD
296+ $ toStreamD s1
416−
417−-- Roughly joinLeft s1 s2 = s1 `difference` s2 + s1 `intersection` s2
418
297
419−-- | Delete first occurrences of those elements from the first stream that are
298+-- | Delete all elements of the first stream from the seconds stream. If an
420−-- present in the second stream. If an element occurs multiple times in the
299+-- element occurs multiple times in the first stream as many occurrences of it
421−-- second stream as many occurrences of it are deleted from the first stream.
300+-- are deleted from the second stream.
422--
301--
423−-- >>> Stream.fold Fold.toList $ Stream.differenceBy (==) (Stream.fromList [1,2,2]) (Stream.fromList [1,2,3])
302+-- >>> Stream.fold Fold.toList $ Stream.deleteInStreamGenericBy (==) (Stream.fromList [1,2,3]) (Stream.fromList [1,2,2])
424-- [2]
303-- [2]
425--
304--
426-- The following laws hold:
305-- The following laws hold:
427--
306--
428−-- @
307+-- > deleteInStreamGenericBy (==) s1 (s1 `append` s2) === s2
429−-- (s1 `serial` s2) `differenceBy eq` s1 === s2
308+-- > deleteInStreamGenericBy (==) s1 (s1 `interleave` s2) === s2
430−-- (s1 `wSerial` s2) `differenceBy eq` s1 === s2
431−-- @
309+--
310+-- Same as the list 'Data.List.//' operation but with argument order flipped.
432--
311--
433−-- Same as the list 'Data.List.//' operation.
312+-- The first stream must be finite and must not block. Second stream is
313+-- processed only after the first stream is fully realized.
434--
314--
435-- Space: O(m) where @m@ is the number of elements in the first stream.
315-- Space: O(m) where @m@ is the number of elements in the first stream.
436--
316--
@@ -438,10 +318,10 @@ intersectBySorted eq s1 =
438-- @n@ is the number of elements in the second stream.
318-- @n@ is the number of elements in the second stream.
439--
319--
440-- /Pre-release/
320-- /Pre-release/
441−{-# INLINE differenceBy #-}
321+{-# INLINE deleteInStreamGenericBy #-}
442−differenceBy :: (Monad m) =>
322+deleteInStreamGenericBy :: Monad m =>
443 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
323 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
444−differenceBy eq s1 s2 =
324+deleteInStreamGenericBy eq s1 s2 =
445 Stream.concatEffect
325 Stream.concatEffect
446 $ do
326 $ do
447 -- This may work well if s1 is small
327 -- This may work well if s1 is small
@@ -449,47 +329,48 @@ differenceBy eq s1 s2 =
449 -- not emitting an element if it was successfully deleted from s2.
329 -- not emitting an element if it was successfully deleted from s2.
450 -- we will need a deleteBy that can return whether the element was
330 -- we will need a deleteBy that can return whether the element was
451 -- deleted or not.
331 -- deleted or not.
452− xs <- Stream.fold Fold.toList s1
332+ xs <- Stream.fold Fold.toList s2
453 let f = Fold.foldl' (flip (List.deleteBy eq)) xs
333 let f = Fold.foldl' (flip (List.deleteBy eq)) xs
454− fmap Stream.fromList $ Stream.fold f s2
334+ fmap Stream.fromList $ Stream.fold f s1
455
335
456−-- | Like 'differenceBy' but works only on sorted streams.
336+-- | A more efficient 'deleteInStreamGenericBy' for streams sorted in ascending order.
457--
337--
458-- Space: O(1)
338-- Space: O(1)
459--
339--
460-- /Unimplemented/
340-- /Unimplemented/
461−{-# INLINE mergeDifferenceBy #-}
341+{-# INLINE deleteInStreamAscBy #-}
462−mergeDifferenceBy :: -- (Monad m) =>
342+deleteInStreamAscBy :: -- (Monad m) =>
463 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
343 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
464−mergeDifferenceBy _eq _s1 _s2 = undefined
344+deleteInStreamAscBy _eq _s1 _s2 = undefined
465
345
346+-- XXX Remove the MonadIO constraint. We can just cache one stream and then
347+-- implement using differenceEqBy.
348+
466−-- | This is essentially an append operation that appends all the extra
349+-- | This essentially appends to the second stream all the occurrences of
467−-- occurrences of elements from the second stream that are not already present
350+-- elements in the first stream that are not already present in the second
468−-- in the first stream.
351+-- stream.
469--
352--
470−-- >>> Stream.fold Fold.toList $ Stream.unionBy (==) (Stream.fromList [1,2,2,4]) (Stream.fromList [1,1,2,3])
353+-- Equivalent to the following except that @s2@ is evaluated only once:
471−-- [1,2,2,4,3]
472--
354--
473−-- Equivalent to the following except that @s1@ is evaluated only once:
355+-- >>> unionWithStreamGenericBy eq s1 s2 = s2 `Stream.append` (Stream.deleteInStreamGenericBy eq s2 s1)
474--
356--
475−-- @
357+-- Example:
476−-- unionBy eq s1 s2 = s1 \`serial` (s2 `differenceBy eq` s1)
477−-- @
478--
358--
479−-- Similar to 'joinOuter' but not the same.
359+-- >>> Stream.fold Fold.toList $ Stream.unionWithStreamGenericBy (==) (Stream.fromList [1,1,2,3]) (Stream.fromList [1,2,2,4])
360+-- [1,2,2,4,3]
480--
361--
481-- Space: O(n)
362-- Space: O(n)
482--
363--
483-- Time: O(m x n)
364-- Time: O(m x n)
484--
365--
485-- /Pre-release/
366-- /Pre-release/
486−{-# INLINE unionBy #-}
367+{-# INLINE unionWithStreamGenericBy #-}
487−unionBy :: MonadIO m =>
368+unionWithStreamGenericBy :: MonadIO m =>
488 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
369 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a
489−unionBy eq s1 s2 =
370+unionWithStreamGenericBy eq s1 s2 =
490 Stream.concatEffect
371 Stream.concatEffect
491 $ do
372 $ do
492− xs <- Stream.fold Fold.toList s2
373+ xs <- Stream.fold Fold.toList s1
493 -- XXX we can use postscanlMAfter' instead of IORef
374 -- XXX we can use postscanlMAfter' instead of IORef
494 ref <- liftIO $ newIORef $! List.nubBy eq xs
375 ref <- liftIO $ newIORef $! List.nubBy eq xs
495 let f x = do
376 let f x = do
@@ -499,14 +380,14 @@ unionBy eq s1 s2 =
499 $ do
380 $ do
500 xs1 <- liftIO $ readIORef ref
381 xs1 <- liftIO $ readIORef ref
501 return $ Stream.fromList xs1
382 return $ Stream.fromList xs1
502− return $ Stream.mapM f s1 <> s3
383+ return $ Stream.mapM f s2 <> s3
503
384
504−-- | Like 'unionBy' but works only on sorted streams.
385+-- | A more efficient 'unionWithStreamGenericBy' for sorted streams.
505--
386--
506-- Space: O(1)
387-- Space: O(1)
507--
388--
508-- /Unimplemented/
389-- /Unimplemented/
509−{-# INLINE mergeUnionBy #-}
390+{-# INLINE unionWithStreamAscBy #-}
510−mergeUnionBy :: -- (Monad m) =>
391+unionWithStreamAscBy :: -- (Monad m) =>
511 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
392 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a
512−mergeUnionBy _eq _s1 _s2 = undefined
393+unionWithStreamAscBy _eq _s1 _s2 = undefined
core/src/Streamly/Internal/Data/Stream/Type.hs+39
@@ -32,6 +32,8 @@ module Streamly.Internal.Data.Stream.Type
32 , crossApply
32 , crossApply
33 , crossApplySnd
33 , crossApplySnd
34 , crossApplyFst
34 , crossApplyFst
35+ , crossWith
36+ , cross
35
37
36 -- * Bind/Concat
38 -- * Bind/Concat
37 , bindWith
39 , bindWith
@@ -387,6 +389,10 @@ fromEffect = fromStreamK . K.fromEffect
387-- Applicative
389-- Applicative
388-------------------------------------------------------------------------------
390-------------------------------------------------------------------------------
389
391
392+-- | Apply a stream of functions to a stream of values and flatten the results.
393+--
394+-- Note that the second stream is evaluated multiple times.
395+--
390{-# INLINE crossApply #-}
396{-# INLINE crossApply #-}
391crossApply :: Monad m => Stream m (a -> b) -> Stream m a -> Stream m b
397crossApply :: Monad m => Stream m (a -> b) -> Stream m a -> Stream m b
392crossApply m1 m2 =
398crossApply m1 m2 =
@@ -402,6 +408,39 @@ crossApplyFst :: Monad m => Stream m a -> Stream m b -> Stream m a
402crossApplyFst m1 m2 =
408crossApplyFst m1 m2 =
403 fromStreamD $ D.crossApplyFst (toStreamD m1) (toStreamD m2)
409 fromStreamD $ D.crossApplyFst (toStreamD m1) (toStreamD m2)
404
410
411+-- |
412+-- Definition:
413+--
414+-- >>> crossWith f m1 m2 = fmap f m1 `Stream.crossApply` m2
415+--
416+-- Note that the second stream is evaluated multiple times.
417+--
418+{-# INLINE crossWith #-}
419+crossWith :: Monad m => (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c
420+crossWith f m1 m2 = fmap f m1 `crossApply` m2
421+
422+-- | Given a @Stream m a@ and @Stream m b@ generate a stream with all possible
423+-- combinations of the tuple @(a, b)@.
424+--
425+-- Definition:
426+--
427+-- >>> cross = Stream.crossWith (,)
428+--
429+-- The second stream is evaluated multiple times. If that is not desired it can
430+-- be cached in an 'Data.Array.Array' and then generated from the array before
431+-- calling this function. Caching may also improve performance if the stream is
432+-- expensive to evaluate.
433+--
434+-- See 'Streamly.Internal.Data.Unfold.cross' for a much faster fused
435+-- alternative.
436+--
437+-- Time: O(m x n)
438+--
439+-- /Pre-release/
440+{-# INLINE cross #-}
441+cross :: Monad m => Stream m a -> Stream m b -> Stream m (a, b)
442+cross = crossWith (,)
443+
405-------------------------------------------------------------------------------
444-------------------------------------------------------------------------------
406-- Bind/Concat
445-- Bind/Concat
407-------------------------------------------------------------------------------
446-------------------------------------------------------------------------------
core/src/Streamly/Internal/Data/Unfold.hs+6
@@ -237,6 +237,7 @@ module Streamly.Internal.Data.Unfold
237 , crossWithM
237 , crossWithM
238 , crossWith
238 , crossWith
239 , cross
239 , cross
240+ , joinInnerGeneric
240 , crossApply
241 , crossApply
241
242
242 -- ** Nesting
243 -- ** Nesting
@@ -829,6 +830,11 @@ dropWhileM f (Unfold step inject) = Unfold step' inject'
829dropWhile :: Monad m => (b -> Bool) -> Unfold m a b -> Unfold m a b
830dropWhile :: Monad m => (b -> Bool) -> Unfold m a b -> Unfold m a b
830dropWhile f = dropWhileM (return . f)
831dropWhile f = dropWhileM (return . f)
831
832
833+{-# INLINE_NORMAL joinInnerGeneric #-}
834+joinInnerGeneric :: Monad m =>
835+ (b -> c -> Bool) -> Unfold m a b -> Unfold m a c -> Unfold m a (b, c)
836+joinInnerGeneric eq s1 s2 = filter (\(a, b) -> a `eq` b) $ cross s1 s2
837+
832------------------------------------------------------------------------------
838------------------------------------------------------------------------------
833-- Exceptions
839-- Exceptions
834------------------------------------------------------------------------------
840------------------------------------------------------------------------------
core/src/Streamly/Internal/Data/Unfold/Type.hs+9−6
@@ -513,14 +513,17 @@ crossWith f = crossWithM (\b c -> return $ f b c)
513
513
514-- | See 'crossWith'.
514-- | See 'crossWith'.
515--
515--
516−-- > cross = crossWith (,)
516+-- Definition:
517--
517--
518−-- To cross the streams from a tuple we can write:
518+-- >>> cross = Unfold.crossWith (,)
519--
519--
520−-- @
520+-- To create a cross product of the streams generated from a tuple we can
521+-- write:
522+--
523+-- >>> :{
521−-- crossProduct :: Monad m => Unfold m a b -> Unfold m c d -> Unfold m (a, c) (b, d)
524+-- cross :: Monad m => Unfold m a b -> Unfold m c d -> Unfold m (a, c) (b, d)
522−-- crossProduct u1 u2 = cross (lmap fst u1) (lmap snd u2)
525+-- cross u1 u2 = Unfold.cross (Unfold.lmap fst u1) (Unfold.lmap snd u2)
523−-- @
526+-- :}
524--
527--
525-- /Pre-release/
528-- /Pre-release/
526{-# INLINE_NORMAL cross #-}
529{-# INLINE_NORMAL cross #-}
src/Streamly/Internal/Data/Stream/Extra.hs+150−14
@@ -5,27 +5,44 @@
5-- Maintainer : streamly@composewell.com
5-- Maintainer : streamly@composewell.com
6-- Stability : experimental
6-- Stability : experimental
7-- Portability : GHC
7-- Portability : GHC
8+--
9+-- Stream operations that require transformers or containers like Set or Map.
10+
11+-- Rename this to Stream.Container?
8
12
9module Streamly.Internal.Data.Stream.Extra
13module Streamly.Internal.Data.Stream.Extra
10 (
14 (
11 nub
15 nub
12− , joinInnerMap
16+
13− , joinLeftMap
17+ -- * Joins for unconstrained types
14− , joinOuterMap
18+ , joinLeftGeneric
19+ , joinOuterGeneric
20+
21+ -- * Joins with Ord constraint
22+ , joinInner
23+ , joinLeft
24+ , joinOuter
15 )
25 )
16where
26where
17
27
18#include "inline.hs"
28#include "inline.hs"
19
29
20import Control.Monad.IO.Class (MonadIO)
30import Control.Monad.IO.Class (MonadIO)
31+import Control.Monad.Trans.State.Strict (get, put)
32+import Data.Function ((&))
33+import Data.Maybe (isJust)
21import Streamly.Internal.Data.Stream.StreamD.Step (Step(..))
34import Streamly.Internal.Data.Stream.StreamD.Step (Step(..))
22import Streamly.Internal.Data.Stream.Type (Stream)
35import Streamly.Internal.Data.Stream.Type (Stream)
36+import Streamly.Internal.Data.Stream.Cross (CrossStream(..))
23
37
24import qualified Data.Map.Strict as Map
38import qualified Data.Map.Strict as Map
25import qualified Data.Set as Set
39import qualified Data.Set as Set
40+import qualified Streamly.Internal.Data.Array.Generic as Array
41+import qualified Streamly.Internal.Data.Array.Mut.Type as MA
26import qualified Streamly.Data.Fold as Fold
42import qualified Streamly.Data.Fold as Fold
27import qualified Streamly.Data.Stream as Stream
43import qualified Streamly.Data.Stream as Stream
28import qualified Streamly.Internal.Data.Stream.StreamD as D
44import qualified Streamly.Internal.Data.Stream.StreamD as D
45+import qualified Streamly.Internal.Data.Stream.Transformer as Stream
29
46
30-- $setup
47-- $setup
31-- >>> :m
48-- >>> :m
@@ -74,10 +91,10 @@ toMap =
74-- Time: O(m + n)
91-- Time: O(m + n)
75--
92--
76-- /Pre-release/
93-- /Pre-release/
77−{-# INLINE joinInnerMap #-}
94+{-# INLINE joinInner #-}
78−joinInnerMap :: (Monad m, Ord k) =>
95+joinInner :: (Monad m, Ord k) =>
79 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, b)
96 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, b)
80−joinInnerMap s1 s2 =
97+joinInner s1 s2 =
81 Stream.concatEffect $ do
98 Stream.concatEffect $ do
82 km <- toMap s2
99 km <- toMap s2
83 pure $ Stream.mapMaybe (joinAB km) s1
100 pure $ Stream.mapMaybe (joinAB km) s1
@@ -89,17 +106,64 @@ joinInnerMap s1 s2 =
89 Just b -> Just (k, a, b)
106 Just b -> Just (k, a, b)
90 Nothing -> Nothing
107 Nothing -> Nothing
91
108
109+-- XXX We can do this concurrently.
110+-- XXX If the second stream is sorted and passed as an Array or a seek capable
111+-- stream then we could use binary search if we have an Ord instance or
112+-- Ordering returning function. The time complexity would then become (m x log
113+-- n).
114+
115+-- | Like 'joinInner' but emit @(a, Just b)@, and additionally, for those @a@'s
116+-- that are not equal to any @b@ emit @(a, Nothing)@.
117+--
118+-- The second stream is evaluated multiple times. If the stream is a
119+-- consume-once stream then the caller should cache it in an 'Data.Array.Array'
120+-- before calling this function. Caching may also improve performance if the
121+-- stream is expensive to evaluate.
122+--
123+-- >>> joinRightGeneric eq = flip (Stream.joinLeftGeneric eq)
124+--
125+-- Space: O(n) assuming the second stream is cached in memory.
126+--
127+-- Time: O(m x n)
128+--
129+-- /Unimplemented/
130+{-# INLINE joinLeftGeneric #-}
131+joinLeftGeneric :: Monad m =>
132+ (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)
133+joinLeftGeneric eq s1 s2 = Stream.evalStateT (return False) $ unCrossStream $ do
134+ a <- CrossStream (Stream.liftInner s1)
135+ -- XXX should we use StreamD monad here?
136+ -- XXX Is there a better way to perform some action at the end of a loop
137+ -- iteration?
138+ CrossStream (Stream.fromEffect $ put False)
139+ let final = Stream.concatEffect $ do
140+ r <- get
141+ if r
142+ then pure Stream.nil
143+ else pure (Stream.fromPure Nothing)
144+ b <- CrossStream (fmap Just (Stream.liftInner s2) <> final)
145+ case b of
146+ Just b1 ->
147+ if a `eq` b1
148+ then do
149+ CrossStream (Stream.fromEffect $ put True)
150+ return (a, Just b1)
151+ else CrossStream Stream.nil
152+ Nothing -> return (a, Nothing)
153+
154+-- XXX rename to joinLeftOrd?
155+
92−-- | Like 'joinLeft' but uses a hashmap for efficiency.
156+-- | A more efficient 'joinLeft' using a hashmap for efficiency.
93--
157--
94-- Space: O(n)
158-- Space: O(n)
95--
159--
96-- Time: O(m + n)
160-- Time: O(m + n)
97--
161--
98-- /Pre-release/
162-- /Pre-release/
99−{-# INLINE joinLeftMap #-}
163+{-# INLINE joinLeft #-}
100−joinLeftMap :: (Ord k, Monad m) =>
164+joinLeft :: (Ord k, Monad m) =>
101 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, Maybe b)
165 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, Maybe b)
102−joinLeftMap s1 s2 =
166+joinLeft s1 s2 =
103 Stream.concatEffect $ do
167 Stream.concatEffect $ do
104 km <- toMap s2
168 km <- toMap s2
105 return $ fmap (joinAB km) s1
169 return $ fmap (joinAB km) s1
@@ -111,10 +175,82 @@ joinLeftMap s1 s2 =
111 Just b -> (k, a, Just b)
175 Just b -> (k, a, Just b)
112 Nothing -> (k, a, Nothing)
176 Nothing -> (k, a, Nothing)
113
177
178+-- XXX We can do this concurrently.
179+
180+-- | Like 'joinLeft' but emits a @(Just a, Just b)@. Like 'joinLeft', for those
181+-- @a@'s that are not equal to any @b@ emit @(Just a, Nothing)@, but
182+-- additionally, for those @b@'s that are not equal to any @a@ emit @(Nothing,
183+-- Just b)@.
184+--
185+-- For space efficiency use the smaller stream as the second stream.
186+--
187+-- Space: O(n)
188+--
189+-- Time: O(m x n)
190+--
191+-- /Pre-release/
192+{-# INLINE joinOuterGeneric #-}
193+joinOuterGeneric :: MonadIO m =>
194+ (a -> b -> Bool)
195+ -> Stream m a
196+ -> Stream m b
197+ -> Stream m (Maybe a, Maybe b)
198+joinOuterGeneric eq s1 s =
199+ Stream.concatEffect $ do
200+ inputArr <- Array.fromStream s
201+ let len = Array.length inputArr
202+ foundArr <-
203+ Stream.fold
204+ (MA.writeN len)
205+ (Stream.fromList (Prelude.replicate len False))
206+ return $ go inputArr foundArr <> leftOver inputArr foundArr
207+
208+ where
209+
210+ leftOver inputArr foundArr =
211+ let stream1 = Array.read inputArr
212+ stream2 = Stream.unfold MA.reader foundArr
213+ in Stream.filter
214+ isJust
215+ ( Stream.zipWith (\x y ->
216+ if y
217+ then Nothing
218+ else Just (Nothing, Just x)
219+ ) stream1 stream2
220+ ) & Stream.catMaybes
221+
222+ evalState = Stream.evalStateT (return False) . unCrossStream
223+
224+ go inputArr foundArr = evalState $ do
225+ a <- CrossStream (Stream.liftInner s1)
226+ -- XXX should we use StreamD monad here?
227+ -- XXX Is there a better way to perform some action at the end of a loop
228+ -- iteration?
229+ CrossStream (Stream.fromEffect $ put False)
230+ let final = Stream.concatEffect $ do
231+ r <- get
232+ if r
233+ then pure Stream.nil
234+ else pure (Stream.fromPure Nothing)
235+ (i, b) <-
236+ let stream = Array.read inputArr
237+ in CrossStream
238+ (Stream.indexed $ fmap Just (Stream.liftInner stream) <> final)
239+
240+ case b of
241+ Just b1 ->
242+ if a `eq` b1
243+ then do
244+ CrossStream (Stream.fromEffect $ put True)
245+ MA.putIndex i True foundArr
246+ return (Just a, Just b1)
247+ else CrossStream Stream.nil
248+ Nothing -> return (Just a, Nothing)
249+
114-- Put the b's that have been paired, in another hash or mutate the hash to set
250-- Put the b's that have been paired, in another hash or mutate the hash to set
115-- a flag. At the end go through @Stream m b@ and find those that are not in that
251-- a flag. At the end go through @Stream m b@ and find those that are not in that
116-- hash to return (Nothing, b).
252-- hash to return (Nothing, b).
117−--
253+
118-- | Like 'joinOuter' but uses a 'Map' for efficiency.
254-- | Like 'joinOuter' but uses a 'Map' for efficiency.
119--
255--
120-- Space: O(m + n)
256-- Space: O(m + n)
@@ -122,11 +258,11 @@ joinLeftMap s1 s2 =
122-- Time: O(m + n)
258-- Time: O(m + n)
123--
259--
124-- /Pre-release/
260-- /Pre-release/
125−{-# INLINE joinOuterMap #-}
261+{-# INLINE joinOuter #-}
126−joinOuterMap ::
262+joinOuter ::
127 (Ord k, MonadIO m) =>
263 (Ord k, MonadIO m) =>
128 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, Maybe a, Maybe b)
264 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, Maybe a, Maybe b)
129−joinOuterMap s1 s2 =
265+joinOuter s1 s2 =
130 Stream.concatEffect $ do
266 Stream.concatEffect $ do
131 km1 <- kvFold s1
267 km1 <- kvFold s1
132 km2 <- kvFold s2
268 km2 <- kvFold s2