on master help sign in
Implement joinLeftHash API
commit582410afda1587005f9a65473d545ad255a225e7 authorRanjeet Kumar Ranjan <ranjeetr@composewell.com> dateWed 5 Jan 2022, 16:35 (5 years ago) parent870958d
3 files changed+79−17
benchmark/Streamly/Benchmark/Prelude/Serial/NestedStream.hs+4 src/Streamly/Internal/Data/Stream/IsStream/Top.hs+28−17 test/Streamly/Test/Prelude/Top.hs+47
benchmark/Streamly/Benchmark/Prelude/Serial/NestedStream.hs+4
@@ -451,6 +451,10 @@ o_n_heap_buffering value =
451451 $ joinWith Internal.joinInner sqrtVal sqrtVal
452452 , benchIOSrc1 "joinInnerMap"
453453 $ joinMapWith Internal.joinInnerMap sqrtVal sqrtVal
454+ , benchIOSrc1 "joinLeft"
455+ $ joinWith Internal.joinLeft sqrtVal sqrtVal
456+ , benchIOSrc1 "joinLeftMap "
457+ $ joinMapWith Internal.joinLeftMap sqrtVal sqrtVal
454458 , benchIOSrc1 "joinOuter"
455459 $ joinWith Internal.joinOuter sqrtVal sqrtVal
456460 , benchIOSrc1 "joinOuterMap"
src/Streamly/Internal/Data/Stream/IsStream/Top.hs+28−17
@@ -39,9 +39,9 @@ module Streamly.Internal.Data.Stream.IsStream.Top
3939 , joinInner
4040 , joinInnerMap
4141 , joinInnerMerge
42− , leftJoin
42+ , joinLeft
4343 , mergeLeftJoin
44− , hashLeftJoin
44+ , joinLeftMap
4545 , joinOuter
4646 , mergeOuterJoin
4747 , joinOuterMap
@@ -291,6 +291,10 @@ joinInner eq s1 s2 = do
291291 ) s2
292292 ) s1
293293
294+-- XXX Generate error if a duplicate insertion is attempted?
295+toMap :: (Monad m, Ord k) => IsStream.SerialT m (k, v) -> m (Map.Map k v)
296+toMap = Stream.foldl' (\kv (k, b) -> Map.insert k b kv) Map.empty
297+
294298-- If the second stream is too big it can be partitioned based on hashes and
295299-- then we can process one parition at a time.
296300--
@@ -313,14 +317,11 @@ joinInnerMap :: (IsStream t, Monad m, Ord k) =>
313317 t m (k, a) -> t m (k, b) -> t m (k, a, b)
314318joinInnerMap s1 s2 =
315319 Stream.concatM $ do
316− km <- kvFold $ IsStream.adapt s2
320+ km <- toMap $ IsStream.adapt s2
317321 pure $ Stream.mapMaybe (joinAB km) s1
318322
319323 where
320324
321− -- XXX Generate error if a duplicate insertion is attempted?
322− kvFold = Stream.foldl' (\kv (k, b) -> Map.insert k b kv) Map.empty
323−
324325 joinAB kvm (k, a) =
325326 case k `Map.lookup` kvm of
326327 Just b -> Just (k, a, b)
@@ -353,7 +354,7 @@ joinInnerMerge = undefined
353354-- stream is expensive to evaluate.
354355--
355356-- @
356−-- rightJoin = flip leftJoin
357+-- rightJoin = flip joinLeft
357358-- @
358359--
359360-- Space: O(n) assuming the second stream is cached in memory.
@@ -361,10 +362,10 @@ joinInnerMerge = undefined
361362-- Time: O(m x n)
362363--
363364-- /Unimplemented/
364−{-# INLINE leftJoin #-}
365−leftJoin :: Monad m =>
365+{-# INLINE joinLeft #-}
366+joinLeft :: Monad m =>
366367 (a -> b -> Bool) -> SerialT m a -> SerialT m b -> SerialT m (a, Maybe b)
367−leftJoin eq s1 s2 = Stream.evalStateT (return False) $ do
368+joinLeft eq s1 s2 = Stream.evalStateT (return False) $ do
368369 a <- Stream.liftInner s1
369370 -- XXX should we use StreamD monad here?
370371 -- XXX Is there a better way to perform some action at the end of a loop
@@ -391,13 +392,23 @@ leftJoin eq s1 s2 = Stream.evalStateT (return False) $ do
391392--
392393-- Time: O(m + n)
393394--
394−-- /Unimplemented/
395−{-# INLINE hashLeftJoin #-}
396−hashLeftJoin :: -- Hashable b =>
397− (a -> b -> Bool) -> t m a -> t m b -> t m (a, Maybe b)
398−hashLeftJoin = undefined
395+-- /Pre-release/
396+{-# INLINE joinLeftMap #-}
397+joinLeftMap :: (IsStream t, Ord k, Monad m) =>
398+ t m (k, a) -> t m (k, b) -> t m (k, a, Maybe b)
399+joinLeftMap s1 s2 =
400+ Stream.concatM $ do
401+ km <- toMap $ IsStream.adapt s2
402+ return $ Stream.map (joinAB km) s1
403+
404+ where
405+
406+ joinAB km (k, a) =
407+ case k `Map.lookup` km of
408+ Just b -> (k, a, Just b)
409+ Nothing -> (k, a, Nothing)
399410
400−-- | Like 'leftJoin' but works only on sorted streams.
411+-- | Like 'joinLeft' but works only on sorted streams.
401412--
402413-- Space: O(1)
403414--
@@ -575,7 +586,7 @@ mergeIntersectBy :: -- (IsStream t, Monad m) =>
575586 (a -> a -> Ordering) -> t m a -> t m a -> t m a
576587mergeIntersectBy _eq _s1 _s2 = undefined
577588
578−-- Roughly leftJoin s1 s2 = s1 `difference` s2 + s1 `intersection` s2
589+-- Roughly joinLeft s1 s2 = s1 `difference` s2 + s1 `intersection` s2
579590
580591-- | Delete first occurrences of those elements from the first stream that are
581592-- present in the second stream. If an element occurs multiple times in the
test/Streamly/Test/Prelude/Top.hs+47
@@ -123,6 +123,51 @@ joinOuterMap =
123123 let v2 = joinOuterList ls0 ls1
124124 assert (sort v1 == sort v2)
125125
126+joinLeftList :: [(Int, Int)] -> [(Int, Int)] -> [(Int, Int, Maybe Int)]
127+joinLeftList ls0 ls1 =
128+ let v = do
129+ i <- ls0
130+ if i `elem` ls1
131+ then return (fst i, fst i, Just (fst i))
132+ else return (fst i, fst i, Nothing)
133+ in v
134+
135+joinLeft :: Property
136+joinLeft =
137+ forAll (listOf (chooseInt (min_value, max_value))) $ \ls0 ->
138+ forAll (listOf (chooseInt (min_value, max_value))) $ \ls1 ->
139+ -- nub the second list as no way to validate using list functions
140+ monadicIO $ action ls0 (nub ls1)
141+
142+ where
143+
144+ action ls0 ls1 = do
145+ v1 <-
146+ run
147+ $ S.toList
148+ $ Top.joinLeft eq (S.fromList ls0) (S.fromList ls1)
149+ let v2 = joinLeftList (map (\a -> (a,a)) ls0) (map (\a -> (a,a)) ls1)
150+ v3 = map (\ (_, x1, x2) -> (x1, x2)) v2
151+ assert (v1 == v3)
152+
153+joinLeftMap :: Property
154+joinLeftMap =
155+ forAll (listOf (chooseInt (min_value, max_value))) $ \ls0 ->
156+ forAll (listOf (chooseInt (min_value, max_value))) $ \ls1 ->
157+ -- nub the second list as no way to validate using list functions
158+ monadicIO $
159+ action (map (\a -> (a,a)) ls0) (map (\a -> (a,a)) (nub ls1))
160+
161+ where
162+
163+ action ls0 ls1 = do
164+ v1 <-
165+ run
166+ $ S.toList
167+ $ Top.joinLeftMap (S.fromList ls0) (S.fromList ls1)
168+ let v2 = joinLeftList ls0 ls1
169+ assert (v1 == v2)
170+
126171-------------------------------------------------------------------------------
127172-- Main
128173-------------------------------------------------------------------------------
@@ -140,3 +185,5 @@ main = hspec $ do
140185 -- XXX currently API is broken https://github.com/composewell/streamly/issues/1032
141186 --prop "joinOuter" Main.joinOuter
142187 prop "joinOuterMap" Main.joinOuterMap
188+ prop "joinLeft" Main.joinLeft
189+ prop "joinLeftMap" Main.joinLeftMap