on master help sign in
Use unfolds in all stream generation ops

Based on a build flag. We want to benchmark how these operations perform when unfolds are used. This is to investigate if we can replace these ops with just unfolds or implement these in terms of unfolds to improve modularity. Increase memory for Split benchmarks for use-unfolds flag.

commitc366e2d5e216206e92572b820a0e24b3fe072f1c authorHarendra Kumar <harendra@composewell.com> dateMon 18 Jul 2022, 01:26 (4 years ago) parentfab76f2
11 files changed+107−38
benchmark/Streamly/Benchmark/Data/Unfold.hs+1 benchmark/streamly-benchmarks.cabal+1−1 core/src/Streamly/Internal/Data/Stream/Bottom.hs+1 core/src/Streamly/Internal/Data/Stream/StreamD/Generate.hs+55 core/src/Streamly/Internal/Data/Stream/StreamD/Type.hs+7 core/src/Streamly/Internal/Data/Unfold.hs−33 core/src/Streamly/Internal/Data/Unfold/Type.hs+20 core/streamly-core.cabal+8 src/Streamly/Data/Unfold.hs+7−2 src/Streamly/Internal/Data/Stream/IsStream/Reduce.hs+6−2 test/Streamly/Test/Data/Unfold.hs+1
benchmark/Streamly/Benchmark/Data/Unfold.hs+1
@@ -28,6 +28,7 @@ import System.IO (Handle, hClose)
2828import System.Random (randomRIO)
2929
3030import qualified Prelude
31+import qualified Streamly.Data.Unfold as UF
3132import qualified Streamly.FileSystem.Handle as FH
3233import qualified Streamly.Internal.Data.Fold as FL
3334import qualified Streamly.Internal.Data.Unfold as UF
benchmark/streamly-benchmarks.cabal+1−1
@@ -223,7 +223,7 @@ benchmark Prelude.Serial
223223 if flag(dev)
224224 ghc-options: +RTS -M3500M -RTS
225225 else
226− ghc-options: +RTS -M2000M -RTS
226+ ghc-options: +RTS -M2500M -RTS
227227
228228
229229benchmark Prelude.WSerial
core/src/Streamly/Internal/Data/Stream/Bottom.hs+1
@@ -79,6 +79,7 @@ import Streamly.Internal.Data.Stream.Type
7979-- >>> import System.IO.Unsafe (unsafePerformIO)
8080-- >>> import Streamly.Internal.Data.Stream as Stream
8181-- >>> import qualified Streamly.Data.Array.Foreign as Array
82+-- >>> import qualified Streamly.Data.Unfold as Unfold
8283-- >>> import qualified Streamly.Internal.Data.Fold as Fold
8384-- >>> import qualified Streamly.Internal.Data.Parser as Parser
8485-- >>> import qualified Streamly.Internal.Data.Unfold as Unfold
core/src/Streamly/Internal/Data/Stream/StreamD/Generate.hs+55
@@ -94,6 +94,11 @@ import Streamly.Internal.Data.Time.Clock
9494import Streamly.Internal.Data.Time.Units
9595 (toAbsTime, AbsTime, toRelTime64, RelTime64)
9696
97+#ifdef USE_UNFOLDS_EVERYWHERE
98+import qualified Streamly.Internal.Data.Unfold as Unfold
99+import qualified Streamly.Internal.Data.Unfold.Enumeration as Unfold
100+#endif
101+
97102import Prelude hiding (iterate, repeat, replicate, takeWhile)
98103import Streamly.Internal.Data.Stream.StreamD.Type
99104
@@ -131,6 +136,9 @@ cons x (Stream step state) = Stream step1 Nothing
131136-- Adapted from vector package
132137{-# INLINE_NORMAL unfoldrM #-}
133138unfoldrM :: Monad m => (s -> m (Maybe (a, s))) -> s -> Stream m a
139+#ifdef USE_UNFOLDS_EVERYWHERE
140+unfoldrM next = unfold (Unfold.unfoldrM next)
141+#else
134142unfoldrM next = Stream step
135143 where
136144 {-# INLINE_LATE step #-}
@@ -139,6 +147,7 @@ unfoldrM next = Stream step
139147 return $ case r of
140148 Just (x, s) -> Yield x s
141149 Nothing -> Stop
150+#endif
142151
143152{-# INLINE_LATE unfoldr #-}
144153unfoldr :: Monad m => (s -> Maybe (a, s)) -> s -> Stream m a
@@ -150,15 +159,26 @@ unfoldr f = unfoldrM (return . f)
150159
151160{-# INLINE_NORMAL repeatM #-}
152161repeatM :: Monad m => m a -> Stream m a
162+#ifdef USE_UNFOLDS_EVERYWHERE
163+repeatM = unfold Unfold.repeatM
164+#else
153165repeatM x = Stream (\_ _ -> x >>= \r -> return $ Yield r ()) ()
166+#endif
154167
155168{-# INLINE_NORMAL repeat #-}
156169repeat :: Monad m => a -> Stream m a
170+#ifdef USE_UNFOLDS_EVERYWHERE
171+repeat x = repeatM (pure x)
172+#else
157173repeat x = Stream (\_ _ -> return $ Yield x ()) ()
174+#endif
158175
159176-- Adapted from the vector package
160177{-# INLINE_NORMAL replicateM #-}
161178replicateM :: forall m a. Monad m => Int -> m a -> Stream m a
179+#ifdef USE_UNFOLDS_EVERYWHERE
180+replicateM n = unfold (Unfold.replicateM n)
181+#else
162182replicateM n p = Stream step n
163183 where
164184 {-# INLINE_LATE step #-}
@@ -167,6 +187,7 @@ replicateM n p = Stream step n
167187 | otherwise = do
168188 x <- p
169189 return $ Yield x (i - 1)
190+#endif
170191
171192{-# INLINE_NORMAL replicate #-}
172193replicate :: Monad m => Int -> a -> Stream m a
@@ -189,10 +210,15 @@ replicate n x = replicateM n (return x)
189210-- enumerateFromStepIntegral is faster for integrals.
190211{-# INLINE_NORMAL enumerateFromStepNum #-}
191212enumerateFromStepNum :: (Monad m, Num a) => a -> a -> Stream m a
213+#ifdef USE_UNFOLDS_EVERYWHERE
214+enumerateFromStepNum from stride =
215+ unfold Unfold.enumerateFromStepNum (from, stride)
216+#else
192217enumerateFromStepNum from stride = Stream step 0
193218 where
194219 {-# INLINE_LATE step #-}
195220 step _ !i = return $ (Yield $! (from + i * stride)) $! (i + 1)
221+#endif
196222
197223{-# INLINE_NORMAL enumerateFromNum #-}
198224enumerateFromNum :: (Monad m, Num a) => a -> Stream m a
@@ -206,6 +232,7 @@ enumerateFromThenNum from next = enumerateFromStepNum from (next - from)
206232-- Enumeration of Integrals
207233------------------------------------------------------------------------------
208234
235+#ifndef USE_UNFOLDS_EVERYWHERE
209236data EnumState a = EnumInit | EnumYield a a a | EnumStop
210237
211238{-# INLINE_NORMAL enumerateFromThenToIntegralUp #-}
@@ -256,6 +283,7 @@ enumerateFromThenToIntegralDn from next to = Stream step EnumInit
256283 else Yield x $ EnumYield (x + stride) stride toMinus
257284
258285 step _ EnumStop = return Stop
286+#endif
259287
260288-- XXX This can perhaps be simplified and written in terms of
261289-- enumeratFromStepIntegral as we have done in unfolds. But anyway we should be
@@ -264,29 +292,44 @@ enumerateFromThenToIntegralDn from next to = Stream step EnumInit
264292enumerateFromThenToIntegral
265293 :: (Monad m, Integral a)
266294 => a -> a -> a -> Stream m a
295+#ifdef USE_UNFOLDS_EVERYWHERE
296+enumerateFromThenToIntegral from next to =
297+ unfold Unfold.enumerateFromThenToIntegral (from, next, to)
298+#else
267299enumerateFromThenToIntegral from next to
268300 | next >= from = enumerateFromThenToIntegralUp from next to
269301 | otherwise = enumerateFromThenToIntegralDn from next to
302+#endif
270303
271304{-# INLINE_NORMAL enumerateFromThenIntegral #-}
272305enumerateFromThenIntegral
273306 :: (Monad m, Integral a, Bounded a)
274307 => a -> a -> Stream m a
308+#ifdef USE_UNFOLDS_EVERYWHERE
309+enumerateFromThenIntegral from next =
310+ unfold Unfold.enumerateFromThenIntegralBounded (from, next)
311+#else
275312enumerateFromThenIntegral from next =
276313 if next > from
277314 then enumerateFromThenToIntegralUp from next maxBound
278315 else enumerateFromThenToIntegralDn from next minBound
316+#endif
279317
280318-- | Can be used to enumerate unbounded integrals. This does not check for
281319-- overflow or underflow for bounded integrals.
282320--
283321{-# INLINE_NORMAL enumerateFromStepIntegral #-}
284322enumerateFromStepIntegral :: (Integral a, Monad m) => a -> a -> Stream m a
323+#ifdef USE_UNFOLDS_EVERYWHERE
324+enumerateFromStepIntegral from stride =
325+ unfold Unfold.enumerateFromStepIntegral (from, stride)
326+#else
285327enumerateFromStepIntegral from stride =
286328 from `seq` stride `seq` Stream step from
287329 where
288330 {-# INLINE_LATE step #-}
289331 step _ !x = return $ Yield x $! (x + stride)
332+#endif
290333
291334-- | Enumerate upwards from @from@ to @to@. We are assuming that "to" is
292335-- constrained by the type to be within max/min bounds.
@@ -353,12 +396,16 @@ times g = Stream step Nothing
353396
354397{-# INLINE_NORMAL fromIndicesM #-}
355398fromIndicesM :: Monad m => (Int -> m a) -> Stream m a
399+#ifdef USE_UNFOLDS_EVERYWHERE
400+fromIndicesM gen = unfold (Unfold.fromIndicesM gen) 0
401+#else
356402fromIndicesM gen = Stream step 0
357403 where
358404 {-# INLINE_LATE step #-}
359405 step _ i = do
360406 x <- gen i
361407 return $ Yield x (i + 1)
408+#endif
362409
363410{-# INLINE fromIndices #-}
364411fromIndices :: Monad m => (Int -> a) -> Stream m a
@@ -385,7 +432,11 @@ generate n gen = generateM n (return . gen)
385432
386433{-# INLINE_NORMAL iterateM #-}
387434iterateM :: Monad m => (a -> m a) -> m a -> Stream m a
435+#ifdef USE_UNFOLDS_EVERYWHERE
436+iterateM step = unfold (Unfold.iterateM step)
437+#else
388438iterateM step = Stream (\_ st -> st >>= \(!x) -> return $ Yield x (step x))
439+#endif
389440
390441{-# INLINE_NORMAL iterate #-}
391442iterate :: Monad m => (a -> a) -> a -> Stream m a
@@ -399,8 +450,12 @@ iterate step st = iterateM (return . step) (return st)
399450-- | Convert a list of monadic actions to a 'Stream'
400451{-# INLINE_LATE fromListM #-}
401452fromListM :: MonadAsync m => [m a] -> Stream m a
453+#ifdef USE_UNFOLDS_EVERYWHERE
454+fromListM = unfold Unfold.fromListM
455+#else
402456fromListM = Stream step
403457 where
404458 {-# INLINE_LATE step #-}
405459 step _ (m:ms) = m >>= \x -> return $ Yield x ms
406460 step _ [] = return Stop
461+#endif
core/src/Streamly/Internal/Data/Stream/StreamD/Type.hs+7
@@ -109,6 +109,9 @@ import Streamly.Internal.Data.Unfold.Type (Unfold(..))
109109
110110import qualified Streamly.Internal.Data.Fold.Type as FL
111111import qualified Streamly.Internal.Data.Stream.StreamK.Type as K
112+#ifdef USE_UNFOLDS_EVERYWHERE
113+import qualified Streamly.Internal.Data.Unfold.Type as Unfold
114+#endif
112115
113116------------------------------------------------------------------------------
114117-- The direct style stream type
@@ -222,11 +225,15 @@ fromEffect m = Stream step True
222225-- | Convert a list of pure values to a 'Stream'
223226{-# INLINE_LATE fromList #-}
224227fromList :: Applicative m => [a] -> Stream m a
228+#ifdef USE_UNFOLDS_EVERYWHERE
229+fromList = unfold Unfold.fromList
230+#else
225231fromList = Stream step
226232 where
227233 {-# INLINE_LATE step #-}
228234 step _ (x:xs) = pure $ Yield x xs
229235 step _ [] = pure Stop
236+#endif
230237
231238------------------------------------------------------------------------------
232239-- Conversions From/To
core/src/Streamly/Internal/Data/Unfold.hs−33
@@ -177,7 +177,6 @@ module Streamly.Internal.Data.Unfold
177177 , fromList
178178 , fromListM
179179
180− , fromStream
181180 , fromStreamK
182181 , fromStreamD
183182
@@ -263,7 +262,6 @@ import Streamly.Internal.Control.Concurrent (MonadRunInIO, MonadAsync, withRunIn
263262import Streamly.Internal.Data.Fold.Type (Fold(..))
264263import Streamly.Internal.Data.IOFinalizer
265264 (newIOFinalizer, runIOFinalizer, clearingIOFinalizer)
266−import Streamly.Internal.Data.Stream.Serial (SerialT(..))
267265import Streamly.Internal.Data.Stream.StreamD.Type (Stream(..), Step(..))
268266import Streamly.Internal.Data.SVar.Type (defState)
269267
@@ -602,20 +600,6 @@ fromStreamK = Unfold step pure
602600 Just (x, xs) -> Yield x xs
603601 Nothing -> Stop) <$> K.uncons stream
604602
605−-- XXX Using Unfold.fromStreamD seems to be faster (using cross product test
606−-- case) than using fromStream even if it is implemented using fromStreamD.
607−-- Check if StreamK to StreamD rewrite rules are working correctly when
608−-- implementing fromStream using fromStreamD.
609−--
610−-- | Convert a stream into an 'Unfold'. Note that a stream converted to an
611−-- 'Unfold' may not be as efficient as an 'Unfold' in some situations.
612−--
613−-- /Since: 0.8.0/
614−--
615−{-# INLINE_NORMAL fromStream #-}
616−fromStream :: Applicative m => Unfold m (SerialT m a) a
617−fromStream = lmap getSerialT fromStreamK
618−
619603-------------------------------------------------------------------------------
620604-- Unfolds
621605-------------------------------------------------------------------------------
@@ -652,23 +636,6 @@ consM action unf = Unfold step inject
652636 Skip s -> Skip (Right (Stream step1 s))
653637 Stop -> Stop) <$> step1 defState st
654638
655−-- XXX Check if "unfold (fromList [1..10])" fuses, if it doesn't we can use
656−-- rewrite rules to rewrite list enumerations to unfold enumerations.
657−--
658−-- | Convert a list of pure values to a 'Stream'
659−--
660−-- /Since: 0.8.0/
661−--
662−{-# INLINE_LATE fromList #-}
663−fromList :: Applicative m => Unfold m [a] a
664−fromList = Unfold step pure
665−
666− where
667−
668− {-# INLINE_LATE step #-}
669− step (x:xs) = pure $ Yield x xs
670− step [] = pure Stop
671−
672639-- | Convert a list of monadic values to a 'Stream'
673640--
674641-- /Since: 0.8.0/
core/src/Streamly/Internal/Data/Unfold/Type.hs+20
@@ -28,6 +28,9 @@ module Streamly.Internal.Data.Unfold.Type
2828 , fromEffect
2929 , fromPure
3030
31+ -- * From Containers
32+ , fromList
33+
3134 -- * Transformations
3235 , lmap
3336 , lmapM
@@ -335,6 +338,23 @@ fromEffect m = Unfold step inject
335338fromPure :: Applicative m => b -> Unfold m a b
336339fromPure = fromEffect Prelude.. pure
337340
341+-- XXX Check if "unfold (fromList [1..10])" fuses, if it doesn't we can use
342+-- rewrite rules to rewrite list enumerations to unfold enumerations.
343+
344+-- | Convert a list of pure values to a 'Stream'
345+--
346+-- /Since: 0.8.0/
347+--
348+{-# INLINE_LATE fromList #-}
349+fromList :: Applicative m => Unfold m [a] a
350+fromList = Unfold step pure
351+
352+ where
353+
354+ {-# INLINE_LATE step #-}
355+ step (x:xs) = pure $ Yield x xs
356+ step [] = pure Stop
357+
338358-- | Outer product discarding the first element.
339359--
340360-- /Unimplemented/
core/streamly-core.cabal+8
@@ -87,6 +87,11 @@ flag use-storable
8787 manual: True
8888 default: False
8989
90+flag use-unfolds
91+ description: Use unfolds for generation everywhere
92+ manual: True
93+ default: False
94+
9095-------------------------------------------------------------------------------
9196-- Common stanzas
9297-------------------------------------------------------------------------------
@@ -103,6 +108,9 @@ common compile-options
103108 if flag(dev)
104109 cpp-options: -DDEVBUILD
105110
111+ if flag(use-unfolds)
112+ cpp-options: -DUSE_UNFOLDS_EVERYWHERE
113+
106114 if flag(use-c-malloc)
107115 cpp-options: -DUSE_C_MALLOC
108116
src/Streamly/Data/Unfold.hs+7−2
@@ -150,11 +150,16 @@ import Prelude hiding
150150 , zipWith
151151 )
152152import Streamly.Internal.Data.Stream.IsStream.Type (IsStream)
153−import Streamly.Internal.Data.Unfold hiding (fromStream)
153+import Streamly.Internal.Data.Unfold
154154
155155import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream
156156import qualified Streamly.Internal.Data.Unfold as Unfold
157157
158+-- XXX Using Unfold.fromStreamD seems to be faster (using cross product test
159+-- case) than using fromStream even if it is implemented using fromStreamD.
160+-- Check if StreamK to StreamD rewrite rules are working correctly when
161+-- implementing fromStream using fromStreamD.
162+
158163-- | Convert a stream into an 'Unfold'. Note that a stream converted to an
159164-- 'Unfold' may not be as efficient as an 'Unfold' in some situations.
160165--
@@ -162,4 +167,4 @@ import qualified Streamly.Internal.Data.Unfold as Unfold
162167--
163168{-# INLINE_NORMAL fromStream #-}
164169fromStream :: (IsStream t, Applicative m) => Unfold m (t m a) a
165−fromStream = lmap IsStream.adapt Unfold.fromStream
170+fromStream = lmap IsStream.toStream Unfold.fromStreamK
src/Streamly/Internal/Data/Stream/IsStream/Reduce.hs+6−2
@@ -180,6 +180,7 @@ import Streamly.Internal.Data.Time.Units
180180 , toAbsTime)
181181
182182import qualified Data.Heap as H
183+import qualified Streamly.Data.Unfold as Unfold
183184import qualified Streamly.Internal.Data.Array.Foreign.Type as A
184185import qualified Streamly.Internal.Data.Fold as FL
185186import qualified Streamly.Internal.Data.IsMap as IsMap
@@ -188,7 +189,6 @@ import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream
188189import qualified Streamly.Internal.Data.Stream.StreamD as D
189190import qualified Streamly.Internal.Data.Stream.IsStream.Expand as Expand
190191import qualified Streamly.Internal.Data.Stream.IsStream.Transform as Transform
191−import qualified Streamly.Internal.Data.Unfold as Unfold
192192
193193import Prelude hiding (concatMap, map)
194194
@@ -1287,6 +1287,10 @@ ejectExpired reset ejectPred extract session@SessionState{..} curTime = do
12871287 assert (IsMap.mapNull mp) (return ())
12881288 return (hp, mp, out, cnt)
12891289
1290+{-# INLINE readSerial #-}
1291+readSerial :: Applicative m => Unfold.Unfold m (IsStream.SerialT m a) a
1292+readSerial = Unfold.fromStream
1293+
12901294-- XXX Use mutable IORef in accumulator
12911295{-# INLINE classifySessionsByGeneric #-}
12921296classifySessionsByGeneric
@@ -1301,7 +1305,7 @@ classifySessionsByGeneric
13011305 -> t m (Key f, b) -- ^ session key, fold result
13021306classifySessionsByGeneric _ tick reset ejectPred tmout
13031307 (Fold step initial extract) input =
1304− Expand.unfoldMany (Unfold.lmap sessionOutputStream Unfold.fromStream)
1308+ Expand.unfoldMany (Unfold.lmap sessionOutputStream readSerial)
13051309 $ scanlMAfter' sstep (return szero) (flush extract)
13061310 $ interjectSuffix tick (return Nothing)
13071311 $ map Just input
test/Streamly/Test/Data/Unfold.hs+1
@@ -13,6 +13,7 @@ import Streamly.Internal.Data.Unfold (Unfold)
1313
1414import qualified Data.List as List
1515import qualified Prelude
16+import qualified Streamly.Data.Unfold as UF
1617import qualified Streamly.Internal.Data.Fold as Fold
1718import qualified Streamly.Internal.Data.Unfold as UF
1819import qualified Streamly.Internal.Data.Stream.IsStream as S