on master help sign in
Make some updates to takeByTime/dropByTime

* Document the precise behavior, some changes were made to the earlier behavior * Make some changes to implementation according to (newly) documented behavior * TakeByTime: perform the time check before generating the element so that we do not drop an element after generation. * TakeByTime now yields at least one element if the duration is non-zero * dropByTime does not check the time after the drop duration is over * Add inspection tests * make the tests for shorter duration, earlier tests took too long

commit290d2f613509f1a71cd8eaeebcf1ce27426f6f1c authorHarendra Kumar <harendra.kumar@gmail.com> dateMon 13 Jan 2020, 01:13 (7 years ago) parent1979370
5 files changed+183−107
benchmark/Linear.hs+8−5 benchmark/Streamly/Benchmark/Prelude.hs+18−8 src/Streamly/Internal/Data/Stream/StreamD.hs+72−42 src/Streamly/Internal/Prelude.hs+40−26 test/Streamly/Test/Internal/Prelude.hs+45−26
benchmark/Linear.hs+8−5
@@ -356,19 +356,22 @@ main = do
356356 [ benchIOSink value "filter-even" (Ops.filterEven 1)
357357 , benchIOSink value "filter-all-out" (Ops.filterAllOut value 1)
358358 , benchIOSink value "filter-all-in" (Ops.filterAllIn value 1)
359+
359360 , benchIOSink value "take-all" (Ops.takeAll value 1)
360−
361− -- XXX Compare this to takeAll
362− , benchIOSink value "takeByTime: All" (Ops.takeByTime (NanoSecond64 maxBound) 1)
363− , benchIOSink value "dropByTime: All" (Ops.dropByTime (NanoSecond64 maxBound) 1)
364−
361+ , benchIOSink value "takeByTime-all"
362+ (Ops.takeByTime (NanoSecond64 maxBound) 1)
365363 , benchIOSink value "takeWhile-true" (Ops.takeWhileTrue value 1)
366364 --, benchIOSink value "takeWhileM-true" (Ops.takeWhileMTrue 1)
365+
366+ -- "drop-one" is dual to "last"
367367 , benchIOSink value "drop-one" (Ops.dropOne 1)
368368 , benchIOSink value "drop-all" (Ops.dropAll value 1)
369+ , benchIOSink value "dropByTime-all"
370+ (Ops.dropByTime (NanoSecond64 maxBound) 1)
369371 , benchIOSink value "dropWhile-true" (Ops.dropWhileTrue value 1)
370372 --, benchIOSink value "dropWhileM-true" (Ops.dropWhileMTrue 1)
371373 , benchIOSink value "dropWhile-false" (Ops.dropWhileFalse value 1)
374+
372375 , benchIOSink value "deleteBy" (Ops.deleteBy value 1)
373376 , benchIOSink value "intersperse" (Ops.intersperse value 1)
374377 , benchIOSink value "insertBy" (Ops.insertBy value 1)
benchmark/Streamly/Benchmark/Prelude.hs+18−8
@@ -383,14 +383,6 @@ filterAllOut,
383383mapMaybeM :: S.MonadAsync m => Int -> Stream m Int -> m ()
384384intersperse :: S.MonadAsync m => Int -> Int -> Stream m Int -> m ()
385385
386−{-# INLINE takeByTime #-}
387−takeByTime :: (MonadIO m, TimeUnit64 t) => t -> Int -> Stream m Int -> m ()
388−takeByTime i n = composeN n (Internal.takeByTime i)
389−
390−{-# INLINE dropByTime #-}
391−dropByTime :: (MonadIO m, TimeUnit64 t) => t -> Int -> Stream m Int -> m ()
392−dropByTime i n = composeN n (Internal.dropByTime i)
393−
394386{-# INLINE mapM #-}
395387{-# INLINE map' #-}
396388{-# INLINE fmap' #-}
@@ -465,6 +457,24 @@ foldrSMap n = composeN n $ Internal.foldrS (\x xs -> x + 1 `S.cons` xs) S.n
465457foldrT n = composeN n $ Internal.foldrT S.cons S.nil
466458foldrTMap n = composeN n $ Internal.foldrT (\x xs -> x + 1 `S.cons` xs) S.nil
467459
460+{-# INLINE takeByTime #-}
461+takeByTime :: NanoSecond64 -> Int -> Stream IO Int -> IO ()
462+takeByTime i n = composeN n (Internal.takeByTime i)
463+
464+#ifdef INSPECTION
465+inspect $ hasNoTypeClasses 'takeByTime
466+-- inspect $ 'takeByTime `hasNoType` ''D.Step
467+#endif
468+
469+{-# INLINE dropByTime #-}
470+dropByTime :: NanoSecond64 -> Int -> Stream IO Int -> IO ()
471+dropByTime i n = composeN n (Internal.dropByTime i)
472+
473+#ifdef INSPECTION
474+inspect $ hasNoTypeClasses 'dropByTime
475+-- inspect $ 'dropByTime `hasNoType` ''D.Step
476+#endif
477+
468478-------------------------------------------------------------------------------
469479-- Pipes
470480-------------------------------------------------------------------------------
src/Streamly/Internal/Data/Stream/StreamD.hs+72−42
@@ -347,6 +347,7 @@ import qualified Prelude
347347import Streamly.Internal.Mutable.Prim.Var
348348 (Prim, Var, readVar, newVar, modifyVar')
349349import Streamly.Internal.Data.Time.Units
350+ (TimeUnit64, toRelTime64, diffAbsTime64)
350351
351352import Streamly.Internal.Data.Atomics (atomicModifyIORefCAS_)
352353import Streamly.Internal.Memory.Array.Types (Array(..))
@@ -4045,51 +4046,80 @@ lastN n = Fold step initial done
40454046-- Time related
40464047------------------------------------------------------------------------------
40474048
4048−-- XXX This does not fuse. Removing the if function results in fusion though.
4049−-- XXX This function will fuse with the use of fusion-plugin.
4050−-- XXX This function also fuses if compiled with -funfolding-use-threshold=500
4049+-- XXX using getTime in the loop can be pretty expensive especially for
4050+-- computations where iterations are lightweight. We have the following
4051+-- options:
4052+--
4053+-- 1) Run a timeout thread updating a flag asynchronously and check that
4054+-- flag here, that way we can have a cheap termination check.
4055+--
4056+-- 2) Use COARSE clock to get time with lower resolution but more efficiently.
4057+--
4058+-- 3) Use rdtscp/rdtsc to get time directly from the processor, compute the
4059+-- termination value of rdtsc in the beginning and then in each iteration just
4060+-- get rdtsc and check if we should terminate.
4061+--
4062+data TakeByTime st s
4063+ = TakeByTimeInit st
4064+ | TakeByTimeCheck st s
4065+ | TakeByTimeYield st s
4066+
40514067{-# INLINE_NORMAL takeByTime #-}
40524068takeByTime :: (MonadIO m, TimeUnit64 t) => t -> Stream m a -> Stream m a
4053−takeByTime lim' (Stream step state) = Stream step' (state, Nothing)
4054− where
4055− lim = toRelTime64 lim'
4056− {-# INLINE_LATE step' #-}
4057− step' _ (s, Nothing) = do
4069+takeByTime duration (Stream step1 state1) = Stream step (TakeByTimeInit state1)
4070+ where
4071+
4072+ lim = toRelTime64 duration
4073+
4074+ {-# INLINE_LATE step #-}
4075+ step _ (TakeByTimeInit _) | lim == 0 = return Stop
4076+ step _ (TakeByTimeInit st) = do
4077+ t0 <- liftIO $ getTime Monotonic
4078+ return $ Skip (TakeByTimeYield st t0)
4079+ step _ (TakeByTimeCheck st t0) = do
40584080 t <- liftIO $ getTime Monotonic
4059− return $ Skip (s, Just t)
4060− step' gst (s, jt@(Just t)) = do
4061− ns <- step gst s
4062− t' <- liftIO $ getTime Monotonic
4063− if diffAbsTime64 t' t > lim
4064− then return Stop
4065− else return $ case ns of
4066− Yield a s' -> Yield a (s', jt)
4067− Skip s' -> Skip (s', jt)
4068− Stop -> Stop
4069−
4070−-- XXX This does not fuse. Removing the if function results in fusion though.
4071−-- XXX This function will fuse with the use of fusion-plugin.
4072−-- XXX This function also fuses if compiled with -funfolding-use-threshold=500
4081+ return $
4082+ if diffAbsTime64 t t0 > lim
4083+ then Stop
4084+ else Skip (TakeByTimeYield st t0)
4085+ step gst (TakeByTimeYield st t0) = do
4086+ r <- step1 gst st
4087+ return $ case r of
4088+ Yield x s -> Yield x (TakeByTimeCheck s t0)
4089+ Skip s -> Skip (TakeByTimeCheck s t0)
4090+ Stop -> Stop
4091+
4092+data DropByTime st s x
4093+ = DropByTimeInit st
4094+ | DropByTimeGen st s
4095+ | DropByTimeCheck st s x
4096+ | DropByTimeYield st
4097+
40734098{-# INLINE_NORMAL dropByTime #-}
40744099dropByTime :: (MonadIO m, TimeUnit64 t) => t -> Stream m a -> Stream m a
4075−dropByTime lim' (Stream step state) = Stream step' (state, Nothing)
4076− where
4077− lim = toRelTime64 lim'
4078− {-# INLINE_LATE step' #-}
4079− step' _ (s, Nothing) = do
4100+dropByTime duration (Stream step1 state1) = Stream step (DropByTimeInit state1)
4101+ where
4102+
4103+ lim = toRelTime64 duration
4104+
4105+ {-# INLINE_LATE step #-}
4106+ step _ (DropByTimeInit st) = do
4107+ t0 <- liftIO $ getTime Monotonic
4108+ return $ Skip (DropByTimeGen st t0)
4109+ step gst (DropByTimeGen st t0) = do
4110+ r <- step1 gst st
4111+ return $ case r of
4112+ Yield x s -> Skip (DropByTimeCheck s t0 x)
4113+ Skip s -> Skip (DropByTimeGen s t0)
4114+ Stop -> Stop
4115+ step _ (DropByTimeCheck st t0 x) = do
40804116 t <- liftIO $ getTime Monotonic
4081− return $ Skip (s, Just t)
4082− step' gst (s, jt@(Just t)) = do
4083− t' <- liftIO $ getTime Monotonic
4084− ns <- step gst s
4085− if diffAbsTime64 t' t <= lim
4086− then return $
4087− case ns of
4088− Yield _ s' -> Skip (s', jt)
4089− Skip s' -> Skip (s', jt)
4090− Stop -> Stop
4091− else return $
4092− case ns of
4093− Yield a s' -> Yield a (s', jt)
4094− Skip s' -> Skip (s', jt)
4095− Stop -> Stop
4117+ if diffAbsTime64 t t0 <= lim
4118+ then return $ Skip $ DropByTimeGen st t0
4119+ else return $ Yield x $ DropByTimeYield st
4120+ step gst (DropByTimeYield st) = do
4121+ r <- step1 gst st
4122+ return $ case r of
4123+ Yield x s -> Yield x (DropByTimeYield s)
4124+ Skip s -> Skip (DropByTimeYield s)
4125+ Stop -> Stop
src/Streamly/Internal/Prelude.hs+40−26
@@ -185,11 +185,13 @@ module Streamly.Internal.Prelude
185185
186186 -- ** Stateful Filters
187187 , take
188+ , takeByTime
188189 -- , takeEnd
189190 , takeWhile
190191 , takeWhileM
191192 -- , takeWhileEnd
192193 , drop
194+ , dropByTime
193195 -- , dropEnd
194196 , dropWhile
195197 , dropWhileM
@@ -413,10 +415,6 @@ module Streamly.Internal.Prelude
413415 -- * Diagnostics
414416 , inspectMode
415417
416− -- * Time related
417− , takeByTime
418− , dropByTime
419−
420418 -- * Deprecated
421419 , K.once
422420 , each
@@ -1876,6 +1874,26 @@ takeWhile p m = fromStreamS $ S.takeWhile p $ toStreamS m
18761874takeWhileM :: (IsStream t, Monad m) => (a -> m Bool) -> t m a -> t m a
18771875takeWhileM p m = fromStreamD $ D.takeWhileM p $ toStreamD m
18781876
1877+-- | @takeByTime duration@ yields stream elements upto specified time
1878+-- @duration@. The duration starts when the stream is evaluated for the first
1879+-- time, before the first element is yielded. The time duration is checked
1880+-- before generating each element, if the duration has expired the stream
1881+-- stops.
1882+--
1883+-- The total time taken in executing the stream is guaranteed to be /at least/
1884+-- @duration@, however, because the duration is checked before generating an
1885+-- element, the upper bound is indeterminate and depends on the time taken in
1886+-- generating and processing the last element.
1887+--
1888+-- No element is yielded if the duration is zero. At least one element is
1889+-- yielded if the duration is non-zero.
1890+--
1891+-- /Internal/
1892+--
1893+{-# INLINE takeByTime #-}
1894+takeByTime ::(MonadIO m, IsStream t, TimeUnit64 d) => d -> t m a -> t m a
1895+takeByTime d = fromStreamD . D.takeByTime d . toStreamD
1896+
18791897-- | Discard first 'n' elements from the stream and take the rest.
18801898--
18811899-- @since 0.1.0
@@ -1898,6 +1916,24 @@ dropWhile p m = fromStreamS $ S.dropWhile p $ toStreamS m
18981916dropWhileM :: (IsStream t, Monad m) => (a -> m Bool) -> t m a -> t m a
18991917dropWhileM p m = fromStreamD $ D.dropWhileM p $ toStreamD m
19001918
1919+-- | @dropByTime duration@ drops stream elements until specified @duration@ has
1920+-- passed. The duration begins when the stream is evaluated for the first
1921+-- time. The time duration is checked /after/ generating a stream element, the
1922+-- element is yielded if the duration has expired otherwise it is dropped.
1923+--
1924+-- The time elapsed before starting to generate the first element is /at most/
1925+-- @duration@, however, because the duration expiry is checked after the
1926+-- element is generated, the lower bound is indeterminate and depends on the
1927+-- time taken in generating an element.
1928+--
1929+-- All elements are yielded if the duration is zero.
1930+--
1931+-- /Internal/
1932+--
1933+{-# INLINE dropByTime #-}
1934+dropByTime ::(MonadIO m, IsStream t, TimeUnit64 d) => d -> t m a -> t m a
1935+dropByTime d = fromStreamD . D.dropByTime d . toStreamD
1936+
19011937------------------------------------------------------------------------------
19021938-- Transformation by Mapping
19031939------------------------------------------------------------------------------
@@ -4195,25 +4231,3 @@ usingStateT s f xs = evalStateT s $ f $ liftInner xs
41954231{-# INLINE runStateT #-}
41964232runStateT :: Monad m => s -> SerialT (StateT s m) a -> SerialT m (s, a)
41974233runStateT s xs = fromStreamD $ D.runStateT s (toStreamD xs)
4198−
4199−------------------------------------------------------------------------------
4200−-- Time related
4201−------------------------------------------------------------------------------
4202−
4203−-- | /takeByTime d/ keeps taking elements from the stream until d duration has
4204−-- passed. We first check the if the duration has expired, if yes, then we
4205−-- 'Stop' else we take the element, executing the corresponding monadic
4206−-- action. The only guarentee is, the element will not be taken (the action
4207−-- will not begin) if the duration has expired. If the duration has expired
4208−-- during the execution of the monadic action, the next element will not be
4209−-- taken but the current action will run to completion.
4210−{-# INLINE takeByTime #-}
4211−takeByTime ::(MonadIO m, IsStream t, TimeUnit64 u) => u -> t m a -> t m a
4212−takeByTime d = fromStreamD . D.takeByTime d . toStreamD
4213−
4214−-- | /dropByTime d/ keeps dropping elements from the stream until d duration has
4215−-- passed. The first element is taken only after the duration has expired.
4216−{-# INLINE dropByTime #-}
4217−dropByTime ::(MonadIO m, IsStream t, TimeUnit64 u) => u -> t m a -> t m a
4218−dropByTime d = fromStreamD . D.dropByTime d . toStreamD
4219−
test/Streamly/Test/Internal/Prelude.hs+45−26
@@ -11,17 +11,18 @@
1111--
1212module Main (main) where
1313
14−import Test.Hspec.QuickCheck
15−import Test.QuickCheck (Property, forAll, choose)
16−import Test.QuickCheck.Monadic (monadicIO, assert, run)
14+import Control.Concurrent (threadDelay)
15+import Control.Monad (when)
1716
1817import Test.Hspec as H
1918
2019import qualified Streamly.Prelude as S
20+import qualified Streamly.Internal.Data.Fold as FL
2121import qualified Streamly.Internal.Prelude as SI
2222
2323import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime)
2424import Streamly.Internal.Data.Time.Units
25+ (AbsTime, NanoSecond64(..), toRelTime64, diffAbsTime64)
2526import Data.Int (Int64)
2627
2728tenPow8 :: Int64
@@ -30,30 +31,48 @@ tenPow8 = 10^(8 :: Int)
3031tenPow7 :: Int64
3132tenPow7 = 10^(7 :: Int)
3233
33−testTakeByTime :: Property
34−testTakeByTime =
35− forAll (choose (1, 10)) $ \(n :: Int64) -> monadicIO $ do
36− let lim = NanoSecond64 $ n * tenPow8
37− let graceTime = NanoSecond64 $ 8 * tenPow7
38− t <- run $ getTime Monotonic
39− t'' <- run $ S.last $ SI.takeByTime lim $ S.repeatM (getTime Monotonic)
40− case t'' of
41− Nothing -> assert True
42− Just t' -> assert $ diffAbsTime64 t' t <= toRelTime64 (lim + graceTime)
43−
44−testDropByTime :: Property
45−testDropByTime =
46− forAll (choose (1, 10)) $ \(n :: Int64) -> monadicIO $ do
47− let lim = NanoSecond64 $ n * tenPow8
48− t <- run $ getTime Monotonic
49− _ <- run $ S.drain $ S.take 1 $ SI.dropByTime lim $ S.repeat ()
50− t' <- run $ getTime Monotonic
51− assert $ diffAbsTime64 t' t >= toRelTime64 lim
34+takeDropTime :: NanoSecond64
35+takeDropTime = NanoSecond64 $ 5 * tenPow8
36+
37+checkTakeDropTime :: (Maybe AbsTime, Maybe AbsTime) -> IO Bool
38+checkTakeDropTime (mt0, mt1) = do
39+ let graceTime = NanoSecond64 $ 8 * tenPow7
40+ case mt0 of
41+ Nothing -> return True
42+ Just t0 ->
43+ case mt1 of
44+ Nothing -> return True
45+ Just t1 -> do
46+ let tMax = toRelTime64 (takeDropTime + graceTime)
47+ let tMin = toRelTime64 (takeDropTime - graceTime)
48+ let t = diffAbsTime64 t1 t0
49+ let r = t >= tMin && t <= tMax
50+ when (not r) $ putStrLn $
51+ "t = " ++ show t ++
52+ " tMin = " ++ show tMin ++
53+ " tMax = " ++ show tMax
54+ return r
55+
56+testTakeByTime :: IO Bool
57+testTakeByTime = do
58+ r <-
59+ S.fold ((,) <$> FL.head <*> FL.last)
60+ $ SI.takeByTime takeDropTime
61+ $ S.repeatM (threadDelay 1000 >> getTime Monotonic)
62+ checkTakeDropTime r
63+
64+testDropByTime :: IO Bool
65+testDropByTime = do
66+ t0 <- getTime Monotonic
67+ mt1 <-
68+ S.head
69+ $ SI.dropByTime takeDropTime
70+ $ S.repeatM (threadDelay 1000 >> getTime Monotonic)
71+ checkTakeDropTime (Just t0, mt1)
5272
5373main :: IO ()
5474main =
5575 hspec $
56− describe "Time combinators" $ do
57− prop "takeByTime" testTakeByTime
58− prop "dropByTime" testDropByTime
59−
76+ describe "Filtering" $ do
77+ it "takeByTime" (testTakeByTime `shouldReturn` True)
78+ it "dropByTime" (testDropByTime `shouldReturn` True)