on master help sign in
Simplify the type of resumable Unfold
commitc93bad01342780bbc3951b60bef24a4cab87eafb authorHarendra Kumar <harendra@composewell.com> dateTue 16 Feb 2021, 17:43 (6 years ago) parent22c83f4
7 files changed+67−74
benchmark/Streamly/Benchmark/Data/Parser.hs+13−11 benchmark/Streamly/Benchmark/Data/Parser/ParserD.hs+1−1 src/Streamly/Internal/Data/Array/Foreign/Mut/Types.hs+1−1 src/Streamly/Internal/Data/Unfold/Resume.hs+14−13 src/Streamly/Internal/Data/Unfold/Resume/Type.hs+12−26 src/Streamly/Internal/Data/Unfold/Source.hs+25−21 test/Streamly/Test/Data/Parser/ParserD.hs+1−1
benchmark/Streamly/Benchmark/Data/Parser.hs+13−11
@@ -245,7 +245,7 @@ parseManyChunksOfSum n inh =
245245{-# INLINE parseManyUnfoldArrays #-}
246246parseManyUnfoldArrays :: Int -> [Array.Array Int] -> IO ()
247247parseManyUnfoldArrays count arrays = do
248− let src = Source.source (arrays, Nothing)
248+ let src = Source.source (Just (arrays, Nothing))
249249 let parser = PR.fromFold (FL.takeLE count FL.drain)
250250 let readSrc =
251251 Source.read
@@ -300,19 +300,19 @@ choice value =
300300-------------------------------------------------------------------------------
301301
302302{-# INLINE parseMany #-}
303−parseMany :: MonadCatch m => SerialT m Int -> m ()
304−parseMany =
303+parseMany :: MonadCatch m => Int -> SerialT m Int -> m ()
304+parseMany n =
305305 S.drain
306306 . S.map getSum
307− . IP.parseMany (PR.fromFold $ FL.takeLE 2 FL.mconcat)
307+ . IP.parseMany (PR.fromFold $ FL.takeLE n FL.mconcat)
308308 . S.map Sum
309309
310310{-# INLINE parseIterate #-}
311−parseIterate :: MonadCatch m => SerialT m Int -> m ()
312−parseIterate =
311+parseIterate :: MonadCatch m => Int -> SerialT m Int -> m ()
312+parseIterate n =
313313 S.drain
314314 . S.map getSum
315− . IP.parseIterate (PR.fromFold . FL.takeLE 2 . FL.sconcat) (Sum 0)
315+ . IP.parseIterate (PR.fromFold . FL.takeLE n . FL.sconcat) (Sum 0)
316316 . S.map Sum
317317
318318-------------------------------------------------------------------------------
@@ -345,8 +345,10 @@ o_1_space_serial value =
345345 , benchIOSink value "teeFst" $ teeFstAllAny value
346346 , benchIOSink value "shortest" $ shortestAllAny value
347347 , benchIOSink value "longest" $ longestAllAny value
348− , benchIOSink value "parseMany" parseMany
349− , benchIOSink value "parseIterate" parseIterate
348+ , benchIOSink value "parseMany (take 1)" (parseMany 1)
349+ , benchIOSink value "parseMany (take all)" (parseMany value)
350+ , benchIOSink value "parseIterate (take 1)" (parseIterate 1)
351+ , benchIOSink value "parseIterate (take all)" (parseIterate value)
350352 ]
351353
352354o_1_space_filesystem :: BenchEnv -> [Benchmark]
@@ -359,9 +361,9 @@ o_1_space_filesystem env =
359361
360362o_1_space_serial_unfold :: Int -> [Array.Array Int] -> [Benchmark]
361363o_1_space_serial_unfold bound arrays =
362− [ bench "parseMany/Unfold/1000 arrays/1 parse"
364+ [ bench "parseMany/Unfold/1000 arrays/take all"
363365 $ nfIO $ parseManyUnfoldArrays bound arrays
364− , bench "parseMany/Unfold/1000 arrays/100000 parses"
366+ , bench "parseMany/Unfold/1000 arrays/take 1"
365367 $ nfIO $ parseManyUnfoldArrays 1 arrays
366368 ]
367369
benchmark/Streamly/Benchmark/Data/Parser/ParserD.hs+1−1
@@ -219,7 +219,7 @@ parseManyGroupsRolling b =
219219{-# INLINE parseManyUnfoldArrays #-}
220220parseManyUnfoldArrays :: Int -> [Array.Array Int] -> IO ()
221221parseManyUnfoldArrays count arrays = do
222− let src = Source.source (arrays, Nothing)
222+ let src = Source.source (Just (arrays, Nothing))
223223 let parser = PR.fromFold (FL.takeLE count FL.drain)
224224 let readSrc =
225225 Source.read
src/Streamly/Internal/Data/Array/Foreign/Mut/Types.hs+1−1
@@ -527,7 +527,7 @@ readResumable = UnfoldR.Unfold step inject extract
527527 (ReadUState fp (p `plusPtr` sizeOf (undefined :: a)))
528528
529529 extract (ReadUState (ForeignPtr end contents) (Ptr p)) =
530− return $ Just $ Array (ForeignPtr p contents) (Ptr end) (Ptr end)
530+ return $ Array (ForeignPtr p contents) (Ptr end) (Ptr end)
531531
532532-- | Unfold an array into a stream.
533533--
src/Streamly/Internal/Data/Unfold/Resume.hs+14−13
@@ -89,7 +89,8 @@ simplify (Unfold step inject _) = Unfold.Unfold step inject
8989-- /Internal/
9090{-# INLINE_NORMAL fromStreamD #-}
9191fromStreamD :: Monad m => Unfold m (Stream m a) a
92−fromStreamD = Unfold step return (return . Just)
92+fromStreamD = Unfold step return return
93+
9394 where
9495
9596 {-# INLINE_LATE step #-}
@@ -115,7 +116,7 @@ parseD
115116 => ParserD.Parser m a b
116117 -> Unfold m (Source s a) a
117118 -> Source s a
118− -> m (b, Maybe (Source s a))
119+ -> m (b, Source s a)
119120parseD
120121 (ParserD.Parser pstep initial extract)
121122 (Unfold ustep uinject uextract)
@@ -153,13 +154,13 @@ parseD
153154 let src0 = Prelude.take n (x:getList buf)
154155 src = Prelude.reverse src0
155156 s1 <- uextract s
156− return (b, fmap (Source.unread src) s1)
157+ return (b, Source.unread src s1)
157158 Error err -> throwM $ ParseError err
158159 Skip s -> go SPEC s buf pst
159160 Stop -> do
160161 b <- extract pst
161162 -- XXX we should return the remaining buffer
162− return (b, Nothing)
163+ return (b, Source.source Nothing)
163164
164165 gobuf !_ s buf (List []) !pst = go SPEC s buf pst
165166 gobuf !_ s buf (List (x:xs)) !pst = do
@@ -184,7 +185,7 @@ parseD
184185 let src0 = Prelude.take n (x:getList buf)
185186 src = Prelude.reverse src0
186187 s1 <- uextract s
187− return (b, fmap (Source.unread src) s1)
188+ return (b, Source.unread src s1)
188189 Error err -> throwM $ ParseError err
189190
190191-- | Parse a resumable unfold returning the parsed value and the state of the
@@ -197,7 +198,7 @@ parse
197198 => ParserK.Parser m a b
198199 -> Unfold m (Source s a) a
199200 -> Source s a
200− -> m (b, Maybe (Source s a))
201+ -> m (b, Source s a)
201202parse = parseD . ParserK.fromParserK
202203
203204-------------------------------------------------------------------------------
@@ -207,17 +208,17 @@ parse = parseD . ParserK.fromParserK
207208{-# INLINE parseManyD #-}
208209parseManyD :: MonadThrow m =>
209210 ParserD.Parser m a b -> Unfold m (Source x a) a -> Unfold m (Source x a) b
210−parseManyD parser reader = Unfold step inject return
211+parseManyD parser reader = Unfold step return return
211212
212213 where
213214
214− inject = return . Just
215−
216215 {-# INLINE_LATE step #-}
217− step (Just src) = do
218− (b, s1) <- parseD parser reader src
219− return $ Yield b s1
220− step Nothing = return Stop
216+ step src = do
217+ if Source.isEmpty src
218+ then return Stop
219+ else do
220+ (b, s1) <- parseD parser reader src
221+ return $ Yield b s1
221222
222223-- | Apply a parser repeatedly on an unfold to generate an unfold of parsed
223224-- values.
src/Streamly/Internal/Data/Unfold/Resume/Type.hs+12−26
@@ -46,22 +46,15 @@ import Prelude hiding (concat)
4646
4747data Unfold m a b =
4848 -- | @Unfold step inject extract@
49− forall s. Unfold (s -> m (Step s b)) (a -> m s) (s -> m (Maybe a))
49+ forall s. Unfold (s -> m (Step s b)) (a -> m s) (s -> m a)
5050
5151------------------------------------------------------------------------------
5252-- Unfolds
5353------------------------------------------------------------------------------
5454
55−-- XXX we do not need a Maybe, we can just use undefined or error. There is no
56−-- state to extract in this case, and it should not be used.
57−--
58−-- However, in certain cases we may not be able to extract, e.g. if we are
59−-- unfolding a Map, we may not be able to resume reading a Map. In such cases
60−-- it may be useful to return a Nothing. Or maybe we do not allow unfolding
61−-- such structures?
6255{-# INLINE nilM #-}
6356nilM :: Monad m => (a -> m c) -> Unfold m a b
64−nilM f = Unfold step return (\_ -> return Nothing)
57+nilM f = Unfold step return return
6558
6659 where
6760
@@ -74,26 +67,26 @@ nil = nilM (\_ -> return ())
7467
7568{-# INLINE unfoldrM #-}
7669unfoldrM :: Monad m => (a -> m (Maybe (b, a))) -> Unfold m a b
77−unfoldrM next = Unfold step return (return . Just)
70+unfoldrM next = Unfold step return return
7871 where
7972 {-# INLINE_LATE step #-}
8073 step st = do
8174 r <- next st
8275 return $ case r of
8376 Just (x, s) -> Yield x s
84− Nothing -> Stop
77+ Nothing -> Stop
8578
8679-- | Convert a list of pure values to a 'Stream'
8780--
8881-- /Internal/
8982{-# INLINE_LATE fromList #-}
9083fromList :: Monad m => Unfold m [a] a
91−fromList = Unfold step inject (return . Just)
84+fromList = Unfold step inject return
9285 where
9386 inject = return
9487 {-# INLINE_LATE step #-}
9588 step (x:xs) = return $ Yield x xs
96− step [] = return Stop
89+ step [] = return Stop
9790
9891------------------------------------------------------------------------------
9992-- Mapping
@@ -106,7 +99,7 @@ fromList = Unfold step inject (return . Just)
10699{-# INLINE_NORMAL lmap #-}
107100lmap :: Functor m => (a -> c) -> (c -> a) -> Unfold m c b -> Unfold m a b
108101lmap f g (Unfold ustep uinject uextract) =
109− Unfold ustep (uinject . f) (fmap (fmap g) . uextract)
102+ Unfold ustep (uinject . f) (fmap g . uextract)
110103
111104------------------------------------------------------------------------------
112105-- Nesting
@@ -115,6 +108,8 @@ lmap f g (Unfold ustep uinject uextract) =
115108{-# ANN type ConcatState Fuse #-}
116109data ConcatState s1 s2 = ConcatOuter s1 | ConcatInner s1 s2
117110
111+-- XXX Use Either a (a,b) for extracted state.
112+--
118113-- | Apply the second unfold to each output element of the first unfold and
119114-- flatten the output in a single stream.
120115--
@@ -152,17 +147,8 @@ concat (Unfold step1 inject1 extract1) (Unfold step2 inject2 extract2) =
152147 Skip s -> Skip (ConcatInner ost s)
153148 Stop -> Skip (ConcatOuter ost)
154149
155− extract (ConcatOuter s1) = do
156− r <- extract1 s1
157− return $ case r of
158− Nothing -> Nothing
159− Just a -> Just (a, Nothing)
150+ extract (ConcatOuter s1) = (,Nothing) <$> extract1 s1
160151 extract (ConcatInner s1 s2) = do
161152 r1 <- extract1 s1
162− case r1 of
163− Nothing -> return Nothing
164− Just a -> do
165− r2 <- extract2 s2
166− return $ case r2 of
167− Nothing -> Just (a, Nothing)
168− Just b -> Just (a, Just b)
153+ r2 <- extract2 s2
154+ return (r1, Just r2)
src/Streamly/Internal/Data/Unfold/Source.hs+25−21
@@ -17,9 +17,12 @@ module Streamly.Internal.Data.Unfold.Source
1717 -- * Creation
1818 , source
1919
20− -- * Reading
21− , read
20+ -- * Transformation
2221 , unread
22+
23+ -- * Consumption
24+ , isEmpty
25+ , read
2326 )
2427where
2528
@@ -32,17 +35,28 @@ import Prelude hiding (read)
3235-- | An unfold seed with some buffered data. It allows us to 'unread' or return
3336-- some data after reading it. Useful in backtracked parsing.
3437--
35−data Source a b = Source [b] a
38+data Source a b = Source [b] (Maybe a)
3639
3740-- | Make a source from a seed value. The buffer would start as empty. You can
3841-- use 'unread' to add to the buffer.
3942--
4043-- /Internal/
41−source :: a -> Source a b
44+source :: Maybe a -> Source a b
4245source = Source []
4346
47+
48+-- | Return some unused data back to the source. The data is prepended (or
49+-- consed) to the source.
50+--
51+-- /Internal/
52+unread :: [b] -> Source a b -> Source a b
53+unread xs (Source ys seed) = Source (xs ++ ys) seed
54+isEmpty :: Source a b -> Bool
55+isEmpty (Source [] Nothing) = True
56+isEmpty _ = False
57+
4458-- | An unfold to read from a 'Source'. If the buffer has data then it is read
45−-- first otherwise the seed is unfolded using the supplied unfold.
59+-- first and then the seed is unfolded using the supplied unfold.
4660--
4761-- /Internal/
4862{-# INLINE_NORMAL read #-}
@@ -51,7 +65,7 @@ read (Unfold step1 inject1 extract1) = Unfold step inject extract
5165
5266 where
5367
54− inject (Source [] a) = do
68+ inject (Source [] (Just a)) = do
5569 s <- inject1 a
5670 return $ Left s
5771 inject (Source xs a) = return $ Right (xs, a)
@@ -63,22 +77,12 @@ read (Unfold step1 inject1 extract1) = Unfold step inject extract
6377 Yield x s1 -> Yield x (Left s1)
6478 Skip s1 -> Skip (Left s1)
6579 Stop -> Stop
66− step (Right ([], _)) = error "Bug: unreachable"
67− step (Right (x:[], a)) = do
80+ step (Right ([], Nothing)) = return Stop
81+ step (Right ([], Just _)) = error "Bug: unreachable"
82+ step (Right (x:[], Just a)) = do
6883 s <- inject1 a
6984 return $ Yield x (Left s)
7085 step (Right (x:xs, a)) = return $ Yield x (Right (xs, a))
7186
72− extract (Left s) = do
73− r <- extract1 s
74− return $ case r of
75− Nothing -> Nothing
76− Just x -> Just $ Source [] x
77− extract (Right (xs, a)) = return $ Just $ Source xs a
78−
79−-- | Return some unused data back to the source. The data is prepended (or
80−-- consed) to the source.
81−--
82−-- /Internal/
83−unread :: [b] -> Source a b -> Source a b
84−unread xs (Source ys seed) = Source (xs ++ ys) seed
87+ extract (Left s) = Source [] . Just <$> extract1 s
88+ extract (Right (xs, a)) = return $ Source xs a
test/Streamly/Test/Data/Parser/ParserD.hs+1−1
@@ -621,7 +621,7 @@ parseUnfold = do
621621 <*> chooseInt (1, len)) $ \(ls, clen, tlen) ->
622622 monadicIO $ do
623623 arrays <- S.toList $ S.arraysOf clen (S.fromList ls)
624− let src = Source.source (arrays, Nothing)
624+ let src = Source.source (Just (arrays, Nothing))
625625 let parser = P.fromFold (FL.takeLE tlen FL.toList)
626626 let readSrc =
627627 Source.read