e81d72f · streamly · git.iko.soy

git.iko.soy/streamly

Allow "Continue 0" case in parser extract

It can happen if the first parser in an alternative does not receive sufficient input but the next parser can work even without input.

commite81d72f3304cabc9d79bf42cc91747af7ed50939 authorHarendra Kumar <harendra@composewell.com> dateThu 27 Oct 2022, 03:26 (4 years ago) parent5fa86e8

8 files changed+36−25

compare with master

unifiedsplit

core/src/Streamly/Internal/Data/Array/Unboxed/Stream.hs+6−5 core/src/Streamly/Internal/Data/Parser/ParserD/Type.hs+12−4 core/src/Streamly/Internal/Data/Parser/ParserK/Type.hs+7−7 core/src/Streamly/Internal/Data/Producer/Source.hs+2−2 core/src/Streamly/Internal/Data/Stream/Eliminate.hs+2 core/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs+2−2 core/src/Streamly/Internal/Data/Stream/StreamD/Nesting.hs+4−4 core/src/Streamly/Internal/Data/Stream/StreamK.hs+1−1

core/src/Streamly/Internal/Data/Array/Unboxed/Stream.hs+6−5

@@ -744,8 +744,8 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

744744 pRes <- extract pst

745745 case pRes of

746746 PR.Partial _ _ -> error "Bug: parseBreak: Partial in extract"

747− PR.Continue 0 _ ->

748− error "parseBreak: extract, Continue 0 creates infinite loop"

747+ PR.Continue 0 s ->

748+ goStop s backBuf

749749 PR.Continue n s -> do

750750 assert (n <= Prelude.length backBuf) (return ())

751751 let (src0, buf1) = splitAt n backBuf

@@ -897,8 +897,8 @@ runArrayParserDBreak

897897 pRes <- extract pst

898898 case pRes of

899899 PR.Partial _ _ -> error "Bug: runArrayParserDBreak: Partial in extract"

900− PR.Continue 0 _ ->

901− error "runArrayParserDBreak: extract, Continue 0 creates infinite loop"

900+ PR.Continue 0 pst1 ->

901+ goStop backBuf pst1

902902 PR.Continue n pst1 -> do

903903 assert

904904 (n <= sum (map Array.length (getList backBuf)))

@@ -1123,7 +1123,8 @@ runArrayFoldManyD

11231123 pRes <- extract pst

11241124 case pRes of

11251125 PR.Partial _ _ -> error "runArrayFoldManyD: Partial in extract"

1126− PR.Continue 0 _ -> error "runArrayFoldManyD: Continue 0 in extract"

1126+ PR.Continue 0 pst1 ->

1127+ return $ D.Skip $ ParseChunksStop backBuf pst1

11271128 PR.Continue n pst1 -> do

11281129 assert (n <= sum (map Array.length backBuf)) (return ())

11291130 let (src0, buf1) = splitAtArrayListRev n backBuf

core/src/Streamly/Internal/Data/Parser/ParserD/Type.hs+12−4

@@ -510,6 +510,14 @@ parseDToK pstep initial extract leftover (0, _) cont = do

510510 Partial _ _ -> error "Bug: parseDToK Partial unreachable"

511511 Continue n pst1 -> return $ K.Continue n (parseCont (return pst1))

512512

513+-- XXX Pass on the stream to parserK and inside parserK drive the parserD

514+-- directly from the stream. That way we can feed full chunks efficiently to

515+-- parserD with fusion. The remaining stream can be passed on to the

516+-- continuation and at the end the leftover stream can be returned.

517+--

518+-- contification of parserD completely breaks fusion. We can avoid that by

519+-- fusing the parserD with the source stream.

520+

513521-- 'Alternative' case. Used count needs to be maintained when inside an

514522-- Alternative.

515523parseDToK pstep initial extract leftover (level, count) cont = do

@@ -569,9 +577,9 @@ toParserK (Parser step initial extract) =

569577{-# INLINE parserDone #-}

570578parserDone :: Monad m => (Int, Int) -> K.Parse b -> m (K.Step m a b)

571579parserDone (0,_) (K.Success n b) = return $ K.Done n b

572−parserDone st (K.Success _ _) =

573− error $ "Bug: fromParserK: inside alternative: " ++ show st

574−parserDone _ (K.Failure e) = return $ K.Error e

580+parserDone (0,_) (K.Failure e) = return $ K.Error e

581+parserDone st _ =

582+ error $ "Bug: fromParserK: parser done inside alternative: " ++ show st

575583

576584-- | Convert a CPS style 'K.Parser' to a direct style 'Parser'.

577585--

@@ -1025,7 +1033,7 @@ alt (Parser stepL initialL extractL) (Parser stepR initialR extractR) =

10251033 IPartial rR -> Continue cnt (AltParseR rR)

10261034 IDone b -> Done cnt b

10271035 IError err -> Error err

1028− Partial _ _ -> error "Bug: serialWith extract 'Partial'"

1036+ Partial _ _ -> error "Bug: alt: extractL 'Partial'"

10291037 Continue n s -> do

10301038 assertM(n == cnt)

10311039 return $ Continue n (AltParseL 0 s)

core/src/Streamly/Internal/Data/Parser/ParserK/Type.hs+7−7

@@ -84,14 +84,14 @@ instance Functor Parse where

8484--

8585newtype Parser m a b = MkParser

8686 { runParser :: forall r.

87− -- The number of elements that were not used by the previous

88− -- consumer and should be carried forward.

87+ -- leftover: the number of elements that were not used by the

88+ -- previous consumer and should be carried forward.

8989 Int

90− -- (nesting level, used elem count). Nesting level is increased

91− -- whenever we enter an Alternative composition and decreased when

92− -- it is done. The used element count is a count of elements

93− -- consumed by the Alternative. If the Alternative fails we need to

94− -- backtrack by this amount.

90+ -- (alt nesting level, alt used elem count). Nesting level is

91+ -- increased whenever we enter an Alternative composition and

92+ -- decreased when it is done. The used element count is a count of

93+ -- elements consumed by the Alternative. If the Alternative fails we

94+ -- need to backtrack by this amount.

9595 --

9696 -- The nesting level is used in parseDToK to optimize the case when

9797 -- we are not in an alternative, in that case we do not need to

core/src/Streamly/Internal/Data/Producer/Source.hs+2−2

@@ -221,8 +221,8 @@ parseD

221221 pRes <- extract pst

222222 case pRes of

223223 Partial _ _ -> error "Bug: parseD: Partial in extract"

224− Continue 0 _ ->

225− error "parseD: extract, Continue 0 creates infinite loop"

224+ Continue 0 pst1 ->

225+ goStop buf pst1

226226 Continue n pst1 -> do

227227 assert (n <= length (getList buf)) (return ())

228228 let (src0, buf1) = splitAt n (getList buf)

core/src/Streamly/Internal/Data/Stream/Eliminate.hs+2

@@ -221,6 +221,8 @@ foldlT f z s = D.foldlT f z (toStreamD s)

221221parseD :: MonadThrow m => PRD.Parser m a b -> Stream m a -> m b

222222parseD p = D.parse p . toStreamD

223223

224+-- XXX Drive directly as parserK rather than converting to parserD first.

225+

224226-- | Parse a stream using the supplied ParserK 'PRK.Parser'.

225227--

226228-- /Internal/

core/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs+2−2

@@ -253,13 +253,13 @@ parseBreak (PRD.Parser pstep initial extract) stream@(Stream step state) = do

253253 PR.Error err -> throwM $ ParseError err

254254

255255 -- This is simplified goExtract

256+ -- XXX Use SPEC?

256257 {-# INLINE goStop #-}

257258 goStop buf pst = do

258259 pRes <- extract pst

259260 case pRes of

260261 PR.Partial _ _ -> error "Bug: parseBreak: Partial in extract"

261− PR.Continue 0 _ ->

262− error "parseBreak: extract, Continue 0 creates infinite loop"

262+ PR.Continue 0 pst1 -> goStop buf pst1

263263 PR.Continue n pst1 -> do

264264 assert (n <= length (getList buf)) (return ())

265265 let (src0, buf1) = splitAt n (getList buf)

core/src/Streamly/Internal/Data/Stream/StreamD/Nesting.hs+4−4

@@ -1300,8 +1300,8 @@ parseMany (PRD.Parser pstep initial extract) (Stream step state) =

13001300 pRes <- extract pst

13011301 case pRes of

13021302 PR.Partial _ _ -> error "Bug: parseMany: Partial in extract"

1303− PR.Continue 0 _ ->

1304− error "parseMany: extract, Continue 0 creates infinite loop"

1303+ PR.Continue 0 pst1 ->

1304+ return $ Skip $ ParseChunksStop buf pst1

13051305 PR.Continue n pst1 -> do

13061306 assert (n <= length buf) (return ())

13071307 let (src0, buf1) = splitAt n buf

@@ -1480,8 +1480,8 @@ parseIterate func seed (Stream step state) =

14801480 pRes <- extract pst

14811481 case pRes of

14821482 PR.Partial _ _ -> error "Bug: parseIterate: Partial in extract"

1483− PR.Continue 0 _ ->

1484− error "parseIterate: extract, Continue 0 creates infinite loop"

1483+ PR.Continue 0 pst1 ->

1484+ return $ Skip $ ConcatParseStop buf pstep pst1 extract

14851485 PR.Continue n pst1 -> do

14861486 assert (n <= length buf) (return ())

14871487 let (src0, buf1) = splitAt n buf

core/src/Streamly/Internal/Data/Stream/StreamK.hs+1−1

@@ -1054,7 +1054,7 @@ parseBreak (PR.Parser pstep initial extract) stream = do

10541054 src = Prelude.reverse src0

10551055 return (b, fromList src)

10561056 PR.Partial _ _ -> error "Bug: parseBreak: Partial in extract"

1057− PR.Continue 0 _ -> error "parseBreak: extract, Continue 0 creates infinite loop"

1057+ PR.Continue 0 s -> goStream nil buf s

10581058 PR.Continue n s -> do

10591059 assertM(n <= length buf)

10601060 let (src0, buf1) = splitAt n buf