@@ -40,9 +40,12 @@
4040-- 'Step' functor provides the following commands to the fold driver
4141-- corresponding to the use cases outlined in the previous para:
4242--
43−-- 1. 'Skip': hold (buffer) the input or go back to a previous position in the stream
44−-- 2. 'Yield', 'Stop': tell how much input is unconsumed
45−-- 3. 'Error': indicates that the parser has failed without a result
43+-- 1. 'Skip': buffer the current input and go back to a previous
44+-- position in the stream
45+-- 2. 'Yield': buffer the current input and go back to a previous
46+-- position in the stream, drop the buffer before that position.
47+-- 3. 'Stop': parser succeeded, returns how much input was leftover
48+-- 4. 'Error': indicates that the parser has failed without a result
4649--
4750-- = How a Parser Works?
4851--
@@ -78,7 +81,7 @@
7881--
7982-- = Error Handling
8083--
81−-- When a parser's @step@ function is invoked it may iterminate by either a
84+-- When a parser's @step@ function is invoked it may terminate by either a
8285-- 'Stop' or an 'Error' return value. In an 'Alternative' composition an error
8386-- return can make the composed parser backtrack and try another parser.
8487--
@@ -159,33 +162,47 @@ import Streamly.Internal.Data.Strict (Tuple3'(..))
159162{-# ANN type Step Fuse #-}
160163data Step s b =
161164 Yield Int s
162− -- ^ @Yield offset state@ indicates that the parser has yielded a new
163− -- result which is a point of no return. The result can be extracted
164− -- using @extract@. The driver drops the buffer except @offset@ elements
165− -- before the current position in stream. The rule is that if a parser
166− -- has yielded at least once it cannot return a failure result.
167−
165+ -- ^ Trim the backtracking buffer keeping only most recent @n@ items.
166+ -- @Yield n state@ indicates that the parser has yielded a new result
167+ -- which is a point of no return. The result can be extracted using
168+ -- @extract@. The driver drops the buffer except @n@ most recent
169+ -- elements. The rule is that if a parser has yielded at least once it
170+ -- cannot return a failure result.
171+
172+ | YieldB Int s
173+ -- ^ Go back by @n@ items in the input buffer and drop the rest.
174+ -- @YieldB n state@ indicates that the parser has yielded a new result
175+ -- which is a point of no return. The result can be extracted using
176+ -- @extract@. The driver moves back the current position of the cursor by
177+ -- @n@ elements and drops any buffer before that.
178+
179+ -- only rewind
168180 | Skip Int s
169− -- ^ @Skip offset state@ indicates that the parser has consumed the current
170− -- input but no new result has been generated. A new @state@ is generated.
181+ -- ^ Go back by @n@ items in the input buffer.
182+ -- @Skip n state@ indicates that the parser has consumed the current input
183+ -- but no new result has been generated. A new @state@ is generated.
171184 -- However, if we use @extract@ on @state@ it will generate a result from
172− -- the previous @Yield@. When @offset@ is non-zero it is a backward offset
173− -- from the current position in the stream from which the driver will feed
174− -- the next input to the parser. The offset cannot be beyond the latest
175− -- point of no return created by @Yield@.
185+ -- the previous @Yield@. When @n@ is non-zero it is a backward offset from
186+ -- the current position in the stream from which the driver will replay the
187+ -- input to the parser. The offset cannot be beyond the latest point of no
188+ -- return created by @Yield@.
176189
177190 | Stop Int b
178− -- ^ @Stop offset result@ asks the driver to stop driving the parser
179− -- because it has reached a fixed point and further input will not change
180− -- the result. @offset@ is the count of unused elements which includes the
181− -- element on which 'Stop' occurred.
191+ -- ^ Trim the backtracking buffer keeping only most recent @n@ items.
192+ -- @Stop n result@ asks the driver to stop driving the parser because it
193+ -- has reached a fixed point and further input will not change the result.
194+ -- @n@ is the count of unused input elements which includes the element on
195+ -- which 'Stop' occurred.
196+
182197 | Error String
183− -- ^ An error makes the parser backtrack to the last checkpoint and try
198+ -- ^ Go back to the start of the backtracking buffer.
199+ -- An error makes the parser backtrack to the last checkpoint and try
184200 -- another alternative.
185201
186202instance Functor (Step s) where
187203 {-# INLINE fmap #-}
188204 fmap _ (Yield n s) = Yield n s
205+ fmap _ (YieldB n s) = YieldB n s
189206 fmap _ (Skip n s) = Skip n s
190207 fmap f (Stop n b) = Stop n (f b)
191208 fmap _ (Error err) = Error err
@@ -282,9 +299,12 @@ splitWith func (Parser stepL initialL extractL)
282299 step (SeqParseL st) a = do
283300 r <- stepL st a
284301 case r of
285− -- Note: this leads to buffering even if we are not in an
286− -- Alternative composition.
302+ -- Note: We need to buffer the input for a possible Alternative
303+ -- e.g. in ((,) <$> p1 <*> p2) <|> p3, if p2 fails we have to
304+ -- backtrack and start running p3. So we need to keep the input
305+ -- buffered until we know that the applicative cannot fail.
287306 Yield _ s -> return $ Skip 0 (SeqParseL s)
307+ YieldB n s -> return $ Skip n (SeqParseL s)
288308 Skip n s -> return $ Skip n (SeqParseL s)
289309 Stop n b -> Skip n <$> (SeqParseR (func b) <$> initialR)
290310 Error err -> return $ Error err
@@ -293,6 +313,7 @@ splitWith func (Parser stepL initialL extractL)
293313 r <- stepR st a
294314 return $ case r of
295315 Yield n s -> Yield n (SeqParseR f s)
316+ YieldB n s -> YieldB n (SeqParseR f s)
296317 Skip n s -> Skip n (SeqParseR f s)
297318 Stop n b -> Stop n (f b)
298319 Error err -> Error err
@@ -328,6 +349,7 @@ split_ (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
328349 -- Note: this leads to buffering even if we are not in an
329350 -- Alternative composition.
330351 Yield _ s -> Skip 0 (SeqAL s)
352+ YieldB n s -> Skip n (SeqAL s)
331353 Skip n s -> Skip n (SeqAL s)
332354 Stop n _ -> Skip n (SeqAR i)
333355 -- XXX should we sequence initialR monadically?
@@ -336,6 +358,7 @@ split_ (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
336358 step (SeqAR st) a = do
337359 (\r -> case r of
338360 Yield n s -> Yield n (SeqAR s)
361+ YieldB n s -> YieldB n (SeqAR s)
339362 Skip n s -> Skip n (SeqAR s)
340363 Stop n b -> Stop n b
341364 Error err -> Error err) <$> stepR st a
@@ -390,6 +413,7 @@ alt (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
390413 r <- stepL st a
391414 case r of
392415 Yield n s -> return $ Yield n (AltParseL 0 s)
416+ YieldB n s -> return $ YieldB n (AltParseL 0 s)
393417 Skip n s -> do
394418 assert (cnt + 1 - n >= 0) (return ())
395419 return $ Skip n (AltParseL (cnt + 1 - n) s)
@@ -402,6 +426,7 @@ alt (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
402426 r <- stepR st a
403427 return $ case r of
404428 Yield n s -> Yield n (AltParseR s)
429+ YieldB n s -> YieldB n (AltParseR s)
405430 Skip n s -> Skip n (AltParseR s)
406431 Stop n b -> Stop n b
407432 Error err -> Error err
@@ -431,14 +456,16 @@ splitMany (Fold fstep finitial fextract) (Parser step1 initial1 extract1) =
431456 let cnt1 = cnt + 1
432457 case r of
433458 Yield _ s -> return $ Skip 0 (Tuple3' s cnt1 fs)
459+ YieldB n s -> do
460+ assert (cnt1 - n >= 0) (return ())
461+ return $ Skip n (Tuple3' s (cnt1 - n) fs)
434462 Skip n s -> do
435463 assert (cnt1 - n >= 0) (return ())
436464 return $ Skip n (Tuple3' s (cnt1 - n) fs)
437465 Stop n b -> do
438466 s <- initial1
439467 fs1 <- fstep fs b
440− -- XXX we need to yield and backtrack here
441− return $ Skip n (Tuple3' s 0 fs1)
468+ return $ YieldB n (Tuple3' s 0 fs1)
442469 Error _ -> do
443470 xs <- fextract fs
444471 return $ Stop cnt1 xs
@@ -471,31 +498,28 @@ splitSome (Fold fstep finitial fextract) (Parser step1 initial1 extract1) =
471498 r <- step1 st a
472499 case r of
473500 Yield _ s -> return $ Skip 0 (Tuple3' s undefined (Left fs))
501+ YieldB n s -> return $ Skip n (Tuple3' s undefined (Left fs))
474502 Skip n s -> return $ Skip n (Tuple3' s undefined (Left fs))
475503 Stop n b -> do
476504 s <- initial1
477505 fs1 <- fstep fs b
478− -- XXX this is also a yield point, we will never fail beyond
479− -- this point. If we do not yield then if an error occurs after
480− -- this then we will backtrack to the previous yield point
481− -- instead of this point which is wrong.
482− --
483− -- so we need a yield with backtrack
484− return $ Skip n (Tuple3' s 0 (Right fs1))
506+ return $ YieldB n (Tuple3' s 0 (Right fs1))
485507 Error err -> return $ Error err
486508 step (Tuple3' st cnt (Right fs)) a = do
487509 r <- step1 st a
488510 let cnt1 = cnt + 1
489511 case r of
490512 Yield _ s -> return $ Yield 0 (Tuple3' s cnt1 (Right fs))
513+ YieldB n s -> do
514+ assert (cnt1 - n >= 0) (return ())
515+ return $ YieldB n (Tuple3' s (cnt1 - n) (Right fs))
491516 Skip n s -> do
492517 assert (cnt1 - n >= 0) (return ())
493518 return $ Skip n (Tuple3' s (cnt1 - n) (Right fs))
494519 Stop n b -> do
495520 s <- initial1
496521 fs1 <- fstep fs b
497− -- XXX we need to yield here but also backtrack
498− return $ Skip n (Tuple3' s 0 (Right fs1))
522+ return $ YieldB n (Tuple3' s 0 (Right fs1))
499523 Error _ -> Stop cnt1 <$> fextract fs
500524
501525 -- XXX The "try" may impact performance if this parser is used as a scan
@@ -575,6 +599,7 @@ concatMap func (Parser stepL initialL extractL) = Parser step initial extract
575599 r <- stepL st a
576600 return $ case r of
577601 Yield _ s -> Skip 0 (ConcatParseL s)
602+ YieldB n s -> Skip n (ConcatParseL s)
578603 Skip n s -> Skip n (ConcatParseL s)
579604 Stop n b -> Skip n (ConcatParseR (func b))
580605 Error err -> Error err
@@ -585,6 +610,8 @@ concatMap func (Parser stepL initialL extractL) = Parser step initial extract
585610 return $ case r of
586611 Yield n s ->
587612 Yield n (ConcatParseR (Parser stepR (return s) extractR))
613+ YieldB n s ->
614+ YieldB n (ConcatParseR (Parser stepR (return s) extractR))
588615 Skip n s ->
589616 Skip n (ConcatParseR (Parser stepR (return s) extractR))
590617 Stop n b -> Stop n b