@@ -40,9 +40,12 @@
40-- 'Step' functor provides the following commands to the fold driver
40-- 'Step' functor provides the following commands to the fold driver
41-- corresponding to the use cases outlined in the previous para:
41-- corresponding to the use cases outlined in the previous para:
43−-- 1. 'Skip': hold (buffer) the input or go back to a previous position in the stream
43+-- 1. 'Skip': buffer the current input and go back to a previous
44−-- 2. 'Yield', 'Stop': tell how much input is unconsumed
44+-- position in the stream
45+-- 2. 'Yield': buffer the current input and go back to a previous
46+-- position in the stream, drop the buffer before that position.
47+-- 3. 'Stop': parser succeeded, returns how much input was leftover
45−-- 3. 'Error': indicates that the parser has failed without a result
48+-- 4. 'Error': indicates that the parser has failed without a result
47-- = How a Parser Works?
50-- = How a Parser Works?
@@ -78,7 +81,7 @@
79-- = Error Handling
82-- = Error Handling
81−-- When a parser's @step@ function is invoked it may iterminate by either a
84+-- When a parser's @step@ function is invoked it may terminate by either a
82-- 'Stop' or an 'Error' return value. In an 'Alternative' composition an error
85-- 'Stop' or an 'Error' return value. In an 'Alternative' composition an error
83-- return can make the composed parser backtrack and try another parser.
86-- return can make the composed parser backtrack and try another parser.
@@ -159,33 +162,47 @@ import Streamly.Internal.Data.Strict (Tuple3'(..))
159{-# ANN type Step Fuse #-}
162{-# ANN type Step Fuse #-}
160data Step s b =
163data Step s b =
161 Yield Int s
164 Yield Int s
165+ -- ^ Trim the backtracking buffer keeping only most recent @n@ items.
162− -- ^ @Yield offset state@ indicates that the parser has yielded a new
166+ -- @Yield n state@ indicates that the parser has yielded a new result
163− -- result which is a point of no return. The result can be extracted
167+ -- which is a point of no return. The result can be extracted using
164− -- using @extract@. The driver drops the buffer except @offset@ elements
168+ -- @extract@. The driver drops the buffer except @n@ most recent
165− -- before the current position in stream. The rule is that if a parser
169+ -- elements. The rule is that if a parser has yielded at least once it
166− -- has yielded at least once it cannot return a failure result.
170+ -- cannot return a failure result.
173+ -- ^ Go back by @n@ items in the input buffer and drop the rest.
174+ -- @YieldB n state@ indicates that the parser has yielded a new result
175+ -- which is a point of no return. The result can be extracted using
176+ -- @extract@. The driver moves back the current position of the cursor by
177+ -- @n@ elements and drops any buffer before that.
168 | Skip Int s
180 | Skip Int s
181+ -- ^ Go back by @n@ items in the input buffer.
169− -- ^ @Skip offset state@ indicates that the parser has consumed the current
182+ -- @Skip n state@ indicates that the parser has consumed the current input
170− -- input but no new result has been generated. A new @state@ is generated.
183+ -- but no new result has been generated. A new @state@ is generated.
171 -- However, if we use @extract@ on @state@ it will generate a result from
184 -- However, if we use @extract@ on @state@ it will generate a result from
172− -- the previous @Yield@. When @offset@ is non-zero it is a backward offset
185+ -- the previous @Yield@. When @n@ is non-zero it is a backward offset from
173− -- from the current position in the stream from which the driver will feed
186+ -- the current position in the stream from which the driver will replay the
174− -- the next input to the parser. The offset cannot be beyond the latest
187+ -- input to the parser. The offset cannot be beyond the latest point of no
175− -- point of no return created by @Yield@.
188+ -- return created by @Yield@.
177 | Stop Int b
190 | Stop Int b
191+ -- ^ Trim the backtracking buffer keeping only most recent @n@ items.
178− -- ^ @Stop offset result@ asks the driver to stop driving the parser
192+ -- @Stop n result@ asks the driver to stop driving the parser because it
179− -- because it has reached a fixed point and further input will not change
193+ -- has reached a fixed point and further input will not change the result.
180− -- the result. @offset@ is the count of unused elements which includes the
194+ -- @n@ is the count of unused input elements which includes the element on
181− -- element on which 'Stop' occurred.
195+ -- which 'Stop' occurred.
182 | Error String
197 | Error String
198+ -- ^ Go back to the start of the backtracking buffer.
183− -- ^ An error makes the parser backtrack to the last checkpoint and try
199+ -- An error makes the parser backtrack to the last checkpoint and try
184 -- another alternative.
200 -- another alternative.
186instance Functor (Step s) where
202instance Functor (Step s) where
187 {-# INLINE fmap #-}
203 {-# INLINE fmap #-}
188 fmap _ (Yield n s) = Yield n s
204 fmap _ (Yield n s) = Yield n s
205+ fmap _ (YieldB n s) = YieldB n s
189 fmap _ (Skip n s) = Skip n s
206 fmap _ (Skip n s) = Skip n s
190 fmap f (Stop n b) = Stop n (f b)
207 fmap f (Stop n b) = Stop n (f b)
191 fmap _ (Error err) = Error err
208 fmap _ (Error err) = Error err
@@ -282,9 +299,12 @@ splitWith func (Parser stepL initialL extractL)
282 step (SeqParseL st) a = do
299 step (SeqParseL st) a = do
283 r <- stepL st a
300 r <- stepL st a
284 case r of
301 case r of
285− -- Note: this leads to buffering even if we are not in an
302+ -- Note: We need to buffer the input for a possible Alternative
286− -- Alternative composition.
303+ -- e.g. in ((,) <$> p1 <*> p2) <|> p3, if p2 fails we have to
304+ -- backtrack and start running p3. So we need to keep the input
305+ -- buffered until we know that the applicative cannot fail.
287 Yield _ s -> return $ Skip 0 (SeqParseL s)
306 Yield _ s -> return $ Skip 0 (SeqParseL s)
307+ YieldB n s -> return $ Skip n (SeqParseL s)
288 Skip n s -> return $ Skip n (SeqParseL s)
308 Skip n s -> return $ Skip n (SeqParseL s)
289 Stop n b -> Skip n <$> (SeqParseR (func b) <$> initialR)
309 Stop n b -> Skip n <$> (SeqParseR (func b) <$> initialR)
290 Error err -> return $ Error err
310 Error err -> return $ Error err
@@ -293,6 +313,7 @@ splitWith func (Parser stepL initialL extractL)
293 r <- stepR st a
313 r <- stepR st a
294 return $ case r of
314 return $ case r of
295 Yield n s -> Yield n (SeqParseR f s)
315 Yield n s -> Yield n (SeqParseR f s)
316+ YieldB n s -> YieldB n (SeqParseR f s)
296 Skip n s -> Skip n (SeqParseR f s)
317 Skip n s -> Skip n (SeqParseR f s)
297 Stop n b -> Stop n (f b)
318 Stop n b -> Stop n (f b)
298 Error err -> Error err
319 Error err -> Error err
@@ -328,6 +349,7 @@ split_ (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
328 -- Note: this leads to buffering even if we are not in an
349 -- Note: this leads to buffering even if we are not in an
329 -- Alternative composition.
350 -- Alternative composition.
330 Yield _ s -> Skip 0 (SeqAL s)
351 Yield _ s -> Skip 0 (SeqAL s)
352+ YieldB n s -> Skip n (SeqAL s)
331 Skip n s -> Skip n (SeqAL s)
353 Skip n s -> Skip n (SeqAL s)
332 Stop n _ -> Skip n (SeqAR i)
354 Stop n _ -> Skip n (SeqAR i)
333 -- XXX should we sequence initialR monadically?
355 -- XXX should we sequence initialR monadically?
@@ -336,6 +358,7 @@ split_ (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
336 step (SeqAR st) a = do
358 step (SeqAR st) a = do
337 (\r -> case r of
359 (\r -> case r of
338 Yield n s -> Yield n (SeqAR s)
360 Yield n s -> Yield n (SeqAR s)
361+ YieldB n s -> YieldB n (SeqAR s)
339 Skip n s -> Skip n (SeqAR s)
362 Skip n s -> Skip n (SeqAR s)
340 Stop n b -> Stop n b
363 Stop n b -> Stop n b
341 Error err -> Error err) <$> stepR st a
364 Error err -> Error err) <$> stepR st a
@@ -390,6 +413,7 @@ alt (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
390 r <- stepL st a
413 r <- stepL st a
391 case r of
414 case r of
392 Yield n s -> return $ Yield n (AltParseL 0 s)
415 Yield n s -> return $ Yield n (AltParseL 0 s)
416+ YieldB n s -> return $ YieldB n (AltParseL 0 s)
393 Skip n s -> do
417 Skip n s -> do
394 assert (cnt + 1 - n >= 0) (return ())
418 assert (cnt + 1 - n >= 0) (return ())
395 return $ Skip n (AltParseL (cnt + 1 - n) s)
419 return $ Skip n (AltParseL (cnt + 1 - n) s)
@@ -402,6 +426,7 @@ alt (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
402 r <- stepR st a
426 r <- stepR st a
403 return $ case r of
427 return $ case r of
404 Yield n s -> Yield n (AltParseR s)
428 Yield n s -> Yield n (AltParseR s)
429+ YieldB n s -> YieldB n (AltParseR s)
405 Skip n s -> Skip n (AltParseR s)
430 Skip n s -> Skip n (AltParseR s)
406 Stop n b -> Stop n b
431 Stop n b -> Stop n b
407 Error err -> Error err
432 Error err -> Error err
@@ -431,14 +456,16 @@ splitMany (Fold fstep finitial fextract) (Parser step1 initial1 extract1) =
431 let cnt1 = cnt + 1
456 let cnt1 = cnt + 1
432 case r of
457 case r of
433 Yield _ s -> return $ Skip 0 (Tuple3' s cnt1 fs)
458 Yield _ s -> return $ Skip 0 (Tuple3' s cnt1 fs)
460+ assert (cnt1 - n >= 0) (return ())
461+ return $ Skip n (Tuple3' s (cnt1 - n) fs)
434 Skip n s -> do
462 Skip n s -> do
435 assert (cnt1 - n >= 0) (return ())
463 assert (cnt1 - n >= 0) (return ())
436 return $ Skip n (Tuple3' s (cnt1 - n) fs)
464 return $ Skip n (Tuple3' s (cnt1 - n) fs)
437 Stop n b -> do
465 Stop n b -> do
438 s <- initial1
466 s <- initial1
439 fs1 <- fstep fs b
467 fs1 <- fstep fs b
440− -- XXX we need to yield and backtrack here
441− return $ Skip n (Tuple3' s 0 fs1)
468+ return $ YieldB n (Tuple3' s 0 fs1)
442 Error _ -> do
469 Error _ -> do
443 xs <- fextract fs
470 xs <- fextract fs
444 return $ Stop cnt1 xs
471 return $ Stop cnt1 xs
@@ -471,31 +498,28 @@ splitSome (Fold fstep finitial fextract) (Parser step1 initial1 extract1) =
471 r <- step1 st a
498 r <- step1 st a
472 case r of
499 case r of
473 Yield _ s -> return $ Skip 0 (Tuple3' s undefined (Left fs))
500 Yield _ s -> return $ Skip 0 (Tuple3' s undefined (Left fs))
501+ YieldB n s -> return $ Skip n (Tuple3' s undefined (Left fs))
474 Skip n s -> return $ Skip n (Tuple3' s undefined (Left fs))
502 Skip n s -> return $ Skip n (Tuple3' s undefined (Left fs))
475 Stop n b -> do
503 Stop n b -> do
476 s <- initial1
504 s <- initial1
477 fs1 <- fstep fs b
505 fs1 <- fstep fs b
478− -- XXX this is also a yield point, we will never fail beyond
479− -- this point. If we do not yield then if an error occurs after
480− -- this then we will backtrack to the previous yield point
481− -- instead of this point which is wrong.
483− -- so we need a yield with backtrack
484− return $ Skip n (Tuple3' s 0 (Right fs1))
506+ return $ YieldB n (Tuple3' s 0 (Right fs1))
485 Error err -> return $ Error err
507 Error err -> return $ Error err
486 step (Tuple3' st cnt (Right fs)) a = do
508 step (Tuple3' st cnt (Right fs)) a = do
487 r <- step1 st a
509 r <- step1 st a
488 let cnt1 = cnt + 1
510 let cnt1 = cnt + 1
489 case r of
511 case r of
490 Yield _ s -> return $ Yield 0 (Tuple3' s cnt1 (Right fs))
512 Yield _ s -> return $ Yield 0 (Tuple3' s cnt1 (Right fs))
514+ assert (cnt1 - n >= 0) (return ())
515+ return $ YieldB n (Tuple3' s (cnt1 - n) (Right fs))
491 Skip n s -> do
516 Skip n s -> do
492 assert (cnt1 - n >= 0) (return ())
517 assert (cnt1 - n >= 0) (return ())
493 return $ Skip n (Tuple3' s (cnt1 - n) (Right fs))
518 return $ Skip n (Tuple3' s (cnt1 - n) (Right fs))
494 Stop n b -> do
519 Stop n b -> do
495 s <- initial1
520 s <- initial1
496 fs1 <- fstep fs b
521 fs1 <- fstep fs b
497− -- XXX we need to yield here but also backtrack
498− return $ Skip n (Tuple3' s 0 (Right fs1))
522+ return $ YieldB n (Tuple3' s 0 (Right fs1))
499 Error _ -> Stop cnt1 <$> fextract fs
523 Error _ -> Stop cnt1 <$> fextract fs
501 -- XXX The "try" may impact performance if this parser is used as a scan
525 -- XXX The "try" may impact performance if this parser is used as a scan
@@ -575,6 +599,7 @@ concatMap func (Parser stepL initialL extractL) = Parser step initial extract
575 r <- stepL st a
599 r <- stepL st a
576 return $ case r of
600 return $ case r of
577 Yield _ s -> Skip 0 (ConcatParseL s)
601 Yield _ s -> Skip 0 (ConcatParseL s)
602+ YieldB n s -> Skip n (ConcatParseL s)
578 Skip n s -> Skip n (ConcatParseL s)
603 Skip n s -> Skip n (ConcatParseL s)
579 Stop n b -> Skip n (ConcatParseR (func b))
604 Stop n b -> Skip n (ConcatParseR (func b))
580 Error err -> Error err
605 Error err -> Error err
@@ -585,6 +610,8 @@ concatMap func (Parser stepL initialL extractL) = Parser step initial extract
585 return $ case r of
610 return $ case r of
586 Yield n s ->
611 Yield n s ->
587 Yield n (ConcatParseR (Parser stepR (return s) extractR))
612 Yield n (ConcatParseR (Parser stepR (return s) extractR))
614+ YieldB n (ConcatParseR (Parser stepR (return s) extractR))
588 Skip n s ->
615 Skip n s ->
589 Skip n (ConcatParseR (Parser stepR (return s) extractR))
616 Skip n (ConcatParseR (Parser stepR (return s) extractR))
590 Stop n b -> Stop n b
617 Stop n b -> Stop n b