@@ -80,11 +80,14 @@ module Streamly.Internal.Data.Parser
8080 , maybe
8181 , either
8282
83− -- * Sequence parsers
83+ -- * Sequence parsers (tokenizers)
8484 --
8585 -- | Parsers chained in series, if one parser terminates the composition
8686 -- terminates.
8787
88+ , lookAhead
89+
90+ -- ** By length
8891 -- | Grab a sequence of input elements without inspecting them
8992 , takeBetween
9093 -- , take -- takeBetween 0 n
@@ -93,30 +96,40 @@ module Streamly.Internal.Data.Parser
9396 , takeP
9497
9598 -- Grab a sequence of input elements by inspecting them
99+ -- ** Exact match
96100 , eqBy
97101 , list
98− , lookAhead
102+
103+ -- ** By predicate
99104 , takeWhileP
100105 , takeWhile
101106 -- $takeWhile
102107 , takeWhile1
103108 , dropWhile
104109
105− -- Separators
106− , sliceSepByP
107− , sliceBeginWith
108− , sliceSepWith
109−
110− -- Quoting and Escaping
111− , escapedSliceSepBy
112− , escapedFrameBy
113−
114− -- Words and grouping
110+ -- ** Separators
111+ -- , takeEndBy
112+ , takeEndBy_
113+ , takeEndByEsc
114+ -- , takeEndByEsc_
115+ , takeStartBy
116+ -- , takeStartBy_
117+ , takeEitherSepBy
115118 , wordBy
119+ -- , wordByEsc
120+
121+ -- ** By comparing
116122 , groupBy
117123 , groupByRolling
118124 , groupByRollingEither
119125
126+ -- ** Framing
127+ -- , takeFramedBy
128+ -- , takeFramedBy_
129+ , takeFramedByEsc
130+ -- , wordByQuoted
131+ -- , wordByQuotedEsc
132+
120133 -- | Unimplemented
121134 --
122135 -- @
@@ -143,6 +156,24 @@ module Streamly.Internal.Data.Parser
143156 -- secondary parser.
144157 , deintercalate
145158
159+ -- *** Special cases
160+ -- | TODO: traditional implmentations of these may be of limited use. For
161+ -- example, consider parsing lines separated by @\\r\\n@. The main parser
162+ -- will have to detect and exclude the sequence @\\r\\n@ anyway so that we
163+ -- can apply the "sep" parser.
164+ --
165+ -- We can instead implement these as special cases of deintercalate.
166+ --
167+ -- @
168+ -- , endBy
169+ -- , sepEndBy
170+ -- , beginBy
171+ -- , sepBeginBy
172+ -- , sepAroundBy
173+ -- @
174+ , sepBy1
175+ , sepBy
176+
146177 -- ** Sequential Alternative
147178 , alt
148179
@@ -167,24 +198,6 @@ module Streamly.Internal.Data.Parser
167198 , manyTill
168199 , manyThen
169200
170− -- ** Special cases
171− -- | TODO: traditional implmentations of these may be of limited use. For
172− -- example, consider parsing lines separated by @\\r\\n@. The main parser
173− -- will have to detect and exclude the sequence @\\r\\n@ anyway so that we
174− -- can apply the "sep" parser.
175− --
176− -- We can instead implement these as special cases of deintercalate.
177− --
178− -- @
179− -- , endBy
180− , sepBy1
181− , sepBy
182− -- , sepEndBy
183− -- , beginBy
184− -- , sepBeginBy
185− -- , sepAroundBy
186− -- @
187−
188201 -- * Distribution
189202 --
190203 -- | A simple and stupid impl would be to just convert the stream to an
@@ -524,12 +537,9 @@ either = D.toParserK . D.either
524537-- @takeBetween@ is the most general take operation, other take operations can
525538-- be defined in terms of takeBetween. For example:
526539--
527−-- @
528−-- take = takeBetween 0 n -- equivalent of take
529−-- take1 = takeBetween 1 n -- equivalent of takeLE1
530−-- takeEQ = takeBetween n n
531−-- takeGE = takeBetween n maxBound
532−-- @
540+-- >>> take n = Parser.takeBetween 0 n
541+-- >>> takeEQ n = Parser.takeBetween n n
542+-- >>> takeGE n = Parser.takeBetween n maxBound
533543--
534544-- /Pre-release/
535545--
@@ -584,10 +594,10 @@ takeGE n = D.toParserK . D.takeGE n
584594-- input. The combinator stops when the condition fails or if the collecting
585595-- parser stops.
586596--
587−-- This is a generalized version of takeWhile, for example 'takeWhile1' can be
588−-- implemented in terms of this:
597+-- Other interesting parsers can be implemented in terms of this parser:
589598--
590−-- >>> takeWhile1 cond p = takeWhile cond (takeBetween 1 maxBound p)
599+-- >>> takeWhile1 cond p = Parser.takeWhileP cond (Parser.takeBetween 1 maxBound p)
600+-- >>> takeWhileBetween cond m n p = Parser.takeWhileP cond (Parser.takeBetween m n p)
591601--
592602-- Stops: when the condition fails or the collecting parser stops.
593603-- Fails: when the collecting parser fails.
@@ -650,89 +660,73 @@ dropWhile p = takeWhile p FL.drain
650660-- Separators
651661-------------------------------------------------------------------------------
652662
653−-- | @sliceSepByP cond parser@ parses a slice of the input using @parser@ until
654−-- @cond@ succeeds or the parser stops.
663+-- | @takeEndBy_ cond parser@ parses a token that ends by a separator chosen by
664+-- the supplied predicate. The separator is dropped.
655665--
656−-- This is a generalized slicing parser which can be used to implement other
657−-- parsers e.g.:
666+-- This can be combined with other parsers to implement other interesting
667+-- parsers as follows:
658668--
659−-- @
660−-- sliceSepByMax cond n p = sliceSepByP cond (take n p)
661−-- sliceSepByBetween cond m n p = sliceSepByP cond (takeBetween m n p)
662−-- @
669+-- >>> takeEndByOrMax cond n p = Parser.takeEndBy_ cond (Parser.fromFold $ Fold.take n p)
670+-- >>> takeEndByBetween cond m n p = Parser.takeEndBy_ cond (Parser.takeBetween m n p)
663671--
664672-- /Pre-release/
665673--
666−{-# INLINE sliceSepByP #-}
667−sliceSepByP ::
668− MonadCatch m =>
669− (a -> Bool) -> Parser m a b -> Parser m a b
670−sliceSepByP cond = D.toParserK . D.sliceSepByP cond . D.fromParserK
674+{-# INLINE takeEndBy_ #-}
675+takeEndBy_ :: MonadCatch m => (a -> Bool) -> Parser m a b -> Parser m a b
676+takeEndBy_ cond = D.toParserK . D.takeEndBy_ cond . D.fromParserK
671677
672−-- | Like 'sliceSepBy' but does not drop the separator element, instead
673−-- separator is emitted as a separate element in the output.
678+-- | Take either the separator or the token. Separator is a Left value and
679+-- token is Right value.
674680--
675681-- /Unimplemented/
676−{-# INLINE sliceSepWith #-}
677−sliceSepWith :: -- MonadCatch m =>
678− (a -> Bool) -> Fold m a b -> Parser m a b
679−sliceSepWith _cond = undefined -- D.toParserK . D.sliceSepBy cond
682+{-# INLINE takeEitherSepBy #-}
683+takeEitherSepBy :: -- MonadCatch m =>
684+ (a -> Bool) -> Fold m (Either a b) c -> Parser m a c
685+takeEitherSepBy _cond = undefined -- D.toParserK . D.takeEitherSepBy cond
680686
681−-- | Collect stream elements until an elements passes the predicate, return the
682−-- last element on which the predicate succeeded back to the input stream. If
683−-- the predicate succeeds on the first element itself then the parser does not
684−-- terminate there. The succeeding element in the leading position
685−-- is treated as a prefix separator which is kept in the output segment.
687+-- | Parse a token that starts with an element chosen by the predicate. The
688+-- parser fails if the input does not start with the selected element.
686689--
687690-- * Stops - when the predicate succeeds in non-leading position.
688−-- * Fails - never.
691+-- * Fails - when the predicate fails in the leading position.
689692--
690−-- S.splitWithPrefix pred f = S.parseMany (PR.sliceBeginWith pred f)
693+-- >>> splitWithPrefix p f = Stream.parseMany (Parser.takeStartBy p f)
691694--
692695-- Examples: -
693696--
694−-- >>> :{
695−-- sliceBeginWithOdd ls = Stream.parse prsr (Stream.fromList ls)
696−-- where prsr = Parser.sliceBeginWith odd Fold.toList
697−-- :}
698−--
699−--
700−-- >>> sliceBeginWithOdd [2, 4, 6, 3]
701−-- *** Exception: sliceBeginWith : slice begins with an element which fails the predicate
697+-- >>> p = Parser.takeStartBy (== ',') Fold.toList
698+-- >>> leadingComma = Stream.parse p . Stream.fromList
699+-- >>> leadingComma "a,b,c,d"
700+-- *** Exception: takeStartBy : token begins with an element which fails the predicate
702701-- ...
703−--
704−-- >>> sliceBeginWithOdd [3, 5, 7, 4]
705−-- [3]
706−--
707−-- >>> sliceBeginWithOdd [3, 4, 6, 8, 5]
708−-- [3,4,6,8]
709−--
710−-- >>> sliceBeginWithOdd []
711−-- []
702+-- >>> leadingComma ",,"
703+-- ","
704+-- >>> leadingComma ",a,b,c,d"
705+-- ",a"
706+-- >>> leadingComma ""
707+-- ""
712708--
713709-- /Pre-release/
714710--
715−{-# INLINE sliceBeginWith #-}
716−sliceBeginWith ::
717− MonadCatch m =>
718− (a -> Bool) -> Fold m a b -> Parser m a b
719−sliceBeginWith cond = D.toParserK . D.sliceBeginWith cond
711+{-# INLINE takeStartBy #-}
712+takeStartBy :: MonadCatch m => (a -> Bool) -> Fold m a b -> Parser m a b
713+takeStartBy cond = D.toParserK . D.takeStartBy cond
720714
721715-------------------------------------------------------------------------------
722716-- Quoting and Escaping
723717-------------------------------------------------------------------------------
724718
725−-- | Like 'sliceSepBy' but the separator elements can be escaped using an
719+-- | Like 'takeEndBy' but the separator elements can be escaped using an
726720-- escape char determined by the second predicate.
727721--
728722-- /Unimplemented/
729−{-# INLINE escapedSliceSepBy #-}
730−escapedSliceSepBy :: -- MonadCatch m =>
723+{-# INLINE takeEndByEsc #-}
724+takeEndByEsc :: -- MonadCatch m =>
731725 (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser m a b
732−escapedSliceSepBy _cond _esc = undefined
733− -- D.toParserK . D.escapedSliceSepBy cond esc
726+takeEndByEsc _cond _esc = undefined
727+ -- D.toParserK . D.takeEndByEsc cond esc
734728
735−-- | @escapedFrameBy begin end escape@ parses a string framed using @begin@ and
729+-- | @takeFramedByEsc begin end escape@ parses a string framed using @begin@ and
736730-- @end@ as the frame begin and end marker elements and @escape@ as an escaping
737731-- element to escape the occurrence of the framing elements within the frame.
738732-- Nested frames are allowed, but nesting is removed when parsing.
@@ -740,26 +734,26 @@ escapedSliceSepBy _cond _esc = undefined
740734-- For example,
741735--
742736-- @
743−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello}"
737+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello}"
744738-- "hello"
745739--
746−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}}"
740+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}}"
747741-- "hello world"
748742--
749−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello \\{world\\}}"
743+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello \\{world\\}}"
750744-- "hello {world}"
751745--
752−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}"
746+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}"
753747-- ParseError "Unterminated '{'"
754748--
755749-- @
756750--
757751-- /Unimplemented/
758−{-# INLINE escapedFrameBy #-}
759−escapedFrameBy :: -- MonadCatch m =>
752+{-# INLINE takeFramedByEsc #-}
753+takeFramedByEsc :: -- MonadCatch m =>
760754 (a -> Bool) -> (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser m a b
761−escapedFrameBy _begin _end _escape _p = undefined
762− -- D.toParserK . D.frameBy begin end escape p
755+takeFramedByEsc _begin _end _escape _p = undefined
756+ -- D.toParserK . D.takeFramedByEsc begin end escape p
763757
764758-------------------------------------------------------------------------------
765759-- Grouping and words