@@ -80,11 +80,14 @@ module Streamly.Internal.Data.Parser
83− -- * Sequence parsers
83+ -- * Sequence parsers (tokenizers)
85 -- | Parsers chained in series, if one parser terminates the composition
85 -- | Parsers chained in series, if one parser terminates the composition
86 -- terminates.
86 -- terminates.
88 -- | Grab a sequence of input elements without inspecting them
91 -- | Grab a sequence of input elements without inspecting them
89 , takeBetween
92 , takeBetween
90 -- , take -- takeBetween 0 n
93 -- , take -- takeBetween 0 n
@@ -93,30 +96,40 @@ module Streamly.Internal.Data.Parser
95 -- Grab a sequence of input elements by inspecting them
98 -- Grab a sequence of input elements by inspecting them
99 , takeWhileP
104 , takeWhileP
100 , takeWhile
105 , takeWhile
101 -- $takeWhile
106 -- $takeWhile
102 , takeWhile1
107 , takeWhile1
103 , dropWhile
108 , dropWhile
105− -- Separators
110+ -- ** Separators
106− , sliceSepByP
111+ -- , takeEndBy
107− , sliceBeginWith
112+ , takeEndBy_
108− , sliceSepWith
113+ , takeEndByEsc
109−
114+ -- , takeEndByEsc_
110− -- Quoting and Escaping
115+ , takeStartBy
111− , escapedSliceSepBy
116+ -- , takeStartBy_
112− , escapedFrameBy
117+ , takeEitherSepBy
114− -- Words and grouping
116 , groupBy
122 , groupBy
117 , groupByRolling
123 , groupByRolling
118 , groupByRollingEither
124 , groupByRollingEither
131+ -- , wordByQuotedEsc
120 -- | Unimplemented
133 -- | Unimplemented
@@ -143,6 +156,24 @@ module Streamly.Internal.Data.Parser
143 -- secondary parser.
156 -- secondary parser.
144 , deintercalate
157 , deintercalate
159+ -- *** Special cases
160+ -- | TODO: traditional implmentations of these may be of limited use. For
161+ -- example, consider parsing lines separated by @\\r\\n@. The main parser
162+ -- will have to detect and exclude the sequence @\\r\\n@ anyway so that we
163+ -- can apply the "sep" parser.
165+ -- We can instead implement these as special cases of deintercalate.
146 -- ** Sequential Alternative
177 -- ** Sequential Alternative
@@ -167,24 +198,6 @@ module Streamly.Internal.Data.Parser
167 , manyTill
198 , manyTill
168 , manyThen
199 , manyThen
171− -- | TODO: traditional implmentations of these may be of limited use. For
172− -- example, consider parsing lines separated by @\\r\\n@. The main parser
173− -- will have to detect and exclude the sequence @\\r\\n@ anyway so that we
174− -- can apply the "sep" parser.
176− -- We can instead implement these as special cases of deintercalate.
188 -- * Distribution
201 -- * Distribution
190 -- | A simple and stupid impl would be to just convert the stream to an
203 -- | A simple and stupid impl would be to just convert the stream to an
@@ -524,12 +537,9 @@ either = D.toParserK . D.either
524-- @takeBetween@ is the most general take operation, other take operations can
537-- @takeBetween@ is the most general take operation, other take operations can
525-- be defined in terms of takeBetween. For example:
538-- be defined in terms of takeBetween. For example:
528−-- take = takeBetween 0 n -- equivalent of take
540+-- >>> take n = Parser.takeBetween 0 n
529−-- take1 = takeBetween 1 n -- equivalent of takeLE1
530−-- takeEQ = takeBetween n n
541+-- >>> takeEQ n = Parser.takeBetween n n
531−-- takeGE = takeBetween n maxBound
542+-- >>> takeGE n = Parser.takeBetween n maxBound
534-- /Pre-release/
544-- /Pre-release/
@@ -584,10 +594,10 @@ takeGE n = D.toParserK . D.takeGE n
584-- input. The combinator stops when the condition fails or if the collecting
594-- input. The combinator stops when the condition fails or if the collecting
585-- parser stops.
595-- parser stops.
587−-- This is a generalized version of takeWhile, for example 'takeWhile1' can be
588−-- implemented in terms of this:
597+-- Other interesting parsers can be implemented in terms of this parser:
590−-- >>> takeWhile1 cond p = takeWhile cond (takeBetween 1 maxBound p)
599+-- >>> takeWhile1 cond p = Parser.takeWhileP cond (Parser.takeBetween 1 maxBound p)
600+-- >>> takeWhileBetween cond m n p = Parser.takeWhileP cond (Parser.takeBetween m n p)
592-- Stops: when the condition fails or the collecting parser stops.
602-- Stops: when the condition fails or the collecting parser stops.
593-- Fails: when the collecting parser fails.
603-- Fails: when the collecting parser fails.
@@ -650,89 +660,73 @@ dropWhile p = takeWhile p FL.drain
650-- Separators
660-- Separators
651-------------------------------------------------------------------------------
661-------------------------------------------------------------------------------
653−-- | @sliceSepByP cond parser@ parses a slice of the input using @parser@ until
663+-- | @takeEndBy_ cond parser@ parses a token that ends by a separator chosen by
654−-- @cond@ succeeds or the parser stops.
664+-- the supplied predicate. The separator is dropped.
656−-- This is a generalized slicing parser which can be used to implement other
666+-- This can be combined with other parsers to implement other interesting
657−-- parsers e.g.:
667+-- parsers as follows:
659−-- @
669+-- >>> takeEndByOrMax cond n p = Parser.takeEndBy_ cond (Parser.fromFold $ Fold.take n p)
660−-- sliceSepByMax cond n p = sliceSepByP cond (take n p)
661−-- sliceSepByBetween cond m n p = sliceSepByP cond (takeBetween m n p)
670+-- >>> takeEndByBetween cond m n p = Parser.takeEndBy_ cond (Parser.takeBetween m n p)
664-- /Pre-release/
672-- /Pre-release/
666−{-# INLINE sliceSepByP #-}
674+{-# INLINE takeEndBy_ #-}
669− (a -> Bool) -> Parser m a b -> Parser m a b
675+takeEndBy_ :: MonadCatch m => (a -> Bool) -> Parser m a b -> Parser m a b
670−sliceSepByP cond = D.toParserK . D.sliceSepByP cond . D.fromParserK
676+takeEndBy_ cond = D.toParserK . D.takeEndBy_ cond . D.fromParserK
672−-- | Like 'sliceSepBy' but does not drop the separator element, instead
678+-- | Take either the separator or the token. Separator is a Left value and
673−-- separator is emitted as a separate element in the output.
679+-- token is Right value.
675-- /Unimplemented/
681-- /Unimplemented/
676−{-# INLINE sliceSepWith #-}
682+{-# INLINE takeEitherSepBy #-}
677−sliceSepWith :: -- MonadCatch m =>
683+takeEitherSepBy :: -- MonadCatch m =>
678− (a -> Bool) -> Fold m a b -> Parser m a b
684+ (a -> Bool) -> Fold m (Either a b) c -> Parser m a c
679−sliceSepWith _cond = undefined -- D.toParserK . D.sliceSepBy cond
685+takeEitherSepBy _cond = undefined -- D.toParserK . D.takeEitherSepBy cond
681−-- | Collect stream elements until an elements passes the predicate, return the
687+-- | Parse a token that starts with an element chosen by the predicate. The
682−-- last element on which the predicate succeeded back to the input stream. If
688+-- parser fails if the input does not start with the selected element.
683−-- the predicate succeeds on the first element itself then the parser does not
684−-- terminate there. The succeeding element in the leading position
685−-- is treated as a prefix separator which is kept in the output segment.
687-- * Stops - when the predicate succeeds in non-leading position.
690-- * Stops - when the predicate succeeds in non-leading position.
688−-- * Fails - never.
691+-- * Fails - when the predicate fails in the leading position.
690−-- S.splitWithPrefix pred f = S.parseMany (PR.sliceBeginWith pred f)
693+-- >>> splitWithPrefix p f = Stream.parseMany (Parser.takeStartBy p f)
692-- Examples: -
695-- Examples: -
694−-- >>> :{
697+-- >>> p = Parser.takeStartBy (== ',') Fold.toList
695−-- sliceBeginWithOdd ls = Stream.parse prsr (Stream.fromList ls)
698+-- >>> leadingComma = Stream.parse p . Stream.fromList
696−-- where prsr = Parser.sliceBeginWith odd Fold.toList
699+-- >>> leadingComma "a,b,c,d"
700−-- >>> sliceBeginWithOdd [2, 4, 6, 3]
701−-- *** Exception: sliceBeginWith : slice begins with an element which fails the predicate
700+-- *** Exception: takeStartBy : token begins with an element which fails the predicate
703−--
702+-- >>> leadingComma ",,"
704−-- >>> sliceBeginWithOdd [3, 5, 7, 4]
706−--
704+-- >>> leadingComma ",a,b,c,d"
707−-- >>> sliceBeginWithOdd [3, 4, 6, 8, 5]
705+-- ",a"
708−-- [3,4,6,8]
706+-- >>> leadingComma ""
710−-- >>> sliceBeginWithOdd []
713-- /Pre-release/
709-- /Pre-release/
715−{-# INLINE sliceBeginWith #-}
711+{-# INLINE takeStartBy #-}
718− (a -> Bool) -> Fold m a b -> Parser m a b
712+takeStartBy :: MonadCatch m => (a -> Bool) -> Fold m a b -> Parser m a b
719−sliceBeginWith cond = D.toParserK . D.sliceBeginWith cond
713+takeStartBy cond = D.toParserK . D.takeStartBy cond
721-------------------------------------------------------------------------------
715-------------------------------------------------------------------------------
722-- Quoting and Escaping
716-- Quoting and Escaping
723-------------------------------------------------------------------------------
717-------------------------------------------------------------------------------
725−-- | Like 'sliceSepBy' but the separator elements can be escaped using an
719+-- | Like 'takeEndBy' but the separator elements can be escaped using an
726-- escape char determined by the second predicate.
720-- escape char determined by the second predicate.
728-- /Unimplemented/
722-- /Unimplemented/
729−{-# INLINE escapedSliceSepBy #-}
723+{-# INLINE takeEndByEsc #-}
730−escapedSliceSepBy :: -- MonadCatch m =>
724+takeEndByEsc :: -- MonadCatch m =>
731 (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser m a b
725 (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser m a b
732−escapedSliceSepBy _cond _esc = undefined
726+takeEndByEsc _cond _esc = undefined
733− -- D.toParserK . D.escapedSliceSepBy cond esc
727+ -- D.toParserK . D.takeEndByEsc cond esc
735−-- | @escapedFrameBy begin end escape@ parses a string framed using @begin@ and
729+-- | @takeFramedByEsc begin end escape@ parses a string framed using @begin@ and
736-- @end@ as the frame begin and end marker elements and @escape@ as an escaping
730-- @end@ as the frame begin and end marker elements and @escape@ as an escaping
737-- element to escape the occurrence of the framing elements within the frame.
731-- element to escape the occurrence of the framing elements within the frame.
738-- Nested frames are allowed, but nesting is removed when parsing.
732-- Nested frames are allowed, but nesting is removed when parsing.
@@ -740,26 +734,26 @@ escapedSliceSepBy _cond _esc = undefined
740-- For example,
734-- For example,
743−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello}"
737+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello}"
744-- "hello"
738-- "hello"
746−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}}"
740+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}}"
747-- "hello world"
741-- "hello world"
749−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello \\{world\\}}"
743+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello \\{world\\}}"
750-- "hello {world}"
744-- "hello {world}"
752−-- > Stream.parse (Parser.escapedFrameBy (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}"
746+-- > Stream.parse (Parser.takeFramedByEsc (== '{') (== '}') (== '\\') Fold.toList) $ Stream.fromList "{hello {world}"
753-- ParseError "Unterminated '{'"
747-- ParseError "Unterminated '{'"
757-- /Unimplemented/
751-- /Unimplemented/
758−{-# INLINE escapedFrameBy #-}
752+{-# INLINE takeFramedByEsc #-}
759−escapedFrameBy :: -- MonadCatch m =>
753+takeFramedByEsc :: -- MonadCatch m =>
760 (a -> Bool) -> (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser m a b
754 (a -> Bool) -> (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser m a b
761−escapedFrameBy _begin _end _escape _p = undefined
755+takeFramedByEsc _begin _end _escape _p = undefined
762− -- D.toParserK . D.frameBy begin end escape p
756+ -- D.toParserK . D.takeFramedByEsc begin end escape p
764-------------------------------------------------------------------------------
758-------------------------------------------------------------------------------
765-- Grouping and words
759-- Grouping and words