git.iko.soy/streamly
on master help sign in
Update Parser and Fold docs
commited37c19b8780800125a83533b112a41d28cbb73e authorHarendra Kumar <harendra@composewell.com> dateFri 17 Nov 2023, 23:30 (3 years ago) parentbcf0ab1
10 files changed+486−296
compare with master
unifiedsplit
core/src/DocTestDataParserK.hs+15 core/src/Streamly/Data/Fold.hs+150−118 core/src/Streamly/Data/Parser.hs+130−60 core/src/Streamly/Data/ParserK.hs+95−3 core/src/Streamly/Internal/Data/Fold/Type.hs+4−2 core/src/Streamly/Internal/Data/Parser.hs+3 core/src/Streamly/Internal/Data/Parser/Type.hs+72−52 core/src/Streamly/Internal/Data/ParserK/Type.hs+14−60 core/src/Streamly/Internal/Data/StreamK.hs+2−1 core/streamly-core.cabal+1
core/src/DocTestDataParserK.hsnew file+15
@@ -0,0 +1,15 @@
1+{- $setup
2+>>> :m
3+>>> import Control.Applicative ((<|>))
4+>>> import Data.Char (isDigit, isAlpha)
5+
6+>>> import Streamly.Data.Parser (Parser)
7+>>> import Streamly.Data.ParserK (ParserK)
8+
9+>>> import qualified Streamly.Data.Parser as Parser
10+>>> import qualified Streamly.Data.ParserK as ParserK
11+
12+For APIs that have not been released yet.
13+
14+>>> import qualified Streamly.Internal.Data.ParserK as ParserK
15+-}
core/src/Streamly/Data/Fold.hs+150−118
@@ -10,6 +10,156 @@
1010-- Fast, composable stream consumers with ability to terminate, supporting
1111-- stream fusion.
1212--
13+-- == Using Folds
14+--
15+-- This module provides elementary folds and fold combinators that can be used
16+-- to consume a stream of data and reduce it to a final value, or transform it
17+-- in a stateful manner using scans. A data stream can be reduced into a stream
18+-- of folded data elements by folding segments of the stream. Fold combinators
19+-- can be used to compose multiple folds in parallel or to create a pipeline of
20+-- folds such that the next fold consumes the result of the previous fold. To
21+-- run these folds on a stream see 'Streamly.Data.Stream.fold',
22+-- 'Streamly.Data.Stream.scan', 'Streamly.Data.Stream.postscan',
23+-- 'Streamly.Data.Stream.scanMaybe', 'Streamly.Data.Stream.foldMany' and other
24+-- operations accepting 'Fold' type as argument "Streamly.Data.Stream".
25+--
26+-- == Reducing a Stream
27+--
28+-- A 'Fold' is a consumer of a stream of values. A fold driver (such as
29+-- 'Streamly.Data.Stream.fold') initializes the fold @accumulator@, runs the
30+-- fold @step@ function in a loop, processing the input stream one element at a
31+-- time and accumulating the result. The loop continues until the fold
32+-- terminates, at which point the accumulated result is returned.
33+--
34+-- For example, a 'sum' Fold represents a stream consumer that adds the values
35+-- in the input stream:
36+--
37+-- >>> Stream.fold Fold.sum $ Stream.fromList [1..100]
38+-- 5050
39+--
40+-- Conceptually, a 'Fold' is a data type that mimics a strict left fold
41+-- ('Data.List.foldl'). The above example is similar to a left fold using
42+-- @(+)@ as the step and @0@ as the initial value of the accumulator:
43+--
44+-- >>> Data.List.foldl' (+) 0 [1..100]
45+-- 5050
46+--
47+-- 'Fold's have an early termination capability e.g. the 'one' fold terminates
48+-- after consuming one element:
49+--
50+-- >>> Stream.fold Fold.one $ Stream.fromList [1..]
51+-- Just 1
52+--
53+-- The above example is similar to the following right fold:
54+--
55+-- >>> Prelude.foldr (\x _ -> Just x) Nothing [1..]
56+-- Just 1
57+--
58+-- 'Fold's can be combined together using combinators. For example, to create a
59+-- fold that sums first two elements in a stream:
60+--
61+-- >>> sumTwo = Fold.take 2 Fold.sum
62+-- >>> Stream.fold sumTwo $ Stream.fromList [1..100]
63+-- 3
64+--
65+-- == Parallel Composition
66+--
67+-- Folds can be combined to run in parallel on the same input. For example, to
68+-- compute the average of numbers in a stream without going through the stream
69+-- twice:
70+--
71+-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
72+-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]
73+-- 50.5
74+--
75+-- Folds can be combined so as to partition the input stream over multiple
76+-- folds. For example, to count even and odd numbers in a stream:
77+--
78+-- >>> split n = if even n then Left n else Right n
79+-- >>> stream = fmap split $ Stream.fromList [1..100]
80+-- >>> countEven = fmap (("Even " ++) . show) Fold.length
81+-- >>> countOdd = fmap (("Odd " ++) . show) Fold.length
82+-- >>> f = Fold.partition countEven countOdd
83+-- >>> Stream.fold f stream
84+-- ("Even 50","Odd 50")
85+--
86+-- == Sequential Composition
87+--
88+-- Terminating folds can be combined to parse the stream serially such that the
89+-- first fold consumes the input until it terminates and the second fold
90+-- consumes the rest of the input until it terminates:
91+--
92+-- >>> f = Fold.splitWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)
93+-- >>> Stream.fold f $ Stream.fromList "header: hello\n"
94+-- ("header: ","hello\n")
95+--
96+-- == Splitting a Stream
97+--
98+-- A 'Fold' can be applied repeatedly on a stream to transform it to a stream
99+-- of fold results. To split a stream on newlines:
100+--
101+-- >>> f = Fold.takeEndBy (== '\n') Fold.toList
102+-- >>> Stream.fold Fold.toList $ Stream.foldMany f $ Stream.fromList "Hello there!\nHow are you\n"
103+-- ["Hello there!\n","How are you\n"]
104+--
105+-- Similarly, we can split the input of a fold too:
106+--
107+-- >>> Stream.fold (Fold.many f Fold.toList) $ Stream.fromList "Hello there!\nHow are you\n"
108+-- ["Hello there!\n","How are you\n"]
109+--
110+-- == Folds vs. Streams
111+--
112+-- We can often use streams or folds to achieve the same goal. However, streams
113+-- are more efficient in composition of producers (e.g.
114+-- 'Data.Stream.append' or 'Data.Stream.mergeBy') whereas folds are
115+-- more efficient in composition of consumers (e.g. 'splitWith', 'partition'
116+-- or 'teeWith').
117+--
118+-- Streams are producers, transformations on streams happen on the output side:
119+--
120+-- >>> :{
121+-- f stream =
122+-- Stream.filter odd stream
123+-- & fmap (+1)
124+-- & Stream.fold Fold.sum
125+-- :}
126+--
127+-- >>> f $ Stream.fromList [1..100 :: Int]
128+-- 2550
129+--
130+-- Folds are stream consumers with an input stream and an output value, stream
131+-- transformations on folds happen on the input side:
132+--
133+-- >>> :{
134+-- f =
135+-- Fold.filter odd
136+-- $ Fold.lmap (+1)
137+-- $ Fold.sum
138+-- :}
139+--
140+-- >>> Stream.fold f $ Stream.fromList [1..100 :: Int]
141+-- 2550
142+--
143+-- Notice the similiarity in the definition of @f@ in both cases, the only
144+-- difference is the composition by @&@ vs @$@ and the use @lmap@ vs @map@, the
145+-- difference is due to output vs input side transformations.
146+--
147+-- == Fusion Limitations
148+--
149+-- Folds support stream fusion for generating loops comparable to the speed of
150+-- C. However, it has some limitations. For fusion to work, the folds must be
151+-- inlined, folds must be statically known and not generated dynamically, folds
152+-- should not be passed recursively.
153+--
154+-- Another limitation is due to the quadratic complexity causing slowdown when
155+-- too many nested compositions are used. Especially, the performance of the
156+-- Applicative instance and splitting operations (e.g. 'splitWith') degrades
157+-- quadratically (O(n^2)) when combined @n@ times, roughly 8 or less sequenced
158+-- operations are fine. For these cases folds can be converted to parsers and
159+-- then used as ParserK.
160+--
161+-- == Experimental APIs
162+--
13163-- Please refer to "Streamly.Internal.Data.Fold" for more functions that have
14164-- not yet been released.
15165
@@ -21,9 +171,6 @@ module Streamly.Data.Fold
21171 --
22172 -- $setup
23173
24− -- * Overview
25− -- $overview
26−
27174 -- * Running A Fold
28175 drive
29176 -- XXX Should we have a stream returning function in fold module?
@@ -258,121 +405,6 @@ import Streamly.Internal.Data.Fold
258405
259406#include "DocTestDataFold.hs"
260407
261−-- $overview
262−--
263−-- A 'Fold' is a consumer of a stream of values. A fold driver (such as
264−-- 'Streamly.Data.Stream.fold') initializes the fold @accumulator@, runs the
265−-- fold @step@ function in a loop, processing the input stream one element at a
266−-- time and accumulating the result. The loop continues until the fold
267−-- terminates, at which point the accumulated result is returned.
268−--
269−-- For example, a 'sum' Fold represents a stream consumer that adds the values
270−-- in the input stream:
271−--
272−-- >>> Stream.fold Fold.sum $ Stream.fromList [1..100]
273−-- 5050
274−--
275−-- Conceptually, a 'Fold' is a data type that mimics a strict left fold
276−-- ('Data.List.foldl'). The above example is similar to a left fold using
277−-- @(+)@ as the step and @0@ as the initial value of the accumulator:
278−--
279−-- >>> Data.List.foldl' (+) 0 [1..100]
280−-- 5050
281−--
282−-- 'Fold's have an early termination capability e.g. the 'one' fold terminates
283−-- after consuming one element:
284−--
285−-- >>> Stream.fold Fold.one $ Stream.fromList [1..]
286−-- Just 1
287−--
288−-- The above example is similar to the following right fold:
289−--
290−-- >>> Prelude.foldr (\x _ -> Just x) Nothing [1..]
291−-- Just 1
292−--
293−-- 'Fold's can be combined together using combinators. For example, to create a
294−-- fold that sums first two elements in a stream:
295−--
296−-- >>> sumTwo = Fold.take 2 Fold.sum
297−-- >>> Stream.fold sumTwo $ Stream.fromList [1..100]
298−-- 3
299−--
300−-- Folds can be combined to run in parallel on the same input. For example, to
301−-- compute the average of numbers in a stream without going through the stream
302−-- twice:
303−--
304−-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)
305−-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]
306−-- 50.5
307−--
308−-- Folds can be combined so as to partition the input stream over multiple
309−-- folds. For example, to count even and odd numbers in a stream:
310−--
311−-- >>> split n = if even n then Left n else Right n
312−-- >>> stream = fmap split $ Stream.fromList [1..100]
313−-- >>> countEven = fmap (("Even " ++) . show) Fold.length
314−-- >>> countOdd = fmap (("Odd " ++) . show) Fold.length
315−-- >>> f = Fold.partition countEven countOdd
316−-- >>> Stream.fold f stream
317−-- ("Even 50","Odd 50")
318−--
319−-- Terminating folds can be combined to parse the stream serially such that the
320−-- first fold consumes the input until it terminates and the second fold
321−-- consumes the rest of the input until it terminates:
322−--
323−-- >>> f = Fold.splitWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)
324−-- >>> Stream.fold f $ Stream.fromList "header: hello\n"
325−-- ("header: ","hello\n")
326−--
327−-- A 'Fold' can be applied repeatedly on a stream to transform it to a stream
328−-- of fold results. To split a stream on newlines:
329−--
330−-- >>> f = Fold.takeEndBy (== '\n') Fold.toList
331−-- >>> Stream.fold Fold.toList $ Stream.foldMany f $ Stream.fromList "Hello there!\nHow are you\n"
332−-- ["Hello there!\n","How are you\n"]
333−--
334−-- Similarly, we can split the input of a fold too:
335−--
336−-- >>> Stream.fold (Fold.many f Fold.toList) $ Stream.fromList "Hello there!\nHow are you\n"
337−-- ["Hello there!\n","How are you\n"]
338−--
339−-- = Folds vs. Streams
340−--
341−-- We can often use streams or folds to achieve the same goal. However, streams
342−-- are more efficient in composition of producers (e.g.
343−-- 'Data.Stream.append' or 'Data.Stream.mergeBy') whereas folds are
344−-- more efficient in composition of consumers (e.g. 'splitWith', 'partition'
345−-- or 'teeWith').
346−--
347−-- Streams are producers, transformations on streams happen on the output side:
348−--
349−-- >>> :{
350−-- f stream =
351−-- Stream.filter odd stream
352−-- & fmap (+1)
353−-- & Stream.fold Fold.sum
354−-- :}
355−--
356−-- >>> f $ Stream.fromList [1..100 :: Int]
357−-- 2550
358−--
359−-- Folds are stream consumers with an input stream and an output value, stream
360−-- transformations on folds happen on the input side:
361−--
362−-- >>> :{
363−-- f =
364−-- Fold.filter odd
365−-- $ Fold.lmap (+1)
366−-- $ Fold.sum
367−-- :}
368−--
369−-- >>> Stream.fold f $ Stream.fromList [1..100 :: Int]
370−-- 2550
371−--
372−-- Notice the similiarity in the definition of @f@ in both cases, the only
373−-- difference is the composition by @&@ vs @$@ and the use @lmap@ vs @map@, the
374−-- difference is due to output vs input side transformations.
375−
376408--------------------------------------------------------------------------------
377409-- Deprecated
378410--------------------------------------------------------------------------------
core/src/Streamly/Data/Parser.hs+130−60
@@ -7,12 +7,137 @@
77-- Stability : pre-release
88-- Portability : GHC
99--
10−-- Fast, composable stream consumers with ability to terminate, backtrack and
11−-- fail, supporting stream fusion. Parsers are a natural extension of
12−-- "Streamly.Data.Fold". Parsers and folds can be interconverted.
10+-- Parsers are stream consumers like folds with the following differences:
1311--
14−-- Please refer to "Streamly.Internal.Data.Parser" for functions that have
15−-- not yet been released.
12+-- * folds cannot fail but parsers can fail and backtrack.
13+-- * folds can be composed as a Tee but parsers cannot.
14+-- * folds can be used for scanning but parsers cannot.
15+-- * folds can be converted to parsers.
16+--
17+-- This module implements parsers with stream fusion which compile to efficient
18+-- loops comparable to the speed of C.
19+--
20+-- == Using Parsers
21+--
22+-- This module provides elementary parsers and parser combinators that can be
23+-- used to parse a stream of data. Additionally, all the folds from the
24+-- "Streamly.Data.Fold" module can be converted to parsers using 'fromFold'.
25+-- All the parsing functionality provided by popular parsing libraries, and
26+-- more is available. Also see "Streamly.Unicode.Parser" module for Char stream
27+-- parsers.
28+--
29+-- A data stream can be transformed to a stream of parsed data elements. Parser
30+-- combinators can be used to create a pipeline of folds or parsers such that
31+-- the next fold or parser consumes the result of the previous parser. See
32+-- 'Streamly.Data.Stream.parse' and 'Streamly.Data.Stream.parseMany' to run
33+-- these parsers on a stream.
34+--
35+-- == Parser vs ParserK
36+--
37+-- There are two functionally equivalent parsing modules,
38+-- "Streamly.Data.Parser" (this module) and "Streamly.Data.ParserK". The latter
39+-- is a CPS based wrapper over the former, and can be used for parsing in
40+-- general. "Streamly.Data.Parser" enables stream fusion and should be
41+-- preferred over "Streamly.Data.ParserK" for high performance stream parsing
42+-- use cases. However, there are a few cases where this module is not
43+-- suitable and ParserK should be used instead.
44+--
45+-- For static fusion, parser combinators have to use strict pattern matching on
46+-- arguments of type Parser. This leads to infinte loop when a parser is
47+-- defined recursively, due to strict evaluation of the recursive call. For
48+-- example, the following implementation loops infinitely because of the
49+-- recursive use of parser @p@ in the @*>@ combinator:
50+--
51+-- >>> import Streamly.Data.Parser (Parser)
52+-- >>> import qualified Streamly.Data.Fold as Fold
53+-- >>> import qualified Streamly.Data.Parser as Parser
54+-- >>> import qualified Streamly.Data.Stream as Stream
55+-- >>> import Control.Applicative ((<|>))
56+--
57+-- >>> :{
58+-- >>> p :: Monad m => Parser Char m String
59+-- >>> p = Parser.satisfy (== '(') *> p <|> Parser.fromFold Fold.toList
60+-- >>> :}
61+--
62+-- Use ParserK when recursive use is required:
63+--
64+-- >>> import Streamly.Data.ParserK (ParserK)
65+-- >>> import qualified Streamly.Data.StreamK as StreamK
66+-- >>> import qualified Streamly.Internal.Data.StreamK as StreamK (parse)
67+-- >>> import qualified Streamly.Internal.Data.ParserK as ParserK (adapt)
68+--
69+-- >>> :{
70+-- >>> p :: Monad m => ParserK Char m String
71+-- >>> p = ParserK.adapt (Parser.satisfy (== '(')) *> p <|> ParserK.adapt (Parser.fromFold Fold.toList)
72+-- >>> :}
73+--
74+-- >>> StreamK.parse p $ StreamK.fromStream $ Stream.fromList "hello"
75+-- Right "hello"
76+--
77+-- For this reason Applicative, Alternative or Monad compositions with
78+-- recursion cannot be used with the 'Parser' type. Alternative type class based
79+-- operations like 'asum' and Alternative based generic parser combinators use
80+-- recursion. Similarly, Applicative type class based operations like
81+-- 'Prelude.sequence' use recursion. Custom implementations of many such
82+-- operations are provided in this module (e.g. 'some', 'many'), and those
83+-- should be used instead.
84+--
85+-- Another limitation of Parser type is due to the quadratic complexity causing
86+-- slowdown when too many nested compositions are used. Especially Applicative,
87+-- Monad, Alternative instances, and sequenced parsing operations (e.g. nested
88+-- 'one', and 'splitWith') degrade the performance quadratically (O(n^2)) when
89+-- combined @n@ times, roughly 8 or less sequenced parsers are fine. READ THE
90+-- DOCS OF APPLICATIVE, MONAD AND ALTERNATIVE INSTANCES.
91+--
92+-- == Streaming Parsers
93+--
94+-- With 'Streamly.Data.ParserK.ParserK' you can use the generic Alternative
95+-- type class based parsers from the
96+-- <https://hackage.haskell.org/package/parser-combinators parser-combinators>
97+-- library or similar. However, we recommend that you use the equivalent
98+-- functionality from this module for better performance and for streaming
99+-- behavior.
100+--
101+-- Firstly, the combinators in this module are faster due to stream fusion.
102+-- Secondly, these are streaming in nature as the results can be passed
103+-- directly to other stream consumers (folds or parsers). The Alternative type
104+-- class based parsers would end up buffering all the results in lists before
105+-- they can be consumed.
106+--
107+-- When recursion or heavy nesting is needed use ParserK.
108+--
109+-- == Error Reporting
110+--
111+-- These parsers do not report the error context (e.g. line number or column).
112+-- This may be supported in future.
113+--
114+-- == Monad Transformer Stack
115+--
116+-- 'MonadTrans' instance is not provided. If the 'Parser' type is the top most
117+-- layer (which should be the case almost always) you can just use 'fromEffect'
118+-- to execute the lower layer monad effects.
119+--
120+-- == Parser vs ParserK Implementation
121+--
122+-- The 'Parser' type represents a stream consumer by composing state as data
123+-- which enables stream fusion. Stream fusion generates a tight loop without
124+-- any constructor allocations between the stages, providing C like performance
125+-- for the loop. Stream fusion works when multiple functions are combined in a
126+-- pipeline statically. Therefore, the operations in this module must be
127+-- inlined and must not be used recursively to allow for stream fusion.
128+--
129+-- The 'ParserK' type represents a stream consumer by composing function calls,
130+-- therefore, a function call overhead is incurred at each composition. It is
131+-- quite fast in general but may be a few times slower than a fused parser.
132+-- However, it allows for scalable dynamic composition especially parsers can
133+-- be used in recursive calls. Using the 'ParserK' type operations like
134+-- 'splitWith' provide linear (O(n)) performance with respect to the number of
135+-- compositions.
136+--
137+-- == Experimental APIs
138+--
139+-- Please refer to "Streamly.Internal.Data.Parser" for functions that have not
140+-- yet been released.
16141--
17142module Streamly.Data.Parser
18143 (
@@ -22,9 +147,6 @@ module Streamly.Data.Parser
22147 --
23148 -- $setup
24149
25− -- * Overview
26− -- $overview
27−
28150 -- * Parser Type
29151 Parser
30152
@@ -123,55 +245,3 @@ import Streamly.Internal.Data.Parser
123245import Prelude hiding (dropWhile, takeWhile, filter)
124246
125247#include "DocTestDataParser.hs"
126−
127−-- $overview
128−--
129−-- Several combinators in this module can be many times faster than CPS based
130−-- parsers because of stream fusion. For example,
131−-- 'Streamly.Internal.Data.Parser.many' combinator in this module is much
132−-- faster than the 'Control.Applicative.many' combinator of
133−-- 'Control.Applicative.Alternative' type class used by CPS based parsers.
134−--
135−-- The use of 'Alternative' type class, in parsers has another drawback.
136−-- Alternative based parsers use plain Haskell lists to collect the results. In
137−-- a strict Monad like IO, the results are necessarily buffered before they can
138−-- be consumed. This may not perform optimally in streaming applications
139−-- processing large amounts of data. Equivalent combinators in this module can
140−-- consume the results of parsing using a 'Fold' or another parser, thus
141−-- providing a scalable and composable consumer.
142−--
143−-- Note that these parsers do not report the error context (e.g. line number or
144−-- column). This may be supported in future.
145−--
146−-- mtl instances are not provided. If the 'Parser' type is the top most layer
147−-- (which should be the case almost always) you can just use 'fromEffect' to
148−-- execute the lower layer monad effects.
149−--
150−-- == Performance Notes
151−--
152−-- The 'Parser' type represents a stream consumer by composing state as data
153−-- which enables stream fusion. Stream fusion generates a tight loop without
154−-- any constructor allocations between the stages, providing C like performance
155−-- for the loop. Stream fusion works when multiple functions are combined in a
156−-- pipeline statically. Therefore, the operations in this module must be
157−-- inlined and must not be used recursively to allow for stream fusion. Note
158−-- that operations like 'sequence', and 'asum' that compose pasrers using
159−-- recursion should be avoided with these parsers. You can use these with the
160−-- 'ParserK' module instead.
161−--
162−-- Using the 'Parser' type, parsing operations like 'one', 'splitWith' etc.
163−-- degrade quadratically (O(n^2)) when combined many times. If you need to
164−-- combine these operations, say more than 8 times in a single loop, then you
165−-- should consider using the continuation style parser type 'ParserK' instead.
166−-- Also, if you need to use these operations in a recursive loop you should use
167−-- 'ParserK' instead.
168−--
169−-- The 'ParserK' type represents a stream consumer by composing function calls,
170−-- therefore, a function call overhead is incurred at each composition. It is
171−-- quite fast in general but may be a few times slower than a fused parser.
172−-- However, it allows for scalable dynamic composition especially parsers can
173−-- be used in recursive calls. Using the 'ParserK' type operations like
174−-- 'splitWith' provide linear (O(n)) performance with respect to the number of
175−-- compositions..
176−--
177−-- 'Parser' and 'ParserK' types can be interconverted.
core/src/Streamly/Data/ParserK.hs+95−3
@@ -6,13 +6,103 @@
66-- Stability : pre-release
77-- Portability : GHC
88--
9−-- Parsers using Continuation Passing Style (CPS). See notes in
10−-- "Streamly.Data.Parser" module to know when to use this module.
9+-- See the general notes about parsing in the "Streamly.Data.Parser" module.
10+-- This module implements a using Continuation Passing Style (CPS) wrapper over
11+-- the "Streamly.Data.Parser" module. It is as fast or faster than attoparsec.
1112--
12−-- To run a 'ParserK' use 'Streamly.Data.StreamK.parseChunks'.
13+-- == Parser vs ParserK
14+--
15+-- 'ParserK' is preferred over 'Streamly.Data.Parser.Parser' when extensive
16+-- applicative, alternative and monadic composition is required, or when
17+-- recursive or dynamic composition of parsers is required. The
18+-- 'Streamly.Data.Parser.Parser' type fuses statically and creates efficient
19+-- loops whereas 'ParserK' uses function call based composition and has
20+-- comparatively larger runtime overhead but it is better suited to the
21+-- specific use cases mentioned above. 'ParserK' also allows to efficient parse
22+-- a stream of arrays, it can also break the input stream into a parse result
23+-- and remaining stream so that the stream can be parsed independently in
24+-- segments.
25+--
26+-- == Using ParserK
27+--
28+-- All the parsers from the "Streamly.Data.Parser" module can be adapted to
29+-- ParserK using the 'Streamly.Data.ParserK.adaptC',
30+-- 'Streamly.Internal.Data.ParserK.adapt', and
31+-- 'Streamly.Internal.Data.ParserK.adaptCG' combinators.
32+--
33+-- 'Streamly.Data.StreamK.parseChunks' runs a parser on a stream of unboxed
34+-- arrays, this is the preferred and most efficient way to parse chunked input.
35+-- The more general 'Streamly.Data.StreamK.parseBreakChunks' function returns
36+-- the remaining stream as well along with the parse result. There are
37+-- 'Streamly.Internal.Data.StreamK.parseChunksGeneric',
38+-- 'Streamly.Internal.Data.StreamK.parseBreakChunksGeneric' as well to run
39+-- parsers on boxed arrays. 'Streamly.Internal.Data.StreamK.parse',
40+-- 'Streamly.Internal.Data.StreamK.parseBreak' run parsers on a stream of
41+-- individual elements instead of stream of arrays.
42+--
43+-- == Monadic Composition
44+--
45+-- Monad composition can be used for lookbehind parsers, we can dynamically
46+-- compose new parsers based on the results of the previously parsed values.
47+--
48+-- >>> :m
49+-- >>> import Control.Applicative ((<|>))
50+-- >>> import Data.Char (isDigit, isAlpha)
51+-- >>> import Streamly.Data.Parser (Parser)
52+-- >>> import Streamly.Data.ParserK (ParserK)
53+-- >>> import qualified Streamly.Data.Parser as Parser
54+-- >>> import qualified Streamly.Data.ParserK as ParserK
55+-- >>> import qualified Streamly.Internal.Data.ParserK as ParserK (adapt)
56+--
57+-- If we have to parse "a9" or "9a" but not "99" or "aa" we can use the
58+-- following non-monadic, backtracking parser:
59+--
60+-- >>> digits p1 p2 = ((:) <$> p1 <*> ((:) <$> p2 <*> pure []))
61+-- >>> :{
62+-- backtracking :: Monad m => ParserK Char m String
63+-- backtracking = ParserK.adapt $
64+-- digits (Parser.satisfy isDigit) (Parser.satisfy isAlpha)
65+-- <|>
66+-- digits (Parser.satisfy isAlpha) (Parser.satisfy isDigit)
67+-- :}
68+--
69+-- We know that if the first parse resulted in a digit at the first place then
70+-- the second parse is going to fail. However, we waste that information and
71+-- parse the first character again in the second parse only to know that it is
72+-- not an alphabetic char. By using lookbehind in a 'Monad' composition we can
73+-- avoid redundant work:
74+--
75+-- >>> data DigitOrAlpha = Digit Char | Alpha Char
76+--
77+-- >>> :{
78+-- lookbehind :: Monad m => ParserK Char m String
79+-- lookbehind = do
80+-- x1 <- ParserK.adapt $
81+-- Digit <$> Parser.satisfy isDigit
82+-- <|> Alpha <$> Parser.satisfy isAlpha
83+-- -- Note: the parse depends on what we parsed already
84+-- x2 <- ParserK.adapt $
85+-- case x1 of
86+-- Digit _ -> Parser.satisfy isAlpha
87+-- Alpha _ -> Parser.satisfy isDigit
88+-- return $ case x1 of
89+-- Digit x -> [x,x2]
90+-- Alpha x -> [x,x2]
91+-- :}
92+--
93+-- == Experimental APIs
94+--
95+-- Please refer to "Streamly.Internal.Data.ParserK" for functions that have
96+-- not yet been released.
1397--
1498module Streamly.Data.ParserK
1599 (
100+ -- * Setup
101+ -- | To execute the code examples provided in this module in ghci, please
102+ -- run the following commands first.
103+ --
104+ -- $setup
105+
16106 -- * Parser Type
17107 ParserK
18108
@@ -41,6 +131,8 @@ import qualified Streamly.Internal.Data.Parser as ParserD
41131
42132import Streamly.Internal.Data.ParserK.Type
43133
134+#include "DocTestDataParserK.hs"
135+
44136{-# DEPRECATED fromFold "Please use \"ParserK.adaptC . Parser.fromFold\" instead." #-}
45137{-# INLINE fromFold #-}
46138fromFold :: (MonadIO m, Unbox a) => Fold m a b -> ParserK (Array a) m b
core/src/Streamly/Internal/Data/Fold/Type.hs+4−2
@@ -844,8 +844,10 @@ data SeqFoldState sl f sr = SeqFoldL !sl | SeqFoldR !f !sr
844844-- complexity, because each composition adds a new branch that each subsequent
845845-- fold's input element has to traverse, therefore, it cannot scale to a large
846846-- number of compositions. After around 100 compositions the performance starts
847−-- dipping rapidly compared to a CPS style implementation. When you need
848−-- scaling use parser monad instead.
847+-- dipping rapidly compared to a CPS style implementation.
848+--
849+-- For larger number of compositions you can convert the fold to a parser and
850+-- use ParserK.
849851--
850852-- /Time: O(n^2) where n is the number of compositions./
851853--
core/src/Streamly/Internal/Data/Parser.hs+3
@@ -10,6 +10,9 @@
1010module Streamly.Internal.Data.Parser
1111 (
1212 -- * Setup
13+ -- | To execute the code examples provided in this module in ghci, please
14+ -- run the following commands first.
15+ --
1316 -- $setup
1417
1518 module Streamly.Internal.Data.Parser.Type
core/src/Streamly/Internal/Data/Parser/Type.hs+72−52
@@ -175,9 +175,6 @@
175175
176176module Streamly.Internal.Data.Parser.Type
177177 (
178− -- * Setup
179− -- $setup
180−
181178 -- * Types
182179 Initial (..)
183180 , Step (..)
@@ -441,6 +438,7 @@ newtype ParseError = ParseError String
441438instance Exception ParseError where
442439 displayException (ParseError err) = err
443440
441+-- | Map a function on the result i.e. on @b@ in @Parser a m b@.
444442instance Functor m => Functor (Parser a m) where
445443 {-# INLINE fmap #-}
446444 fmap f (Parser step1 initial1 extract) =
@@ -507,26 +505,30 @@ data SeqParseState sl f sr = SeqParseL !sl | SeqParseR !f !sr
507505-- 'Streamly.Data.Stream.append', it splits the streams using two parsers and
508506-- zips the results.
509507
510−-- | Sequential parser application.
508+-- | Sequential parser application. Apply two parsers sequentially to an input
509+-- stream. The first parser runs and processes the input, the remaining input
510+-- is then passed to the second parser. If both parsers succeed, their outputs
511+-- are combined using the supplied function. If either parser fails, the
512+-- operation fails.
511513--
512−-- Apply two parsers sequentially to an input stream. The first parser runs and
513−-- processes the input, the remaining input is then passed to the second
514−-- parser. If both parsers succeed, their outputs are combined using the
515−-- supplied function. If either parser fails, the operation fails.
514+-- This combinator delivers high performance by stream fusion but it comes with
515+-- some limitations. For those cases use the 'Applicative' instance of
516+-- 'Streamly.Data.ParserK.ParserK'.
516517--
517−-- This implementation is strict in the second argument, therefore, the
518−-- following will fail:
518+-- CAVEAT 1: NO RECURSION. This function is strict in both arguments. As a
519+-- result, if a parser is defined recursively using this, it may cause an
520+-- infintie loop. The following example checks the strictness:
519521--
520−-- >>> Stream.parse (Parser.splitWith const (Parser.satisfy (> 0)) undefined) $ Stream.fromList [1]
522+-- >>> p = Parser.splitWith const (Parser.satisfy (> 0)) undefined
523+-- >>> Stream.parse p $ Stream.fromList [1]
521524-- *** Exception: Prelude.undefined
522525-- ...
523526--
524−-- Although this implementation allows stream fusion, it has quadratic
525−-- complexity, making it suitable only for a small number of compositions.
526−-- As a thumb rule use it for less than 8 compositions, use ParserK otherwise.
527+-- CAVEAT 2: QUADRATIC TIME COMPLEXITY. Static composition is fast due to
528+-- stream fusion, but it works well only for limited (e.g. up to 8)
529+-- compositions, use "Streamly.Data.ParserK" for larger compositions.
527530--
528−-- Below are some common idioms that can be expressed using 'splitWith' and
529−-- other parser primitives:
531+-- Below are some common idioms that can be expressed using 'splitWith':
530532--
531533-- >>> span p f1 f2 = Parser.splitWith (,) (Parser.takeWhile p f1) (Parser.fromFold f2)
532534-- >>> spanBy eq f1 f2 = Parser.splitWith (,) (Parser.groupBy eq f1) (Parser.fromFold f2)
@@ -615,7 +617,12 @@ splitWith func (Parser stepL initialL extractL)
615617-- We can use the compiler to automatically annotate accumulators, terminating
616618-- folds, non-failing parsers and failing parsers.
617619
618−-- | Works correctly only if both the parsers are guaranteed to never fail.
620+-- | Better performance 'splitWith' for non-failing parsers.
621+--
622+-- Does not work correctly for parsers that can fail.
623+--
624+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.
625+--
619626{-# INLINE noErrorUnsafeSplitWith #-}
620627noErrorUnsafeSplitWith :: Monad m
621628 => (a -> b -> c) -> Parser x m a -> Parser x m b -> Parser x m c
@@ -686,6 +693,8 @@ data SeqAState sl sr = SeqAL !sl | SeqAR !sr
686693-- second parser. The output of the parser is the output of the second parser.
687694-- The operation fails if any of the parsers fail.
688695--
696+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.
697+--
689698-- This implementation is strict in the second argument, therefore, the
690699-- following will fail:
691700--
@@ -693,10 +702,6 @@ data SeqAState sl sr = SeqAL !sl | SeqAR !sr
693702-- *** Exception: Prelude.undefined
694703-- ...
695704--
696−-- Although this implementation allows stream fusion, it has quadratic
697−-- complexity, making it suitable only for a small number of compositions.
698−-- As a thumb rule use it for less than 8 compositions, use ParserK otherwise.
699−--
700705-- /Pre-release/
701706--
702707{-# INLINE split_ #-}
@@ -752,7 +757,12 @@ split_ (Parser stepL initialL extractL) (Parser stepR initialR extractR) =
752757 Partial _ _ -> error "split_: Partial"
753758 Continue n s -> return $ Continue n (SeqAL s)
754759
755−-- For backtracking folds
760+-- | Better performance 'split_' for non-failing parsers.
761+--
762+-- Does not work correctly for parsers that can fail.
763+--
764+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.
765+--
756766{-# INLINE noErrorUnsafeSplit_ #-}
757767noErrorUnsafeSplit_ :: Monad m => Parser x m a -> Parser x m b -> Parser x m b
758768noErrorUnsafeSplit_
@@ -807,7 +817,11 @@ noErrorUnsafeSplit_
807817 Partial _ _ -> error "split_: Partial"
808818 Continue n s -> return $ Continue n (SeqAL s)
809819
810−-- | 'Applicative' form of 'splitWith'.
820+-- | READ THE CAVEATS in 'splitWith' before using this instance.
821+--
822+-- >>> pure = Parser.fromPure
823+-- >>> (<*>) = Parser.splitWith id
824+-- >>> (*>) = Parser.split_
811825instance Monad m => Applicative (Parser a m) where
812826 {-# INLINE pure #-}
813827 pure = fromPure
@@ -833,20 +847,26 @@ data AltParseState sl sr = AltParseL !Int !sl | AltParseR !sr
833847-- each subsequent alternative's input element has to go through, therefore, it
834848-- cannot scale to a large number of compositions
835849
836−-- | Sequential alternative. The input is first passed to the first parser, and
850+-- | Sequential alternative. The input is first passed to the first parser,
837851-- if it succeeds, the result is returned. However, if the first parser fails,
838−-- the parser driver backtracks and tries the same input on the second parser,
839−-- returning the result if it succeeds.
852+-- the parser driver backtracks and tries the same input on the second
853+-- (alternative) parser, returning the result if it succeeds.
854+--
855+-- This combinator delivers high performance by stream fusion but it comes with
856+-- some limitations. For those cases use the 'Alternative' instance of
857+-- 'Streamly.Data.ParserK.ParserK'.
840858--
841−-- Note: This implementation is not lazy in the second argument. The following
842−-- will fail:
859+-- CAVEAT 1: NO RECURSION. This function is strict in both arguments. As a
860+-- result, if a parser is defined recursively using this, it may cause an
861+-- infintie loop. The following example checks the strictness:
843862--
844−-- >> Stream.parse (Parser.satisfy (> 0) `Parser.alt` undefined) $ Stream.fromList [1..10]
863+-- >>> p = Parser.satisfy (> 0) `Parser.alt` undefined
864+-- >>> Stream.parse p $ Stream.fromList [1..10]
845865-- *** Exception: Prelude.undefined
846866--
847−-- Although this implementation allows stream fusion, it has quadratic
848−-- complexity, making it suitable only for a small number of compositions.
849−-- As a thumb rule use it for less than 8 compositions, use ParserK otherwise.
867+-- CAVEAT 2: QUADRATIC TIME COMPLEXITY. Static composition is fast due to
868+-- stream fusion, but it works well only for limited (e.g. up to 8)
869+-- compositions, use "Streamly.Data.ParserK" for larger compositions.
850870--
851871-- /Time Complexity:/ O(n^2) where n is the number of compositions.
852872--
@@ -1176,23 +1196,12 @@ dieM err = Parser undefined (IError <$> err) undefined
11761196-- Note: With the direct style parser type, the list in "some" and "many" is
11771197-- accumulated strictly, it cannot be consumed lazily.
11781198
1179−-- | Sequential alternative. The input is first passed to the first parser, and
1180−-- if it succeeds, the result is returned. However, if the first parser fails,
1181−-- the parser driver backtracks and tries the same input on the second parser,
1182−-- returning the result if it succeeds.
1183−--
1184−-- Note: The implementation of '<|>' is not lazy in the second
1185−-- argument. The following code will fail:
1186−--
1187−-- >>> Stream.parse (Parser.satisfy (> 0) <|> undefined) $ Stream.fromList [1..10]
1188−-- *** Exception: Prelude.undefined
1189−-- ...
1199+-- | READ THE CAVEATS in 'alt' before using this instance.
11901200--
1191−-- WARNING! this is not suitable for large scale use. As a thumb rule stream
1192−-- fusion works well for less than 8 compositions of this operation, otherwise
1193−-- consider using 'ParserK'. Do not use recursive parser implementations based
1194−-- on this Alternative instance.
1195−
1201+-- >>> empty = Parser.die "empty"
1202+-- >>> (<|>) = Parser.alt
1203+-- >>> many = flip Parser.many Fold.toList
1204+-- >>> some = flip Parser.some Fold.toList
11961205instance Monad m => Alternative (Parser a m) where
11971206 {-# INLINE empty #-}
11981207 empty = die "empty"
@@ -1211,8 +1220,13 @@ data ConcatParseState sl m a b =
12111220 ConcatParseL !sl
12121221 | forall s. ConcatParseR (s -> a -> m (Step s b)) s (s -> m (Step s b))
12131222
1223+-- XXX Does it fuse completely? Check and update, it cannot fuse the
1224+-- dynamically generated parser.
1225+
12141226-- | Map a 'Parser' returning function on the result of a 'Parser'.
12151227--
1228+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.
1229+--
12161230-- /Pre-release/
12171231--
12181232{-# INLINE concatMap #-}
@@ -1282,6 +1296,12 @@ concatMap func (Parser stepL initialL extractL) = Parser step initial extract
12821296 Partial _ _ -> error "concatMap: extract Partial"
12831297 Continue n s -> return $ Continue n (ConcatParseL s)
12841298
1299+-- | Better performance 'concatMap' for non-failing parsers.
1300+--
1301+-- Does not work correctly for parsers that can fail.
1302+--
1303+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.
1304+--
12851305{-# INLINE noErrorUnsafeConcatMap #-}
12861306noErrorUnsafeConcatMap :: Monad m =>
12871307 (b -> Parser a m c) -> Parser a m b -> Parser a m c
@@ -1354,11 +1374,9 @@ noErrorUnsafeConcatMap func (Parser stepL initialL extractL) =
13541374-- for small number of compositions but for a scalable implementation we need a
13551375-- CPS version.
13561376
1357−-- | See documentation of 'Streamly.Internal.Data.Parser.ParserK.Type.Parser'.
1377+-- | READ THE CAVEATS in 'concatMap' before using this instance.
13581378--
1359−-- Although this implementation allows stream fusion, it has quadratic
1360−-- complexity, making it suitable only for a small number of compositions. As a
1361−-- thumb rule use it for less than 8 compositions, use 'ParserK' otherwise.
1379+-- >>> (>>=) = flip Parser.concatMap
13621380--
13631381instance Monad m => Monad (Parser a m) where
13641382 {-# INLINE return #-}
@@ -1370,6 +1388,7 @@ instance Monad m => Monad (Parser a m) where
13701388 {-# INLINE (>>) #-}
13711389 (>>) = (*>)
13721390
1391+-- | >>> fail = Parser.die
13731392instance Monad m => Fail.MonadFail (Parser a m) where
13741393 {-# INLINE fail #-}
13751394 fail = die
@@ -1385,6 +1404,7 @@ instance Monad m => MonadPlus (Parser a m) where
13851404 mplus = alt
13861405-}
13871406
1407+-- | >>> liftIO = Parser.fromEffect . liftIO
13881408instance (MonadIO m) => MonadIO (Parser a m) where
13891409 {-# INLINE liftIO #-}
13901410 liftIO = fromEffect . liftIO
core/src/Streamly/Internal/Data/ParserK/Type.hs+14−60
@@ -168,8 +168,8 @@ newtype ParserK a m b = MkParser
168168-------------------------------------------------------------------------------
169169
170170-- XXX rewrite this using ParserD, expose rmapM from ParserD.
171−-- | Maps a function over the output of the parser.
172−--
171+
172+-- | Map a function on the result i.e. on @b@ in @Parser a m b@.
173173instance Functor m => Functor (ParserK a m) where
174174 {-# INLINE fmap #-}
175175 fmap f parser = MkParser $ \k n st arr ->
@@ -181,7 +181,7 @@ instance Functor m => Functor (ParserK a m) where
181181-------------------------------------------------------------------------------
182182
183183-- This is the dual of stream "fromPure".
184−--
184+
185185-- | A parser that always yields a pure value without consuming any input.
186186--
187187-- /Pre-release/
@@ -199,9 +199,11 @@ fromEffect :: Monad m => m b -> ParserK a m b
199199fromEffect eff =
200200 MkParser $ \k n st arr -> eff >>= \b -> k (Success n b) st arr
201201
202−-- | 'Applicative' form of 'Streamly.Internal.Data.Parser.splitWith'. Note that
203−-- this operation does not fuse, use 'Streamly.Internal.Data.Parser.splitWith'
204−-- when fusion is important.
202+-- | @f \<$> p1 \<*> p2@ applies parsers p1 and p2 sequentially to an input
203+-- stream. The first parser runs and processes the input, the remaining input
204+-- is then passed to the second parser. If both parsers succeed, their outputs
205+-- are applied to the function @f@. If either parser fails, the operation
206+-- fails.
205207--
206208instance Monad m => Applicative (ParserK a m) where
207209 {-# INLINE pure #-}
@@ -243,48 +245,8 @@ instance Monad m => Applicative (ParserK a m) where
243245die :: String -> ParserK a m b
244246die err = MkParser (\k n st arr -> k (Failure n err) st arr)
245247
246−-- | Monad composition can be used for lookbehind parsers, we can make the
247−-- future parses depend on the previously parsed values.
248−--
249−-- If we have to parse "a9" or "9a" but not "99" or "aa" we can use the
250−-- following parser:
251−--
252−-- @
253−-- backtracking :: MonadCatch m => PR.Parser Char m String
254−-- backtracking =
255−-- sequence [PR.satisfy isDigit, PR.satisfy isAlpha]
256−-- '<|>'
257−-- sequence [PR.satisfy isAlpha, PR.satisfy isDigit]
258−-- @
259−--
260−-- We know that if the first parse resulted in a digit at the first place then
261−-- the second parse is going to fail. However, we waste that information and
262−-- parse the first character again in the second parse only to know that it is
263−-- not an alphabetic char. By using lookbehind in a 'Monad' composition we can
264−-- avoid redundant work:
265−--
266−-- @
267−-- data DigitOrAlpha = Digit Char | Alpha Char
268−--
269−-- lookbehind :: MonadCatch m => PR.Parser Char m String
270−-- lookbehind = do
271−-- x1 \<- Digit '<$>' PR.satisfy isDigit
272−-- '<|>' Alpha '<$>' PR.satisfy isAlpha
273−--
274−-- -- Note: the parse depends on what we parsed already
275−-- x2 <- case x1 of
276−-- Digit _ -> PR.satisfy isAlpha
277−-- Alpha _ -> PR.satisfy isDigit
278−--
279−-- return $ case x1 of
280−-- Digit x -> [x,x2]
281−-- Alpha x -> [x,x2]
282−-- @
283−--
284−-- See also 'Streamly.Internal.Data.Parser.concatMap'. This monad instance
285−-- does not fuse, use 'Streamly.Internal.Data.Parser.concatMap' when you need
286−-- fusion.
287−--
248+-- | Monad composition can be used for lookbehind parsers, we can dynamically
249+-- compose new parsers based on the results of the previously parsed values.
288250instance Monad m => Monad (ParserK a m) where
289251 {-# INLINE return #-}
290252 return = pure
@@ -316,18 +278,10 @@ instance MonadIO m => MonadIO (ParserK a m) where
316278-- Alternative
317279-------------------------------------------------------------------------------
318280
319−-- | 'Alternative' form of 'Streamly.Internal.Data.Parser.alt'. Backtrack and
320−-- run the second parser if the first one fails.
321−--
322−-- The "some" and "many" operations of alternative accumulate results in a pure
323−-- list which is not scalable and streaming. Instead use
324−-- 'Streamly.Internal.Data.Parser.some' and
325−-- 'Streamly.Internal.Data.Parser.many' for fusible operations with composable
326−-- accumulation of results.
327−--
328−-- See also 'Streamly.Internal.Data.Parser.alt'. This 'Alternative' instance
329−-- does not fuse, use 'Streamly.Internal.Data.Parser.alt' when you need
330−-- fusion.
281+-- | @p1 \<|> p2@ passes the input to parser p1, if it succeeds, the result is
282+-- returned. However, if p1 fails, the parser driver backtracks and tries the
283+-- same input on the alternative parser p2, returning the result if it
284+-- succeeds.
331285--
332286instance Monad m => Alternative (ParserK a m) where
333287 {-# INLINE empty #-}
core/src/Streamly/Internal/Data/StreamK.hs+2−1
@@ -1331,7 +1331,8 @@ parserDone (ParserK.Failure n e) _ _ = pure $ ParserK.Error n e
13311331-- using parseBreakChunks. We can also use parseBreak as an alternative to the
13321332-- monad instance of ParserD.
13331333
1334−-- | Run a 'ParserK' over a chunked 'StreamK' and return the rest of the Stream.
1334+-- | Run a 'ParserK' over a chunked 'StreamK' and return the parse result and
1335+-- the remaining Stream.
13351336{-# INLINE_NORMAL parseBreakChunks #-}
13361337parseBreakChunks
13371338 :: (Monad m, Unbox a)
core/streamly-core.cabal+1
@@ -50,6 +50,7 @@ extra-source-files:
5050 src/DocTestDataMutArray.hs
5151 src/DocTestDataMutArrayGeneric.hs
5252 src/DocTestDataParser.hs
53+ src/DocTestDataParserK.hs
5354 src/DocTestDataStream.hs
5455 src/DocTestDataStreamK.hs
5556 src/DocTestDataUnfold.hs