git.iko.soy/typeable/attoparsec
on master help sign in
.gitignore+1
since splitsnapshots
unifiedsplit
@@ -3,3 +3,4 @@
3/dist/
3/dist/
4.cabal-sandbox
4.cabal-sandbox
5cabal.sandbox.config
5cabal.sandbox.config
6+*.sw[op]
.hgignore+5−1
@@ -1,11 +1,15 @@
1−^(?:\.cabal-sandbox|benchmarks/\.cabal-sandbox|cabal-dev|dist)$
2\.(?:aux|eventlog|h[ip]|hs.html|log|[mt]ix|[oa]|orig|prof|ps|rej|swp)$
1\.(?:aux|eventlog|h[ip]|hs.html|log|[mt]ix|[oa]|orig|prof|ps|rej|swp)$
3~$
2~$
3+^(?:.+/|)\.cabal-sandbox/
4+^(?:.+/|)dist/
4benchmarks/Arse
5benchmarks/Arse
5benchmarks/dist
6benchmarks/dist
6benchmarks/med.txt
7benchmarks/med.txt
7benchmarks/tiny
8benchmarks/tiny
9+benchmarks/warp-[0-9]*
8cabal.sandbox.config
10cabal.sandbox.config
11+examples/c-http
12+examples/http_parser.[ch]
9hpc.*\.html$
13hpc.*\.html$
10
14
11syntax: glob
15syntax: glob
.hgtags+14
@@ -47,3 +47,17 @@ ec23283e88c00af34c3144a8775ca4358daea3f7 0.10.4.0
476f4b317a12acdd05d67fd9ec55d3defaf37a01eb 0.11.1.0
476f4b317a12acdd05d67fd9ec55d3defaf37a01eb 0.11.1.0
4848809d7716468fe2bbc47b191b1d64358daaf6d1 0.11.2.1
4848809d7716468fe2bbc47b191b1d64358daaf6d1 0.11.2.1
49856a772f83a6e382bbcfe53e221a5a92623b107c 0.11.3.0
49856a772f83a6e382bbcfe53e221a5a92623b107c 0.11.3.0
50+a3c7a521fd58595d6a470ab890cc54a80dd45bd7 0.11.3.1
51+38326ce07aca6c8c3a014b0d454d2a5bfdbd6a52 0.11.3.2
52+694039f4fc157a8ca5dce37959391615b6c3f535 0.11.3.3
53+1fe92d1838fb1b1afcee1ca18d4970cf7e01956e 0.11.3.4
54+68203a4cb847da905fa44b5aa1942b99d5867ab4 0.12.0.0
55+c9c2b05707b2fffcd749d5817e3e0d04e6ad6d0d 0.12.1.0
56+9157c2b4c35aa5f683b4ac51e6ff52c696bc22e2 0.12.1.1
57+682ba515f7ba62500ad79371b37b90932e0c0e3d 0.12.1.2
58+2066a72b79b478f59a9c7f1839c522fc59ba9af7 0.12.1.3
59+ef875ad3bebef2b6fb1bd7063de6a46a06d0d4b4 0.12.1.4
60+dc1fcff28d13a508f677c7b06af6420877842493 0.12.1.5
61+013e04e4526bfd5e9e4a611b42e6f03c413bea91 0.12.1.6
62+0ddbce995b77b8db391b4d6519f13feddf4b5bf3 0.13.0.0
63+46c16463ec20ad46c9cd0bfb1050c4268f34b749 0.13.0.1
.travis.ymlnew file+161
@@ -0,0 +1,161 @@
1+# This Travis job script has been generated by a script via
2+#
3+# haskell-ci '--tests-jobs=>=7.4' '--no-benchmarks' 'attoparsec.cabal'
4+#
5+# For more information, see https://github.com/haskell-CI/haskell-ci
6+#
7+# version: 0.3.20190815
8+#
9+language: c
10+dist: xenial
11+git:
12+ # whether to recursively clone submodules
13+ submodules: false
14+cache:
15+ directories:
16+ - $HOME/.cabal/packages
17+ - $HOME/.cabal/store
18+before_cache:
19+ - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log
20+ # remove files that are regenerated by 'cabal update'
21+ - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.*
22+ - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json
23+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache
24+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar
25+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx
26+ - rm -rfv $CABALHOME/packages/head.hackage
27+matrix:
28+ include:
29+ - compiler: ghc-8.8.1
30+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.8.1","cabal-install-3.0"]}}
31+ - compiler: ghc-8.6.5
32+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.6.5","cabal-install-2.4"]}}
33+ - compiler: ghc-8.4.4
34+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.4.4","cabal-install-2.4"]}}
35+ - compiler: ghc-8.2.2
36+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.2.2","cabal-install-2.4"]}}
37+ - compiler: ghc-8.0.2
38+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.0.2","cabal-install-2.4"]}}
39+ - compiler: ghc-7.10.3
40+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.10.3","cabal-install-2.4"]}}
41+ - compiler: ghc-7.8.4
42+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.8.4","cabal-install-2.4"]}}
43+ - compiler: ghc-7.6.3
44+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.6.3","cabal-install-2.4"]}}
45+ - compiler: ghc-7.4.2
46+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.4.2","cabal-install-2.4"]}}
47+ - compiler: ghc-7.2.2
48+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.2.2","cabal-install-2.4"]}}
49+ - compiler: ghc-7.0.4
50+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.0.4","cabal-install-2.4"]}}
51+before_install:
52+ - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//')
53+ - WITHCOMPILER="-w $HC"
54+ - HCPKG="$HC-pkg"
55+ - unset CC
56+ - CABAL=/opt/ghc/bin/cabal
57+ - CABALHOME=$HOME/.cabal
58+ - export PATH="$CABALHOME/bin:$PATH"
59+ - TOP=$(pwd)
60+ - "HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\\d+)\\.(\\d+)\\.(\\d+)(\\.(\\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))')"
61+ - echo $HCNUMVER
62+ - CABAL="$CABAL -vnormal+nowrap+markoutput"
63+ - set -o pipefail
64+ - |
65+ echo 'function blue(s) { printf "\033[0;34m" s "\033[0m " }' >> .colorful.awk
66+ echo 'BEGIN { state = "output"; }' >> .colorful.awk
67+ echo '/^-----BEGIN CABAL OUTPUT-----$/ { state = "cabal" }' >> .colorful.awk
68+ echo '/^-----END CABAL OUTPUT-----$/ { state = "output" }' >> .colorful.awk
69+ echo '!/^(-----BEGIN CABAL OUTPUT-----|-----END CABAL OUTPUT-----)/ {' >> .colorful.awk
70+ echo ' if (state == "cabal") {' >> .colorful.awk
71+ echo ' print blue($0)' >> .colorful.awk
72+ echo ' } else {' >> .colorful.awk
73+ echo ' print $0' >> .colorful.awk
74+ echo ' }' >> .colorful.awk
75+ echo '}' >> .colorful.awk
76+ - cat .colorful.awk
77+ - |
78+ color_cabal_output () {
79+ awk -f $TOP/.colorful.awk
80+ }
81+ - echo text | color_cabal_output
82+install:
83+ - ${CABAL} --version
84+ - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"
85+ - TEST=--enable-tests
86+ - if [ $HCNUMVER -lt 70400 ] ; then TEST=--disable-tests ; fi
87+ - BENCH=--enable-benchmarks
88+ - BENCH=--disable-benchmarks
89+ - HEADHACKAGE=false
90+ - rm -f $CABALHOME/config
91+ - |
92+ echo "verbose: normal +nowrap +markoutput" >> $CABALHOME/config
93+ echo "remote-build-reporting: anonymous" >> $CABALHOME/config
94+ echo "write-ghc-environment-files: always" >> $CABALHOME/config
95+ echo "remote-repo-cache: $CABALHOME/packages" >> $CABALHOME/config
96+ echo "logs-dir: $CABALHOME/logs" >> $CABALHOME/config
97+ echo "world-file: $CABALHOME/world" >> $CABALHOME/config
98+ echo "extra-prog-path: $CABALHOME/bin" >> $CABALHOME/config
99+ echo "symlink-bindir: $CABALHOME/bin" >> $CABALHOME/config
100+ echo "installdir: $CABALHOME/bin" >> $CABALHOME/config
101+ echo "build-summary: $CABALHOME/logs/build.log" >> $CABALHOME/config
102+ echo "store-dir: $CABALHOME/store" >> $CABALHOME/config
103+ echo "install-dirs user" >> $CABALHOME/config
104+ echo " prefix: $CABALHOME" >> $CABALHOME/config
105+ echo "repository hackage.haskell.org" >> $CABALHOME/config
106+ echo " url: http://hackage.haskell.org/" >> $CABALHOME/config
107+ - cat $CABALHOME/config
108+ - rm -fv cabal.project cabal.project.local cabal.project.freeze
109+ - travis_retry ${CABAL} v2-update -v
110+ # Generate cabal.project
111+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
112+ - touch cabal.project
113+ - |
114+ echo "packages: ." >> cabal.project
115+ - |
116+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(attoparsec)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
117+ - cat cabal.project || true
118+ - cat cabal.project.local || true
119+ - if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi
120+ - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH} | color_cabal_output
121+ - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
122+ - rm cabal.project.freeze
123+ - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} --dep -j2 all | color_cabal_output
124+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --dep -j2 all | color_cabal_output
125+script:
126+ - DISTDIR=$(mktemp -d /tmp/dist-test.XXXX)
127+ # Packaging...
128+ - ${CABAL} v2-sdist all | color_cabal_output
129+ # Unpacking...
130+ - mv dist-newstyle/sdist/*.tar.gz ${DISTDIR}/
131+ - cd ${DISTDIR} || false
132+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec tar -xvf '{}' \;
133+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \;
134+ - PKGDIR_attoparsec="$(find . -maxdepth 1 -type d -regex '.*/attoparsec-[0-9.]*')"
135+ # Generate cabal.project
136+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
137+ - touch cabal.project
138+ - |
139+ echo "packages: ${PKGDIR_attoparsec}" >> cabal.project
140+ - |
141+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(attoparsec)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
142+ - cat cabal.project || true
143+ - cat cabal.project.local || true
144+ # Building...
145+ # this builds all libraries and executables (without tests/benchmarks)
146+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all | color_cabal_output
147+ # Building with tests and benchmarks...
148+ # build & run tests, build benchmarks
149+ - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} all | color_cabal_output
150+ # Testing...
151+ - if [ $HCNUMVER -ge 70400 ] ; then ${CABAL} v2-test $WITHCOMPILER ${TEST} ${BENCH} all | color_cabal_output ; fi
152+ # cabal check...
153+ - (cd ${PKGDIR_attoparsec} && ${CABAL} -vnormal check)
154+ # haddock...
155+ - ${CABAL} v2-haddock $WITHCOMPILER ${TEST} ${BENCH} all | color_cabal_output
156+ # Building without installed constraints for packages in global-db...
157+ - rm -f cabal.project.local
158+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all | color_cabal_output
159+
160+# REGENDATA ["--tests-jobs=>=7.4","--no-benchmarks","attoparsec.cabal"]
161+# EOF
Data/Attoparsec.hs+1−1
@@ -1,6 +1,6 @@
1-- |
1-- |
2-- Module : Data.Attoparsec
2-- Module : Data.Attoparsec
3−-- Copyright : Bryan O'Sullivan 2007-2011
3+-- Copyright : Bryan O'Sullivan 2007-2015
4-- License : BSD3
4-- License : BSD3
5--
5--
6-- Maintainer : bos@serpentine.com
6-- Maintainer : bos@serpentine.com
Data/Attoparsec/ByteString.hs+34−19
@@ -1,6 +1,10 @@
1+{-# LANGUAGE CPP #-}
2+#if __GLASGOW_HASKELL__ >= 702
3+{-# LANGUAGE Trustworthy #-}
4+#endif
1-- |
5-- |
2-- Module : Data.Attoparsec.ByteString
6-- Module : Data.Attoparsec.ByteString
3−-- Copyright : Bryan O'Sullivan 2007-2011
7+-- Copyright : Bryan O'Sullivan 2007-2015
4-- License : BSD3
8-- License : BSD3
5--
9--
6-- Maintainer : bos@serpentine.com
10-- Maintainer : bos@serpentine.com
@@ -59,6 +63,7 @@ module Data.Attoparsec.ByteString
59 , I.skipWhile
63 , I.skipWhile
60 , I.take
64 , I.take
61 , I.scan
65 , I.scan
66+ , I.runScanner
62 , I.takeWhile
67 , I.takeWhile
63 , I.takeWhile1
68 , I.takeWhile1
64 , I.takeTill
69 , I.takeTill
@@ -92,6 +97,7 @@ module Data.Attoparsec.ByteString
92 ) where
97 ) where
93
98
94import Data.Attoparsec.Combinator
99import Data.Attoparsec.Combinator
100+import Data.List (intercalate)
95import qualified Data.Attoparsec.ByteString.Internal as I
101import qualified Data.Attoparsec.ByteString.Internal as I
96import qualified Data.Attoparsec.Internal as I
102import qualified Data.Attoparsec.Internal as I
97import qualified Data.ByteString as B
103import qualified Data.ByteString as B
@@ -100,50 +106,51 @@ import qualified Data.Attoparsec.Internal.Types as T
100
106
101-- $parsec
107-- $parsec
102--
108--
103−-- Compared to Parsec 3, Attoparsec makes several tradeoffs. It is
109+-- Compared to Parsec 3, attoparsec makes several tradeoffs. It is
104-- not intended for, or ideal for, all possible uses.
110-- not intended for, or ideal for, all possible uses.
105--
111--
106−-- * While Attoparsec can consume input incrementally, Parsec cannot.
112+-- * While attoparsec can consume input incrementally, Parsec cannot.
107-- Incremental input is a huge deal for efficient and secure network
113-- Incremental input is a huge deal for efficient and secure network
108-- and system programming, since it gives much more control to users
114-- and system programming, since it gives much more control to users
109-- of the library over matters such as resource usage and the I/O
115-- of the library over matters such as resource usage and the I/O
110-- model to use.
116-- model to use.
111--
117--
112−-- * Much of the performance advantage of Attoparsec is gained via
118+-- * Much of the performance advantage of attoparsec is gained via
113-- high-performance parsers such as 'I.takeWhile' and 'I.string'.
119-- high-performance parsers such as 'I.takeWhile' and 'I.string'.
114-- If you use complicated combinators that return lists of bytes or
120-- If you use complicated combinators that return lists of bytes or
115-- characters, there is less performance difference between the two
121-- characters, there is less performance difference between the two
116-- libraries.
122-- libraries.
117--
123--
118−-- * Unlike Parsec 3, Attoparsec does not support being used as a
124+-- * Unlike Parsec 3, attoparsec does not support being used as a
119-- monad transformer.
125-- monad transformer.
120--
126--
121−-- * Attoparsec is specialised to deal only with strict 'B.ByteString'
127+-- * attoparsec is specialised to deal only with strict 'B.ByteString'
122-- input. Efficiency concerns rule out both lists and lazy
128-- input. Efficiency concerns rule out both lists and lazy
123-- bytestrings. The usual use for lazy bytestrings would be to
129-- bytestrings. The usual use for lazy bytestrings would be to
124-- allow consumption of very large input without a large footprint.
130-- allow consumption of very large input without a large footprint.
125−-- For this need, Attoparsec's incremental input provides an
131+-- For this need, attoparsec's incremental input provides an
126-- excellent substitute, with much more control over when input
132-- excellent substitute, with much more control over when input
127-- takes place. If you must use lazy bytestrings, see the
133-- takes place. If you must use lazy bytestrings, see the
128-- "Data.Attoparsec.ByteString.Lazy" module, which feeds lazy chunks
134-- "Data.Attoparsec.ByteString.Lazy" module, which feeds lazy chunks
129-- to a regular parser.
135-- to a regular parser.
130--
136--
131-- * Parsec parsers can produce more helpful error messages than
137-- * Parsec parsers can produce more helpful error messages than
132−-- Attoparsec parsers. This is a matter of focus: Attoparsec avoids
138+-- attoparsec parsers. This is a matter of focus: attoparsec avoids
133-- the extra book-keeping in favour of higher performance.
139-- the extra book-keeping in favour of higher performance.
134
140
135-- $incremental
141-- $incremental
136--
142--
137−-- Attoparsec supports incremental input, meaning that you can feed it
143+-- attoparsec supports incremental input, meaning that you can feed it
138-- a bytestring that represents only part of the expected total amount
144-- a bytestring that represents only part of the expected total amount
139-- of data to parse. If your parser reaches the end of a fragment of
145-- of data to parse. If your parser reaches the end of a fragment of
140-- input and could consume more input, it will suspend parsing and
146-- input and could consume more input, it will suspend parsing and
141-- return a 'T.Partial' continuation.
147-- return a 'T.Partial' continuation.
142--
148--
143−-- Supplying the 'T.Partial' continuation with another bytestring will
149+-- Supplying the 'T.Partial' continuation with a bytestring will
144−-- resume parsing at the point where it was suspended. You must be
150+-- resume parsing at the point where it was suspended, with the
151+-- bytestring you supplied used as new input at the end of the
145−-- prepared for the result of the resumed parse to be another
152+-- existing input. You must be prepared for the result of the resumed
146−-- 'T.Partial' continuation.
153+-- parse to be another 'T.Partial' continuation.
147--
154--
148-- To indicate that you have no more input, supply the 'T.Partial'
155-- To indicate that you have no more input, supply the 'T.Partial'
149-- continuation with an empty bytestring.
156-- continuation with an empty bytestring.
@@ -155,12 +162,19 @@ import qualified Data.Attoparsec.Internal.Types as T
155-- If you do not need support for incremental input, consider using
162-- If you do not need support for incremental input, consider using
156-- the 'I.parseOnly' function to run your parser. It will never
163-- the 'I.parseOnly' function to run your parser. It will never
157-- prompt for more input.
164-- prompt for more input.
165+--
166+-- /Note/: incremental input does /not/ imply that attoparsec will
167+-- release portions of its internal state for garbage collection as it
168+-- proceeds. Its internal representation is equivalent to a single
169+-- 'ByteString': if you feed incremental input to a parser, it will
170+-- require memory proportional to the amount of input you supply.
171+-- (This is necessary to support arbitrary backtracking.)
158
172
159-- $performance
173-- $performance
160--
174--
161−-- If you write an Attoparsec-based parser carefully, it can be
175+-- If you write an attoparsec-based parser carefully, it can be
162−-- realistic to expect it to perform within a factor of 2 of a
176+-- realistic to expect it to perform similarly to a hand-rolled C
163−-- hand-rolled C parser (measuring megabytes parsed per second).
177+-- parser (measuring megabytes parsed per second).
164--
178--
165-- To actually achieve high performance, there are a few guidelines
179-- To actually achieve high performance, there are a few guidelines
166-- that it is useful to follow.
180-- that it is useful to follow.
@@ -210,6 +224,7 @@ maybeResult _ = Nothing
210-- | Convert a 'Result' value to an 'Either' value. A 'T.Partial'
224-- | Convert a 'Result' value to an 'Either' value. A 'T.Partial'
211-- result is treated as failure.
225-- result is treated as failure.
212eitherResult :: Result r -> Either String r
226eitherResult :: Result r -> Either String r
213−eitherResult (T.Done _ r) = Right r
227+eitherResult (T.Done _ r) = Right r
214−eitherResult (T.Fail _ _ msg) = Left msg
228+eitherResult (T.Fail _ [] msg) = Left msg
229+eitherResult (T.Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
215−eitherResult _ = Left "Result: incomplete input"
230+eitherResult _ = Left "Result: incomplete input"
Data/Attoparsec/ByteString/Buffer.hsnew file+156
@@ -0,0 +1,156 @@
1+{-# LANGUAGE BangPatterns #-}
2+-- |
3+-- Module : Data.Attoparsec.ByteString.Buffer
4+-- Copyright : Bryan O'Sullivan 2007-2015
5+-- License : BSD3
6+--
7+-- Maintainer : bos@serpentine.com
8+-- Stability : experimental
9+-- Portability : GHC
10+--
11+-- An "immutable" buffer that supports cheap appends.
12+--
13+-- A Buffer is divided into an immutable read-only zone, followed by a
14+-- mutable area that we've preallocated, but not yet written to.
15+--
16+-- We overallocate at the end of a Buffer so that we can cheaply
17+-- append. Since a user of an existing Buffer cannot see past the end
18+-- of its immutable zone into the data that will change during an
19+-- append, this is safe.
20+--
21+-- Once we run out of space at the end of a Buffer, we do the usual
22+-- doubling of the buffer size.
23+--
24+-- The fact of having a mutable buffer really helps with performance,
25+-- but it does have a consequence: if someone misuses the Partial API
26+-- that attoparsec uses by calling the same continuation repeatedly
27+-- (which never makes sense in practice), they could overwrite data.
28+--
29+-- Since the API *looks* pure, it should *act* pure, too, so we use
30+-- two generation counters (one mutable, one immutable) to track the
31+-- number of appends to a mutable buffer. If the counters ever get out
32+-- of sync, someone is appending twice to a mutable buffer, so we
33+-- duplicate the entire buffer in order to preserve the immutability
34+-- of its older self.
35+--
36+-- While we could go a step further and gain protection against API
37+-- abuse on a multicore system, by use of an atomic increment
38+-- instruction to bump the mutable generation counter, that would be
39+-- very expensive, and feels like it would also be in the realm of the
40+-- ridiculous. Clients should never call a continuation more than
41+-- once; we lack a linear type system that could enforce this; and
42+-- there's only so far we should go to accommodate broken uses.
43+
44+module Data.Attoparsec.ByteString.Buffer
45+ (
46+ Buffer
47+ , buffer
48+ , unbuffer
49+ , pappend
50+ , length
51+ , unsafeIndex
52+ , substring
53+ , unsafeDrop
54+ ) where
55+
56+import Control.Exception (assert)
57+import Data.ByteString.Internal (ByteString(..), memcpy, nullForeignPtr)
58+import Data.Attoparsec.Internal.Fhthagn (inlinePerformIO)
59+import Data.List (foldl1')
60+import Data.Monoid as Mon (Monoid(..))
61+import Data.Semigroup (Semigroup(..))
62+import Data.Word (Word8)
63+import Foreign.ForeignPtr (ForeignPtr, withForeignPtr)
64+import Foreign.Ptr (castPtr, plusPtr)
65+import Foreign.Storable (peek, peekByteOff, poke, sizeOf)
66+import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
67+import Prelude hiding (length)
68+
69+-- If _cap is zero, this buffer is empty.
70+data Buffer = Buf {
71+ _fp :: {-# UNPACK #-} !(ForeignPtr Word8)
72+ , _off :: {-# UNPACK #-} !Int
73+ , _len :: {-# UNPACK #-} !Int
74+ , _cap :: {-# UNPACK #-} !Int
75+ , _gen :: {-# UNPACK #-} !Int
76+ }
77+
78+instance Show Buffer where
79+ showsPrec p = showsPrec p . unbuffer
80+
81+-- | The initial 'Buffer' has no mutable zone, so we can avoid all
82+-- copies in the (hopefully) common case of no further input being fed
83+-- to us.
84+buffer :: ByteString -> Buffer
85+buffer (PS fp off len) = Buf fp off len len 0
86+
87+unbuffer :: Buffer -> ByteString
88+unbuffer (Buf fp off len _ _) = PS fp off len
89+
90+instance Semigroup Buffer where
91+ (Buf _ _ _ 0 _) <> b = b
92+ a <> (Buf _ _ _ 0 _) = a
93+ buf <> (Buf fp off len _ _) = append buf fp off len
94+
95+instance Monoid Buffer where
96+ mempty = Buf nullForeignPtr 0 0 0 0
97+
98+ mappend = (<>)
99+
100+ mconcat [] = Mon.mempty
101+ mconcat xs = foldl1' mappend xs
102+
103+pappend :: Buffer -> ByteString -> Buffer
104+pappend (Buf _ _ _ 0 _) bs = buffer bs
105+pappend buf (PS fp off len) = append buf fp off len
106+
107+append :: Buffer -> ForeignPtr a -> Int -> Int -> Buffer
108+append (Buf fp0 off0 len0 cap0 gen0) !fp1 !off1 !len1 =
109+ inlinePerformIO . withForeignPtr fp0 $ \ptr0 ->
110+ withForeignPtr fp1 $ \ptr1 -> do
111+ let genSize = sizeOf (0::Int)
112+ newlen = len0 + len1
113+ gen <- if gen0 == 0
114+ then return 0
115+ else peek (castPtr ptr0)
116+ if gen == gen0 && newlen <= cap0
117+ then do
118+ let newgen = gen + 1
119+ poke (castPtr ptr0) newgen
120+ memcpy (ptr0 `plusPtr` (off0+len0))
121+ (ptr1 `plusPtr` off1)
122+ (fromIntegral len1)
123+ return (Buf fp0 off0 newlen cap0 newgen)
124+ else do
125+ let newcap = newlen * 2
126+ fp <- mallocPlainForeignPtrBytes (newcap + genSize)
127+ withForeignPtr fp $ \ptr_ -> do
128+ let ptr = ptr_ `plusPtr` genSize
129+ newgen = 1
130+ poke (castPtr ptr_) newgen
131+ memcpy ptr (ptr0 `plusPtr` off0) (fromIntegral len0)
132+ memcpy (ptr `plusPtr` len0) (ptr1 `plusPtr` off1)
133+ (fromIntegral len1)
134+ return (Buf fp genSize newlen newcap newgen)
135+
136+length :: Buffer -> Int
137+length (Buf _ _ len _ _) = len
138+{-# INLINE length #-}
139+
140+unsafeIndex :: Buffer -> Int -> Word8
141+unsafeIndex (Buf fp off len _ _) i = assert (i >= 0 && i < len) .
142+ inlinePerformIO . withForeignPtr fp $ flip peekByteOff (off+i)
143+{-# INLINE unsafeIndex #-}
144+
145+substring :: Int -> Int -> Buffer -> ByteString
146+substring s l (Buf fp off len _ _) =
147+ assert (s >= 0 && s <= len) .
148+ assert (l >= 0 && l <= len-s) $
149+ PS fp (off+s) l
150+{-# INLINE substring #-}
151+
152+unsafeDrop :: Int -> Buffer -> ByteString
153+unsafeDrop s (Buf fp off len _ _) =
154+ assert (s >= 0 && s <= len) $
155+ PS fp (off+s) (len-s)
156+{-# INLINE unsafeDrop #-}
Data/Attoparsec/ByteString/Char8.hs+59−70
@@ -1,10 +1,13 @@
1−{-# LANGUAGE BangPatterns, FlexibleInstances, TypeFamilies,
1+{-# LANGUAGE BangPatterns, CPP, FlexibleInstances, TypeFamilies,
2 TypeSynonymInstances, GADTs #-}
2 TypeSynonymInstances, GADTs #-}
3+#if __GLASGOW_HASKELL__ >= 702
4+{-# LANGUAGE Trustworthy #-} -- Imports internal modules
5+#endif
3−{-# OPTIONS_GHC -fno-warn-orphans #-}
6+{-# OPTIONS_GHC -fno-warn-orphans -fno-warn-warnings-deprecations #-}
4
7
5-- |
8-- |
6-- Module : Data.Attoparsec.ByteString.Char8
9-- Module : Data.Attoparsec.ByteString.Char8
7−-- Copyright : Bryan O'Sullivan 2007-2011
10+-- Copyright : Bryan O'Sullivan 2007-2015
8-- License : BSD3
11-- License : BSD3
9--
12--
10-- Maintainer : bos@serpentine.com
13-- Maintainer : bos@serpentine.com
@@ -27,9 +30,10 @@ module Data.Attoparsec.ByteString.Char8
27
30
28 -- * Running parsers
31 -- * Running parsers
29 , A.parse
32 , A.parse
33+ , A.feed
30 , A.parseOnly
34 , A.parseOnly
31− , A.parseTest
32 , A.parseWith
35 , A.parseWith
36+ , A.parseTest
33
37
34 -- ** Result conversion
38 -- ** Result conversion
35 , A.maybeResult
39 , A.maybeResult
@@ -66,7 +70,7 @@ module Data.Attoparsec.ByteString.Char8
66
70
67 -- * Efficient string handling
71 -- * Efficient string handling
68 , I.string
72 , I.string
69− , stringCI
73+ , I.stringCI
70 , skipSpace
74 , skipSpace
71 , skipWhile
75 , skipWhile
72 , I.take
76 , I.take
@@ -123,7 +127,11 @@ module Data.Attoparsec.ByteString.Char8
123 , I.atEnd
127 , I.atEnd
124 ) where
128 ) where
125
129
130+#if !MIN_VERSION_base(4,8,0)
126−import Control.Applicative (pure, (*>), (<*), (<$>), (<|>))
131+import Control.Applicative (pure, (*>), (<*), (<$>))
132+import Data.Word (Word)
133+#endif
134+import Control.Applicative ((<|>))
127import Control.Monad (void, when)
135import Control.Monad (void, when)
128import Data.Attoparsec.ByteString.FastSet (charClass, memberChar)
136import Data.Attoparsec.ByteString.FastSet (charClass, memberChar)
129import Data.Attoparsec.ByteString.Internal (Parser)
137import Data.Attoparsec.ByteString.Internal (Parser)
@@ -133,9 +141,9 @@ import Data.Bits (Bits, (.|.), shiftL)
133import Data.ByteString.Internal (c2w, w2c)
141import Data.ByteString.Internal (c2w, w2c)
134import Data.Int (Int8, Int16, Int32, Int64)
142import Data.Int (Int8, Int16, Int32, Int64)
135import Data.String (IsString(..))
143import Data.String (IsString(..))
136−import Data.Scientific (Scientific, coefficient, base10Exponent)
144+import Data.Scientific (Scientific)
137−import qualified Data.Scientific as Sci (scientific)
145+import qualified Data.Scientific as Sci
138−import Data.Word (Word8, Word16, Word32, Word64, Word)
146+import Data.Word (Word8, Word16, Word32, Word64)
139import Prelude hiding (takeWhile)
147import Prelude hiding (takeWhile)
140import qualified Data.Attoparsec.ByteString as A
148import qualified Data.Attoparsec.ByteString as A
141import qualified Data.Attoparsec.ByteString.Internal as I
149import qualified Data.Attoparsec.ByteString.Internal as I
@@ -161,16 +169,6 @@ instance (a ~ B.ByteString) => IsString (Parser a) where
161-- currency sign in ISO-8859-1). Haskell 'Char' values above U+00FF
169-- currency sign in ISO-8859-1). Haskell 'Char' values above U+00FF
162-- are truncated, so e.g. U+1D6B7 is truncated to the byte @0xB7@.
170-- are truncated, so e.g. U+1D6B7 is truncated to the byte @0xB7@.
163
171
164−-- ASCII-specific but fast, oh yes.
165−toLower :: Word8 -> Word8
166−toLower w | w >= 65 && w <= 90 = w + 32
167− | otherwise = w
168−
169−-- | Satisfy a literal string, ignoring case.
170−stringCI :: B.ByteString -> Parser B.ByteString
171−stringCI = I.stringTransform (B8.map toLower)
172−{-# INLINE stringCI #-}
173−
174-- | Consume input as long as the predicate returns 'True', and return
172-- | Consume input as long as the predicate returns 'True', and return
175-- the consumed input.
173-- the consumed input.
176--
174--
@@ -234,7 +232,7 @@ isDigit c = c >= '0' && c <= '9'
234
232
235-- | A fast digit predicate.
233-- | A fast digit predicate.
236isDigit_w8 :: Word8 -> Bool
234isDigit_w8 :: Word8 -> Bool
237−isDigit_w8 w = w >= 48 && w <= 57
235+isDigit_w8 w = w - 48 <= 9
238{-# INLINE isDigit_w8 #-}
236{-# INLINE isDigit_w8 #-}
239
237
240-- | Match any character.
238-- | Match any character.
@@ -271,7 +269,7 @@ isSpace c = (c == ' ') || ('\t' <= c && c <= '\r')
271
269
272-- | Fast 'Word8' predicate for matching ASCII space characters.
270-- | Fast 'Word8' predicate for matching ASCII space characters.
273isSpace_w8 :: Word8 -> Bool
271isSpace_w8 :: Word8 -> Bool
274−isSpace_w8 w = (w == 32) || (9 <= w && w <= 13)
272+isSpace_w8 w = w == 32 || w - 9 <= 4
275{-# INLINE isSpace_w8 #-}
273{-# INLINE isSpace_w8 #-}
276
274
277
275
@@ -446,9 +444,8 @@ hexadecimal = B8.foldl' step 0 `fmap` I.takeWhile1 isHexDigit
446
444
447-- | Parse and decode an unsigned decimal number.
445-- | Parse and decode an unsigned decimal number.
448decimal :: Integral a => Parser a
446decimal :: Integral a => Parser a
449−decimal = B8.foldl' step 0 `fmap` I.takeWhile1 isDig
447+decimal = B8.foldl' step 0 `fmap` I.takeWhile1 isDigit_w8
450− where isDig w = w >= 48 && w <= 57
451− step a w = a * 10 + fromIntegral (w - 48)
448+ where step a w = a * 10 + fromIntegral (w - 48)
452{-# SPECIALISE decimal :: Parser Int #-}
449{-# SPECIALISE decimal :: Parser Int #-}
453{-# SPECIALISE decimal :: Parser Int8 #-}
450{-# SPECIALISE decimal :: Parser Int8 #-}
454{-# SPECIALISE decimal :: Parser Int16 #-}
451{-# SPECIALISE decimal :: Parser Int16 #-}
@@ -474,6 +471,24 @@ signed p = (negate <$> (char8 '-' *> p))
474 <|> (char8 '+' *> p)
471 <|> (char8 '+' *> p)
475 <|> p
472 <|> p
476
473
474+-- | Parse a rational number.
475+--
476+-- The syntax accepted by this parser is the same as for 'double'.
477+--
478+-- /Note/: this parser is not safe for use with inputs from untrusted
479+-- sources. An input with a suitably large exponent such as
480+-- @"1e1000000000"@ will cause a huge 'Integer' to be allocated,
481+-- resulting in what is effectively a denial-of-service attack.
482+--
483+-- In most cases, it is better to use 'double' or 'scientific'
484+-- instead.
485+rational :: Fractional a => Parser a
486+{-# SPECIALIZE rational :: Parser Double #-}
487+{-# SPECIALIZE rational :: Parser Float #-}
488+{-# SPECIALIZE rational :: Parser Rational #-}
489+{-# SPECIALIZE rational :: Parser Scientific #-}
490+rational = scientifically realToFrac
491+
477-- | Parse a rational number.
492-- | Parse a rational number.
478--
493--
479-- This parser accepts an optional leading sign character, followed by
494-- This parser accepts an optional leading sign character, followed by
@@ -501,56 +516,30 @@ signed p = (negate <$> (char8 '-' *> p))
501--
516--
502-- This function does not accept string representations of \"NaN\" or
517-- This function does not accept string representations of \"NaN\" or
503-- \"Infinity\".
518-- \"Infinity\".
504−rational :: Fractional a => Parser a
505−{-# SPECIALIZE rational :: Parser Double #-}
506−{-# SPECIALIZE rational :: Parser Float #-}
507−{-# SPECIALIZE rational :: Parser Rational #-}
508−{-# SPECIALIZE rational :: Parser Scientific #-}
509−rational = scientifically realToFrac
510−
511−-- | Parse a rational number.
512−--
513−-- The syntax accepted by this parser is the same as for 'rational'.
514−--
515−-- /Note/: This function is almost ten times faster than 'rational',
516−-- but is slightly less accurate.
517−--
518−-- The 'Double' type supports about 16 decimal places of accuracy.
519−-- For 94.2% of numbers, this function and 'rational' give identical
520−-- results, but for the remaining 5.8%, this function loses precision
521−-- around the 15th decimal place. For 0.001% of numbers, this
522−-- function will lose precision at the 13th or 14th decimal place.
523−--
524−-- This function does not accept string representations of \"NaN\" or
525−-- \"Infinity\".
526double :: Parser Double
519double :: Parser Double
527−double = rational
520+double = scientifically Sci.toRealFloat
528
521
529-- | Parse a number, attempting to preserve both speed and precision.
522-- | Parse a number, attempting to preserve both speed and precision.
530--
523--
531−-- The syntax accepted by this parser is the same as for 'rational'.
524+-- The syntax accepted by this parser is the same as for 'double'.
532−--
533−-- /Note/: This function is almost ten times faster than 'rational'.
534−-- On integral inputs, it gives perfectly accurate answers, and on
535−-- floating point inputs, it is slightly less accurate than
536−-- 'rational'.
537−--
538−-- This function does not accept string representations of \"NaN\" or
539−-- \"
540number :: Parser Number
525number :: Parser Number
541number = scientifically $ \s ->
526number = scientifically $ \s ->
542− let e = base10Exponent s
527+ let e = Sci.base10Exponent s
543− c = coefficient s
528+ c = Sci.coefficient s
544 in if e >= 0
529 in if e >= 0
545 then I (c * 10 ^ e)
530 then I (c * 10 ^ e)
546− else D (fromInteger c / 10 ^ negate e)
531+ else D (Sci.toRealFloat s)
532+{-# DEPRECATED number "Use 'scientific' instead." #-}
547
533
548-- | Parse a scientific number.
534-- | Parse a scientific number.
549--
535--
550−-- The syntax accepted by this parser is the same as for 'rational'.
536+-- The syntax accepted by this parser is the same as for 'double'.
551scientific :: Parser Scientific
537scientific :: Parser Scientific
552scientific = scientifically id
538scientific = scientifically id
553
539
540+-- A strict pair
541+data SP = SP !Integer {-# UNPACK #-}!Int
542+
554{-# INLINE scientifically #-}
543{-# INLINE scientifically #-}
555scientifically :: (Scientific -> a) -> Parser a
544scientifically :: (Scientific -> a) -> Parser a
556scientifically h = do
545scientifically h = do
@@ -563,21 +552,21 @@ scientifically h = do
563
552
564 n <- decimal
553 n <- decimal
565
554
566− let f fracDigits = Sci.scientific (B8.foldl' step n fracDigits)
555+ let f fracDigits = SP (B8.foldl' step n fracDigits)
567− (negate $ B8.length fracDigits)
556+ (negate $ B8.length fracDigits)
568 step a w = a * 10 + fromIntegral (w - 48)
557 step a w = a * 10 + fromIntegral (w - 48)
569
558
570 dotty <- I.peekWord8
559 dotty <- I.peekWord8
571 -- '.' -> ascii 46
560 -- '.' -> ascii 46
572− s <- case dotty of
561+ SP c e <- case dotty of
573− Just 46 -> I.anyWord8 *> (f <$> I.takeWhile isDigit_w8)
562+ Just 46 -> I.anyWord8 *> (f <$> I.takeWhile isDigit_w8)
574− _ -> pure (Sci.scientific n 0)
563+ _ -> pure (SP n 0)
575
564
576− let !signedCoeff | positive = coefficient s
565+ let !signedCoeff | positive = c
577− | otherwise = negate $ coefficient s
566+ | otherwise = -c
578
567
579 let littleE = 101
568 let littleE = 101
580 bigE = 69
569 bigE = 69
581− (I.satisfy (\c -> c == littleE || c == bigE) *>
570+ (I.satisfy (\ex -> ex == littleE || ex == bigE) *>
582− fmap (h . Sci.scientific signedCoeff . (base10Exponent s +)) (signed decimal)) <|>
571+ fmap (h . Sci.scientific signedCoeff . (e +)) (signed decimal)) <|>
583− return (h $ Sci.scientific signedCoeff (base10Exponent s))
572+ return (h $ Sci.scientific signedCoeff e)
Data/Attoparsec/ByteString/FastSet.hs+1−1
@@ -3,7 +3,7 @@
3-----------------------------------------------------------------------------
3-----------------------------------------------------------------------------
4-- |
4-- |
5-- Module : Data.Attoparsec.ByteString.FastSet
5-- Module : Data.Attoparsec.ByteString.FastSet
6−-- Copyright : Bryan O'Sullivan 2008
6+-- Copyright : Bryan O'Sullivan 2007-2015
7-- License : BSD3
7-- License : BSD3
8--
8--
9-- Maintainer : bos@serpentine.com
9-- Maintainer : bos@serpentine.com
Data/Attoparsec/ByteString/Internal.hs+157−157
@@ -1,15 +1,15 @@
1−{-# LANGUAGE BangPatterns, Rank2Types, OverloadedStrings,
1+{-# LANGUAGE BangPatterns, CPP, GADTs, OverloadedStrings, RankNTypes,
2− RecordWildCards, MagicHash, UnboxedTuples #-}
2+ RecordWildCards #-}
3-- |
3-- |
4-- Module : Data.Attoparsec.ByteString.Internal
4-- Module : Data.Attoparsec.ByteString.Internal
5−-- Copyright : Bryan O'Sullivan 2007-2011
5+-- Copyright : Bryan O'Sullivan 2007-2015
6-- License : BSD3
6-- License : BSD3
7--
7--
8-- Maintainer : bos@serpentine.com
8-- Maintainer : bos@serpentine.com
9-- Stability : experimental
9-- Stability : experimental
10-- Portability : unknown
10-- Portability : unknown
11--
11--
12−-- Simple, efficient parser combinators for 'B.ByteString' strings,
12+-- Simple, efficient parser combinators for 'ByteString' strings,
13-- loosely based on the Parsec library.
13-- loosely based on the Parsec library.
14
14
15module Data.Attoparsec.ByteString.Internal
15module Data.Attoparsec.ByteString.Internal
@@ -47,7 +47,7 @@ module Data.Attoparsec.ByteString.Internal
47 -- * Efficient string handling
47 -- * Efficient string handling
48 , skipWhile
48 , skipWhile
49 , string
49 , string
50− , stringTransform
50+ , stringCI
51 , take
51 , take
52 , scan
52 , scan
53 , runScanner
53 , runScanner
@@ -66,17 +66,25 @@ module Data.Attoparsec.ByteString.Internal
66 , atEnd
66 , atEnd
67 ) where
67 ) where
68
68
69+#if !MIN_VERSION_base(4,8,0)
69−import Control.Applicative ((<|>), (<$>))
70+import Control.Applicative ((<$>))
71+#endif
72+import Control.Applicative ((<|>))
70import Control.Monad (when)
73import Control.Monad (when)
74+import Data.Attoparsec.ByteString.Buffer (Buffer, buffer)
71import Data.Attoparsec.ByteString.FastSet (charClass, memberWord8)
75import Data.Attoparsec.ByteString.FastSet (charClass, memberWord8)
72import Data.Attoparsec.Combinator ((<?>))
76import Data.Attoparsec.Combinator ((<?>))
77+import Data.Attoparsec.Internal
78+import Data.Attoparsec.Internal.Fhthagn (inlinePerformIO)
73import Data.Attoparsec.Internal.Types hiding (Parser, Failure, Success)
79import Data.Attoparsec.Internal.Types hiding (Parser, Failure, Success)
80+import Data.ByteString (ByteString)
81+import Data.List (intercalate)
74import Data.Word (Word8)
82import Data.Word (Word8)
75import Foreign.ForeignPtr (withForeignPtr)
83import Foreign.ForeignPtr (withForeignPtr)
76import Foreign.Ptr (castPtr, minusPtr, plusPtr)
84import Foreign.Ptr (castPtr, minusPtr, plusPtr)
77import Foreign.Storable (Storable(peek, sizeOf))
85import Foreign.Storable (Storable(peek, sizeOf))
78import Prelude hiding (getChar, succ, take, takeWhile)
86import Prelude hiding (getChar, succ, take, takeWhile)
79−import Data.ByteString (ByteString)
87+import qualified Data.Attoparsec.ByteString.Buffer as Buf
80import qualified Data.Attoparsec.Internal.Types as T
88import qualified Data.Attoparsec.Internal.Types as T
81import qualified Data.ByteString as B8
89import qualified Data.ByteString as B8
82import qualified Data.ByteString.Char8 as B
90import qualified Data.ByteString.Char8 as B
@@ -84,13 +92,10 @@ import qualified Data.ByteString.Internal as B
84import qualified Data.ByteString.Lazy as L
92import qualified Data.ByteString.Lazy as L
85import qualified Data.ByteString.Unsafe as B
93import qualified Data.ByteString.Unsafe as B
86
94
87−import GHC.Base (realWorld#)
88−import GHC.IO (IO(IO))
89−
90−type Parser = T.Parser B.ByteString
95+type Parser = T.Parser ByteString
91−type Result = IResult B.ByteString
96+type Result = IResult ByteString
92−type Failure r = T.Failure B.ByteString r
97+type Failure r = T.Failure ByteString Buffer r
93−type Success a r = T.Success B.ByteString a r
98+type Success a r = T.Success ByteString Buffer a r
94
99
95-- | The parser @satisfy p@ succeeds for any byte for which the
100-- | The parser @satisfy p@ succeeds for any byte for which the
96-- predicate @p@ returns 'True'. Returns the byte that is actually
101-- predicate @p@ returns 'True'. Returns the byte that is actually
@@ -100,8 +105,7 @@ type Success a r = T.Success B.ByteString a r
100-- > where isDigit w = w >= 48 && w <= 57
105-- > where isDigit w = w >= 48 && w <= 57
101satisfy :: (Word8 -> Bool) -> Parser Word8
106satisfy :: (Word8 -> Bool) -> Parser Word8
102satisfy p = do
107satisfy p = do
103− c <- ensure 1
108+ h <- peekWord8'
104− let !h = B.unsafeHead c
105 if p h
109 if p h
106 then advance 1 >> return h
110 then advance 1 >> return h
107 else fail "satisfy"
111 else fail "satisfy"
@@ -114,8 +118,8 @@ satisfy p = do
114-- > where isDigit w = w >= 48 && w <= 57
118-- > where isDigit w = w >= 48 && w <= 57
115skip :: (Word8 -> Bool) -> Parser ()
119skip :: (Word8 -> Bool) -> Parser ()
116skip p = do
120skip p = do
117− s <- ensure 1
121+ h <- peekWord8'
118− if p (B.unsafeHead s)
122+ if p h
119 then advance 1
123 then advance 1
120 else fail "skip"
124 else fail "skip"
121
125
@@ -124,8 +128,8 @@ skip p = do
124-- parser returns the transformed byte that was parsed.
128-- parser returns the transformed byte that was parsed.
125satisfyWith :: (Word8 -> a) -> (a -> Bool) -> Parser a
129satisfyWith :: (Word8 -> a) -> (a -> Bool) -> Parser a
126satisfyWith f p = do
130satisfyWith f p = do
127− s <- ensure 1
131+ h <- peekWord8'
128− let c = f $! B.unsafeHead s
132+ let c = f h
129 if p c
133 if p c
130 then advance 1 >> return c
134 then advance 1 >> return c
131 else fail "satisfyWith"
135 else fail "satisfyWith"
@@ -137,22 +141,15 @@ storable = hack undefined
137 hack :: Storable b => b -> Parser b
141 hack :: Storable b => b -> Parser b
138 hack dummy = do
142 hack dummy = do
139 (fp,o,_) <- B.toForeignPtr `fmap` take (sizeOf dummy)
143 (fp,o,_) <- B.toForeignPtr `fmap` take (sizeOf dummy)
140− return . B.inlinePerformIO . withForeignPtr fp $ \p ->
144+ return . inlinePerformIO . withForeignPtr fp $ \p ->
141 peek (castPtr $ p `plusPtr` o)
145 peek (castPtr $ p `plusPtr` o)
142
146
143−-- | Consume @n@ bytes of input, but succeed only if the predicate
147+-- | Consume exactly @n@ bytes of input.
144−-- returns 'True'.
145−takeWith :: Int -> (B.ByteString -> Bool) -> Parser B.ByteString
148+take :: Int -> Parser ByteString
146−takeWith n0 p = do
149+take n0 = do
147 let n = max n0 0
150 let n = max n0 0
148 s <- ensure n
151 s <- ensure n
149− if p s
150− then advance n >> return s
152+ advance n >> return s
151− else fail "takeWith"
152−
153−-- | Consume exactly @n@ bytes of input.
154−take :: Int -> Parser B.ByteString
155−take n = takeWith n (const True)
156{-# INLINE take #-}
153{-# INLINE take #-}
157
154
158-- | @string s@ parses a sequence of bytes that identically match
155-- | @string s@ parses a sequence of bytes that identically match
@@ -168,16 +165,62 @@ take n = takeWith n (const True)
168--
165--
169-- The reason for its failure is that the first branch is a
166-- The reason for its failure is that the first branch is a
170-- partial match, and will consume the letters @\'f\'@ and @\'o\'@
167-- partial match, and will consume the letters @\'f\'@ and @\'o\'@
171−-- before failing. In Attoparsec, the above parser will /succeed/ on
168+-- before failing. In attoparsec, the above parser will /succeed/ on
172-- that input, because the failed first branch will consume nothing.
169-- that input, because the failed first branch will consume nothing.
173−string :: B.ByteString -> Parser B.ByteString
170+string :: ByteString -> Parser ByteString
174−string s = takeWith (B.length s) (==s)
171+string s = string_ (stringSuspended id) id s
175{-# INLINE string #-}
172{-# INLINE string #-}
176
173
177−stringTransform :: (B.ByteString -> B.ByteString) -> B.ByteString
174+-- ASCII-specific but fast, oh yes.
178− -> Parser B.ByteString
175+toLower :: Word8 -> Word8
179−stringTransform f s = takeWith (B.length s) ((==f s) . f)
176+toLower w | w >= 65 && w <= 90 = w + 32
180−{-# INLINE stringTransform #-}
177+ | otherwise = w
178+
179+-- | Satisfy a literal string, ignoring case.
180+stringCI :: ByteString -> Parser ByteString
181+stringCI s = string_ (stringSuspended lower) lower s
182+ where lower = B8.map toLower
183+{-# INLINE stringCI #-}
184+
185+string_ :: (forall r. ByteString -> ByteString -> Buffer -> Pos -> More
186+ -> Failure r -> Success ByteString r -> Result r)
187+ -> (ByteString -> ByteString)
188+ -> ByteString -> Parser ByteString
189+string_ suspended f s0 = T.Parser $ \t pos more lose succ ->
190+ let n = B.length s
191+ s = f s0
192+ in if lengthAtLeast pos n t
193+ then let t' = substring pos (Pos n) t
194+ in if s == f t'
195+ then succ t (pos + Pos n) more t'
196+ else lose t pos more [] "string"
197+ else let t' = Buf.unsafeDrop (fromPos pos) t
198+ in if f t' `B.isPrefixOf` s
199+ then suspended s (B.drop (B.length t') s) t pos more lose succ
200+ else lose t pos more [] "string"
201+{-# INLINE string_ #-}
202+
203+stringSuspended :: (ByteString -> ByteString)
204+ -> ByteString -> ByteString -> Buffer -> Pos -> More
205+ -> Failure r
206+ -> Success ByteString r
207+ -> Result r
208+stringSuspended f s0 s t pos more lose succ =
209+ runParser (demandInput_ >>= go) t pos more lose succ
210+ where go s'0 = T.Parser $ \t' pos' more' lose' succ' ->
211+ let m = B.length s
212+ s' = f s'0
213+ n = B.length s'
214+ in if n >= m
215+ then if B.unsafeTake m s' == s
216+ then let o = Pos (B.length s0)
217+ in succ' t' (pos' + o) more'
218+ (substring pos' o t')
219+ else lose' t' pos' more' [] "string"
220+ else if s' == B.unsafeTake n s
221+ then stringSuspended f s0 (B.unsafeDrop n s)
222+ t' pos' more' lose' succ'
223+ else lose' t' pos' more' [] "string"
181
224
182-- | Skip past input for as long as the predicate returns 'True'.
225-- | Skip past input for as long as the predicate returns 'True'.
183skipWhile :: (Word8 -> Bool) -> Parser ()
226skipWhile :: (Word8 -> Bool) -> Parser ()
@@ -185,11 +228,8 @@ skipWhile p = go
185 where
228 where
186 go = do
229 go = do
187 t <- B8.takeWhile p <$> get
230 t <- B8.takeWhile p <$> get
188− advance (B.length t)
231+ continue <- inputSpansChunks (B.length t)
189− eoc <- endOfChunk
232+ when continue go
190− when eoc $ do
191− input <- wantInput
192− when input go
193{-# INLINE skipWhile #-}
233{-# INLINE skipWhile #-}
194
234
195-- | Consume input as long as the predicate returns 'False'
235-- | Consume input as long as the predicate returns 'False'
@@ -202,7 +242,7 @@ skipWhile p = go
202-- combinators such as 'Control.Applicative.many', because such
242-- combinators such as 'Control.Applicative.many', because such
203-- parsers loop until a failure occurs. Careless use will thus result
243-- parsers loop until a failure occurs. Careless use will thus result
204-- in an infinite loop.
244-- in an infinite loop.
205−takeTill :: (Word8 -> Bool) -> Parser B.ByteString
245+takeTill :: (Word8 -> Bool) -> Parser ByteString
206takeTill p = takeWhile (not . p)
246takeTill p = takeWhile (not . p)
207{-# INLINE takeTill #-}
247{-# INLINE takeTill #-}
208
248
@@ -216,23 +256,27 @@ takeTill p = takeWhile (not . p)
216-- combinators such as 'Control.Applicative.many', because such
256-- combinators such as 'Control.Applicative.many', because such
217-- parsers loop until a failure occurs. Careless use will thus result
257-- parsers loop until a failure occurs. Careless use will thus result
218-- in an infinite loop.
258-- in an infinite loop.
219−takeWhile :: (Word8 -> Bool) -> Parser B.ByteString
259+takeWhile :: (Word8 -> Bool) -> Parser ByteString
220−takeWhile p = (B.concat . reverse) `fmap` go []
260+takeWhile p = do
261+ s <- B8.takeWhile p <$> get
262+ continue <- inputSpansChunks (B.length s)
263+ if continue
264+ then takeWhileAcc p [s]
265+ else return s
266+{-# INLINE takeWhile #-}
267+
268+takeWhileAcc :: (Word8 -> Bool) -> [ByteString] -> Parser ByteString
269+takeWhileAcc p = go
221 where
270 where
222 go acc = do
271 go acc = do
223 s <- B8.takeWhile p <$> get
272 s <- B8.takeWhile p <$> get
224− advance (B.length s)
273+ continue <- inputSpansChunks (B.length s)
225− eoc <- endOfChunk
274+ if continue
226− if eoc
227− then do
228− input <- wantInput
229− if input
230− then go (s:acc)
275+ then go (s:acc)
231− else return (s:acc)
276+ else return $ concatReverse (s:acc)
232− else return (s:acc)
277+{-# INLINE takeWhileAcc #-}
233−{-# INLINE takeWhile #-}
234
278
235−takeRest :: Parser [B.ByteString]
279+takeRest :: Parser [ByteString]
236takeRest = go []
280takeRest = go []
237 where
281 where
238 go acc = do
282 go acc = do
@@ -245,7 +289,7 @@ takeRest = go []
245 else return (reverse acc)
289 else return (reverse acc)
246
290
247-- | Consume all remaining input and return it as a single string.
291-- | Consume all remaining input and return it as a single string.
248−takeByteString :: Parser B.ByteString
292+takeByteString :: Parser ByteString
249takeByteString = B.concat `fmap` takeRest
293takeByteString = B.concat `fmap` takeRest
250
294
251-- | Consume all remaining input and return it as a single string.
295-- | Consume all remaining input and return it as a single string.
@@ -254,7 +298,7 @@ takeLazyByteString = L.fromChunks `fmap` takeRest
254
298
255data T s = T {-# UNPACK #-} !Int s
299data T s = T {-# UNPACK #-} !Int s
256
300
257−scan_ :: (s -> [B.ByteString] -> Parser r) -> s -> (s -> Word8 -> Maybe s)
301+scan_ :: (s -> [ByteString] -> Parser r) -> s -> (s -> Word8 -> Maybe s)
258 -> Parser r
302 -> Parser r
259scan_ f s0 p = go [] s0
303scan_ f s0 p = go [] s0
260 where
304 where
@@ -275,14 +319,9 @@ scan_ f s0 p = go [] s0
275 bs <- get
319 bs <- get
276 let T i s' = inlinePerformIO $ scanner bs
320 let T i s' = inlinePerformIO $ scanner bs
277 !h = B.unsafeTake i bs
321 !h = B.unsafeTake i bs
278− advance i
322+ continue <- inputSpansChunks i
279− eoc <- endOfChunk
323+ if continue
280− if eoc
281− then do
282− input <- wantInput
283− if input
284− then go (h:acc) s'
324+ then go (h:acc) s'
285− else f s' (h:acc)
286 else f s' (h:acc)
325 else f s' (h:acc)
287{-# INLINE scan_ #-}
326{-# INLINE scan_ #-}
288
327
@@ -298,17 +337,14 @@ scan_ f s0 p = go [] s0
298-- combinators such as 'Control.Applicative.many', because such
337-- combinators such as 'Control.Applicative.many', because such
299-- parsers loop until a failure occurs. Careless use will thus result
338-- parsers loop until a failure occurs. Careless use will thus result
300-- in an infinite loop.
339-- in an infinite loop.
301−scan :: s -> (s -> Word8 -> Maybe s) -> Parser B.ByteString
340+scan :: s -> (s -> Word8 -> Maybe s) -> Parser ByteString
302−scan = scan_ $ \_ chunks ->
341+scan = scan_ $ \_ chunks -> return $! concatReverse chunks
303− case chunks of
304− [x] -> return x
305− xs -> return $! B.concat $ reverse xs
306{-# INLINE scan #-}
342{-# INLINE scan #-}
307
343
308-- | Like 'scan', but generalized to return the final state of the
344-- | Like 'scan', but generalized to return the final state of the
309-- scanner.
345-- scanner.
310−runScanner :: s -> (s -> Word8 -> Maybe s) -> Parser (B.ByteString, s)
346+runScanner :: s -> (s -> Word8 -> Maybe s) -> Parser (ByteString, s)
311−runScanner = scan_ $ \s xs -> return (B.concat (reverse xs), s)
347+runScanner = scan_ $ \s xs -> let !sx = concatReverse xs in return (sx, s)
312{-# INLINE runScanner #-}
348{-# INLINE runScanner #-}
313
349
314-- | Consume input as long as the predicate returns 'True', and return
350-- | Consume input as long as the predicate returns 'True', and return
@@ -317,7 +353,7 @@ runScanner = scan_ $ \s xs -> return (B.concat (reverse xs), s)
317-- This parser requires the predicate to succeed on at least one byte
353-- This parser requires the predicate to succeed on at least one byte
318-- of input: it will fail if the predicate never returns 'True' or if
354-- of input: it will fail if the predicate never returns 'True' or if
319-- there is no input left.
355-- there is no input left.
320−takeWhile1 :: (Word8 -> Bool) -> Parser B.ByteString
356+takeWhile1 :: (Word8 -> Bool) -> Parser ByteString
321takeWhile1 p = do
357takeWhile1 p = do
322 (`when` demandInput) =<< endOfChunk
358 (`when` demandInput) =<< endOfChunk
323 s <- B8.takeWhile p <$> get
359 s <- B8.takeWhile p <$> get
@@ -328,8 +364,9 @@ takeWhile1 p = do
328 advance len
364 advance len
329 eoc <- endOfChunk
365 eoc <- endOfChunk
330 if eoc
366 if eoc
331− then (s<>) `fmap` takeWhile p
367+ then takeWhileAcc p [s]
332 else return s
368 else return s
369+{-# INLINE takeWhile1 #-}
333
370
334-- | Match any byte in a set.
371-- | Match any byte in a set.
335--
372--
@@ -377,13 +414,13 @@ notWord8 c = satisfy (/= c) <?> "not " ++ show c
377peekWord8 :: Parser (Maybe Word8)
414peekWord8 :: Parser (Maybe Word8)
378peekWord8 = T.Parser $ \t pos@(Pos pos_) more _lose succ ->
415peekWord8 = T.Parser $ \t pos@(Pos pos_) more _lose succ ->
379 case () of
416 case () of
380− _| pos_ < B.length t ->
417+ _| pos_ < Buf.length t ->
381− let !w = B.unsafeIndex t pos_
418+ let !w = Buf.unsafeIndex t pos_
382 in succ t pos more (Just w)
419 in succ t pos more (Just w)
383 | more == Complete ->
420 | more == Complete ->
384 succ t pos more Nothing
421 succ t pos more Nothing
385 | otherwise ->
422 | otherwise ->
386− let succ' t' pos' more' = let !w = B.unsafeIndex t pos_
423+ let succ' t' pos' more' = let !w = Buf.unsafeIndex t' pos_
387 in succ t' pos' more' (Just w)
424 in succ t' pos' more' (Just w)
388 lose' t' pos' more' = succ t' pos' more' Nothing
425 lose' t' pos' more' = succ t' pos' more' Nothing
389 in prompt t pos more lose' succ'
426 in prompt t pos more lose' succ'
@@ -392,9 +429,11 @@ peekWord8 = T.Parser $ \t pos@(Pos pos_) more _lose succ ->
392-- | Match any byte, to perform lookahead. Does not consume any
429-- | Match any byte, to perform lookahead. Does not consume any
393-- input, but will fail if end of input has been reached.
430-- input, but will fail if end of input has been reached.
394peekWord8' :: Parser Word8
431peekWord8' :: Parser Word8
395−peekWord8' = do
432+peekWord8' = T.Parser $ \t pos more lose succ ->
396− s <- ensure 1
433+ if lengthAtLeast pos 1 t
397− return $! B.unsafeHead s
434+ then succ t pos more (Buf.unsafeIndex t (fromPos pos))
435+ else let succ' t' pos' more' bs' = succ t' pos' more' $! B.unsafeHead bs'
436+ in ensureSuspended 1 t pos more lose succ'
398{-# INLINE peekWord8' #-}
437{-# INLINE peekWord8' #-}
399
438
400-- | Match either a single newline character @\'\\n\'@, or a carriage
439-- | Match either a single newline character @\'\\n\'@, or a carriage
@@ -404,51 +443,62 @@ endOfLine = (word8 10 >> return ()) <|> (string "\r\n" >> return ())
404
443
405-- | Terminal failure continuation.
444-- | Terminal failure continuation.
406failK :: Failure a
445failK :: Failure a
407−failK t (Pos pos) _more stack msg = Fail (B.unsafeDrop pos t) stack msg
446+failK t (Pos pos) _more stack msg = Fail (Buf.unsafeDrop pos t) stack msg
408{-# INLINE failK #-}
447{-# INLINE failK #-}
409
448
410-- | Terminal success continuation.
449-- | Terminal success continuation.
411successK :: Success a a
450successK :: Success a a
412−successK t (Pos pos) _more a = Done (B.unsafeDrop pos t) a
451+successK t (Pos pos) _more a = Done (Buf.unsafeDrop pos t) a
413{-# INLINE successK #-}
452{-# INLINE successK #-}
414
453
415-- | Run a parser.
454-- | Run a parser.
416−parse :: Parser a -> B.ByteString -> Result a
455+parse :: Parser a -> ByteString -> Result a
417−parse m s = T.runParser m s (Pos 0) Incomplete failK successK
456+parse m s = T.runParser m (buffer s) (Pos 0) Incomplete failK successK
418{-# INLINE parse #-}
457{-# INLINE parse #-}
419
458
420-- | Run a parser that cannot be resupplied via a 'Partial' result.
459-- | Run a parser that cannot be resupplied via a 'Partial' result.
460+--
461+-- This function does not force a parser to consume all of its input.
462+-- Instead, any residual input will be discarded. To force a parser
463+-- to consume all of its input, use something like this:
464+--
465+-- @
466+--'parseOnly' (myParser 'Control.Applicative.<*' 'endOfInput')
467+-- @
421−parseOnly :: Parser a -> B.ByteString -> Either String a
468+parseOnly :: Parser a -> ByteString -> Either String a
422−parseOnly m s = case T.runParser m s (Pos 0) Complete failK successK of
469+parseOnly m s = case T.runParser m (buffer s) (Pos 0) Complete failK successK of
470+ Fail _ [] err -> Left err
423− Fail _ _ err -> Left err
471+ Fail _ ctxs err -> Left (intercalate " > " ctxs ++ ": " ++ err)
424− Done _ a -> Right a
472+ Done _ a -> Right a
425− _ -> error "parseOnly: impossible error!"
473+ _ -> error "parseOnly: impossible error!"
426{-# INLINE parseOnly #-}
474{-# INLINE parseOnly #-}
427
475
428−-- | Just like unsafePerformIO, but we inline it. Big performance gains as
429−-- it exposes lots of things to further inlining. /Very unsafe/. In
430−-- particular, you should do no memory allocation inside an
431−-- 'inlinePerformIO' block. On Hugs this is just @unsafePerformIO@.
432−inlinePerformIO :: IO a -> a
433−inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r
434−{-# INLINE inlinePerformIO #-}
435−
436get :: Parser ByteString
476get :: Parser ByteString
437get = T.Parser $ \t pos more _lose succ ->
477get = T.Parser $ \t pos more _lose succ ->
438− succ t pos more (B.unsafeDrop (fromPos pos) t)
478+ succ t pos more (Buf.unsafeDrop (fromPos pos) t)
439{-# INLINE get #-}
479{-# INLINE get #-}
440
480
441endOfChunk :: Parser Bool
481endOfChunk :: Parser Bool
442endOfChunk = T.Parser $ \t pos more _lose succ ->
482endOfChunk = T.Parser $ \t pos more _lose succ ->
443− succ t pos more (fromPos pos == B.length t)
483+ succ t pos more (fromPos pos == Buf.length t)
444{-# INLINE endOfChunk #-}
484{-# INLINE endOfChunk #-}
445
485
486+inputSpansChunks :: Int -> Parser Bool
487+inputSpansChunks i = T.Parser $ \t pos_ more _lose succ ->
488+ let pos = pos_ + Pos i
489+ in if fromPos pos < Buf.length t || more == Complete
490+ then succ t pos more False
491+ else let lose' t' pos' more' = succ t' pos' more' False
492+ succ' t' pos' more' = succ t' pos' more' True
493+ in prompt t pos more lose' succ'
494+{-# INLINE inputSpansChunks #-}
495+
446advance :: Int -> Parser ()
496advance :: Int -> Parser ()
447advance n = T.Parser $ \t pos more _lose succ ->
497advance n = T.Parser $ \t pos more _lose succ ->
448 succ t (pos + Pos n) more ()
498 succ t (pos + Pos n) more ()
449{-# INLINE advance #-}
499{-# INLINE advance #-}
450
500
451−ensureSuspended :: Int -> ByteString -> Pos -> More
501+ensureSuspended :: Int -> Buffer -> Pos -> More
452 -> Failure r
502 -> Failure r
453 -> Success ByteString r
503 -> Success ByteString r
454 -> Result r
504 -> Result r
@@ -467,52 +517,8 @@ ensure n = T.Parser $ \t pos more lose succ ->
467 then succ t pos more (substring pos (Pos n) t)
517 then succ t pos more (substring pos (Pos n) t)
468 -- The uncommon case is kept out-of-line to reduce code size:
518 -- The uncommon case is kept out-of-line to reduce code size:
469 else ensureSuspended n t pos more lose succ
519 else ensureSuspended n t pos more lose succ
470−-- Non-recursive so the bounds check can be inlined:
471{-# INLINE ensure #-}
520{-# INLINE ensure #-}
472
521
473−-- | Ask for input. If we receive any, pass it to a success
474−-- continuation, otherwise to a failure continuation.
475−prompt :: ByteString -> Pos -> More
476− -> (ByteString -> Pos -> More -> IResult ByteString r)
477− -> (ByteString -> Pos -> More -> IResult ByteString r)
478− -> IResult ByteString r
479−prompt t pos _more lose succ = Partial $ \s ->
480− if B.null s
481− then lose t pos Complete
482− else succ (t <> s) pos Incomplete
483−
484−-- | Immediately demand more input via a 'Partial' continuation
485−-- result.
486−demandInput :: Parser ()
487−demandInput = T.Parser $ \t pos more lose succ ->
488− case more of
489− Complete -> lose t pos more [] "not enough input"
490− _ -> let lose' t' pos' more' = lose t' pos' more' [] "not enough input"
491− succ' t' pos' more' = succ t' pos' more' ()
492− in prompt t pos more lose' succ'
493−
494−-- | This parser always succeeds. It returns 'True' if any input is
495−-- available either immediately or on demand, and 'False' if the end
496−-- of all input has been reached.
497−wantInput :: Parser Bool
498−wantInput = T.Parser $ \t pos more _lose succ ->
499− case () of
500− _ | lengthAtLeast pos 1 t -> succ t pos more True
501− | more == Complete -> succ t pos more False
502− | otherwise -> let lose' t' pos' more' = succ t' pos' more' False
503− succ' t' pos' more' = succ t' pos' more' True
504− in prompt t pos more lose' succ'
505−-- | Match only if all input has been consumed.
506−endOfInput :: Parser ()
507−endOfInput = T.Parser $ \t pos more lose succ ->
508− case () of
509− _| lengthAtLeast pos 1 t -> lose t pos more [] "endOfInput"
510− | more == Complete -> succ t pos more ()
511− | otherwise ->
512− let lose' t' pos' more' _ctx _msg = succ t' pos' more' ()
513− succ' t' pos' more' _a = lose t' pos' more' [] "endOfInput"
514− in runParser demandInput t pos more lose' succ'
515−
516-- | Return both the result of a parse and the portion of the input
522-- | Return both the result of a parse and the portion of the input
517-- that was consumed while it was being parsed.
523-- that was consumed while it was being parsed.
518match :: Parser a -> Parser (ByteString, a)
524match :: Parser a -> Parser (ByteString, a)
@@ -521,16 +527,10 @@ match p = T.Parser $ \t pos more lose succ ->
521 succ t' pos' more' (substring pos (pos'-pos) t', a)
527 succ t' pos' more' (substring pos (pos'-pos) t', a)
522 in runParser p t pos more lose succ'
528 in runParser p t pos more lose succ'
523
529
524−-- | Return an indication of whether the end of input has been
525−-- reached.
526−atEnd :: Parser Bool
527−atEnd = not <$> wantInput
528−{-# INLINE atEnd #-}
529−
530−lengthAtLeast :: Pos -> Int -> ByteString -> Bool
530+lengthAtLeast :: Pos -> Int -> Buffer -> Bool
531−lengthAtLeast (Pos pos) n bs = B.length bs >= pos + n
531+lengthAtLeast (Pos pos) n bs = Buf.length bs >= pos + n
532{-# INLINE lengthAtLeast #-}
532{-# INLINE lengthAtLeast #-}
533
533
534−substring :: Pos -> Pos -> ByteString -> ByteString
534+substring :: Pos -> Pos -> Buffer -> ByteString
535−substring (Pos pos) (Pos n) bs = B.unsafeTake n (B.unsafeDrop pos bs)
535+substring (Pos pos) (Pos n) = Buf.substring pos n
536{-# INLINE substring #-}
536{-# INLINE substring #-}
Data/Attoparsec/ByteString/Lazy.hs+10−3
@@ -1,6 +1,11 @@
1+{-# LANGUAGE CPP #-}
2+#if __GLASGOW_HASKELL__ >= 702
3+{-# LANGUAGE Trustworthy #-} -- Imports internal modules
4+#endif
5+
1-- |
6-- |
2-- Module : Data.Attoparsec.ByteString.Lazy
7-- Module : Data.Attoparsec.ByteString.Lazy
3−-- Copyright : Bryan O'Sullivan 2010, 2011
8+-- Copyright : Bryan O'Sullivan 2007-2015
4-- License : BSD3
9-- License : BSD3
5--
10--
6-- Maintainer : bos@serpentine.com
11-- Maintainer : bos@serpentine.com
@@ -36,6 +41,7 @@ module Data.Attoparsec.ByteString.Lazy
36
41
37import Control.DeepSeq (NFData(rnf))
42import Control.DeepSeq (NFData(rnf))
38import Data.ByteString.Lazy.Internal (ByteString(..), chunk)
43import Data.ByteString.Lazy.Internal (ByteString(..), chunk)
44+import Data.List (intercalate)
39import qualified Data.ByteString as B
45import qualified Data.ByteString as B
40import qualified Data.Attoparsec.ByteString as A
46import qualified Data.Attoparsec.ByteString as A
41import qualified Data.Attoparsec.Internal.Types as T
47import qualified Data.Attoparsec.Internal.Types as T
@@ -99,5 +105,6 @@ maybeResult _ = Nothing
99
105
100-- | Convert a 'Result' value to an 'Either' value.
106-- | Convert a 'Result' value to an 'Either' value.
101eitherResult :: Result r -> Either String r
107eitherResult :: Result r -> Either String r
102−eitherResult (Done _ r) = Right r
108+eitherResult (Done _ r) = Right r
103−eitherResult (Fail _ _ msg) = Left msg
109+eitherResult (Fail _ [] msg) = Left msg
110+eitherResult (Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
Data/Attoparsec/Char8.hs+1−1
@@ -1,6 +1,6 @@
1-- |
1-- |
2-- Module : Data.Attoparsec.Char8
2-- Module : Data.Attoparsec.Char8
3−-- Copyright : Bryan O'Sullivan 2007-2011
3+-- Copyright : Bryan O'Sullivan 2007-2015
4-- License : BSD3
4-- License : BSD3
5--
5--
6-- Maintainer : bos@serpentine.com
6-- Maintainer : bos@serpentine.com
Data/Attoparsec/Combinator.hs+34−16
@@ -1,7 +1,10 @@
1−{-# LANGUAGE BangPatterns #-}
1+{-# LANGUAGE BangPatterns, CPP #-}
2+#if __GLASGOW_HASKELL__ >= 702
3+{-# LANGUAGE Trustworthy #-} -- Imports internal modules
4+#endif
2-- |
5-- |
3-- Module : Data.Attoparsec.Combinator
6-- Module : Data.Attoparsec.Combinator
4−-- Copyright : Daan Leijen 1999-2001, Bryan O'Sullivan 2009-2010
7+-- Copyright : Daan Leijen 1999-2001, Bryan O'Sullivan 2007-2015
5-- License : BSD3
8-- License : BSD3
6--
9--
7-- Maintainer : bos@serpentine.com
10-- Maintainer : bos@serpentine.com
@@ -30,14 +33,21 @@ module Data.Attoparsec.Combinator
30 , skipMany1
33 , skipMany1
31 , eitherP
34 , eitherP
32 , feed
35 , feed
36+ , satisfyElem
37+ , endOfInput
38+ , atEnd
39+ , lookAhead
33 ) where
40 ) where
34
41
42+#if !MIN_VERSION_base(4,8,0)
43+import Control.Applicative (Applicative(..), (<$>))
44+import Data.Monoid (Monoid(mappend))
45+#endif
35−import Control.Applicative (Alternative(..), Applicative(..), empty, liftA2,
46+import Control.Applicative (Alternative(..), empty, liftA2, many, (<|>))
36− many, (<|>), (*>), (<$>))
37import Control.Monad (MonadPlus(..))
47import Control.Monad (MonadPlus(..))
38import Data.Attoparsec.Internal.Types (Parser(..), IResult(..))
48import Data.Attoparsec.Internal.Types (Parser(..), IResult(..))
49+import Data.Attoparsec.Internal (endOfInput, atEnd, satisfyElem)
39import Data.ByteString (ByteString)
50import Data.ByteString (ByteString)
40−import Data.Monoid (Monoid(mappend))
41import Data.Text (Text)
51import Data.Text (Text)
42import qualified Data.Attoparsec.Zepto as Z
52import qualified Data.Attoparsec.Zepto as Z
43import Prelude hiding (succ)
53import Prelude hiding (succ)
@@ -46,15 +56,15 @@ import Prelude hiding (succ)
46-- input appears to have been consumed.
56-- input appears to have been consumed.
47--
57--
48-- This combinator is provided for compatibility with Parsec.
58-- This combinator is provided for compatibility with Parsec.
49−-- Attoparsec parsers always backtrack on failure.
59+-- attoparsec parsers always backtrack on failure.
50−try :: Parser t a -> Parser t a
60+try :: Parser i a -> Parser i a
51try p = p
61try p = p
52{-# INLINE try #-}
62{-# INLINE try #-}
53
63
54-- | Name the parser, in case failure occurs.
64-- | Name the parser, in case failure occurs.
55−(<?>) :: Parser t a
65+(<?>) :: Parser i a
56 -> String -- ^ the name to use if parsing fails
66 -> String -- ^ the name to use if parsing fails
57− -> Parser t a
67+ -> Parser i a
58p <?> msg0 = Parser $ \t pos more lose succ ->
68p <?> msg0 = Parser $ \t pos more lose succ ->
59 let lose' t' pos' more' strs msg = lose t' pos' more' (msg0:strs) msg
69 let lose' t' pos' more' strs msg = lose t' pos' more' (msg0:strs) msg
60 in runParser p t pos more lose' succ
70 in runParser p t pos more lose' succ
@@ -66,7 +76,8 @@ infix 0 <?>
66-- action.
76-- action.
67choice :: Alternative f => [f a] -> f a
77choice :: Alternative f => [f a] -> f a
68choice = foldr (<|>) empty
78choice = foldr (<|>) empty
69−{-# SPECIALIZE choice :: [Parser ByteString a] -> Parser ByteString a #-}
79+{-# SPECIALIZE choice :: [Parser ByteString a]
80+ -> Parser ByteString a #-}
70{-# SPECIALIZE choice :: [Parser Text a] -> Parser Text a #-}
81{-# SPECIALIZE choice :: [Parser Text a] -> Parser Text a #-}
71{-# SPECIALIZE choice :: [Z.Parser a] -> Z.Parser a #-}
82{-# SPECIALIZE choice :: [Z.Parser a] -> Z.Parser a #-}
72
83
@@ -121,7 +132,7 @@ many1' p = liftM2' (:) p (many' p)
121-- | @sepBy p sep@ applies /zero/ or more occurrences of @p@, separated
132-- | @sepBy p sep@ applies /zero/ or more occurrences of @p@, separated
122-- by @sep@. Returns a list of the values returned by @p@.
133-- by @sep@. Returns a list of the values returned by @p@.
123--
134--
124−-- > commaSep p = p `sepBy` (symbol ",")
135+-- > commaSep p = p `sepBy` (char ',')
125sepBy :: Alternative f => f a -> f s -> f [a]
136sepBy :: Alternative f => f a -> f s -> f [a]
126sepBy p s = liftA2 (:) p ((s *> sepBy1 p s) <|> pure []) <|> pure []
137sepBy p s = liftA2 (:) p ((s *> sepBy1 p s) <|> pure []) <|> pure []
127{-# SPECIALIZE sepBy :: Parser ByteString a -> Parser ByteString s
138{-# SPECIALIZE sepBy :: Parser ByteString a -> Parser ByteString s
@@ -133,7 +144,7 @@ sepBy p s = liftA2 (:) p ((s *> sepBy1 p s) <|> pure []) <|> pure []
133-- by @sep@. Returns a list of the values returned by @p@. The value
144-- by @sep@. Returns a list of the values returned by @p@. The value
134-- returned by @p@ is forced to WHNF.
145-- returned by @p@ is forced to WHNF.
135--
146--
136−-- > commaSep p = p `sepBy'` (symbol ",")
147+-- > commaSep p = p `sepBy'` (char ',')
137sepBy' :: (MonadPlus m) => m a -> m s -> m [a]
148sepBy' :: (MonadPlus m) => m a -> m s -> m [a]
138sepBy' p s = scan `mplus` return []
149sepBy' p s = scan `mplus` return []
139 where scan = liftM2' (:) p ((s >> sepBy1' p s) `mplus` return [])
150 where scan = liftM2' (:) p ((s >> sepBy1' p s) `mplus` return [])
@@ -145,7 +156,7 @@ sepBy' p s = scan `mplus` return []
145-- | @sepBy1 p sep@ applies /one/ or more occurrences of @p@, separated
156-- | @sepBy1 p sep@ applies /one/ or more occurrences of @p@, separated
146-- by @sep@. Returns a list of the values returned by @p@.
157-- by @sep@. Returns a list of the values returned by @p@.
147--
158--
148−-- > commaSep p = p `sepBy1` (symbol ",")
159+-- > commaSep p = p `sepBy1` (char ',')
149sepBy1 :: Alternative f => f a -> f s -> f [a]
160sepBy1 :: Alternative f => f a -> f s -> f [a]
150sepBy1 p s = scan
161sepBy1 p s = scan
151 where scan = liftA2 (:) p ((s *> scan) <|> pure [])
162 where scan = liftA2 (:) p ((s *> scan) <|> pure [])
@@ -158,7 +169,7 @@ sepBy1 p s = scan
158-- by @sep@. Returns a list of the values returned by @p@. The value
169-- by @sep@. Returns a list of the values returned by @p@. The value
159-- returned by @p@ is forced to WHNF.
170-- returned by @p@ is forced to WHNF.
160--
171--
161−-- > commaSep p = p `sepBy1'` (symbol ",")
172+-- > commaSep p = p `sepBy1'` (char ',')
162sepBy1' :: (MonadPlus m) => m a -> m s -> m [a]
173sepBy1' :: (MonadPlus m) => m a -> m s -> m [a]
163sepBy1' p s = scan
174sepBy1' p s = scan
164 where scan = liftM2' (:) p ((s >> scan) `mplus` return [])
175 where scan = liftM2' (:) p ((s >> scan) `mplus` return [])
@@ -230,8 +241,15 @@ eitherP a b = (Left <$> a) <|> (Right <$> b)
230
241
231-- | If a parser has returned a 'T.Partial' result, supply it with more
242-- | If a parser has returned a 'T.Partial' result, supply it with more
232-- input.
243-- input.
233−feed :: Monoid t => IResult t r -> t -> IResult t r
244+feed :: Monoid i => IResult i r -> i -> IResult i r
234−feed f@(Fail _ _ _) _ = f
245+feed (Fail t ctxs msg) d = Fail (mappend t d) ctxs msg
235feed (Partial k) d = k d
246feed (Partial k) d = k d
236feed (Done t r) d = Done (mappend t d) r
247feed (Done t r) d = Done (mappend t d) r
237{-# INLINE feed #-}
248{-# INLINE feed #-}
249+
250+-- | Apply a parser without consuming any input.
251+lookAhead :: Parser i a -> Parser i a
252+lookAhead p = Parser $ \t pos more lose succ ->
253+ let succ' t' _pos' more' = succ t' pos more'
254+ in runParser p t pos more lose succ'
255+{-# INLINE lookAhead #-}
Data/Attoparsec/Internal.hs+143−5
@@ -1,7 +1,7 @@
1−{-# LANGUAGE BangPatterns #-}
1+{-# LANGUAGE BangPatterns, CPP, ScopedTypeVariables #-}
2-- |
2-- |
3-- Module : Data.Attoparsec.Internal
3-- Module : Data.Attoparsec.Internal
4−-- Copyright : Bryan O'Sullivan 2012
4+-- Copyright : Bryan O'Sullivan 2007-2015
5-- License : BSD3
5-- License : BSD3
6--
6--
7-- Maintainer : bos@serpentine.com
7-- Maintainer : bos@serpentine.com
@@ -12,11 +12,24 @@
12-- library.
12-- library.
13
13
14module Data.Attoparsec.Internal
14module Data.Attoparsec.Internal
15− (
16− compareResults
15+ ( compareResults
16+ , prompt
17+ , demandInput
18+ , demandInput_
19+ , wantInput
20+ , endOfInput
21+ , atEnd
22+ , satisfyElem
23+ , concatReverse
17 ) where
24 ) where
18
25
26+#if !MIN_VERSION_base(4,8,0)
27+import Control.Applicative ((<$>))
28+import Data.Monoid (Monoid, mconcat)
29+#endif
19import Data.Attoparsec.Internal.Types
30import Data.Attoparsec.Internal.Types
31+import Data.ByteString (ByteString)
32+import Data.Text (Text)
20import Prelude hiding (succ)
33import Prelude hiding (succ)
21
34
22-- | Compare two 'IResult' values for equality.
35-- | Compare two 'IResult' values for equality.
@@ -24,10 +37,135 @@ import Prelude hiding (succ)
24-- If both 'IResult's are 'Partial', the result will be 'Nothing', as
37-- If both 'IResult's are 'Partial', the result will be 'Nothing', as
25-- they are incomplete and hence their equality cannot be known.
38-- they are incomplete and hence their equality cannot be known.
26-- (This is why there is no 'Eq' instance for 'IResult'.)
39-- (This is why there is no 'Eq' instance for 'IResult'.)
27−compareResults :: (Eq t, Eq r) => IResult t r -> IResult t r -> Maybe Bool
40+compareResults :: (Eq i, Eq r) => IResult i r -> IResult i r -> Maybe Bool
28compareResults (Fail t0 ctxs0 msg0) (Fail t1 ctxs1 msg1) =
41compareResults (Fail t0 ctxs0 msg0) (Fail t1 ctxs1 msg1) =
29 Just (t0 == t1 && ctxs0 == ctxs1 && msg0 == msg1)
42 Just (t0 == t1 && ctxs0 == ctxs1 && msg0 == msg1)
30compareResults (Done t0 r0) (Done t1 r1) =
43compareResults (Done t0 r0) (Done t1 r1) =
31 Just (t0 == t1 && r0 == r1)
44 Just (t0 == t1 && r0 == r1)
32compareResults (Partial _) (Partial _) = Nothing
45compareResults (Partial _) (Partial _) = Nothing
33compareResults _ _ = Just False
46compareResults _ _ = Just False
47+
48+-- | Ask for input. If we receive any, pass the augmented input to a
49+-- success continuation, otherwise to a failure continuation.
50+prompt :: Chunk t
51+ => State t -> Pos -> More
52+ -> (State t -> Pos -> More -> IResult t r)
53+ -> (State t -> Pos -> More -> IResult t r)
54+ -> IResult t r
55+prompt t pos _more lose succ = Partial $ \s ->
56+ if nullChunk s
57+ then lose t pos Complete
58+ else succ (pappendChunk t s) pos Incomplete
59+{-# SPECIALIZE prompt :: State ByteString -> Pos -> More
60+ -> (State ByteString -> Pos -> More
61+ -> IResult ByteString r)
62+ -> (State ByteString -> Pos -> More
63+ -> IResult ByteString r)
64+ -> IResult ByteString r #-}
65+{-# SPECIALIZE prompt :: State Text -> Pos -> More
66+ -> (State Text -> Pos -> More -> IResult Text r)
67+ -> (State Text -> Pos -> More -> IResult Text r)
68+ -> IResult Text r #-}
69+
70+-- | Immediately demand more input via a 'Partial' continuation
71+-- result.
72+demandInput :: Chunk t => Parser t ()
73+demandInput = Parser $ \t pos more lose succ ->
74+ case more of
75+ Complete -> lose t pos more [] "not enough input"
76+ _ -> let lose' _ pos' more' = lose t pos' more' [] "not enough input"
77+ succ' t' pos' more' = succ t' pos' more' ()
78+ in prompt t pos more lose' succ'
79+{-# SPECIALIZE demandInput :: Parser ByteString () #-}
80+{-# SPECIALIZE demandInput :: Parser Text () #-}
81+
82+-- | Immediately demand more input via a 'Partial' continuation
83+-- result. Return the new input.
84+demandInput_ :: Chunk t => Parser t t
85+demandInput_ = Parser $ \t pos more lose succ ->
86+ case more of
87+ Complete -> lose t pos more [] "not enough input"
88+ _ -> Partial $ \s ->
89+ if nullChunk s
90+ then lose t pos Complete [] "not enough input"
91+ else succ (pappendChunk t s) pos more s
92+{-# SPECIALIZE demandInput_ :: Parser ByteString ByteString #-}
93+{-# SPECIALIZE demandInput_ :: Parser Text Text #-}
94+
95+-- | This parser always succeeds. It returns 'True' if any input is
96+-- available either immediately or on demand, and 'False' if the end
97+-- of all input has been reached.
98+wantInput :: forall t . Chunk t => Parser t Bool
99+wantInput = Parser $ \t pos more _lose succ ->
100+ case () of
101+ _ | pos < atBufferEnd (undefined :: t) t -> succ t pos more True
102+ | more == Complete -> succ t pos more False
103+ | otherwise -> let lose' t' pos' more' = succ t' pos' more' False
104+ succ' t' pos' more' = succ t' pos' more' True
105+ in prompt t pos more lose' succ'
106+{-# INLINE wantInput #-}
107+
108+-- | Match only if all input has been consumed.
109+endOfInput :: forall t . Chunk t => Parser t ()
110+endOfInput = Parser $ \t pos more lose succ ->
111+ case () of
112+ _| pos < atBufferEnd (undefined :: t) t -> lose t pos more [] "endOfInput"
113+ | more == Complete -> succ t pos more ()
114+ | otherwise ->
115+ let lose' t' pos' more' _ctx _msg = succ t' pos' more' ()
116+ succ' t' pos' more' _a = lose t' pos' more' [] "endOfInput"
117+ in runParser demandInput t pos more lose' succ'
118+{-# SPECIALIZE endOfInput :: Parser ByteString () #-}
119+{-# SPECIALIZE endOfInput :: Parser Text () #-}
120+
121+-- | Return an indication of whether the end of input has been
122+-- reached.
123+atEnd :: Chunk t => Parser t Bool
124+atEnd = not <$> wantInput
125+{-# INLINE atEnd #-}
126+
127+satisfySuspended :: forall t r . Chunk t
128+ => (ChunkElem t -> Bool)
129+ -> State t -> Pos -> More
130+ -> Failure t (State t) r
131+ -> Success t (State t) (ChunkElem t) r
132+ -> IResult t r
133+satisfySuspended p t pos more lose succ =
134+ runParser (demandInput >> go) t pos more lose succ
135+ where go = Parser $ \t' pos' more' lose' succ' ->
136+ case bufferElemAt (undefined :: t) pos' t' of
137+ Just (e, l) | p e -> succ' t' (pos' + Pos l) more' e
138+ | otherwise -> lose' t' pos' more' [] "satisfyElem"
139+ Nothing -> runParser (demandInput >> go) t' pos' more' lose' succ'
140+{-# SPECIALIZE satisfySuspended :: (ChunkElem ByteString -> Bool)
141+ -> State ByteString -> Pos -> More
142+ -> Failure ByteString (State ByteString) r
143+ -> Success ByteString (State ByteString)
144+ (ChunkElem ByteString) r
145+ -> IResult ByteString r #-}
146+{-# SPECIALIZE satisfySuspended :: (ChunkElem Text -> Bool)
147+ -> State Text -> Pos -> More
148+ -> Failure Text (State Text) r
149+ -> Success Text (State Text)
150+ (ChunkElem Text) r
151+ -> IResult Text r #-}
152+
153+-- | The parser @satisfyElem p@ succeeds for any chunk element for which the
154+-- predicate @p@ returns 'True'. Returns the element that is
155+-- actually parsed.
156+satisfyElem :: forall t . Chunk t
157+ => (ChunkElem t -> Bool) -> Parser t (ChunkElem t)
158+satisfyElem p = Parser $ \t pos more lose succ ->
159+ case bufferElemAt (undefined :: t) pos t of
160+ Just (e, l) | p e -> succ t (pos + Pos l) more e
161+ | otherwise -> lose t pos more [] "satisfyElem"
162+ Nothing -> satisfySuspended p t pos more lose succ
163+{-# INLINE satisfyElem #-}
164+
165+-- | Concatenate a monoid after reversing its elements. Used to
166+-- glue together a series of textual chunks that have been accumulated
167+-- \"backwards\".
168+concatReverse :: Monoid m => [m] -> m
169+concatReverse [x] = x
170+concatReverse xs = mconcat (reverse xs)
171+{-# INLINE concatReverse #-}
Data/Attoparsec/Internal/Fhthagn.hsnew file+18
@@ -0,0 +1,18 @@
1+{-# LANGUAGE BangPatterns, Rank2Types, OverloadedStrings,
2+ RecordWildCards, MagicHash, UnboxedTuples #-}
3+
4+module Data.Attoparsec.Internal.Fhthagn
5+ (
6+ inlinePerformIO
7+ ) where
8+
9+import GHC.Base (realWorld#)
10+import GHC.IO (IO(IO))
11+
12+-- | Just like unsafePerformIO, but we inline it. Big performance gains as
13+-- it exposes lots of things to further inlining. /Very unsafe/. In
14+-- particular, you should do no memory allocation inside an
15+-- 'inlinePerformIO' block. On Hugs this is just @unsafePerformIO@.
16+inlinePerformIO :: IO a -> a
17+inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r
18+{-# INLINE inlinePerformIO #-}
Data/Attoparsec/Internal/Types.hs+148−69
@@ -1,8 +1,8 @@
1−{-# LANGUAGE BangPatterns, GeneralizedNewtypeDeriving, OverloadedStrings,
1+{-# LANGUAGE CPP, BangPatterns, GeneralizedNewtypeDeriving, OverloadedStrings,
2 Rank2Types, RecordWildCards, TypeFamilies #-}
2 Rank2Types, RecordWildCards, TypeFamilies #-}
3-- |
3-- |
4-- Module : Data.Attoparsec.Internal.Types
4-- Module : Data.Attoparsec.Internal.Types
5−-- Copyright : Bryan O'Sullivan 2007-2014
5+-- Copyright : Bryan O'Sullivan 2007-2015
6-- License : BSD3
6-- License : BSD3
7--
7--
8-- Maintainer : bos@serpentine.com
8-- Maintainer : bos@serpentine.com
@@ -15,63 +15,82 @@
15module Data.Attoparsec.Internal.Types
15module Data.Attoparsec.Internal.Types
16 (
16 (
17 Parser(..)
17 Parser(..)
18+ , State
18 , Failure
19 , Failure
19 , Success
20 , Success
20 , Pos(..)
21 , Pos(..)
21 , IResult(..)
22 , IResult(..)
22 , More(..)
23 , More(..)
23 , (<>)
24 , (<>)
25+ , Chunk(..)
24 ) where
26 ) where
25
27
26−import Control.Applicative (Alternative(..), Applicative(..), (<$>))
28+import Control.Applicative as App (Applicative(..), (<$>))
29+import Control.Applicative (Alternative(..))
27import Control.DeepSeq (NFData(rnf))
30import Control.DeepSeq (NFData(rnf))
28import Control.Monad (MonadPlus(..))
31import Control.Monad (MonadPlus(..))
32+import qualified Control.Monad.Fail as Fail (MonadFail(..))
29−import Data.Monoid (Monoid(..))
33+import Data.Monoid as Mon (Monoid(..))
34+import Data.Semigroup (Semigroup(..))
35+import Data.Word (Word8)
36+import Data.ByteString (ByteString)
37+import qualified Data.ByteString as BS
38+import Data.ByteString.Internal (w2c)
39+import Data.Text (Text)
40+import qualified Data.Text as Text
41+import Data.Text.Unsafe (Iter(..))
30−import Prelude hiding (getChar, succ)
42+import Prelude hiding (succ)
43+import qualified Data.Attoparsec.ByteString.Buffer as B
44+import qualified Data.Attoparsec.Text.Buffer as T
31
45
32newtype Pos = Pos { fromPos :: Int }
46newtype Pos = Pos { fromPos :: Int }
33 deriving (Eq, Ord, Show, Num)
47 deriving (Eq, Ord, Show, Num)
34
48
35−-- | The result of a parse. This is parameterised over the type @t@
49+-- | The result of a parse. This is parameterised over the type @i@
36-- of string that was processed.
50-- of string that was processed.
37--
51--
38-- This type is an instance of 'Functor', where 'fmap' transforms the
52-- This type is an instance of 'Functor', where 'fmap' transforms the
39-- value in a 'Done' result.
53-- value in a 'Done' result.
54+data IResult i r =
40−data IResult t r = Fail t [String] String
55+ Fail i [String] String
41− -- ^ The parse failed. The 't' parameter is the
56+ -- ^ The parse failed. The @i@ parameter is the input that had
42− -- input that had not yet been consumed when the
57+ -- not yet been consumed when the failure occurred. The
43− -- failure occurred. The @[@'String'@]@ is a list of
44− -- contexts in which the error occurred. The
58+ -- @[@'String'@]@ is a list of contexts in which the error
45− -- 'String' is the message describing the error, if
59+ -- occurred. The 'String' is the message describing the error, if
46− -- any.
60+ -- any.
47− | Partial (t -> IResult t r)
61+ | Partial (i -> IResult i r)
48− -- ^ Supply this continuation with more input so that
62+ -- ^ Supply this continuation with more input so that the parser
49− -- the parser can resume. To indicate that no more
63+ -- can resume. To indicate that no more input is available, pass
50− -- input is available, use an empty string.
64+ -- an empty string to the continuation.
65+ --
66+ -- __Note__: if you get a 'Partial' result, do not call its
67+ -- continuation more than once.
51− | Done t r
68+ | Done i r
52− -- ^ The parse succeeded. The 't' parameter is the
69+ -- ^ The parse succeeded. The @i@ parameter is the input that had
53− -- input that had not yet been consumed (if any) when
70+ -- not yet been consumed (if any) when the parse succeeded.
54− -- the parse succeeded.
71+
55−
56−instance (Show t, Show r) => Show (IResult t r) where
72+instance (Show i, Show r) => Show (IResult i r) where
73+ showsPrec d ir = showParen (d > 10) $
74+ case ir of
57− show (Fail t stk msg) =
75+ (Fail t stk msg) -> showString "Fail" . f t . f stk . f msg
58− unwords [ "Fail", show t, show stk, show msg]
59− show (Partial _) = "Partial _"
76+ (Partial _) -> showString "Partial _"
60− show (Done t r) = unwords ["Done", show t, show r]
77+ (Done t r) -> showString "Done" . f t . f r
61−
78+ where f :: Show a => a -> ShowS
79+ f x = showChar ' ' . showsPrec 11 x
80+
62−instance (NFData t, NFData r) => NFData (IResult t r) where
81+instance (NFData i, NFData r) => NFData (IResult i r) where
63 rnf (Fail t stk msg) = rnf t `seq` rnf stk `seq` rnf msg
82 rnf (Fail t stk msg) = rnf t `seq` rnf stk `seq` rnf msg
64 rnf (Partial _) = ()
83 rnf (Partial _) = ()
65 rnf (Done t r) = rnf t `seq` rnf r
84 rnf (Done t r) = rnf t `seq` rnf r
66 {-# INLINE rnf #-}
85 {-# INLINE rnf #-}
67
86
68−instance Functor (IResult t) where
87+instance Functor (IResult i) where
69 fmap _ (Fail t stk msg) = Fail t stk msg
88 fmap _ (Fail t stk msg) = Fail t stk msg
70 fmap f (Partial k) = Partial (fmap f . k)
89 fmap f (Partial k) = Partial (fmap f . k)
71 fmap f (Done t r) = Done t (f r)
90 fmap f (Done t r) = Done t (f r)
72
91
73−-- | The core parser type. This is parameterised over the type @t@ of
92+-- | The core parser type. This is parameterised over the type @i@
74−-- string being processed.
93+-- of string being processed.
75--
94--
76-- This type is an instance of the following classes:
95-- This type is an instance of the following classes:
77--
96--
@@ -84,36 +103,45 @@ instance Functor (IResult t) where
84-- 'mplus' executes the right-hand parser if the left-hand one
103-- 'mplus' executes the right-hand parser if the left-hand one
85-- fails. When the parser on the right executes, the input is reset
104-- fails. When the parser on the right executes, the input is reset
86-- to the same state as the parser on the left started with. (In
105-- to the same state as the parser on the left started with. (In
87−-- other words, Attoparsec is a backtracking parser that supports
106+-- other words, attoparsec is a backtracking parser that supports
88-- arbitrary lookahead.)
107-- arbitrary lookahead.)
89--
108--
90-- * 'Alternative', which follows 'MonadPlus'.
109-- * 'Alternative', which follows 'MonadPlus'.
91−newtype Parser t a = Parser {
110+newtype Parser i a = Parser {
92− runParser :: forall r. t -> Pos -> More
111+ runParser :: forall r.
112+ State i -> Pos -> More
93− -> Failure t r
113+ -> Failure i (State i) r
94− -> Success t a r
114+ -> Success i (State i) a r
95− -> IResult t r
115+ -> IResult i r
96 }
116 }
97
117
118+type family State i
119+type instance State ByteString = B.Buffer
120+type instance State Text = T.Buffer
121+
98−type Failure t r = t -> Pos -> More -> [String] -> String
122+type Failure i t r = t -> Pos -> More -> [String] -> String
99− -> IResult t r
123+ -> IResult i r
100−type Success t a r = t -> Pos -> More -> a -> IResult t r
124+type Success i t a r = t -> Pos -> More -> a -> IResult i r
101
125
102-- | Have we read all available input?
126-- | Have we read all available input?
103data More = Complete | Incomplete
127data More = Complete | Incomplete
104 deriving (Eq, Show)
128 deriving (Eq, Show)
105
129
106−instance Monoid More where
130+instance Semigroup More where
107− mappend c@Complete _ = c
131+ c@Complete <> _ = c
108− mappend _ m = m
132+ _ <> m = m
109− mempty = Incomplete
110
133
134+instance Mon.Monoid More where
135+ mappend = (<>)
136+ mempty = Incomplete
137+
111−instance Monad (Parser t) where
138+instance Monad (Parser i) where
112− fail err = Parser $ \t pos more lose _succ -> lose t pos more [] msg
139+#if !(MIN_VERSION_base(4,13,0))
113− where msg = "Failed reading: " ++ err
140+ fail = Fail.fail
114 {-# INLINE fail #-}
141 {-# INLINE fail #-}
142+#endif
115
143
116− return v = Parser $ \t pos more _lose succ -> succ t pos more v
144+ return = App.pure
117 {-# INLINE return #-}
145 {-# INLINE return #-}
118
146
119 m >>= k = Parser $ \t !pos more lose succ ->
147 m >>= k = Parser $ \t !pos more lose succ ->
@@ -121,50 +149,59 @@ instance Monad (Parser t) where
121 in runParser m t pos more lose succ'
149 in runParser m t pos more lose succ'
122 {-# INLINE (>>=) #-}
150 {-# INLINE (>>=) #-}
123
151
152+ (>>) = (*>)
153+ {-# INLINE (>>) #-}
154+
155+
156+instance Fail.MonadFail (Parser i) where
157+ fail err = Parser $ \t pos more lose _succ -> lose t pos more [] msg
158+ where msg = "Failed reading: " ++ err
159+ {-# INLINE fail #-}
160+
124−plus :: (Monoid t) => Parser t a -> Parser t a -> Parser t a
161+plus :: Parser i a -> Parser i a -> Parser i a
125plus f g = Parser $ \t pos more lose succ ->
162plus f g = Parser $ \t pos more lose succ ->
126 let lose' t' _pos' more' _ctx _msg = runParser g t' pos more' lose succ
163 let lose' t' _pos' more' _ctx _msg = runParser g t' pos more' lose succ
127 in runParser f t pos more lose' succ
164 in runParser f t pos more lose' succ
128
165
129−instance (Monoid t) => MonadPlus (Parser t) where
166+instance MonadPlus (Parser i) where
130 mzero = fail "mzero"
167 mzero = fail "mzero"
131 {-# INLINE mzero #-}
168 {-# INLINE mzero #-}
132 mplus = plus
169 mplus = plus
133
170
134−instance Functor (Parser t) where
171+instance Functor (Parser i) where
135 fmap f p = Parser $ \t pos more lose succ ->
172 fmap f p = Parser $ \t pos more lose succ ->
136 let succ' t' pos' more' a = succ t' pos' more' (f a)
173 let succ' t' pos' more' a = succ t' pos' more' (f a)
137 in runParser p t pos more lose succ'
174 in runParser p t pos more lose succ'
138 {-# INLINE fmap #-}
175 {-# INLINE fmap #-}
139
176
140−apP :: Parser t (a -> b) -> Parser t a -> Parser t b
177+apP :: Parser i (a -> b) -> Parser i a -> Parser i b
141apP d e = do
178apP d e = do
142 b <- d
179 b <- d
143 a <- e
180 a <- e
144 return (b a)
181 return (b a)
145{-# INLINE apP #-}
182{-# INLINE apP #-}
146
183
147−instance Applicative (Parser t) where
184+instance Applicative (Parser i) where
148− pure = return
185+ pure v = Parser $ \t !pos more _lose succ -> succ t pos more v
149 {-# INLINE pure #-}
186 {-# INLINE pure #-}
150 (<*>) = apP
187 (<*>) = apP
151 {-# INLINE (<*>) #-}
188 {-# INLINE (<*>) #-}
152−
189+ m *> k = m >>= \_ -> k
153− -- These definitions are equal to the defaults, but this
154− -- way the optimizer doesn't have to work so hard to figure
155− -- that out.
156− (*>) = (>>)
157 {-# INLINE (*>) #-}
190 {-# INLINE (*>) #-}
158− x <* y = x >>= \a -> y >> return a
191+ x <* y = x >>= \a -> y >> pure a
159 {-# INLINE (<*) #-}
192 {-# INLINE (<*) #-}
160
193
194+instance Semigroup (Parser i a) where
195+ (<>) = plus
196+ {-# INLINE (<>) #-}
197+
161−instance (Monoid t) => Monoid (Parser t a) where
198+instance Monoid (Parser i a) where
162 mempty = fail "mempty"
199 mempty = fail "mempty"
163 {-# INLINE mempty #-}
200 {-# INLINE mempty #-}
164− mappend = plus
201+ mappend = (<>)
165 {-# INLINE mappend #-}
202 {-# INLINE mappend #-}
166
203
167−instance (Monoid t) => Alternative (Parser t) where
204+instance Alternative (Parser i) where
168 empty = fail "empty"
205 empty = fail "empty"
169 {-# INLINE empty #-}
206 {-# INLINE empty #-}
170
207
@@ -172,8 +209,9 @@ instance (Monoid t) => Alternative (Parser t) where
172 {-# INLINE (<|>) #-}
209 {-# INLINE (<|>) #-}
173
210
174 many v = many_v
211 many v = many_v
212+ where
175− where many_v = some_v <|> pure []
213+ many_v = some_v <|> pure []
176− some_v = (:) <$> v <*> many_v
214+ some_v = (:) <$> v <*> many_v
177 {-# INLINE many #-}
215 {-# INLINE many #-}
178
216
179 some v = some_v
217 some v = some_v
@@ -182,6 +220,47 @@ instance (Monoid t) => Alternative (Parser t) where
182 some_v = (:) <$> v <*> many_v
220 some_v = (:) <$> v <*> many_v
183 {-# INLINE some #-}
221 {-# INLINE some #-}
184
222
185−(<>) :: (Monoid m) => m -> m -> m
223+-- | A common interface for input chunks.
186−(<>) = mappend
224+class Monoid c => Chunk c where
225+ type ChunkElem c
226+ -- | Test if the chunk is empty.
227+ nullChunk :: c -> Bool
228+ -- | Append chunk to a buffer.
229+ pappendChunk :: State c -> c -> State c
230+ -- | Position at the end of a buffer. The first argument is ignored.
231+ atBufferEnd :: c -> State c -> Pos
232+ -- | Return the buffer element at the given position along with its length.
233+ bufferElemAt :: c -> Pos -> State c -> Maybe (ChunkElem c, Int)
234+ -- | Map an element to the corresponding character.
235+ -- The first argument is ignored.
236+ chunkElemToChar :: c -> ChunkElem c -> Char
237+
238+instance Chunk ByteString where
239+ type ChunkElem ByteString = Word8
240+ nullChunk = BS.null
187−{-# INLINE (<>) #-}
241+ {-# INLINE nullChunk #-}
242+ pappendChunk = B.pappend
243+ {-# INLINE pappendChunk #-}
244+ atBufferEnd _ = Pos . B.length
245+ {-# INLINE atBufferEnd #-}
246+ bufferElemAt _ (Pos i) buf
247+ | i < B.length buf = Just (B.unsafeIndex buf i, 1)
248+ | otherwise = Nothing
249+ {-# INLINE bufferElemAt #-}
250+ chunkElemToChar _ = w2c
251+ {-# INLINE chunkElemToChar #-}
252+
253+instance Chunk Text where
254+ type ChunkElem Text = Char
255+ nullChunk = Text.null
256+ {-# INLINE nullChunk #-}
257+ pappendChunk = T.pappend
258+ {-# INLINE pappendChunk #-}
259+ atBufferEnd _ = Pos . T.length
260+ {-# INLINE atBufferEnd #-}
261+ bufferElemAt _ (Pos i) buf
262+ | i < T.length buf = let Iter c l = T.iter buf i in Just (c, l)
263+ | otherwise = Nothing
264+ {-# INLINE bufferElemAt #-}
265+ chunkElemToChar _ = id
266+ {-# INLINE chunkElemToChar #-}
Data/Attoparsec/Lazy.hs+1−1
@@ -1,6 +1,6 @@
1-- |
1-- |
2-- Module : Data.Attoparsec.Lazy
2-- Module : Data.Attoparsec.Lazy
3−-- Copyright : Bryan O'Sullivan 2010
3+-- Copyright : Bryan O'Sullivan 2007-2015
4-- License : BSD3
4-- License : BSD3
5--
5--
6-- Maintainer : bos@serpentine.com
6-- Maintainer : bos@serpentine.com
Data/Attoparsec/Number.hs+15−2
@@ -1,16 +1,25 @@
1{-# LANGUAGE DeriveDataTypeable #-}
1{-# LANGUAGE DeriveDataTypeable #-}
2-- |
2-- |
3-- Module : Data.Attoparsec.Number
3-- Module : Data.Attoparsec.Number
4−-- Copyright : Bryan O'Sullivan 2011
4+-- Copyright : Bryan O'Sullivan 2007-2015
5-- License : BSD3
5-- License : BSD3
6--
6--
7-- Maintainer : bos@serpentine.com
7-- Maintainer : bos@serpentine.com
8-- Stability : experimental
8-- Stability : experimental
9-- Portability : unknown
9-- Portability : unknown
10--
10--
11+-- This module is deprecated, and both the module and 'Number' type
12+-- will be removed in the next major release. Use the
13+-- <http://hackage.haskell.org/package/scientific scientific> package
14+-- and the 'Data.Scientific.Scientific' type instead.
15+--
11-- A simple number type, useful for parsing both exact and inexact
16-- A simple number type, useful for parsing both exact and inexact
12-- quantities without losing much precision.
17-- quantities without losing much precision.
13−module Data.Attoparsec.Number ( Number(..) ) where
18+module Data.Attoparsec.Number
19+ {-# DEPRECATED "This module will be removed in the next major release." #-}
20+ (
21+ Number(..)
22+ ) where
14
23
15import Control.DeepSeq (NFData(rnf))
24import Control.DeepSeq (NFData(rnf))
16import Data.Data (Data)
25import Data.Data (Data)
@@ -19,9 +28,13 @@ import Data.Typeable (Typeable)
19
28
20-- | A numeric type that can represent integers accurately, and
29-- | A numeric type that can represent integers accurately, and
21-- floating point numbers to the precision of a 'Double'.
30-- floating point numbers to the precision of a 'Double'.
31+--
32+-- /Note/: this type is deprecated, and will be removed in the next
33+-- major release. Use the 'Data.Scientific.Scientific' type instead.
22data Number = I !Integer
34data Number = I !Integer
23 | D {-# UNPACK #-} !Double
35 | D {-# UNPACK #-} !Double
24 deriving (Typeable, Data)
36 deriving (Typeable, Data)
37+{-# DEPRECATED Number "Use Scientific instead." #-}
25
38
26instance Show Number where
39instance Show Number where
27 show (I a) = show a
40 show (I a) = show a
Data/Attoparsec/Text.hs+79−65
@@ -1,8 +1,12 @@
1−{-# LANGUAGE BangPatterns, FlexibleInstances, TypeSynonymInstances #-}
1+{-# LANGUAGE BangPatterns, CPP, FlexibleInstances, TypeSynonymInstances #-}
2+#if __GLASGOW_HASKELL__ >= 702
3+{-# LANGUAGE Trustworthy #-} -- Imports internal modules
4+#endif
5+{-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
2
6
3-- |
7-- |
4-- Module : Data.Attoparsec.Text
8-- Module : Data.Attoparsec.Text
5−-- Copyright : Bryan O'Sullivan 2011
9+-- Copyright : Bryan O'Sullivan 2007-2015
6-- License : BSD3
10-- License : BSD3
7--
11--
8-- Maintainer : bos@serpentine.com
12-- Maintainer : bos@serpentine.com
@@ -68,6 +72,7 @@ module Data.Attoparsec.Text
68 , skipSpace
72 , skipSpace
69 , I.skipWhile
73 , I.skipWhile
70 , I.scan
74 , I.scan
75+ , I.runScanner
71 , I.take
76 , I.take
72 , I.takeWhile
77 , I.takeWhile
73 , I.takeWhile1
78 , I.takeWhile1
@@ -121,17 +126,22 @@ module Data.Attoparsec.Text
121 , I.atEnd
126 , I.atEnd
122 ) where
127 ) where
123
128
129+#if !MIN_VERSION_base(4,8,0)
124−import Control.Applicative (pure, (<$>), (*>), (<*), (<|>))
130+import Control.Applicative (pure, (*>), (<*), (<$>))
131+import Data.Word (Word)
132+#endif
133+import Control.Applicative ((<|>))
125import Data.Attoparsec.Combinator
134import Data.Attoparsec.Combinator
126import Data.Attoparsec.Number (Number(..))
135import Data.Attoparsec.Number (Number(..))
127−import Data.Scientific (Scientific, coefficient, base10Exponent)
136+import Data.Scientific (Scientific)
128−import qualified Data.Scientific as Sci (scientific)
137+import qualified Data.Scientific as Sci
129import Data.Attoparsec.Text.Internal (Parser, Result, parse, takeWhile1)
138import Data.Attoparsec.Text.Internal (Parser, Result, parse, takeWhile1)
130import Data.Bits (Bits, (.|.), shiftL)
139import Data.Bits (Bits, (.|.), shiftL)
131import Data.Char (isAlpha, isDigit, isSpace, ord)
140import Data.Char (isAlpha, isDigit, isSpace, ord)
132import Data.Int (Int8, Int16, Int32, Int64)
141import Data.Int (Int8, Int16, Int32, Int64)
142+import Data.List (intercalate)
133import Data.Text (Text)
143import Data.Text (Text)
134−import Data.Word (Word8, Word16, Word32, Word64, Word)
144+import Data.Word (Word8, Word16, Word32, Word64)
135import qualified Data.Attoparsec.Internal as I
145import qualified Data.Attoparsec.Internal as I
136import qualified Data.Attoparsec.Internal.Types as T
146import qualified Data.Attoparsec.Internal.Types as T
137import qualified Data.Attoparsec.Text.Internal as I
147import qualified Data.Attoparsec.Text.Internal as I
@@ -139,49 +149,50 @@ import qualified Data.Text as T
139
149
140-- $parsec
150-- $parsec
141--
151--
142−-- Compared to Parsec 3, Attoparsec makes several tradeoffs. It is
152+-- Compared to Parsec 3, attoparsec makes several tradeoffs. It is
143-- not intended for, or ideal for, all possible uses.
153-- not intended for, or ideal for, all possible uses.
144--
154--
145−-- * While Attoparsec can consume input incrementally, Parsec cannot.
155+-- * While attoparsec can consume input incrementally, Parsec cannot.
146-- Incremental input is a huge deal for efficient and secure network
156-- Incremental input is a huge deal for efficient and secure network
147-- and system programming, since it gives much more control to users
157-- and system programming, since it gives much more control to users
148-- of the library over matters such as resource usage and the I/O
158-- of the library over matters such as resource usage and the I/O
149-- model to use.
159-- model to use.
150--
160--
151−-- * Much of the performance advantage of Attoparsec is gained via
161+-- * Much of the performance advantage of attoparsec is gained via
152-- high-performance parsers such as 'I.takeWhile' and 'I.string'.
162-- high-performance parsers such as 'I.takeWhile' and 'I.string'.
153-- If you use complicated combinators that return lists of
163-- If you use complicated combinators that return lists of
154-- characters, there is less performance difference between the two
164-- characters, there is less performance difference between the two
155-- libraries.
165-- libraries.
156--
166--
157−-- * Unlike Parsec 3, Attoparsec does not support being used as a
167+-- * Unlike Parsec 3, attoparsec does not support being used as a
158-- monad transformer.
168-- monad transformer.
159--
169--
160−-- * Attoparsec is specialised to deal only with strict 'Text'
170+-- * attoparsec is specialised to deal only with strict 'Text'
161-- input. Efficiency concerns rule out both lists and lazy text.
171-- input. Efficiency concerns rule out both lists and lazy text.
162-- The usual use for lazy text would be to allow consumption of very
172-- The usual use for lazy text would be to allow consumption of very
163-- large input without a large footprint. For this need,
173-- large input without a large footprint. For this need,
164−-- Attoparsec's incremental input provides an excellent substitute,
174+-- attoparsec's incremental input provides an excellent substitute,
165-- with much more control over when input takes place. If you must
175-- with much more control over when input takes place. If you must
166-- use lazy text, see the 'Lazy' module, which feeds lazy chunks to
176-- use lazy text, see the 'Lazy' module, which feeds lazy chunks to
167-- a regular parser.
177-- a regular parser.
168--
178--
169-- * Parsec parsers can produce more helpful error messages than
179-- * Parsec parsers can produce more helpful error messages than
170−-- Attoparsec parsers. This is a matter of focus: Attoparsec avoids
180+-- attoparsec parsers. This is a matter of focus: attoparsec avoids
171-- the extra book-keeping in favour of higher performance.
181-- the extra book-keeping in favour of higher performance.
172
182
173-- $incremental
183-- $incremental
174--
184--
175−-- Attoparsec supports incremental input, meaning that you can feed it
185+-- attoparsec supports incremental input, meaning that you can feed it
176-- a 'Text' that represents only part of the expected total amount
186-- a 'Text' that represents only part of the expected total amount
177-- of data to parse. If your parser reaches the end of a fragment of
187-- of data to parse. If your parser reaches the end of a fragment of
178-- input and could consume more input, it will suspend parsing and
188-- input and could consume more input, it will suspend parsing and
179-- return a 'T.Partial' continuation.
189-- return a 'T.Partial' continuation.
180--
190--
181-- Supplying the 'T.Partial' continuation with another string will
191-- Supplying the 'T.Partial' continuation with another string will
182−-- resume parsing at the point where it was suspended. You must be
192+-- resume parsing at the point where it was suspended, with the string
193+-- you supplied used as new input at the end of the existing
183−-- prepared for the result of the resumed parse to be another
194+-- input. You must be prepared for the result of the resumed parse to
184−-- 'Partial' continuation.
195+-- be another 'Partial' continuation.
185--
196--
186-- To indicate that you have no more input, supply the 'Partial'
197-- To indicate that you have no more input, supply the 'Partial'
187-- continuation with an 'T.empty' 'Text'.
198-- continuation with an 'T.empty' 'Text'.
@@ -193,12 +204,19 @@ import qualified Data.Text as T
193-- If you do not need support for incremental input, consider using
204-- If you do not need support for incremental input, consider using
194-- the 'I.parseOnly' function to run your parser. It will never
205-- the 'I.parseOnly' function to run your parser. It will never
195-- prompt for more input.
206-- prompt for more input.
207+--
208+-- /Note/: incremental input does /not/ imply that attoparsec will
209+-- release portions of its internal state for garbage collection as it
210+-- proceeds. Its internal representation is equivalent to a single
211+-- 'Text': if you feed incremental input to an a parser, it will
212+-- require memory proportional to the amount of input you supply.
213+-- (This is necessary to support arbitrary backtracking.)
196
214
197-- $performance
215-- $performance
198--
216--
199−-- If you write an Attoparsec-based parser carefully, it can be
217+-- If you write an attoparsec-based parser carefully, it can be
200−-- realistic to expect it to perform within a factor of 2 of a
218+-- realistic to expect it to perform similarly to a hand-rolled C
201−-- hand-rolled C parser (measuring megabytes parsed per second).
219+-- parser (measuring megabytes parsed per second).
202--
220--
203-- To actually achieve high performance, there are a few guidelines
221-- To actually achieve high performance, there are a few guidelines
204-- that it is useful to follow.
222-- that it is useful to follow.
@@ -248,9 +266,10 @@ maybeResult _ = Nothing
248-- | Convert a 'Result' value to an 'Either' value. A 'Partial' result
266-- | Convert a 'Result' value to an 'Either' value. A 'Partial' result
249-- is treated as failure.
267-- is treated as failure.
250eitherResult :: Result r -> Either String r
268eitherResult :: Result r -> Either String r
251−eitherResult (T.Done _ r) = Right r
269+eitherResult (T.Done _ r) = Right r
252−eitherResult (T.Fail _ _ msg) = Left msg
270+eitherResult (T.Fail _ [] msg) = Left msg
271+eitherResult (T.Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
253−eitherResult _ = Left "Result: incomplete input"
272+eitherResult _ = Left "Result: incomplete input"
254
273
255-- | A predicate that matches either a carriage return @\'\\r\'@ or
274-- | A predicate that matches either a carriage return @\'\\r\'@ or
256-- newline @\'\\n\'@ character.
275-- newline @\'\\n\'@ character.
@@ -323,6 +342,24 @@ signed p = (negate <$> (I.char '-' *> p))
323 <|> (I.char '+' *> p)
342 <|> (I.char '+' *> p)
324 <|> p
343 <|> p
325
344
345+-- | Parse a rational number.
346+--
347+-- The syntax accepted by this parser is the same as for 'double'.
348+--
349+-- /Note/: this parser is not safe for use with inputs from untrusted
350+-- sources. An input with a suitably large exponent such as
351+-- @"1e1000000000"@ will cause a huge 'Integer' to be allocated,
352+-- resulting in what is effectively a denial-of-service attack.
353+--
354+-- In most cases, it is better to use 'double' or 'scientific'
355+-- instead.
356+rational :: Fractional a => Parser a
357+{-# SPECIALIZE rational :: Parser Double #-}
358+{-# SPECIALIZE rational :: Parser Float #-}
359+{-# SPECIALIZE rational :: Parser Rational #-}
360+{-# SPECIALIZE rational :: Parser Scientific #-}
361+rational = scientifically realToFrac
362+
326-- | Parse a rational number.
363-- | Parse a rational number.
327--
364--
328-- This parser accepts an optional leading sign character, followed by
365-- This parser accepts an optional leading sign character, followed by
@@ -350,56 +387,33 @@ signed p = (negate <$> (I.char '-' *> p))
350--
387--
351-- This function does not accept string representations of \"NaN\" or
388-- This function does not accept string representations of \"NaN\" or
352-- \"Infinity\".
389-- \"Infinity\".
353−rational :: Fractional a => Parser a
354−{-# SPECIALIZE rational :: Parser Double #-}
355−{-# SPECIALIZE rational :: Parser Float #-}
356−{-# SPECIALIZE rational :: Parser Rational #-}
357−{-# SPECIALIZE rational :: Parser Scientific #-}
358−rational = scientifically realToFrac
359−
360−-- | Parse a rational number.
361−--
362−-- The syntax accepted by this parser is the same as for 'rational'.
363−--
364−-- /Note/: This function is almost ten times faster than 'rational',
365−-- but is slightly less accurate.
366−--
367−-- The 'Double' type supports about 16 decimal places of accuracy.
368−-- For 94.2% of numbers, this function and 'rational' give identical
369−-- results, but for the remaining 5.8%, this function loses precision
370−-- around the 15th decimal place. For 0.001% of numbers, this
371−-- function will lose precision at the 13th or 14th decimal place.
372−--
373−-- This function does not accept string representations of \"NaN\" or
374−-- \"Infinity\".
375double :: Parser Double
390double :: Parser Double
376−double = rational
391+double = scientifically Sci.toRealFloat
377
392
378-- | Parse a number, attempting to preserve both speed and precision.
393-- | Parse a number, attempting to preserve both speed and precision.
379--
394--
380−-- The syntax accepted by this parser is the same as for 'rational'.
395+-- The syntax accepted by this parser is the same as for 'double'.
381−--
382−-- /Note/: This function is almost ten times faster than 'rational'.
383−-- On integral inputs, it gives perfectly accurate answers, and on
384−-- floating point inputs, it is slightly less accurate than
385−-- 'rational'.
386--
396--
387-- This function does not accept string representations of \"NaN\" or
397-- This function does not accept string representations of \"NaN\" or
388-- \"Infinity\".
398-- \"Infinity\".
389number :: Parser Number
399number :: Parser Number
390number = scientifically $ \s ->
400number = scientifically $ \s ->
391− let e = base10Exponent s
401+ let e = Sci.base10Exponent s
392− c = coefficient s
402+ c = Sci.coefficient s
393 in if e >= 0
403 in if e >= 0
394 then I (c * 10 ^ e)
404 then I (c * 10 ^ e)
395− else D (fromInteger c / 10 ^ negate e)
405+ else D (Sci.toRealFloat s)
406+{-# DEPRECATED number "Use 'scientific' instead." #-}
396
407
397-- | Parse a scientific number.
408-- | Parse a scientific number.
398--
409--
399−-- The syntax accepted by this parser is the same as for 'rational'.
410+-- The syntax accepted by this parser is the same as for 'double'.
400scientific :: Parser Scientific
411scientific :: Parser Scientific
401scientific = scientifically id
412scientific = scientifically id
402
413
414+-- A strict pair
415+data SP = SP !Integer {-# UNPACK #-}!Int
416+
403{-# INLINE scientifically #-}
417{-# INLINE scientifically #-}
404scientifically :: (Scientific -> a) -> Parser a
418scientifically :: (Scientific -> a) -> Parser a
405scientifically h = do
419scientifically h = do
@@ -408,19 +422,19 @@ scientifically h = do
408
422
409 n <- decimal
423 n <- decimal
410
424
411− let f fracDigits = Sci.scientific (T.foldl' step n fracDigits)
425+ let f fracDigits = SP (T.foldl' step n fracDigits)
412− (negate $ T.length fracDigits)
426+ (negate $ T.length fracDigits)
413 step a c = a * 10 + fromIntegral (ord c - 48)
427 step a c = a * 10 + fromIntegral (ord c - 48)
414
428
415− s <- (I.satisfy (=='.') *> (f <$> I.takeWhile isDigit)) <|>
429+ SP c e <- (I.satisfy (=='.') *> (f <$> I.takeWhile isDigit)) <|>
416− pure (Sci.scientific n 0)
430+ pure (SP n 0)
417
431
418− let !signedCoeff | positive = coefficient s
432+ let !signedCoeff | positive = c
419− | otherwise = negate $ coefficient s
433+ | otherwise = -c
420
434
421− (I.satisfy (\c -> c == 'e' || c == 'E') *>
435+ (I.satisfy (\w -> w == 'e' || w == 'E') *>
422− fmap (h . Sci.scientific signedCoeff . (base10Exponent s +)) (signed decimal)) <|>
436+ fmap (h . Sci.scientific signedCoeff . (e +)) (signed decimal)) <|>
423− return (h $ Sci.scientific signedCoeff (base10Exponent s))
437+ return (h $ Sci.scientific signedCoeff e)
424
438
425-- | Parse a single digit, as recognised by 'isDigit'.
439-- | Parse a single digit, as recognised by 'isDigit'.
426digit :: Parser Char
440digit :: Parser Char
Data/Attoparsec/Text/Buffer.hsnew file+172
@@ -0,0 +1,172 @@
1+{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, RecordWildCards,
2+ UnboxedTuples #-}
3+
4+-- |
5+-- Module : Data.Attoparsec.Text.Buffer
6+-- Copyright : Bryan O'Sullivan 2007-2015
7+-- License : BSD3
8+--
9+-- Maintainer : bos@serpentine.com
10+-- Stability : experimental
11+-- Portability : GHC
12+--
13+-- An immutable buffer that supports cheap appends.
14+
15+-- A Buffer is divided into an immutable read-only zone, followed by a
16+-- mutable area that we've preallocated, but not yet written to.
17+--
18+-- We overallocate at the end of a Buffer so that we can cheaply
19+-- append. Since a user of an existing Buffer cannot see past the end
20+-- of its immutable zone into the data that will change during an
21+-- append, this is safe.
22+--
23+-- Once we run out of space at the end of a Buffer, we do the usual
24+-- doubling of the buffer size.
25+
26+module Data.Attoparsec.Text.Buffer
27+ (
28+ Buffer
29+ , buffer
30+ , unbuffer
31+ , unbufferAt
32+ , length
33+ , pappend
34+ , iter
35+ , iter_
36+ , substring
37+ , dropWord16
38+ ) where
39+
40+import Control.Exception (assert)
41+import Data.Bits (shiftR)
42+import Data.List (foldl1')
43+import Data.Monoid as Mon (Monoid(..))
44+import Data.Semigroup (Semigroup(..))
45+import Data.Text ()
46+import Data.Text.Internal (Text(..))
47+import Data.Text.Internal.Encoding.Utf16 (chr2)
48+import Data.Text.Internal.Unsafe.Char (unsafeChr)
49+import Data.Text.Unsafe (Iter(..))
50+import Foreign.Storable (sizeOf)
51+import GHC.Base (Int(..), indexIntArray#, unsafeCoerce#, writeIntArray#)
52+import GHC.ST (ST(..), runST)
53+import Prelude hiding (length)
54+import qualified Data.Text.Array as A
55+
56+-- If _cap is zero, this buffer is empty.
57+data Buffer = Buf {
58+ _arr :: {-# UNPACK #-} !A.Array
59+ , _off :: {-# UNPACK #-} !Int
60+ , _len :: {-# UNPACK #-} !Int
61+ , _cap :: {-# UNPACK #-} !Int
62+ , _gen :: {-# UNPACK #-} !Int
63+ }
64+
65+instance Show Buffer where
66+ showsPrec p = showsPrec p . unbuffer
67+
68+-- | The initial 'Buffer' has no mutable zone, so we can avoid all
69+-- copies in the (hopefully) common case of no further input being fed
70+-- to us.
71+buffer :: Text -> Buffer
72+buffer (Text arr off len) = Buf arr off len len 0
73+
74+unbuffer :: Buffer -> Text
75+unbuffer (Buf arr off len _ _) = Text arr off len
76+
77+unbufferAt :: Int -> Buffer -> Text
78+unbufferAt s (Buf arr off len _ _) =
79+ assert (s >= 0 && s <= len) $
80+ Text arr (off+s) (len-s)
81+
82+instance Semigroup Buffer where
83+ (Buf _ _ _ 0 _) <> b = b
84+ a <> (Buf _ _ _ 0 _) = a
85+ buf <> (Buf arr off len _ _) = append buf arr off len
86+ {-# INLINE (<>) #-}
87+
88+instance Monoid Buffer where
89+ mempty = Buf A.empty 0 0 0 0
90+ {-# INLINE mempty #-}
91+
92+ mappend = (<>)
93+
94+ mconcat [] = Mon.mempty
95+ mconcat xs = foldl1' (<>) xs
96+
97+pappend :: Buffer -> Text -> Buffer
98+pappend (Buf _ _ _ 0 _) t = buffer t
99+pappend buf (Text arr off len) = append buf arr off len
100+
101+append :: Buffer -> A.Array -> Int -> Int -> Buffer
102+append (Buf arr0 off0 len0 cap0 gen0) !arr1 !off1 !len1 = runST $ do
103+ let woff = sizeOf (0::Int) `shiftR` 1
104+ newlen = len0 + len1
105+ !gen = if gen0 == 0 then 0 else readGen arr0
106+ if gen == gen0 && newlen <= cap0
107+ then do
108+ let newgen = gen + 1
109+ marr <- unsafeThaw arr0
110+ writeGen marr newgen
111+ A.copyI marr (off0+len0) arr1 off1 (off0+newlen)
112+ arr2 <- A.unsafeFreeze marr
113+ return (Buf arr2 off0 newlen cap0 newgen)
114+ else do
115+ let newcap = newlen * 2
116+ newgen = 1
117+ marr <- A.new (newcap + woff)
118+ writeGen marr newgen
119+ A.copyI marr woff arr0 off0 (woff+len0)
120+ A.copyI marr (woff+len0) arr1 off1 (woff+newlen)
121+ arr2 <- A.unsafeFreeze marr
122+ return (Buf arr2 woff newlen newcap newgen)
123+
124+length :: Buffer -> Int
125+length (Buf _ _ len _ _) = len
126+{-# INLINE length #-}
127+
128+substring :: Int -> Int -> Buffer -> Text
129+substring s l (Buf arr off len _ _) =
130+ assert (s >= 0 && s <= len) .
131+ assert (l >= 0 && l <= len-s) $
132+ Text arr (off+s) l
133+{-# INLINE substring #-}
134+
135+dropWord16 :: Int -> Buffer -> Text
136+dropWord16 s (Buf arr off len _ _) =
137+ assert (s >= 0 && s <= len) $
138+ Text arr (off+s) (len-s)
139+{-# INLINE dropWord16 #-}
140+
141+-- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-16
142+-- array, returning the current character and the delta to add to give
143+-- the next offset to iterate at.
144+iter :: Buffer -> Int -> Iter
145+iter (Buf arr off _ _ _) i
146+ | m < 0xD800 || m > 0xDBFF = Iter (unsafeChr m) 1
147+ | otherwise = Iter (chr2 m n) 2
148+ where m = A.unsafeIndex arr j
149+ n = A.unsafeIndex arr k
150+ j = off + i
151+ k = j + 1
152+{-# INLINE iter #-}
153+
154+-- | /O(1)/ Iterate one step through a UTF-16 array, returning the
155+-- delta to add to give the next offset to iterate at.
156+iter_ :: Buffer -> Int -> Int
157+iter_ (Buf arr off _ _ _) i | m < 0xD800 || m > 0xDBFF = 1
158+ | otherwise = 2
159+ where m = A.unsafeIndex arr (off+i)
160+{-# INLINE iter_ #-}
161+
162+unsafeThaw :: A.Array -> ST s (A.MArray s)
163+unsafeThaw A.Array{..} = ST $ \s# ->
164+ (# s#, A.MArray (unsafeCoerce# aBA) #)
165+
166+readGen :: A.Array -> Int
167+readGen a = case indexIntArray# (A.aBA a) 0# of r# -> I# r#
168+
169+writeGen :: A.MArray s -> Int -> ST s ()
170+writeGen a (I# gen#) = ST $ \s0# ->
171+ case writeIntArray# (A.maBA a) 0# gen# s0# of
172+ s1# -> (# s1#, () #)
Data/Attoparsec/Text/FastSet.hs+85−27
@@ -1,16 +1,20 @@
1−-----------------------------------------------------------------------------
1+{-# LANGUAGE BangPatterns #-}
2+
3+------------------------------------------------------------------------------
2-- |
4-- |
3-- Module : Data.Attoparsec.FastSet
5-- Module : Data.Attoparsec.FastSet
4−-- Copyright : Felipe Lessa 2010, Bryan O'Sullivan 2008
6+-- Copyright : Felipe Lessa 2010, Bryan O'Sullivan 2007-2015
5-- License : BSD3
7-- License : BSD3
6--
8--
7-- Maintainer : felipe.lessa@gmail.com
9-- Maintainer : felipe.lessa@gmail.com
8-- Stability : experimental
10-- Stability : experimental
9-- Portability : unknown
11-- Portability : unknown
10--
12--
11−-- Fast set membership tests for 'Char' values. The set
13+-- Fast set membership tests for 'Char' values. We test for
14+-- membership using a hashtable implemented with Robin Hood
12−-- representation is unboxed for efficiency. We test for
15+-- collision resolution. The set representation is unboxed,
13−-- membership using a binary search.
16+-- and the characters and hashes interleaved, for efficiency.
17+--
14--
18--
15-----------------------------------------------------------------------------
19-----------------------------------------------------------------------------
16module Data.Attoparsec.Text.FastSet
20module Data.Attoparsec.Text.FastSet
@@ -26,38 +30,92 @@ module Data.Attoparsec.Text.FastSet
26 , charClass
30 , charClass
27 ) where
31 ) where
28
32
29−import Data.List (sort)
33+import Data.Bits ((.|.), (.&.), shiftR)
34+import Data.Function (on)
35+import Data.List (sort, sortBy)
30import qualified Data.Array.Base as AB
36import qualified Data.Array.Base as AB
31import qualified Data.Array.Unboxed as A
37import qualified Data.Array.Unboxed as A
32import qualified Data.Text as T
38import qualified Data.Text as T
33
39
34−newtype FastSet = FastSet (A.UArray Int Char)
40+data FastSet = FastSet {
35− deriving (Eq, Ord, Show)
41+ table :: {-# UNPACK #-} !(A.UArray Int Int)
42+ , mask :: {-# UNPACK #-} !Int
43+ }
36
44
37−-- | Create a set.
45+data Entry = Entry {
38−set :: T.Text -> FastSet
46+ key :: {-# UNPACK #-} !Char
39−set t = mkSet (T.length t) (sort $ T.unpack t)
47+ , initialIndex :: {-# UNPACK #-} !Int
48+ , index :: {-# UNPACK #-} !Int
49+ }
50+
51+offset :: Entry -> Int
52+offset e = index e - initialIndex e
53+
54+resolveCollisions :: [Entry] -> [Entry]
55+resolveCollisions [] = []
56+resolveCollisions [e] = [e]
57+resolveCollisions (a:b:entries) = a' : resolveCollisions (b' : entries)
58+ where (a', b')
59+ | index a < index b = (a, b)
60+ | offset a < offset b = (b { index=index a }, a { index=index a + 1 })
61+ | otherwise = (a, b { index=index a + 1 })
40
62
41−fromList :: [Char] -> FastSet
63+pad :: Int -> [Entry] -> [Entry]
42−fromList cs = mkSet (length cs) (sort cs)
64+pad = go 0
65+ where -- ensure that we pad enough so that lookups beyond the
66+ -- last hash in the table fall within the array
67+ go !_ !m [] = replicate (max 1 m + 1) empty
68+ go k m (e:entries) = map (const empty) [k..i - 1] ++ e :
69+ go (i + 1) (m + i - k - 1) entries
70+ where i = index e
71+ empty = Entry '\0' maxBound 0
43
72
44−mkSet :: Int -> [Char] -> FastSet
73+nextPowerOf2 :: Int -> Int
74+nextPowerOf2 0 = 1
75+nextPowerOf2 x = go (x - 1) 1
76+ where go y 32 = y + 1
77+ go y k = go (y .|. (y `shiftR` k)) $ k * 2
78+
79+fastHash :: Char -> Int
80+fastHash = fromEnum
81+
82+fromList :: String -> FastSet
45−mkSet l = FastSet . A.listArray (0,l-1)
83+fromList s = FastSet (AB.listArray (0, length interleaved - 1) interleaved)
84+ mask'
85+ where s' = ordNub (sort s)
86+ l = length s'
87+ mask' = nextPowerOf2 ((5 * l) `div` 4) - 1
88+ entries = pad mask' .
89+ resolveCollisions .
90+ sortBy (compare `on` initialIndex) .
91+ zipWith (\c i -> Entry c i i) s' .
92+ map ((.&. mask') . fastHash) $ s'
93+ interleaved = concatMap (\e -> [fromEnum $ key e, initialIndex e])
94+ entries
95+
96+ordNub :: Eq a => [a] -> [a]
97+ordNub [] = []
98+ordNub (y:ys) = go y ys
99+ where go x (z:zs)
100+ | x == z = go x zs
101+ | otherwise = x : go z zs
102+ go x [] = [x]
103+
104+set :: T.Text -> FastSet
105+set = fromList . T.unpack
46
106
47-- | Check the set for membership.
107-- | Check the set for membership.
48member :: Char -> FastSet -> Bool
108member :: Char -> FastSet -> Bool
49−member c (FastSet a) = uncurry search (A.bounds a)
109+member c a = go (2 * i)
50− where search lo hi
110+ where i = fastHash c .&. mask a
51− | hi < lo = False
111+ lookupAt j b = (i' <= i) && (c == c' || b)
52− | otherwise =
112+ where c' = toEnum $ AB.unsafeAt (table a) j
53− let mid = (lo + hi) `quot` 2
113+ i' = AB.unsafeAt (table a) $ j + 1
54− in case compare c (AB.unsafeAt a mid) of
114+ go j = lookupAt j . lookupAt (j + 2) . lookupAt (j + 4) .
55− GT -> search (mid + 1) hi
115+ lookupAt (j + 6) . go $ j + 8
56− LT -> search lo (mid - 1)
57− _ -> True
58
116
59charClass :: String -> FastSet
117charClass :: String -> FastSet
60charClass = fromList . go
118charClass = fromList . go
61− where go (a:'-':b:xs) = [a..b] ++ go xs
119+ where go (a:'-':b:xs) = [a..b] ++ go xs
62− go (x:xs) = x : go xs
120+ go (x:xs) = x : go xs
63− go _ = ""
121+ go _ = ""
Data/Attoparsec/Text/Internal.hs+130−123
@@ -3,14 +3,14 @@
3{-# OPTIONS_GHC -fno-warn-orphans #-}
3{-# OPTIONS_GHC -fno-warn-orphans #-}
4-- |
4-- |
5-- Module : Data.Attoparsec.Text.Internal
5-- Module : Data.Attoparsec.Text.Internal
6−-- Copyright : Bryan O'Sullivan 2011
6+-- Copyright : Bryan O'Sullivan 2007-2015
7-- License : BSD3
7-- License : BSD3
8--
8--
9-- Maintainer : bos@serpentine.com
9-- Maintainer : bos@serpentine.com
10-- Stability : experimental
10-- Stability : experimental
11-- Portability : unknown
11-- Portability : unknown
12--
12--
13−-- Simple, efficient parser combinators for 'T.Text' strings, loosely
13+-- Simple, efficient parser combinators for 'Text' strings, loosely
14-- based on the Parsec library.
14-- based on the Parsec library.
15
15
16module Data.Attoparsec.Text.Internal
16module Data.Attoparsec.Text.Internal
@@ -65,11 +65,15 @@ module Data.Attoparsec.Text.Internal
65 , atEnd
65 , atEnd
66 ) where
66 ) where
67
67
68−import Control.Applicative ((<|>), (<$>))
68+import Control.Applicative ((<|>), (<$>), pure, (*>))
69import Control.Monad (when)
69import Control.Monad (when)
70import Data.Attoparsec.Combinator ((<?>))
70import Data.Attoparsec.Combinator ((<?>))
71+import Data.Attoparsec.Internal
71import Data.Attoparsec.Internal.Types hiding (Parser, Failure, Success)
72import Data.Attoparsec.Internal.Types hiding (Parser, Failure, Success)
72−import Data.Char (chr, ord)
73+import qualified Data.Attoparsec.Text.Buffer as Buf
74+import Data.Attoparsec.Text.Buffer (Buffer, buffer)
75+import Data.Char (isAsciiUpper, isAsciiLower, toUpper, toLower)
76+import Data.List (intercalate)
73import Data.String (IsString(..))
77import Data.String (IsString(..))
74import Data.Text.Internal (Text(..))
78import Data.Text.Internal (Text(..))
75import Prelude hiding (getChar, succ, take, takeWhile)
79import Prelude hiding (getChar, succ, take, takeWhile)
@@ -81,8 +85,8 @@ import qualified Data.Text.Unsafe as T
81
85
82type Parser = T.Parser Text
86type Parser = T.Parser Text
83type Result = IResult Text
87type Result = IResult Text
84−type Failure r = T.Failure Text r
88+type Failure r = T.Failure Text Buffer r
85−type Success a r = T.Success Text a r
89+type Success a r = T.Success Text Buffer a r
86
90
87instance (a ~ Text) => IsString (Parser a) where
91instance (a ~ Text) => IsString (Parser a) where
88 fromString = string . T.pack
92 fromString = string . T.pack
@@ -153,12 +157,49 @@ take n = takeWith (max n 0) (const True)
153--
157--
154-- The reason for its failure is that the first branch is a
158-- The reason for its failure is that the first branch is a
155-- partial match, and will consume the letters @\'f\'@ and @\'o\'@
159-- partial match, and will consume the letters @\'f\'@ and @\'o\'@
156−-- before failing. In Attoparsec, the above parser will /succeed/ on
160+-- before failing. In attoparsec, the above parser will /succeed/ on
157-- that input, because the failed first branch will consume nothing.
161-- that input, because the failed first branch will consume nothing.
158string :: Text -> Parser Text
162string :: Text -> Parser Text
159−string s = takeWith (T.length s) (==s)
163+string s = string_ (stringSuspended id) id s
160{-# INLINE string #-}
164{-# INLINE string #-}
161
165
166+string_ :: (forall r. Text -> Text -> Buffer -> Pos -> More
167+ -> Failure r -> Success Text r -> Result r)
168+ -> (Text -> Text)
169+ -> Text -> Parser Text
170+string_ suspended f s0 = T.Parser $ \t pos more lose succ ->
171+ let s = f s0
172+ ft = f (Buf.unbufferAt (fromPos pos) t)
173+ in case T.commonPrefixes s ft of
174+ Nothing
175+ | T.null s -> succ t pos more T.empty
176+ | T.null ft -> suspended s s t pos more lose succ
177+ | otherwise -> lose t pos more [] "string"
178+ Just (pfx,ssfx,tsfx)
179+ | T.null ssfx -> let l = Pos (T.lengthWord16 pfx)
180+ in succ t (pos + l) more (substring pos l t)
181+ | not (T.null tsfx) -> lose t pos more [] "string"
182+ | otherwise -> suspended s ssfx t pos more lose succ
183+{-# INLINE string_ #-}
184+
185+stringSuspended :: (Text -> Text)
186+ -> Text -> Text -> Buffer -> Pos -> More
187+ -> Failure r
188+ -> Success Text r
189+ -> Result r
190+stringSuspended f s000 s0 t0 pos0 more0 lose0 succ0 =
191+ runParser (demandInput_ >>= go) t0 pos0 more0 lose0 succ0
192+ where
193+ go s' = T.Parser $ \t pos more lose succ ->
194+ let s = f s'
195+ in case T.commonPrefixes s0 s of
196+ Nothing -> lose t pos more [] "string"
197+ Just (_pfx,ssfx,tsfx)
198+ | T.null ssfx -> let l = Pos (T.lengthWord16 s000)
199+ in succ t (pos + l) more (substring pos l t)
200+ | T.null tsfx -> stringSuspended f s000 ssfx t pos more lose succ
201+ | otherwise -> lose t pos more [] "string"
202+
162-- | Satisfy a literal string, ignoring case.
203-- | Satisfy a literal string, ignoring case.
163--
204--
164-- Note: this function is currently quite inefficient. Unicode case
205-- Note: this function is currently quite inefficient. Unicode case
@@ -181,34 +222,24 @@ stringCI s = go 0
181
222
182-- | Satisfy a literal string, ignoring case for characters in the ASCII range.
223-- | Satisfy a literal string, ignoring case for characters in the ASCII range.
183asciiCI :: Text -> Parser Text
224asciiCI :: Text -> Parser Text
184−asciiCI input = do
225+asciiCI s = fmap fst $ match $ T.foldr ((*>) . asciiCharCI) (pure ()) s
185− (k,t) <- ensure n
186− if asciiToLower t == s
187− then advance k >> return t
188− else fail "asciiCI"
189− where
190− n = T.length input
191− s = asciiToLower input
192−
193− -- convert letters in the ASCII range to lower-case
194− asciiToLower = T.map f
195− where
196− offset = ord 'a' - ord 'A'
197− f c | 'A' <= c && c <= 'Z' = chr (ord c + offset)
198− | otherwise = c
199{-# INLINE asciiCI #-}
226{-# INLINE asciiCI #-}
200
227
228+asciiCharCI :: Char -> Parser Char
229+asciiCharCI c
230+ | isAsciiUpper c = char c <|> char (toLower c)
231+ | isAsciiLower c = char c <|> char (toUpper c)
232+ | otherwise = char c
233+{-# INLINE asciiCharCI #-}
234+
201-- | Skip past input for as long as the predicate returns 'True'.
235-- | Skip past input for as long as the predicate returns 'True'.
202skipWhile :: (Char -> Bool) -> Parser ()
236skipWhile :: (Char -> Bool) -> Parser ()
203skipWhile p = go
237skipWhile p = go
204 where
238 where
205 go = do
239 go = do
206 t <- T.takeWhile p <$> get
240 t <- T.takeWhile p <$> get
207− advance (lengthOf t)
241+ continue <- inputSpansChunks (size t)
208− eoc <- endOfChunk
242+ when continue go
209− when eoc $ do
210− input <- wantInput
211− when input go
212{-# INLINE skipWhile #-}
243{-# INLINE skipWhile #-}
213
244
214-- | Consume input as long as the predicate returns 'False'
245-- | Consume input as long as the predicate returns 'False'
@@ -236,19 +267,25 @@ takeTill p = takeWhile (not . p)
236-- parsers loop until a failure occurs. Careless use will thus result
267-- parsers loop until a failure occurs. Careless use will thus result
237-- in an infinite loop.
268-- in an infinite loop.
238takeWhile :: (Char -> Bool) -> Parser Text
269takeWhile :: (Char -> Bool) -> Parser Text
239−takeWhile p = (T.concat . reverse) `fmap` go []
270+takeWhile p = do
271+ h <- T.takeWhile p <$> get
272+ continue <- inputSpansChunks (size h)
273+ -- only use slow concat path if necessary
274+ if continue
275+ then takeWhileAcc p [h]
276+ else return h
277+{-# INLINE takeWhile #-}
278+
279+takeWhileAcc :: (Char -> Bool) -> [Text] -> Parser Text
280+takeWhileAcc p = go
240 where
281 where
241 go acc = do
282 go acc = do
242 h <- T.takeWhile p <$> get
283 h <- T.takeWhile p <$> get
243− advance (lengthOf h)
284+ continue <- inputSpansChunks (size h)
244− eoc <- endOfChunk
285+ if continue
245− if eoc
246− then do
247− input <- wantInput
248− if input
249− then go (h:acc)
286+ then go (h:acc)
250− else return (h:acc)
287+ else return $ concatReverse (h:acc)
251− else return (h:acc)
288+{-# INLINE takeWhileAcc #-}
252
289
253takeRest :: Parser [Text]
290takeRest :: Parser [Text]
254takeRest = go []
291takeRest = go []
@@ -258,7 +295,7 @@ takeRest = go []
258 if input
295 if input
259 then do
296 then do
260 s <- get
297 s <- get
261− advance (lengthOf s)
298+ advance (size s)
262 go (s:acc)
299 go (s:acc)
263 else return (reverse acc)
300 else return (reverse acc)
264
301
@@ -271,7 +308,7 @@ takeLazyText :: Parser L.Text
271takeLazyText = L.fromChunks `fmap` takeRest
308takeLazyText = L.fromChunks `fmap` takeRest
272
309
273data Scan s = Continue s
310data Scan s = Continue s
274− | Finished {-# UNPACK #-} !Int T.Text
311+ | Finished s {-# UNPACK #-} !Int Text
275
312
276scan_ :: (s -> [Text] -> Parser r) -> s -> (s -> Char -> Maybe s) -> Parser r
313scan_ :: (s -> [Text] -> Parser r) -> s -> (s -> Char -> Maybe s) -> Parser r
277scan_ f s0 p = go [] s0
314scan_ f s0 p = go [] s0
@@ -280,18 +317,17 @@ scan_ f s0 p = go [] s0
280 case T.uncons t of
317 case T.uncons t of
281 Just (c,t') -> case p s c of
318 Just (c,t') -> case p s c of
282 Just s' -> scanner s' (n+1) t'
319 Just s' -> scanner s' (n+1) t'
283− Nothing -> Finished n t
320+ Nothing -> Finished s n t
284 Nothing -> Continue s
321 Nothing -> Continue s
285 go acc s = do
322 go acc s = do
286 input <- get
323 input <- get
287 case scanner s 0 input of
324 case scanner s 0 input of
288− Continue s' -> do advance (lengthOf input)
325+ Continue s' -> do continue <- inputSpansChunks (size input)
289− more <- wantInput
326+ if continue
290− if more
291 then go (input : acc) s'
327 then go (input : acc) s'
292 else f s' (input : acc)
328 else f s' (input : acc)
293− Finished n t -> do advance (lengthOf input - lengthOf t)
329+ Finished s' n t -> do advance (size input - size t)
294− f s (T.take n input : acc)
330+ f s' (T.take n input : acc)
295{-# INLINE scan_ #-}
331{-# INLINE scan_ #-}
296
332
297-- | A stateful scanner. The predicate consumes and transforms a
333-- | A stateful scanner. The predicate consumes and transforms a
@@ -307,16 +343,13 @@ scan_ f s0 p = go [] s0
307-- parsers loop until a failure occurs. Careless use will thus result
343-- parsers loop until a failure occurs. Careless use will thus result
308-- in an infinite loop.
344-- in an infinite loop.
309scan :: s -> (s -> Char -> Maybe s) -> Parser Text
345scan :: s -> (s -> Char -> Maybe s) -> Parser Text
310−scan = scan_ $ \_ chunks ->
346+scan = scan_ $ \_ chunks -> return $! concatReverse chunks
311− case chunks of
312− [x] -> return x
313− xs -> return . T.concat . reverse $ xs
314{-# INLINE scan #-}
347{-# INLINE scan #-}
315
348
316-- | Like 'scan', but generalized to return the final state of the
349-- | Like 'scan', but generalized to return the final state of the
317-- scanner.
350-- scanner.
318runScanner :: s -> (s -> Char -> Maybe s) -> Parser (Text, s)
351runScanner :: s -> (s -> Char -> Maybe s) -> Parser (Text, s)
319−runScanner = scan_ $ \s xs -> return (T.concat (reverse xs), s)
352+runScanner = scan_ $ \s xs -> let !sx = concatReverse xs in return (sx, s)
320{-# INLINE runScanner #-}
353{-# INLINE runScanner #-}
321
354
322-- | Consume input as long as the predicate returns 'True', and return
355-- | Consume input as long as the predicate returns 'True', and return
@@ -329,13 +362,14 @@ takeWhile1 :: (Char -> Bool) -> Parser Text
329takeWhile1 p = do
362takeWhile1 p = do
330 (`when` demandInput) =<< endOfChunk
363 (`when` demandInput) =<< endOfChunk
331 h <- T.takeWhile p <$> get
364 h <- T.takeWhile p <$> get
332− let len = lengthOf h
365+ let size' = size h
333− when (len == 0) $ fail "takeWhile1"
366+ when (size' == 0) $ fail "takeWhile1"
334− advance len
367+ advance size'
335 eoc <- endOfChunk
368 eoc <- endOfChunk
336 if eoc
369 if eoc
337− then (h<>) `fmap` takeWhile p
370+ then takeWhileAcc p [h]
338 else return h
371 else return h
372+{-# INLINE takeWhile1 #-}
339
373
340-- | Match any character in a set.
374-- | Match any character in a set.
341--
375--
@@ -384,13 +418,13 @@ peekChar :: Parser (Maybe Char)
384peekChar = T.Parser $ \t pos more _lose succ ->
418peekChar = T.Parser $ \t pos more _lose succ ->
385 case () of
419 case () of
386 _| pos < lengthOf t ->
420 _| pos < lengthOf t ->
387− let T.Iter !c _ = T.iter t (fromPos pos)
421+ let T.Iter !c _ = Buf.iter t (fromPos pos)
388 in succ t pos more (Just c)
422 in succ t pos more (Just c)
389 | more == Complete ->
423 | more == Complete ->
390 succ t pos more Nothing
424 succ t pos more Nothing
391 | otherwise ->
425 | otherwise ->
392 let succ' t' pos' more' =
426 let succ' t' pos' more' =
393− let T.Iter !c _ = T.iter t' (fromPos pos')
427+ let T.Iter !c _ = Buf.iter t' (fromPos pos')
394 in succ t' pos' more' (Just c)
428 in succ t' pos' more' (Just c)
395 lose' t' pos' more' = succ t' pos' more' Nothing
429 lose' t' pos' more' = succ t' pos' more' Nothing
396 in prompt t pos more lose' succ'
430 in prompt t pos more lose' succ'
@@ -411,30 +445,39 @@ endOfLine = (char '\n' >> return ()) <|> (string "\r\n" >> return ())
411
445
412-- | Terminal failure continuation.
446-- | Terminal failure continuation.
413failK :: Failure a
447failK :: Failure a
414−failK t (Pos pos) _more stack msg = Fail (T.dropWord16 pos t) stack msg
448+failK t (Pos pos) _more stack msg = Fail (Buf.dropWord16 pos t) stack msg
415{-# INLINE failK #-}
449{-# INLINE failK #-}
416
450
417-- | Terminal success continuation.
451-- | Terminal success continuation.
418successK :: Success a a
452successK :: Success a a
419−successK t (Pos pos) _more a = Done (T.dropWord16 pos t) a
453+successK t (Pos pos) _more a = Done (Buf.dropWord16 pos t) a
420{-# INLINE successK #-}
454{-# INLINE successK #-}
421
455
422-- | Run a parser.
456-- | Run a parser.
423parse :: Parser a -> Text -> Result a
457parse :: Parser a -> Text -> Result a
424−parse m s = runParser m s 0 Incomplete failK successK
458+parse m s = runParser m (buffer s) 0 Incomplete failK successK
425{-# INLINE parse #-}
459{-# INLINE parse #-}
426
460
427-- | Run a parser that cannot be resupplied via a 'Partial' result.
461-- | Run a parser that cannot be resupplied via a 'Partial' result.
462+--
463+-- This function does not force a parser to consume all of its input.
464+-- Instead, any residual input will be discarded. To force a parser
465+-- to consume all of its input, use something like this:
466+--
467+-- @
468+--'parseOnly' (myParser 'Control.Applicative.<*' 'endOfInput')
469+-- @
428parseOnly :: Parser a -> Text -> Either String a
470parseOnly :: Parser a -> Text -> Either String a
429−parseOnly m s = case runParser m s 0 Complete failK successK of
471+parseOnly m s = case runParser m (buffer s) 0 Complete failK successK of
472+ Fail _ [] err -> Left err
430− Fail _ _ err -> Left err
473+ Fail _ ctxs err -> Left (intercalate " > " ctxs ++ ": " ++ err)
431− Done _ a -> Right a
474+ Done _ a -> Right a
432− _ -> error "parseOnly: impossible error!"
475+ _ -> error "parseOnly: impossible error!"
433{-# INLINE parseOnly #-}
476{-# INLINE parseOnly #-}
434
477
435get :: Parser Text
478get :: Parser Text
436get = T.Parser $ \t pos more _lose succ ->
479get = T.Parser $ \t pos more _lose succ ->
437− succ t pos more (T.dropWord16 (fromPos pos) t)
480+ succ t pos more (Buf.dropWord16 (fromPos pos) t)
438{-# INLINE get #-}
481{-# INLINE get #-}
439
482
440endOfChunk :: Parser Bool
483endOfChunk :: Parser Bool
@@ -442,11 +485,21 @@ endOfChunk = T.Parser $ \t pos more _lose succ ->
442 succ t pos more (pos == lengthOf t)
485 succ t pos more (pos == lengthOf t)
443{-# INLINE endOfChunk #-}
486{-# INLINE endOfChunk #-}
444
487
488+inputSpansChunks :: Pos -> Parser Bool
489+inputSpansChunks i = T.Parser $ \t pos_ more _lose succ ->
490+ let pos = pos_ + i
491+ in if pos < lengthOf t || more == Complete
492+ then succ t pos more False
493+ else let lose' t' pos' more' = succ t' pos' more' False
494+ succ' t' pos' more' = succ t' pos' more' True
495+ in prompt t pos more lose' succ'
496+{-# INLINE inputSpansChunks #-}
497+
445advance :: Pos -> Parser ()
498advance :: Pos -> Parser ()
446advance n = T.Parser $ \t pos more _lose succ -> succ t (pos+n) more ()
499advance n = T.Parser $ \t pos more _lose succ -> succ t (pos+n) more ()
447{-# INLINE advance #-}
500{-# INLINE advance #-}
448
501
449−ensureSuspended :: Int -> Text -> Pos -> More
502+ensureSuspended :: Int -> Buffer -> Pos -> More
450 -> Failure r -> Success (Pos, Text) r
503 -> Failure r -> Success (Pos, Text) r
451 -> Result r
504 -> Result r
452ensureSuspended n t pos more lose succ =
505ensureSuspended n t pos more lose succ =
@@ -464,53 +517,8 @@ ensure n = T.Parser $ \t pos more lose succ ->
464 Just n' -> succ t pos more (n', substring pos n' t)
517 Just n' -> succ t pos more (n', substring pos n' t)
465 -- The uncommon case is kept out-of-line to reduce code size:
518 -- The uncommon case is kept out-of-line to reduce code size:
466 Nothing -> ensureSuspended n t pos more lose succ
519 Nothing -> ensureSuspended n t pos more lose succ
467−-- Non-recursive so the bounds check can be inlined:
468{-# INLINE ensure #-}
520{-# INLINE ensure #-}
469
521
470−-- | Ask for input. If we receive any, pass it to a success
471−-- continuation, otherwise to a failure continuation.
472−prompt :: Text -> Pos -> More
473− -> (Text -> Pos -> More -> IResult Text r)
474− -> (Text -> Pos -> More -> IResult Text r)
475− -> Result r
476−prompt t pos _more lose succ = Partial $ \s ->
477− if T.null s
478− then lose t pos Complete
479− else succ (t <> s) pos Incomplete
480−
481−-- | Immediately demand more input via a 'Partial' continuation
482−-- result.
483−demandInput :: Parser ()
484−demandInput = T.Parser $ \t pos more lose succ ->
485− case more of
486− Complete -> lose t pos more [] "not enough input"
487− _ -> let lose' t' pos' more' = lose t' pos' more' [] "not enough input"
488− succ' t' pos' more' = succ t' pos' more' ()
489− in prompt t pos more lose' succ'
490−
491−-- | This parser always succeeds. It returns 'True' if any input is
492−-- available either immediately or on demand, and 'False' if the end
493−-- of all input has been reached.
494−wantInput :: Parser Bool
495−wantInput = T.Parser $ \t pos more _lose succ ->
496− case () of
497− _ | pos < lengthOf t -> succ t pos more True
498− | more == Complete -> succ t pos more False
499− | otherwise -> let lose' t' pos' more' = succ t' pos' more' False
500− succ' t' pos' more' = succ t' pos' more' True
501− in prompt t pos more lose' succ'
502−
503−-- | Match only if all input has been consumed.
504−endOfInput :: Parser ()
505−endOfInput = T.Parser $ \t pos more lose succ ->
506− case () of
507− _| pos < lengthOf t -> lose t pos more [] "endOfInput"
508− | more == Complete -> succ t pos more ()
509− | otherwise ->
510− let lose' t' pos' more' _ctx _msg = succ t' pos' more' ()
511− succ' t' pos' more' _a = lose t' pos' more' [] "endOfInput"
512− in runParser demandInput t pos more lose' succ'
513−
514-- | Return both the result of a parse and the portion of the input
522-- | Return both the result of a parse and the portion of the input
515-- that was consumed while it was being parsed.
523-- that was consumed while it was being parsed.
516match :: Parser a -> Parser (Text, a)
524match :: Parser a -> Parser (Text, a)
@@ -519,24 +527,23 @@ match p = T.Parser $ \t pos more lose succ ->
519 (substring pos (pos'-pos) t', a)
527 (substring pos (pos'-pos) t', a)
520 in runParser p t pos more lose succ'
528 in runParser p t pos more lose succ'
521
529
522−-- | Return an indication of whether the end of input has been
530+-- | Ensure that at least @n@ code points of input are available.
523−-- reached.
531+-- Returns the number of words consumed while traversing.
524−atEnd :: Parser Bool
525−atEnd = not <$> wantInput
526−{-# INLINE atEnd #-}
527−
528−lengthAtLeast :: Pos -> Int -> Text -> Maybe Pos
532+lengthAtLeast :: Pos -> Int -> Buffer -> Maybe Pos
529lengthAtLeast pos n t = go 0 (fromPos pos)
533lengthAtLeast pos n t = go 0 (fromPos pos)
530 where go i !p
534 where go i !p
531 | i == n = Just (Pos p - pos)
535 | i == n = Just (Pos p - pos)
532 | p == len = Nothing
536 | p == len = Nothing
533− | otherwise = go (i+1) (p + T.iter_ t p)
537+ | otherwise = go (i+1) (p + Buf.iter_ t p)
534 Pos len = lengthOf t
538 Pos len = lengthOf t
535{-# INLINE lengthAtLeast #-}
539{-# INLINE lengthAtLeast #-}
536
540
537−substring :: Pos -> Pos -> Text -> Text
541+substring :: Pos -> Pos -> Buffer -> Text
538−substring (Pos pos) (Pos n) t = T.takeWord16 n (T.dropWord16 pos t)
542+substring (Pos pos) (Pos n) = Buf.substring pos n
539{-# INLINE substring #-}
543{-# INLINE substring #-}
540
544
541−lengthOf :: Text -> Pos
545+lengthOf :: Buffer -> Pos
542−lengthOf (Text _arr _off len) = Pos len
546+lengthOf = Pos . Buf.length
547+
548+size :: Text -> Pos
549+size (Text _ _ l) = Pos l
Data/Attoparsec/Text/Lazy.hs+13−10
@@ -1,7 +1,12 @@
1+{-# LANGUAGE CPP #-}
2+#if __GLASGOW_HASKELL__ >= 702
3+{-# LANGUAGE Trustworthy #-} -- Imports internal modules
4+#endif
5+
1{-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
6{-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
2-- |
7-- |
3-- Module : Data.Attoparsec.Text.Lazy
8-- Module : Data.Attoparsec.Text.Lazy
4−-- Copyright : Bryan O'Sullivan 2011
9+-- Copyright : Bryan O'Sullivan 2007-2015
5-- License : BSD3
10-- License : BSD3
6--
11--
7-- Maintainer : bos@serpentine.com
12-- Maintainer : bos@serpentine.com
@@ -36,6 +41,7 @@ module Data.Attoparsec.Text.Lazy
36 ) where
41 ) where
37
42
38import Control.DeepSeq (NFData(rnf))
43import Control.DeepSeq (NFData(rnf))
44+import Data.List (intercalate)
39import Data.Text.Lazy.Internal (Text(..), chunk)
45import Data.Text.Lazy.Internal (Text(..), chunk)
40import qualified Data.Attoparsec.Internal.Types as T
46import qualified Data.Attoparsec.Internal.Types as T
41import qualified Data.Attoparsec.Text as A
47import qualified Data.Attoparsec.Text as A
@@ -45,20 +51,16 @@ import Data.Attoparsec.Text hiding (IResult(..), Result, eitherResult,
45
51
46-- | The result of a parse.
52-- | The result of a parse.
47data Result r = Fail Text [String] String
53data Result r = Fail Text [String] String
48− -- ^ The parse failed. The 'ByteString' is the input
54+ -- ^ The parse failed. The 'Text' is the input
49 -- that had not yet been consumed when the failure
55 -- that had not yet been consumed when the failure
50 -- occurred. The @[@'String'@]@ is a list of contexts
56 -- occurred. The @[@'String'@]@ is a list of contexts
51 -- in which the error occurred. The 'String' is the
57 -- in which the error occurred. The 'String' is the
52 -- message describing the error, if any.
58 -- message describing the error, if any.
53 | Done Text r
59 | Done Text r
54− -- ^ The parse succeeded. The 'ByteString' is the
60+ -- ^ The parse succeeded. The 'Text' is the
55 -- input that had not yet been consumed (if any) when
61 -- input that had not yet been consumed (if any) when
56 -- the parse succeeded.
62 -- the parse succeeded.
57−
63+ deriving (Show)
58−instance Show r => Show (Result r) where
59− show (Fail bs stk msg) =
60− "Fail " ++ show bs ++ " " ++ show stk ++ " " ++ show msg
61− show (Done bs r) = "Done " ++ show bs ++ " " ++ show r
62
64
63instance NFData r => NFData (Result r) where
65instance NFData r => NFData (Result r) where
64 rnf (Fail bs ctxs msg) = rnf bs `seq` rnf ctxs `seq` rnf msg
66 rnf (Fail bs ctxs msg) = rnf bs `seq` rnf ctxs `seq` rnf msg
@@ -94,5 +96,6 @@ maybeResult _ = Nothing
94
96
95-- | Convert a 'Result' value to an 'Either' value.
97-- | Convert a 'Result' value to an 'Either' value.
96eitherResult :: Result r -> Either String r
98eitherResult :: Result r -> Either String r
97−eitherResult (Done _ r) = Right r
99+eitherResult (Done _ r) = Right r
98−eitherResult (Fail _ _ msg) = Left msg
100+eitherResult (Fail _ [] msg) = Left msg
101+eitherResult (Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
Data/Attoparsec/Types.hs+3−2
@@ -1,6 +1,6 @@
1-- |
1-- |
2-- Module : Data.Attoparsec.Types
2-- Module : Data.Attoparsec.Types
3−-- Copyright : Bryan O'Sullivan 2011
3+-- Copyright : Bryan O'Sullivan 2007-2015
4-- License : BSD3
4-- License : BSD3
5--
5--
6-- Maintainer : bos@serpentine.com
6-- Maintainer : bos@serpentine.com
@@ -14,6 +14,7 @@ module Data.Attoparsec.Types
14 (
14 (
15 Parser
15 Parser
16 , IResult(..)
16 , IResult(..)
17+ , Chunk(chunkElemToChar)
17 ) where
18 ) where
18
19
19−import Data.Attoparsec.Internal.Types (Parser(..), IResult(..))
20+import Data.Attoparsec.Internal.Types (Parser(..), IResult(..), Chunk(..))
Data/Attoparsec/Zepto.hs+88−46
@@ -1,8 +1,12 @@
1−{-# LANGUAGE BangPatterns, MagicHash, UnboxedTuples #-}
1+{-# LANGUAGE CPP #-}
2+#if __GLASGOW_HASKELL__ >= 702
3+{-# LANGUAGE Trustworthy #-} -- Data.ByteString.Unsafe
4+#endif
5+{-# LANGUAGE BangPatterns #-}
2
6
3-- |
7-- |
4-- Module : Data.Attoparsec.Zepto
8-- Module : Data.Attoparsec.Zepto
5−-- Copyright : Bryan O'Sullivan 2011
9+-- Copyright : Bryan O'Sullivan 2007-2015
6-- License : BSD3
10-- License : BSD3
7--
11--
8-- Maintainer : bos@serpentine.com
12-- Maintainer : bos@serpentine.com
@@ -12,112 +16,150 @@
12-- A tiny, highly specialized combinator parser for 'B.ByteString'
16-- A tiny, highly specialized combinator parser for 'B.ByteString'
13-- strings.
17-- strings.
14--
18--
15−-- While the main Attoparsec module generally performs well, this
19+-- While the main attoparsec module generally performs well, this
16-- module is particularly fast for simple non-recursive loops that
20-- module is particularly fast for simple non-recursive loops that
17-- should not normally result in failed parses.
21-- should not normally result in failed parses.
18--
22--
19-- /Warning/: on more complex inputs involving recursion or failure,
23-- /Warning/: on more complex inputs involving recursion or failure,
20-- parsers based on this module may be as much as /ten times slower/
24-- parsers based on this module may be as much as /ten times slower/
21−-- than regular Attoparsec! You should /only/ use this module when you
25+-- than regular attoparsec! You should /only/ use this module when you
22-- have benchmarks that prove that its use speeds your code up.
26-- have benchmarks that prove that its use speeds your code up.
23module Data.Attoparsec.Zepto
27module Data.Attoparsec.Zepto
24 (
28 (
25 Parser
29 Parser
30+ , ZeptoT
26 , parse
31 , parse
32+ , parseT
27 , atEnd
33 , atEnd
28 , string
34 , string
29 , take
35 , take
30 , takeWhile
36 , takeWhile
31 ) where
37 ) where
32
38
33−import Data.Word (Word8)
34import Control.Applicative
39import Control.Applicative
35−import Control.Monad
40+import Control.Monad (MonadPlus(..), ap)
36−import Data.Monoid
37−import qualified Data.ByteString as B
41+import qualified Control.Monad.Fail as Fail
38−import qualified Data.ByteString.Unsafe as B
42+import Control.Monad.IO.Class (MonadIO(..))
39import Data.ByteString (ByteString)
43import Data.ByteString (ByteString)
44+import Data.Functor.Identity (Identity(runIdentity))
45+import Data.Monoid as Mon (Monoid(..))
46+import Data.Semigroup (Semigroup(..))
47+import Data.Word (Word8)
40import Prelude hiding (take, takeWhile)
48import Prelude hiding (take, takeWhile)
49+import qualified Data.ByteString as B
50+import qualified Data.ByteString.Unsafe as B
41
51
42newtype S = S {
52newtype S = S {
43 input :: ByteString
53 input :: ByteString
44 }
54 }
45
55
46data Result a = Fail String
56data Result a = Fail String
47− | OK !a
57+ | OK !a S
48
58
49-- | A simple parser.
59-- | A simple parser.
50--
60--
51-- This monad is strict in its state, and the monadic bind operator
61-- This monad is strict in its state, and the monadic bind operator
52-- ('>>=') evaluates each result to weak head normal form before
62-- ('>>=') evaluates each result to weak head normal form before
53-- passing it along.
63-- passing it along.
54−newtype Parser a = Parser {
64+newtype ZeptoT m a = Parser {
55− runParser :: S -> (# Result a, S #)
65+ runParser :: S -> m (Result a)
56 }
66 }
57
67
68+type Parser a = ZeptoT Identity a
69+
58−instance Functor Parser where
70+instance Monad m => Functor (ZeptoT m) where
59− fmap f m = Parser $ \s -> case runParser m s of
71+ fmap f m = Parser $ \s -> do
72+ result <- runParser m s
73+ case result of
60− (# OK a, s' #) -> (# OK (f a), s' #)
74+ OK a s' -> return (OK (f a) s')
61− (# Fail err, s' #) -> (# Fail err, s' #)
75+ Fail err -> return (Fail err)
62 {-# INLINE fmap #-}
76 {-# INLINE fmap #-}
63
77
64−instance Monad Parser where
78+instance MonadIO m => MonadIO (ZeptoT m) where
65− return a = Parser $ \s -> (# OK a, s #)
79+ liftIO act = Parser $ \s -> do
80+ result <- liftIO act
81+ return (OK result s)
82+ {-# INLINE liftIO #-}
83+
84+instance Monad m => Monad (ZeptoT m) where
85+ return = pure
66 {-# INLINE return #-}
86 {-# INLINE return #-}
67
87
68− m >>= k = Parser $ \s -> case runParser m s of
88+ m >>= k = Parser $ \s -> do
89+ result <- runParser m s
90+ case result of
69− (# OK a, s' #) -> runParser (k a) s'
91+ OK a s' -> runParser (k a) s'
70− (# Fail err, s' #) -> (# Fail err, s' #)
92+ Fail err -> return (Fail err)
71 {-# INLINE (>>=) #-}
93 {-# INLINE (>>=) #-}
72
94
95+#if !(MIN_VERSION_base(4,13,0))
73− fail msg = Parser $ \s -> (# Fail msg, s #)
96+ fail = Fail.fail
97+ {-# INLINE fail #-}
98+#endif
74
99
75−instance MonadPlus Parser where
100+instance Monad m => Fail.MonadFail (ZeptoT m) where
101+ fail msg = Parser $ \_ -> return (Fail msg)
102+ {-# INLINE fail #-}
103+
104+instance Monad m => MonadPlus (ZeptoT m) where
76 mzero = fail "mzero"
105 mzero = fail "mzero"
77 {-# INLINE mzero #-}
106 {-# INLINE mzero #-}
78
107
79− mplus a b = Parser $ \s ->
108+ mplus a b = Parser $ \s -> do
80− case runParser a s of
109+ result <- runParser a s
110+ case result of
81− (# ok@(OK _), s' #) -> (# ok, s' #)
111+ ok@(OK _ _) -> return ok
82− (# _, _ #) -> case runParser b s of
112+ _ -> runParser b s
83− (# ok@(OK _), s'' #) -> (# ok, s'' #)
84− (# err, s'' #) -> (# err, s'' #)
85 {-# INLINE mplus #-}
113 {-# INLINE mplus #-}
86
114
87−instance Applicative Parser where
115+instance (Monad m) => Applicative (ZeptoT m) where
88− pure = return
116+ pure a = Parser $ \s -> return (OK a s)
89 {-# INLINE pure #-}
117 {-# INLINE pure #-}
90 (<*>) = ap
118 (<*>) = ap
91 {-# INLINE (<*>) #-}
119 {-# INLINE (<*>) #-}
92
120
93−gets :: (S -> a) -> Parser a
121+gets :: Monad m => (S -> a) -> ZeptoT m a
94−gets f = Parser $ \s -> (# OK (f s), s #)
122+gets f = Parser $ \s -> return (OK (f s) s)
95{-# INLINE gets #-}
123{-# INLINE gets #-}
96
124
97−put :: S -> Parser ()
125+put :: Monad m => S -> ZeptoT m ()
98−put s = Parser $ \_ -> (# OK (), s #)
126+put s = Parser $ \_ -> return (OK () s)
99{-# INLINE put #-}
127{-# INLINE put #-}
100
128
101-- | Run a parser.
129-- | Run a parser.
102parse :: Parser a -> ByteString -> Either String a
130parse :: Parser a -> ByteString -> Either String a
103−parse p bs = case runParser p (S bs) of
131+parse p bs = case runIdentity (runParser p (S bs)) of
104− (# OK a, _ #) -> Right a
132+ (OK a _) -> Right a
105− (# Fail err, _ #) -> Left err
133+ (Fail err) -> Left err
106−
134+{-# INLINE parse #-}
107−instance Monoid (Parser a) where
135+
136+-- | Run a parser on top of the given base monad.
137+parseT :: Monad m => ZeptoT m a -> ByteString -> m (Either String a)
138+parseT p bs = do
139+ result <- runParser p (S bs)
140+ case result of
141+ OK a _ -> return (Right a)
142+ Fail err -> return (Left err)
143+{-# INLINE parseT #-}
144+
145+instance Monad m => Semigroup (ZeptoT m a) where
146+ (<>) = mplus
147+ {-# INLINE (<>) #-}
148+
149+instance Monad m => Mon.Monoid (ZeptoT m a) where
108 mempty = fail "mempty"
150 mempty = fail "mempty"
109 {-# INLINE mempty #-}
151 {-# INLINE mempty #-}
110− mappend = mplus
152+ mappend = (<>)
111 {-# INLINE mappend #-}
153 {-# INLINE mappend #-}
112
154
113−instance Alternative Parser where
155+instance Monad m => Alternative (ZeptoT m) where
114 empty = fail "empty"
156 empty = fail "empty"
115 {-# INLINE empty #-}
157 {-# INLINE empty #-}
116 (<|>) = mplus
158 (<|>) = mplus
117 {-# INLINE (<|>) #-}
159 {-# INLINE (<|>) #-}
118
160
119-- | Consume input while the predicate returns 'True'.
161-- | Consume input while the predicate returns 'True'.
120−takeWhile :: (Word8 -> Bool) -> Parser ByteString
162+takeWhile :: Monad m => (Word8 -> Bool) -> ZeptoT m ByteString
121takeWhile p = do
163takeWhile p = do
122 (h,t) <- gets (B.span p . input)
164 (h,t) <- gets (B.span p . input)
123 put (S t)
165 put (S t)
@@ -125,7 +167,7 @@ takeWhile p = do
125{-# INLINE takeWhile #-}
167{-# INLINE takeWhile #-}
126
168
127-- | Consume @n@ bytes of input.
169-- | Consume @n@ bytes of input.
128−take :: Int -> Parser ByteString
170+take :: Monad m => Int -> ZeptoT m ByteString
129take !n = do
171take !n = do
130 s <- gets input
172 s <- gets input
131 if B.length s >= n
173 if B.length s >= n
@@ -134,7 +176,7 @@ take !n = do
134{-# INLINE take #-}
176{-# INLINE take #-}
135
177
136-- | Match a string exactly.
178-- | Match a string exactly.
137−string :: ByteString -> Parser ()
179+string :: Monad m => ByteString -> ZeptoT m ()
138string s = do
180string s = do
139 i <- gets input
181 i <- gets input
140 if s `B.isPrefixOf` i
182 if s `B.isPrefixOf` i
@@ -143,7 +185,7 @@ string s = do
143{-# INLINE string #-}
185{-# INLINE string #-}
144
186
145-- | Indicate whether the end of the input has been reached.
187-- | Indicate whether the end of the input has been reached.
146−atEnd :: Parser Bool
188+atEnd :: Monad m => ZeptoT m Bool
147atEnd = do
189atEnd = do
148 i <- gets input
190 i <- gets input
149 return $! B.null i
191 return $! B.null i
README.markdown−6
@@ -16,12 +16,6 @@ Master [git repository](https://github.com/bos/attoparsec):
16
16
17* `git clone git://github.com/bos/attoparsec.git`
17* `git clone git://github.com/bos/attoparsec.git`
18
18
19−There's also a [Mercurial mirror](https://bitbucket.org/bos/attoparsec):
20−
21−* `hg clone https://bitbucket.org/bos/attoparsec`
22−
23−(You can create and contribute changes using either Mercurial or git.)
24−
25Authors
19Authors
26-------
20-------
27
21
attoparsec.cabal+93−28
@@ -1,12 +1,13 @@
1name: attoparsec
1name: attoparsec
2−version: 0.12.0.0
2+version: 0.13.2.2
3+x-revision: 3
3license: BSD3
4license: BSD3
4license-file: LICENSE
5license-file: LICENSE
5category: Text, Parsing
6category: Text, Parsing
6author: Bryan O'Sullivan <bos@serpentine.com>
7author: Bryan O'Sullivan <bos@serpentine.com>
7−maintainer: Bryan O'Sullivan <bos@serpentine.com>
8+maintainer: Bryan O'Sullivan <bos@serpentine.com>, Ben Gamari <ben@smart-cactus.org>
8stability: experimental
9stability: experimental
9−tested-with: GHC == 7.0, GHC == 7.2, GHC == 7.4, GHC == 7.6, GHC == 7.8
10+tested-with: GHC == 7.0.4, GHC == 7.2.2, GHC == 7.4.2, GHC ==7.6.3, GHC ==7.8.4, GHC ==7.10.3, GHC ==8.0.2, GHC ==8.2.2, GHC==8.4.4, GHC==8.6.5, GHC==8.8.1
10synopsis: Fast combinator parsing for bytestrings and text
11synopsis: Fast combinator parsing for bytestrings and text
11cabal-version: >= 1.8
12cabal-version: >= 1.8
12homepage: https://github.com/bos/attoparsec
13homepage: https://github.com/bos/attoparsec
@@ -21,6 +22,7 @@ extra-source-files:
21 benchmarks/*.cabal
22 benchmarks/*.cabal
22 benchmarks/*.hs
23 benchmarks/*.hs
23 benchmarks/*.txt
24 benchmarks/*.txt
25+ benchmarks/json-data/*.json
24 benchmarks/Makefile
26 benchmarks/Makefile
25 benchmarks/med.txt.bz2
27 benchmarks/med.txt.bz2
26 changelog.md
28 changelog.md
@@ -28,9 +30,8 @@ extra-source-files:
28 examples/*.hs
30 examples/*.hs
29 examples/Makefile
31 examples/Makefile
30 tests/*.hs
32 tests/*.hs
31− tests/Makefile
32 tests/QC/*.hs
33 tests/QC/*.hs
33− tests/TestFastSet.hs
34+ tests/QC/IPv6/*.hs
34
35
35Flag developer
36Flag developer
36 Description: Whether to build the library in development mode
37 Description: Whether to build the library in development mode
@@ -39,12 +40,21 @@ Flag developer
39
40
40library
41library
41 build-depends: array,
42 build-depends: array,
42− base >= 4.2 && < 5,
43+ base >= 4.3 && < 5,
43 bytestring,
44 bytestring,
44 containers,
45 containers,
45 deepseq,
46 deepseq,
46− text >= 0.11.3.1,
47− scientific >= 0.2
47+ scientific >= 0.3.1 && < 0.4,
48+ transformers >= 0.2 && (< 0.4 || >= 0.4.1.0) && < 0.6,
49+ text >= 1.1.1.3
50+ if impl(ghc < 7.4)
51+ build-depends:
52+ bytestring < 0.10.4.0
53+
54+ if !impl(ghc >= 8.0)
55+ -- Data.Semigroup && Control.Monad.Fail are available in base-4.9+
56+ build-depends: fail == 4.9.*,
57+ semigroups >=0.16.1 && <0.20
48
58
49 exposed-modules: Data.Attoparsec
59 exposed-modules: Data.Attoparsec
50 Data.Attoparsec.ByteString
60 Data.Attoparsec.ByteString
@@ -52,16 +62,19 @@ library
52 Data.Attoparsec.ByteString.Lazy
62 Data.Attoparsec.ByteString.Lazy
53 Data.Attoparsec.Char8
63 Data.Attoparsec.Char8
54 Data.Attoparsec.Combinator
64 Data.Attoparsec.Combinator
65+ Data.Attoparsec.Internal
66+ Data.Attoparsec.Internal.Types
55 Data.Attoparsec.Lazy
67 Data.Attoparsec.Lazy
56 Data.Attoparsec.Number
68 Data.Attoparsec.Number
57 Data.Attoparsec.Text
69 Data.Attoparsec.Text
58 Data.Attoparsec.Text.Lazy
70 Data.Attoparsec.Text.Lazy
59 Data.Attoparsec.Types
71 Data.Attoparsec.Types
60 Data.Attoparsec.Zepto
72 Data.Attoparsec.Zepto
61− other-modules: Data.Attoparsec.ByteString.FastSet
73+ other-modules: Data.Attoparsec.ByteString.Buffer
74+ Data.Attoparsec.ByteString.FastSet
62 Data.Attoparsec.ByteString.Internal
75 Data.Attoparsec.ByteString.Internal
63− Data.Attoparsec.Internal
76+ Data.Attoparsec.Internal.Fhthagn
64− Data.Attoparsec.Internal.Types
77+ Data.Attoparsec.Text.Buffer
65 Data.Attoparsec.Text.FastSet
78 Data.Attoparsec.Text.FastSet
66 Data.Attoparsec.Text.Internal
79 Data.Attoparsec.Text.Internal
67 ghc-options: -O2 -Wall
80 ghc-options: -O2 -Wall
@@ -72,12 +85,37 @@ library
72
85
73test-suite tests
86test-suite tests
74 type: exitcode-stdio-1.0
87 type: exitcode-stdio-1.0
75− hs-source-dirs: tests
88+ hs-source-dirs: tests .
76 main-is: QC.hs
89 main-is: QC.hs
77− other-modules: QC.ByteString
90+ other-modules: QC.Buffer
91+ QC.ByteString
78 QC.Combinator
92 QC.Combinator
79 QC.Common
93 QC.Common
94+ QC.IPv6.Internal
95+ QC.IPv6.Types
96+ QC.Rechunked
97+ QC.Simple
80 QC.Text
98 QC.Text
99+ QC.Text.FastSet
100+ QC.Text.Regressions
101+
102+ other-modules: Data.Attoparsec.ByteString
103+ Data.Attoparsec.ByteString.Buffer
104+ Data.Attoparsec.ByteString.Char8
105+ Data.Attoparsec.ByteString.FastSet
106+ Data.Attoparsec.ByteString.Internal
107+ Data.Attoparsec.ByteString.Lazy
108+ Data.Attoparsec.Combinator
109+ Data.Attoparsec.Internal
110+ Data.Attoparsec.Internal.Fhthagn
111+ Data.Attoparsec.Internal.Types
112+ Data.Attoparsec.Number
113+ Data.Attoparsec.Text
114+ Data.Attoparsec.Text.Buffer
115+ Data.Attoparsec.Text.FastSet
116+ Data.Attoparsec.Text.Internal
117+ Data.Attoparsec.Text.Lazy
118+ Data.Attoparsec.Zepto
81
119
82 ghc-options:
120 ghc-options:
83 -Wall -threaded -rtsopts
121 -Wall -threaded -rtsopts
@@ -86,44 +124,71 @@ test-suite tests
86 ghc-options: -Werror
124 ghc-options: -Werror
87
125
88 build-depends:
126 build-depends:
89− attoparsec,
127+ array,
90− base >= 4 && < 5,
128+ base,
91 bytestring,
129 bytestring,
130+ deepseq >= 1.1,
92− QuickCheck >= 2.7,
131+ QuickCheck >= 2.13.2 && < 2.14,
93− test-framework >= 0.8.0.2,
132+ quickcheck-unicode,
94− test-framework-quickcheck2 >= 0.3.0.3,
133+ scientific,
134+ tasty >= 0.11,
135+ tasty-quickcheck >= 0.8,
95− text
136+ text,
137+ transformers >= 0.2 && (< 0.4 || >= 0.4.1.0) && < 0.6,
138+ vector
139+
140+ if !impl(ghc >= 8.0)
141+ -- Data.Semigroup && Control.Monad.Fail are available in base-4.9+
142+ build-depends: fail == 4.9.*,
143+ semigroups >=0.16.1 && <0.20
96
144
97benchmark benchmarks
145benchmark benchmarks
98 type: exitcode-stdio-1.0
146 type: exitcode-stdio-1.0
99− hs-source-dirs: benchmarks, .
147+ hs-source-dirs: benchmarks benchmarks/warp-3.0.1.1 .
100− ghc-options: -O2 -Wall
148+ ghc-options: -O2 -Wall -rtsopts
101 main-is: Benchmarks.hs
149 main-is: Benchmarks.hs
102 other-modules:
150 other-modules:
103 Common
151 Common
104 HeadersByteString
152 HeadersByteString
153+ HeadersByteString.Atto
105 HeadersText
154 HeadersText
106 Links
155 Links
156+ Network.Wai.Handler.Warp.ReadInt
157+ Network.Wai.Handler.Warp.RequestHeader
107 Numbers
158 Numbers
159+ Sets
160+ TextFastSet
161+ Warp
108 ghc-options: -O2 -Wall
162 ghc-options: -O2 -Wall
109
163
110 if flag(developer)
164 if flag(developer)
111 ghc-options: -Werror
165 ghc-options: -Werror
112
166
113 build-depends:
167 build-depends:
114− base == 4.*,
115 array,
168 array,
169+ base == 4.*,
116− bytestring,
170+ bytestring >= 0.10.4.0,
171+ case-insensitive,
172+ containers,
117− criterion >= 0.8.1.0,
173+ criterion >= 1.0,
118 deepseq >= 1.1,
174 deepseq >= 1.1,
175+ directory,
176+ filepath,
177+ ghc-prim,
178+ http-types,
119 parsec >= 3.1.2,
179 parsec >= 3.1.2,
120 scientific,
180 scientific,
121− text
181+ text >= 1.1.1.0,
182+ transformers >= 0.2 && (< 0.4 || >= 0.4.1.0) && < 0.6,
183+ unordered-containers,
184+ vector
185+
186+ if !impl(ghc >= 8.0)
187+ -- Data.Semigroup && Control.Monad.Fail are available in base-4.9+
188+ build-depends: fail == 4.9.*,
189+ semigroups >=0.16.1 && <0.19
122
190
123source-repository head
191source-repository head
124 type: git
192 type: git
125 location: https://github.com/bos/attoparsec
193 location: https://github.com/bos/attoparsec
126
194
127−source-repository head
128− type: mercurial
129− location: https://bitbucket.org/bos/attoparsec
benchmarks/Aeson.hs+41−6
@@ -10,18 +10,25 @@ module Aeson
10import Data.ByteString.Builder
10import Data.ByteString.Builder
11 (Builder, byteString, toLazyByteString, charUtf8, word8)
11 (Builder, byteString, toLazyByteString, charUtf8, word8)
12
12
13+#if !MIN_VERSION_base(4,8,0)
13−import Control.Applicative ((*>), (<$>), (<*), liftA2, pure)
14+import Control.Applicative ((*>), (<$>), (<*), pure)
15+import Data.Monoid (mappend, mempty)
16+#endif
17+
18+import Common (pathTo)
19+import Control.Applicative (liftA2)
20+import Control.DeepSeq (NFData(..))
14import Control.Monad (forM)
21import Control.Monad (forM)
15−import Data.Aeson.Types (Result(..), Value(..))
16import Data.Attoparsec.ByteString.Char8 (Parser, char, endOfInput, scientific,
22import Data.Attoparsec.ByteString.Char8 (Parser, char, endOfInput, scientific,
17 skipSpace, string)
23 skipSpace, string)
18import Data.Bits ((.|.), shiftL)
24import Data.Bits ((.|.), shiftL)
19import Data.ByteString (ByteString)
25import Data.ByteString (ByteString)
20import Data.Char (chr)
26import Data.Char (chr)
21−import Data.Monoid (mappend, mempty)
27+import Data.List (sort)
28+import Data.Scientific (Scientific)
22import Data.Text (Text)
29import Data.Text (Text)
23import Data.Text.Encoding (decodeUtf8')
30import Data.Text.Encoding (decodeUtf8')
24−import Data.Vector as Vector (Vector, fromList)
31+import Data.Vector as Vector (Vector, foldl', fromList)
25import Data.Word (Word8)
32import Data.Word (Word8)
26import System.Directory (getDirectoryContents)
33import System.Directory (getDirectoryContents)
27import System.FilePath ((</>), dropExtension)
34import System.FilePath ((</>), dropExtension)
@@ -50,6 +57,34 @@ import Criterion.Main
50#define C_n 110
57#define C_n 110
51#define C_t 116
58#define C_t 116
52
59
60+data Result a = Error String
61+ | Success a
62+ deriving (Eq, Show)
63+
64+
65+-- | A JSON \"object\" (key\/value map).
66+type Object = H.HashMap Text Value
67+
68+-- | A JSON \"array\" (sequence).
69+type Array = Vector Value
70+
71+-- | A JSON value represented as a Haskell value.
72+data Value = Object !Object
73+ | Array !Array
74+ | String !Text
75+ | Number !Scientific
76+ | Bool !Bool
77+ | Null
78+ deriving (Eq, Show)
79+
80+instance NFData Value where
81+ rnf (Object o) = rnf o
82+ rnf (Array a) = Vector.foldl' (\x y -> rnf y `seq` x) () a
83+ rnf (String s) = rnf s
84+ rnf (Number n) = rnf n
85+ rnf (Bool b) = rnf b
86+ rnf Null = ()
87+
53-- | Parse a top-level JSON value. This must be either an object or
88-- | Parse a top-level JSON value. This must be either an object or
54-- an array, per RFC 4627.
89-- an array, per RFC 4627.
55--
90--
@@ -310,8 +345,8 @@ toByteString = L.toStrict . toLazyByteString
310
345
311aeson :: IO Benchmark
346aeson :: IO Benchmark
312aeson = do
347aeson = do
313− let path = "json-data"
348+ path <- pathTo "json-data"
314− names <- filter (`notElem` [".", ".."]) <$> getDirectoryContents path
349+ names <- sort . filter (`notElem` [".", ".."]) <$> getDirectoryContents path
315 benches <- forM names $ \name -> do
350 benches <- forM names $ \name -> do
316 bs <- B.readFile (path </> name)
351 bs <- B.readFile (path </> name)
317 return . bench (dropExtension name) $ nf (A.parseOnly jsonEOF') bs
352 return . bench (dropExtension name) $ nf (A.parseOnly jsonEOF') bs
benchmarks/Benchmarks.hs+15
@@ -1,5 +1,6 @@
1{-# LANGUAGE BangPatterns, CPP #-}
1{-# LANGUAGE BangPatterns, CPP #-}
2
2
3+import Common ()
3import Control.Applicative (many)
4import Control.Applicative (many)
4import Criterion.Main (bench, bgroup, defaultMain, nf)
5import Criterion.Main (bench, bgroup, defaultMain, nf)
5import Data.Bits
6import Data.Bits
@@ -10,7 +11,9 @@ import Numbers (numbers)
10import Common (chunksOf)
11import Common (chunksOf)
11import Text.Parsec.Text ()
12import Text.Parsec.Text ()
12import Text.Parsec.Text.Lazy ()
13import Text.Parsec.Text.Lazy ()
14+import qualified Warp
13import qualified Aeson
15import qualified Aeson
16+import qualified Genome
14import qualified Data.Attoparsec.ByteString as AB
17import qualified Data.Attoparsec.ByteString as AB
15import qualified Data.Attoparsec.ByteString.Char8 as AC
18import qualified Data.Attoparsec.ByteString.Char8 as AC
16import qualified Data.Attoparsec.ByteString.Lazy as ABL
19import qualified Data.Attoparsec.ByteString.Lazy as ABL
@@ -25,6 +28,7 @@ import qualified HeadersByteString
25import qualified HeadersText
28import qualified HeadersText
26import qualified Links
29import qualified Links
27import qualified Text.Parsec as P
30import qualified Text.Parsec as P
31+import qualified Sets
28
32
29main :: IO ()
33main :: IO ()
30main = do
34main = do
@@ -67,6 +71,14 @@ main = do
67 , bench "isAlpha_ascii" $ nf (ABL.parse (AC.takeWhile AC.isAlpha_ascii)) bl
71 , bench "isAlpha_ascii" $ nf (ABL.parse (AC.takeWhile AC.isAlpha_ascii)) bl
68 , bench "isAlpha_iso8859_15" $
72 , bench "isAlpha_iso8859_15" $
69 nf (ABL.parse (AC.takeWhile AC.isAlpha_iso8859_15)) bl
73 nf (ABL.parse (AC.takeWhile AC.isAlpha_iso8859_15)) bl
74+ , bench "T isAlpha" $ nf (AT.parse (AT.takeWhile isAlpha)) t
75+ , bench "TL isAlpha" $ nf (ATL.parse (AT.takeWhile isAlpha)) tl
76+ ]
77+ , bgroup "takeWhile1" [
78+ bench "isAlpha" $ nf (ABL.parse (AC.takeWhile1 isAlpha)) bl
79+ , bench "isAlpha_ascii" $ nf (ABL.parse (AC.takeWhile1 AC.isAlpha_ascii)) bl
80+ , bench "T isAlpha" $ nf (AT.parse (AT.takeWhile1 isAlpha)) t
81+ , bench "TL isAlpha" $ nf (ATL.parse (AT.takeWhile1 isAlpha)) tl
70 ]
82 ]
71 , bench "word32LE" $ nf (AB.parse word32LE) b
83 , bench "word32LE" $ nf (AB.parse word32LE) b
72 , bgroup "scan" [
84 , bgroup "scan" [
@@ -74,10 +86,13 @@ main = do
74 , bench "long" $ nf (AB.parse quotedString) b
86 , bench "long" $ nf (AB.parse quotedString) b
75 ]
87 ]
76 , aeson
88 , aeson
89+ , Genome.genome
77 , headersBS
90 , headersBS
78 , headersT
91 , headersT
79 , Links.links
92 , Links.links
80 , numbers
93 , numbers
94+ , Sets.benchmarks
95+ , Warp.benchmarks
81 ]
96 ]
82
97
83-- Benchmarks bind and (potential) bounds-check merging.
98-- Benchmarks bind and (potential) bounds-check merging.
benchmarks/Common.hs+25−1
@@ -1,10 +1,21 @@
1{-# LANGUAGE CPP #-}
1{-# LANGUAGE CPP #-}
2{-# OPTIONS_GHC -fno-warn-orphans #-}
2{-# OPTIONS_GHC -fno-warn-orphans #-}
3
3
4−module Common (chunksOf) where
4+module Common (
5+ chunksOf
6+ , pathTo
7+ , rechunkBS
8+ , rechunkT
9+ ) where
5
10
6import Control.DeepSeq (NFData(rnf))
11import Control.DeepSeq (NFData(rnf))
12+import System.Directory (doesDirectoryExist)
13+import System.FilePath ((</>))
7import Text.Parsec (ParseError)
14import Text.Parsec (ParseError)
15+import qualified Data.ByteString as B
16+import qualified Data.ByteString.Lazy as BL
17+import qualified Data.Text as T
18+import qualified Data.Text.Lazy as TL
8
19
9#if !MIN_VERSION_bytestring(0,10,0)
20#if !MIN_VERSION_bytestring(0,10,0)
10import Data.ByteString.Internal (ByteString(..))
21import Data.ByteString.Internal (ByteString(..))
@@ -21,3 +32,16 @@ chunksOf k = go
21 where go xs = case splitAt k xs of
32 where go xs = case splitAt k xs of
22 ([],_) -> []
33 ([],_) -> []
23 (y, ys) -> y : go ys
34 (y, ys) -> y : go ys
35+
36+rechunkBS :: Int -> B.ByteString -> BL.ByteString
37+rechunkBS n = BL.fromChunks . map B.pack . chunksOf n . B.unpack
38+
39+rechunkT :: Int -> T.Text -> TL.Text
40+rechunkT n = TL.fromChunks . map T.pack . chunksOf n . T.unpack
41+
42+pathTo :: String -> IO FilePath
43+pathTo wat = do
44+ exists <- doesDirectoryExist "benchmarks"
45+ return $ if exists
46+ then "benchmarks" </> wat
47+ else wat
benchmarks/Genome.hsnew file+75
@@ -0,0 +1,75 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+module Genome
4+ (
5+ genome
6+ ) where
7+
8+import Control.Applicative
9+import Criterion.Main
10+import Data.ByteString (ByteString)
11+import qualified Data.ByteString.Char8 as B8
12+import qualified Data.ByteString.Lazy.Char8 as L8
13+import Data.Attoparsec.ByteString.Char8 as B
14+import qualified Data.Attoparsec.ByteString.Lazy as BL
15+import Data.Text (Text)
16+import qualified Data.Text as T
17+import qualified Data.Text.Lazy as TL
18+import Data.Attoparsec.Text as T
19+import qualified Data.Attoparsec.Text.Lazy as TL
20+import Common (rechunkBS, rechunkT)
21+
22+genome :: Benchmark
23+genome = bgroup "genome" [
24+ bgroup "bytestring" [
25+ bench "s" $ nf (map (B.parse searchBS)) (B8.tails geneB)
26+ , bench "l" $ nf (map (BL.parse searchBS)) (L8.tails geneBL)
27+ , bgroup "CI" [
28+ bench "s" $ nf (map (B.parse searchBSCI)) (B8.tails geneB)
29+ , bench "l" $ nf (map (BL.parse searchBSCI)) (L8.tails geneBL)
30+ ]
31+ ]
32+ , bgroup "text" [
33+ bench "s" $ nf (map (T.parse searchT)) (T.tails geneT)
34+ , bench "l" $ nf (map (TL.parse searchT)) (TL.tails geneTL)
35+ , bgroup "CI" [
36+ bench "s" $ nf (map (T.parse searchTCI)) (T.tails geneT)
37+ , bench "l" $ nf (map (TL.parse searchTCI)) (TL.tails geneTL)
38+ ]
39+ ]
40+ ]
41+ where geneB = B8.pack gene
42+ geneBL = rechunkBS 4 geneB
43+ geneT = T.pack gene
44+ geneTL = rechunkT 4 geneT
45+
46+searchBS :: B.Parser ByteString
47+searchBS = "caac" *> ("aaca" <|> "aact")
48+
49+searchBSCI :: B.Parser ByteString
50+searchBSCI = B.stringCI "CAAC" *> (B.stringCI "AACA" <|> B.stringCI "AACT")
51+
52+searchT :: T.Parser Text
53+searchT = "caac" *> ("aaca" <|> "aact")
54+
55+searchTCI :: T.Parser Text
56+searchTCI = T.asciiCI "CAAC" *> (T.asciiCI "AACA" <|> T.asciiCI "AACT")
57+
58+-- Dictyostelium discoideum developmental protein DG1094 (gacT) gene,
59+-- partial cds. http://www.ncbi.nlm.nih.gov/nuccore/AF081586.1
60+
61+gene :: String
62+gene = "atcgatttagaaagatacaaagatagaaccatcaataataaacaagagaagagagcaagt\
63+ \agagatattaataaagagattgaaagagagattgaaaagaagagattatcaccaagagaa\
64+ \agattaaatttatttggtctttcttcctcatcttcatcagtgaattcaacattaacaaga\
65+ \tctacagcaaatattatctctacaatagacggtagtggaggtagtaatcgtaatagtaaa\
66+ \aattatggtaatggctcatcctcctcctcaaatagaagatatagtaatactattaatcaa\
67+ \caattacaaatgcaattacaacaacttcaaatccaacaacaacaatatcaacaaactcaa\
68+ \caatctcaaataccattacaatatcaacaacaacaacagcaacaacaacaacaaaccact\
69+ \acaactacaactacatcaagtggtagtaatagattctcttcaaatagatataaaccagtt\
70+ \gatcttacacaatcatcttcaaactttcgttattcacgtgaaatttatgatgatgattat\
71+ \tattcaaataataatttaatgatgtttggtaatgagcaaccaaatcaaacaccaatttct\
72+ \gtatcatcttcatctgcattcacacgtcaaagatctcaaagttgctttgaaccagagaat\
73+ \cttgtattgctacaacaacaatatcaacaatatcaacaacaacaacaacaacaacaacaa\
74+ \attccattccaagcaaatccacaatatagtaatgctgttattgaacaaaaattggatcaa\
75+ \attagagataccattaataatttacatagagataaccgagtctctaga"
benchmarks/HeadersByteString.hs+27−37
@@ -1,47 +1,37 @@
1−{-# LANGUAGE OverloadedStrings #-}
2−{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
3module HeadersByteString (headers) where
1module HeadersByteString (headers) where
4
2
5−import Common ()
3+import Common (pathTo, rechunkBS)
6−import Control.Applicative
7−import Criterion.Main (bench, bgroup, nf)
4+import Criterion.Main (bench, bgroup, nf, nfIO)
8import Criterion.Types (Benchmark)
5import Criterion.Types (Benchmark)
6+import HeadersByteString.Atto (request, response)
7+import Network.Wai.Handler.Warp.RequestHeader (parseHeaderLines)
9import qualified Data.Attoparsec.ByteString.Char8 as B
8import qualified Data.Attoparsec.ByteString.Char8 as B
9+import qualified Data.Attoparsec.ByteString.Lazy as BL
10import qualified Data.ByteString.Char8 as B
10import qualified Data.ByteString.Char8 as B
11
11
12−header = do
12+-- Note: In the benchmarks for parsing an http request
13− name <- B.takeWhile1 (B.inClass "a-zA-Z0-9_-") <* B.char ':' <* B.skipSpace
13+-- from a strict bytestring, we consider warp's implementation,
14− body <- (:) <$> bodyLine <*> many (B.takeWhile1 B.isSpace *> bodyLine)
14+-- which has highly optimized code for handling the first
15− return (name, body)
15+-- line in particular. It's treatment of the headers
16−
16+-- is more relaxed from the treatment they are given by the
17−bodyLine = B.takeTill (\c -> c == '\r' || c == '\n') <* B.endOfLine
17+-- attoparsec parser benchmarked here. Consequently, it
18−
18+-- is should not be possible to match its performance since
19−requestLine =
19+-- it accepts header names with disallowed characters.
20− (,,) <$>
21− (method <* B.skipSpace) <*>
22− (B.takeTill B.isSpace <* B.skipSpace) <*>
23− httpVersion
24− where method = "GET" <|> "POST"
25−
26−httpVersion = "HTTP/" *> ((,) <$> (int <* B.char '.') <*> int)
27−
28−responseLine = (,,) <$>
29− (httpVersion <* B.skipSpace) <*>
30− (int <* B.skipSpace) <*>
31− bodyLine
32−
33−int :: B.Parser Int
34−int = B.decimal
35−
36−request = (,) <$> (requestLine <* B.endOfLine) <*> many header
37−
38−response = (,) <$> responseLine <*> many header
39
20
40headers :: IO Benchmark
21headers :: IO Benchmark
41headers = do
22headers = do
42− req <- B.readFile "http-request.txt"
23+ req <- B.readFile =<< pathTo "http-request.txt"
43− resp <- B.readFile "http-response.txt"
24+ resp <- B.readFile =<< pathTo "http-response.txt"
25+ let reql = rechunkBS 4 req
26+ respl = rechunkBS 4 resp
44− return $ bgroup "headersBS" [
27+ return $ bgroup "headers" [
28+ bgroup "B" [
45− bench "request" $ nf (B.parseOnly request) req
29+ bench "request" $ nf (B.parseOnly request) req
30+ , bench "warp" $ nfIO (parseHeaderLines (B.split '\n' req))
46− , bench "response" $ nf (B.parseOnly response) resp
31+ , bench "response" $ nf (B.parseOnly response) resp
32+ ]
33+ , bgroup "BL" [
34+ bench "request" $ nf (BL.parse request) reql
35+ , bench "response" $ nf (BL.parse response) respl
36+ ]
47 ]
37 ]
benchmarks/HeadersByteString/Atto.hsnew file+57
@@ -0,0 +1,57 @@
1+{-# LANGUAGE BangPatterns, OverloadedStrings #-}
2+{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-orphans #-}
3+module HeadersByteString.Atto
4+ (
5+ request
6+ , response
7+ ) where
8+
9+import Control.Applicative
10+import Control.DeepSeq (NFData(..))
11+import Network.HTTP.Types.Version (HttpVersion, http11)
12+import qualified Data.Attoparsec.ByteString.Char8 as B
13+import qualified Data.ByteString.Char8 as B
14+
15+instance NFData HttpVersion where
16+ rnf !_ = ()
17+
18+isHeaderChar :: Char -> Bool
19+isHeaderChar c =
20+ (c >= 'a' && c <= 'z') ||
21+ (c >= 'A' && c <= 'Z') ||
22+ (c >= '0' && c <= '9') ||
23+ (c == '_') ||
24+ (c == '-')
25+
26+header = do
27+ name <- B.takeWhile1 isHeaderChar <* B.char ':' <* B.skipSpace
28+ body <- bodyLine
29+ return (name, body)
30+
31+bodyLine = B.takeTill (\c -> c == '\r' || c == '\n') <* B.endOfLine
32+
33+requestLine = do
34+ m <- (B.takeTill B.isSpace <* B.char ' ')
35+ (p,q) <- B.break (=='?') <$> (B.takeTill B.isSpace <* B.char ' ')
36+ v <- httpVersion
37+ return (m,p,q,v)
38+
39+httpVersion = http11 <$ "HTTP/1.1"
40+
41+responseLine = (,,) <$>
42+ (httpVersion <* B.skipSpace) <*>
43+ (int <* B.skipSpace) <*>
44+ bodyLine
45+
46+int :: B.Parser Int
47+int = B.decimal
48+
49+request = (,) <$> (requestLine <* B.endOfLine) <*> manyheader
50+
51+response = (,) <$> responseLine <*> many header
52+
53+manyheader = do
54+ c <- B.peekChar'
55+ if c == '\r' || c == '\n'
56+ then return []
57+ else (:) <$> header <*> manyheader
benchmarks/HeadersText.hs+15−5
@@ -2,11 +2,13 @@
2{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
2{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
3module HeadersText (headers) where
3module HeadersText (headers) where
4
4
5+import Common (pathTo, rechunkT)
5import Control.Applicative
6import Control.Applicative
6import Criterion.Main (bench, bgroup, nf)
7import Criterion.Main (bench, bgroup, nf)
7import Criterion.Types (Benchmark)
8import Criterion.Types (Benchmark)
8import Data.Char (isSpace)
9import Data.Char (isSpace)
9import qualified Data.Attoparsec.Text as T
10import qualified Data.Attoparsec.Text as T
11+import qualified Data.Attoparsec.Text.Lazy as TL
10import qualified Data.Text.IO as T
12import qualified Data.Text.IO as T
11
13
12header = do
14header = do
@@ -39,9 +41,17 @@ response = (,) <$> responseLine <*> many header
39
41
40headers :: IO Benchmark
42headers :: IO Benchmark
41headers = do
43headers = do
42− req <- T.readFile "http-request.txt"
44+ req <- T.readFile =<< pathTo "http-request.txt"
43− resp <- T.readFile "http-response.txt"
45+ resp <- T.readFile =<< pathTo "http-response.txt"
46+ let reql = rechunkT 4 req
47+ respl = rechunkT 4 resp
44− return $ bgroup "headersT" [
48+ return $ bgroup "headers" [
49+ bgroup "T" [
45− bench "request" $ nf (T.parseOnly request) req
50+ bench "request" $ nf (T.parseOnly request) req
46− , bench "response" $ nf (T.parseOnly response) resp
51+ , bench "response" $ nf (T.parseOnly response) resp
52+ ]
53+ , bgroup "TL" [
54+ bench "request" $ nf (TL.parse request) reql
55+ , bench "response" $ nf (TL.parse response) respl
56+ ]
47 ]
57 ]
benchmarks/Links.hs+2−2
@@ -5,8 +5,8 @@ module Links (links) where
5import Control.Applicative
5import Control.Applicative
6import Control.DeepSeq (NFData(..))
6import Control.DeepSeq (NFData(..))
7import Criterion.Main (Benchmark, bench, nf)
7import Criterion.Main (Benchmark, bench, nf)
8−import Data.Attoparsec as A
8+import Data.Attoparsec.ByteString as A
9−import Data.Attoparsec.Char8 as A8
9+import Data.Attoparsec.ByteString.Char8 as A8
10import Data.ByteString.Char8 as B8
10import Data.ByteString.Char8 as B8
11
11
12data Link = Link {
12data Link = Link {
benchmarks/Numbers.hs+1
@@ -1,4 +1,5 @@
1{-# LANGUAGE BangPatterns #-}
1{-# LANGUAGE BangPatterns #-}
2+{-# OPTIONS_GHC -fno-warn-deprecations #-}
2
3
3module Numbers (numbers) where
4module Numbers (numbers) where
4
5
benchmarks/Sets.hsnew file+21
@@ -0,0 +1,21 @@
1+module Sets (benchmarks) where
2+
3+import Criterion
4+import Data.Char (ord)
5+import qualified Data.Attoparsec.Text.FastSet as FastSet
6+import qualified TextFastSet
7+import qualified Data.HashSet as HashSet
8+import qualified Data.IntSet as IntSet
9+
10+smallSet :: String
11+smallSet = ['a'..'z'] ++ ['A'..'Z'] ++ ['0'..'9']
12+
13+benchmarks :: Benchmark
14+benchmarks = bgroup "sets" [
15+ bench "Fast" $ whnf (FastSet.member '*') (FastSet.fromList smallSet)
16+ , bench "Hash" $ whnf (HashSet.member '*') (HashSet.fromList smallSet)
17+ , bench "Int" $ whnf (IntSet.member (ord '*'))
18+ (IntSet.fromList (map ord smallSet))
19+ , bench "TextFast" $ whnf (TextFastSet.member '*')
20+ (TextFastSet.fromList smallSet)
21+ ]
benchmarks/TextFastSet.hsnew file+121
@@ -0,0 +1,121 @@
1+{-# LANGUAGE BangPatterns #-}
2+
3+------------------------------------------------------------------------------
4+-- |
5+-- Module : Data.Attoparsec.FastSet
6+-- Copyright : Felipe Lessa 2010, Bryan O'Sullivan 2007-2015
7+-- License : BSD3
8+--
9+-- Maintainer : felipe.lessa@gmail.com
10+-- Stability : experimental
11+-- Portability : unknown
12+--
13+-- Fast set membership tests for 'Char' values. We test for
14+-- membership using a hashtable implemented with Robin Hood
15+-- collision resolution. The set representation is unboxed,
16+-- and the characters and hashes interleaved, for efficiency.
17+--
18+--
19+-----------------------------------------------------------------------------
20+module TextFastSet
21+ (
22+ -- * Data type
23+ FastSet
24+ -- * Construction
25+ , fromList
26+ , set
27+ -- * Lookup
28+ , member
29+ -- * Handy interface
30+ , charClass
31+ ) where
32+
33+import Data.Bits ((.|.), (.&.), shiftR)
34+import Data.Function (on)
35+import Data.List (sort, sortBy)
36+import qualified Data.Array.Base as AB
37+import qualified Data.Array.Unboxed as A
38+import qualified Data.Text as T
39+
40+data FastSet = FastSet {
41+ table :: {-# UNPACK #-} !(A.UArray Int Int)
42+ , mask :: {-# UNPACK #-} !Int
43+ }
44+
45+data Entry = Entry {
46+ key :: {-# UNPACK #-} !Char
47+ , initialIndex :: {-# UNPACK #-} !Int
48+ , index :: {-# UNPACK #-} !Int
49+ }
50+
51+offset :: Entry -> Int
52+offset e = index e - initialIndex e
53+
54+resolveCollisions :: [Entry] -> [Entry]
55+resolveCollisions [] = []
56+resolveCollisions [e] = [e]
57+resolveCollisions (a:b:entries) = a' : resolveCollisions (b' : entries)
58+ where (a', b')
59+ | index a < index b = (a, b)
60+ | offset a < offset b = (b { index=index a }, a { index=index a + 1 })
61+ | otherwise = (a, b { index=index a + 1 })
62+
63+pad :: Int -> [Entry] -> [Entry]
64+pad = go 0
65+ where -- ensure that we pad enough so that lookups beyond the
66+ -- last hash in the table fall within the array
67+ go !_ !m [] = replicate (max 1 m + 1) empty
68+ go k m (e:entries) = map (const empty) [k..i - 1] ++ e :
69+ go (i + 1) (m + i - k - 1) entries
70+ where i = index e
71+ empty = Entry '\0' maxBound 0
72+
73+nextPowerOf2 :: Int -> Int
74+nextPowerOf2 0 = 1
75+nextPowerOf2 x = go (x - 1) 1
76+ where go y 32 = y + 1
77+ go y k = go (y .|. (y `shiftR` k)) $ k * 2
78+
79+fastHash :: Char -> Int
80+fastHash = fromEnum
81+
82+fromList :: String -> FastSet
83+fromList s = FastSet (AB.listArray (0, length interleaved - 1) interleaved)
84+ mask'
85+ where s' = ordNub (sort s)
86+ l = length s'
87+ mask' = nextPowerOf2 ((5 * l) `div` 4) - 1
88+ entries = pad mask' .
89+ resolveCollisions .
90+ sortBy (compare `on` initialIndex) .
91+ zipWith (\c i -> Entry c i i) s' .
92+ map ((.&. mask') . fastHash) $ s'
93+ interleaved = concatMap (\e -> [fromEnum $ key e, initialIndex e])
94+ entries
95+
96+ordNub :: Eq a => [a] -> [a]
97+ordNub [] = []
98+ordNub (y:ys) = go y ys
99+ where go x (z:zs)
100+ | x == z = go x zs
101+ | otherwise = x : go z zs
102+ go x [] = [x]
103+
104+set :: T.Text -> FastSet
105+set = fromList . T.unpack
106+
107+-- | Check the set for membership.
108+member :: Char -> FastSet -> Bool
109+member c a = go (2 * i)
110+ where i = fastHash c .&. mask a
111+ lookupAt j b = (i' <= i) && (c == c' || b)
112+ where c' = toEnum $ AB.unsafeAt (table a) j
113+ i' = AB.unsafeAt (table a) $ j + 1
114+ go j = lookupAt j . lookupAt (j + 2) . lookupAt (j + 4) .
115+ lookupAt (j + 6) . go $ j + 8
116+
117+charClass :: String -> FastSet
118+charClass = fromList . go
119+ where go (a:'-':b:xs) = [a..b] ++ go xs
120+ go (x:xs) = x : go xs
121+ go _ = ""
benchmarks/Warp.hsnew file+17
@@ -0,0 +1,17 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+module Warp (benchmarks) where
4+
5+import Criterion.Main (bench, bgroup, nf)
6+import Criterion.Types (Benchmark)
7+import Data.ByteString (ByteString)
8+import Network.Wai.Handler.Warp.ReadInt (readInt)
9+import qualified Data.Attoparsec.ByteString.Char8 as B
10+
11+benchmarks :: Benchmark
12+benchmarks = bgroup "warp" [
13+ bgroup "decimal" [
14+ bench "warp" $ nf (readInt :: ByteString -> Int) "31337"
15+ , bench "atto" $ nf (B.parse (B.decimal :: B.Parser Int)) "31337"
16+ ]
17+ ]
benchmarks/attoparsec-benchmarks.cabal+14−5
@@ -11,22 +11,31 @@ executable attoparsec-benchmarks
11 other-modules:
11 other-modules:
12 Common
12 Common
13 HeadersByteString
13 HeadersByteString
14+ HeadersByteString.Atto
14 HeadersText
15 HeadersText
15 Links
16 Links
16 Numbers
17 Numbers
18+ Network.Wai.Handler.Warp.ReadInt
19+ Sets
20+ TextFastSet
21+ Warp
17− hs-source-dirs: .. .
22+ hs-source-dirs: .. . warp-3.0.1.1
18− ghc-options: -O2 -Wall
23+ ghc-options: -O2 -Wall -rtsopts
19 build-depends:
24 build-depends:
20− aeson,
21 array,
25 array,
22 base == 4.*,
26 base == 4.*,
23 bytestring >= 0.10.4.0,
27 bytestring >= 0.10.4.0,
28+ case-insensitive,
29+ containers,
24− criterion >= 0.8.1.0,
30+ criterion >= 1.0,
25 deepseq >= 1.1,
31 deepseq >= 1.1,
26 directory,
32 directory,
27 filepath,
33 filepath,
34+ ghc-prim,
35+ http-types,
28 parsec >= 3.1.2,
36 parsec >= 3.1.2,
29− scientific,
37+ scientific >= 0.3.1,
30 text >= 1.1.1.0,
38 text >= 1.1.1.0,
39+ transformers,
31 unordered-containers,
40 unordered-containers,
32 vector
41 vector
benchmarks/http-request.txt+3−8
@@ -1,14 +1,9 @@
1GET / HTTP/1.1
1GET / HTTP/1.1
2Host: twitter.com
2Host: twitter.com
3−Accept: text/html, application/xhtml+xml, application/xml; q=0.9,
3+Accept: text/html, application/xhtml+xml, application/xml; q=0.9, image/webp, */*; q=0.8
4− image/webp, */*; q=0.8
5Accept-Encoding: gzip,deflate,sdch
4Accept-Encoding: gzip,deflate,sdch
6Accept-Language: en-GB,en-US;q=0.8,en;q=0.6
5Accept-Language: en-GB,en-US;q=0.8,en;q=0.6
7Cache-Control: max-age=0
6Cache-Control: max-age=0
8−Cookie: guest_id=v1%3A139; _twitter_sess=BAh7CSIKZmxhc2hJQz-e1e1;
7+Cookie: guest_id=v1%3A139; _twitter_sess=BAh7CSIKZmxhc2hJQz-e1e1; __utma=43838368.452555194.1399611824.1; __utmb=43838368; __utmc=43838368; __utmz=1399611824.1.1.utmcsr=(direct)|utmcmd=(none)
9− __utma=43838368.452555194.1399611824.1; __utmb=43838368;
10− __utmc=43838368; __utmz=1399611824.1.1.utmcsr=(direct)|utmcmd=(none)
11−User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5)
12− AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.86
8+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.86 Safari/537.36
13− Safari/537.36
14
9
benchmarks/http-response.txt+2−4
@@ -3,10 +3,8 @@ Date: Fri, 09 May 2014 04:24:57 GMT
3Expires: -1
3Expires: -1
4Cache-Control: private, max-age=0
4Cache-Control: private, max-age=0
5Content-Type: text/html; charset=ISO-8859-1
5Content-Type: text/html; charset=ISO-8859-1
6−Set-Cookie: PREF=ID=1a871afc4240db5e:FF:LM=1399609497:S=e5MZ0itEekjVwNcU;
6+Set-Cookie: PREF=ID=1a871afc4240db5e:FF:LM=1399609497:S=e5MZ0itEekjVwNcU; expires=Sun, 08-May-2016 04:24:57 GMT; path=/; domain=.google.com
7− expires=Sun, 08-May-2016 04:24:57 GMT; path=/; domain=.google.com
8−Set-Cookie: NID=67=-WlfaDG6VSEPk7abAjrK98HBSoCD2ID6JKkUR95tEumzDmg7Fc8pSQ8;
7+Set-Cookie: NID=67=-WlfaDG6VSEPk7abAjrK98HBSoCD2ID6JKkUR95tEumzDmg7Fc8pSQ8; expires=Sat, 08-Nov-2014 04:24:57 GMT; path=/; domain=.google.com; HttpOnly
9− expires=Sat, 08-Nov-2014 04:24:57 GMT; path=/; domain=.google.com; HttpOnly
10P3P: CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info."
8P3P: CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info."
11Server: gws
9Server: gws
12X-XSS-Protection: 1; mode=block
10X-XSS-Protection: 1; mode=block
benchmarks/warp-3.0.1.1/LICENSEnew file+20
@@ -0,0 +1,20 @@
1+Copyright (c) 2012 Michael Snoyman, http://www.yesodweb.com/
2+
3+Permission is hereby granted, free of charge, to any person obtaining
4+a copy of this software and associated documentation files (the
5+"Software"), to deal in the Software without restriction, including
6+without limitation the rights to use, copy, modify, merge, publish,
7+distribute, sublicense, and/or sell copies of the Software, and to
8+permit persons to whom the Software is furnished to do so, subject to
9+the following conditions:
10+
11+The above copyright notice and this permission notice shall be
12+included in all copies or substantial portions of the Software.
13+
14+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
benchmarks/warp-3.0.1.1/Network/Wai/Handler/Warp/ReadInt.hsnew file+67
@@ -0,0 +1,67 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+{-# LANGUAGE MagicHash #-}
3+{-# LANGUAGE BangPatterns #-}
4+
5+-- Copyright : Erik de Castro Lopo <erikd@mega-nerd.com>
6+-- License : BSD3
7+
8+module Network.Wai.Handler.Warp.ReadInt (
9+ readInt
10+ , readInt64
11+ ) where
12+
13+-- This function lives in its own file because the MagicHash pragma interacts
14+-- poorly with the CPP pragma.
15+
16+import Data.ByteString (ByteString)
17+import qualified Data.ByteString as S
18+import Data.Int (Int64)
19+import GHC.Prim
20+import GHC.Types
21+import GHC.Word
22+
23+{-# INLINE readInt #-}
24+readInt :: Integral a => ByteString -> a
25+readInt bs = fromIntegral $ readInt64 bs
26+
27+-- This function is used to parse the Content-Length field of HTTP headers and
28+-- is a performance hot spot. It should only be replaced with something
29+-- significantly and provably faster.
30+--
31+-- It needs to be able work correctly on 32 bit CPUs for file sizes > 2G so we
32+-- use Int64 here and then make a generic 'readInt' that allows conversion to
33+-- Int and Integer.
34+
35+{-# NOINLINE readInt64 #-}
36+readInt64 :: ByteString -> Int64
37+readInt64 bs = S.foldl' (\ !i !c -> i * 10 + fromIntegral (mhDigitToInt c)) 0
38+ $ S.takeWhile isDigit bs
39+
40+data Table = Table !Addr#
41+
42+{-# NOINLINE mhDigitToInt #-}
43+mhDigitToInt :: Word8 -> Int
44+mhDigitToInt (W8# i) = I# (word2Int# (indexWord8OffAddr# addr (word2Int# i)))
45+ where
46+ !(Table addr) = table
47+ table :: Table
48+ table = Table
49+ "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
50+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
51+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
52+ \\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x00\x00\x00\x00\x00\x00\
53+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
54+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
55+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
56+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
57+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
58+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
59+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
60+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
61+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
62+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
63+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
64+ \\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"#
65+
66+isDigit :: Word8 -> Bool
67+isDigit w = w >= 48 && w <= 57
benchmarks/warp-3.0.1.1/Network/Wai/Handler/Warp/RequestHeader.hsnew file+172
@@ -0,0 +1,172 @@
1+{-# LANGUAGE DeriveDataTypeable #-}
2+{-# LANGUAGE OverloadedStrings #-}
3+{-# LANGUAGE BangPatterns #-}
4+
5+module Network.Wai.Handler.Warp.RequestHeader (
6+ parseHeaderLines
7+ , parseByteRanges
8+ ) where
9+
10+import Control.Exception (Exception, throwIO)
11+import Control.Monad (when)
12+import Data.Typeable (Typeable)
13+import qualified Data.ByteString as S
14+import qualified Data.ByteString.Char8 as B (unpack, readInteger)
15+import Data.ByteString.Internal (ByteString(..), memchr)
16+import qualified Data.CaseInsensitive as CI
17+import Data.Word (Word8)
18+import Foreign.ForeignPtr (withForeignPtr)
19+import Foreign.Ptr (Ptr, plusPtr, minusPtr, nullPtr)
20+import Foreign.Storable (peek)
21+import qualified Network.HTTP.Types as H
22+-- import Network.Wai.Handler.Warp.Types
23+import qualified Network.HTTP.Types.Header as HH
24+-- $setup
25+-- >>> :set -XOverloadedStrings
26+
27+data InvalidRequest = NotEnoughLines [String]
28+ | BadFirstLine String
29+ | NonHttp
30+ | IncompleteHeaders
31+ | ConnectionClosedByPeer
32+ | OverLargeHeader
33+ deriving (Eq, Typeable, Show)
34+
35+instance Exception InvalidRequest
36+
37+----------------------------------------------------------------
38+
39+parseHeaderLines :: [ByteString]
40+ -> IO (H.Method
41+ ,ByteString -- Path
42+ ,ByteString -- Path, parsed
43+ ,ByteString -- Query
44+ ,H.HttpVersion
45+ ,H.RequestHeaders
46+ )
47+parseHeaderLines [] = throwIO $ NotEnoughLines []
48+parseHeaderLines (firstLine:otherLines) = do
49+ (method, path', query, httpversion) <- parseRequestLine firstLine
50+ let path = H.extractPath path'
51+ hdr = map parseHeader otherLines
52+ return (method, path', path, query, httpversion, hdr)
53+
54+----------------------------------------------------------------
55+
56+-- |
57+--
58+-- >>> parseRequestLine "GET / HTTP/1.1"
59+-- ("GET","/","",HTTP/1.1)
60+-- >>> parseRequestLine "POST /cgi/search.cgi?key=foo HTTP/1.0"
61+-- ("POST","/cgi/search.cgi","?key=foo",HTTP/1.0)
62+-- >>> parseRequestLine "GET "
63+-- *** Exception: Warp: Invalid first line of request: "GET "
64+-- >>> parseRequestLine "GET /NotHTTP UNKNOWN/1.1"
65+-- *** Exception: Warp: Request line specified a non-HTTP request
66+parseRequestLine :: ByteString
67+ -> IO (H.Method
68+ ,ByteString -- Path
69+ ,ByteString -- Query
70+ ,H.HttpVersion)
71+parseRequestLine requestLine@(PS fptr off len) = withForeignPtr fptr $ \ptr -> do
72+ when (len < 14) $ throwIO baderr
73+ let methodptr = ptr `plusPtr` off
74+ limptr = methodptr `plusPtr` len
75+ lim0 = fromIntegral len
76+
77+ pathptr0 <- memchr methodptr 32 lim0 -- ' '
78+ when (pathptr0 == nullPtr || (limptr `minusPtr` pathptr0) < 11) $
79+ throwIO baderr
80+ let pathptr = pathptr0 `plusPtr` 1
81+ lim1 = fromIntegral (limptr `minusPtr` pathptr0)
82+
83+ httpptr0 <- memchr pathptr 32 lim1 -- ' '
84+ when (httpptr0 == nullPtr || (limptr `minusPtr` httpptr0) < 9) $
85+ throwIO baderr
86+ let httpptr = httpptr0 `plusPtr` 1
87+ lim2 = fromIntegral (httpptr0 `minusPtr` pathptr)
88+
89+ checkHTTP httpptr
90+ !hv <- httpVersion httpptr
91+ queryptr <- memchr pathptr 63 lim2 -- '?'
92+
93+ let !method = bs ptr methodptr pathptr0
94+ !path
95+ | queryptr == nullPtr = bs ptr pathptr httpptr0
96+ | otherwise = bs ptr pathptr queryptr
97+ !query
98+ | queryptr == nullPtr = S.empty
99+ | otherwise = bs ptr queryptr httpptr0
100+
101+ return (method,path,query,hv)
102+ where
103+ baderr = BadFirstLine $ B.unpack requestLine
104+ check :: Ptr Word8 -> Int -> Word8 -> IO ()
105+ check p n w = do
106+ w0 <- peek $ p `plusPtr` n
107+ when (w0 /= w) $ throwIO NonHttp
108+ checkHTTP httpptr = do
109+ check httpptr 0 72 -- 'H'
110+ check httpptr 1 84 -- 'T'
111+ check httpptr 2 84 -- 'T'
112+ check httpptr 3 80 -- 'P'
113+ check httpptr 4 47 -- '/'
114+ check httpptr 6 46 -- '.'
115+ httpVersion httpptr = do
116+ major <- peek $ httpptr `plusPtr` 5
117+ minor <- peek $ httpptr `plusPtr` 7
118+ return $ if major == (49 :: Word8) && minor == (49 :: Word8) then
119+ H.http11
120+ else
121+ H.http10
122+ bs ptr p0 p1 = PS fptr o l
123+ where
124+ o = p0 `minusPtr` ptr
125+ l = p1 `minusPtr` p0
126+
127+----------------------------------------------------------------
128+
129+-- |
130+--
131+-- >>> parseHeader "Content-Length:47"
132+-- ("Content-Length","47")
133+-- >>> parseHeader "Accept-Ranges: bytes"
134+-- ("Accept-Ranges","bytes")
135+-- >>> parseHeader "Host: example.com:8080"
136+-- ("Host","example.com:8080")
137+-- >>> parseHeader "NoSemiColon"
138+-- ("NoSemiColon","")
139+
140+parseHeader :: ByteString -> H.Header
141+parseHeader s =
142+ let (k, rest) = S.break (== 58) s -- ':'
143+ rest' = S.dropWhile (\c -> c == 32 || c == 9) $ S.drop 1 rest
144+ in (CI.mk k, rest')
145+
146+parseByteRanges :: S.ByteString -> Maybe HH.ByteRanges
147+parseByteRanges bs1 = do
148+ bs2 <- stripPrefix "bytes=" bs1
149+ (r, bs3) <- range bs2
150+ ranges (r:) bs3
151+ where
152+ range bs2 =
153+ case stripPrefix "-" bs2 of
154+ Just bs3 -> do
155+ (i, bs4) <- B.readInteger bs3
156+ Just (HH.ByteRangeSuffix i, bs4)
157+ Nothing -> do
158+ (i, bs3) <- B.readInteger bs2
159+ bs4 <- stripPrefix "-" bs3
160+ case B.readInteger bs4 of
161+ Nothing -> Just (HH.ByteRangeFrom i, bs4)
162+ Just (j, bs5) -> Just (HH.ByteRangeFromTo i j, bs5)
163+ ranges front bs3 =
164+ case stripPrefix "," bs3 of
165+ Nothing -> Just (front [])
166+ Just bs4 -> do
167+ (r, bs5) <- range bs4
168+ ranges (front . (r:)) bs5
169+
170+ stripPrefix x y
171+ | x `S.isPrefixOf` y = Just (S.drop (S.length x) y)
172+ | otherwise = Nothing
changelog.md+93
@@ -1,3 +1,96 @@
1+0.13.2.1
2+
3+* Improved performance of `Data.Attoparsec.Text.asciiCI`
4+
5+0.13.2.0
6+
7+* `pure` is now strict in `Position`
8+
9+0.13.1.0
10+
11+* `runScanner` now correctly returns the final state
12+ (https://github.com/bos/attoparsec/issues/105).
13+* `Parser`, `ZeptoT`, `Buffer`, and `More` now expose `Semigroup` instances.
14+* `Parser`, and `ZeptoT` now expose `MonadFail` instances.
15+
16+0.13.0.2
17+
18+* Restore the fast specialised character set implementation for Text
19+* Move testsuite from test-framework to tasty
20+* Performance optimization of takeWhile and takeWhile1
21+
22+0.13.0.1
23+
24+* Fixed a bug in the implementations of inClass and notInClass for
25+ Text (https://github.com/bos/attoparsec/issues/103)
26+
27+0.13.0.0
28+
29+* Made the parser type in the Zepto module a monad transformer
30+ (needed by aeson's string unescaping parser).
31+
32+0.12.1.6
33+
34+* Fixed a case folding bug in the ByteString version of stringCI.
35+
36+0.12.1.5
37+
38+* Fixed an indexing bug in the new Text implementation of string,
39+ reported by Michel Boucey.
40+
41+0.12.1.4
42+
43+* Fixed a case where the string parser would consume an unnecessary
44+ amount of input before failing a match, when it could bail much
45+ earlier (https://github.com/bos/attoparsec/issues/97)
46+
47+* Added more context to error messages
48+ (https://github.com/bos/attoparsec/pull/79)
49+
50+0.12.1.3
51+
52+* Fixed incorrect tracking of Text lengths
53+ (https://github.com/bos/attoparsec/issues/80)
54+
55+0.12.1.2
56+
57+* Fixed the incorrect tracking of capacity if the initial buffer was
58+ empty (https://github.com/bos/attoparsec/issues/75)
59+
60+0.12.1.1
61+
62+* Fixed a data corruption bug that occurred under some circumstances
63+ if a buffer grew after prompting for more input
64+ (https://github.com/bos/attoparsec/issues/74)
65+
66+0.12.1.0
67+
68+* Now compatible with GHC 7.9
69+
70+* Reintroduced the Chunk class, used by the parsers package
71+
72+0.12.0.0
73+
74+* A new internal representation makes almost all real-world parsers
75+ faster, sometimes by big margins. For example, parsing JSON data
76+ with aeson is now up to 70% faster. These performance improvements
77+ also come with reduced memory consumption and some new capabilities.
78+
79+* The new match combinator gives both the result of a parse and the
80+ input that it matched.
81+
82+* The test suite has doubled in size. This made it possible to switch
83+ to the new internal representation with a decent degree of
84+ confidence that everything was more or less working.
85+
86+* The benchmark suite now contains a small family of benchmarks taken
87+ from real-world uses of attoparsec.
88+
89+* A few types that ought to have been private now are.
90+
91+* A few obsolete modules and functions have been marked as deprecated.
92+ They will be removed from the next major release.
93+
10.11.3.0
940.11.3.0
2
95
3* New function scientific is compatible with rational, but parses
96* New function scientific is compatible with rational, but parses
examples/TestRFC2616.hs → examples/Atto_RFC2616.hsrenamed+16−8
@@ -1,21 +1,28 @@
1{-# LANGUAGE BangPatterns #-}
1{-# LANGUAGE BangPatterns #-}
2−import RFC2616
2+
3−import Control.Monad (forM_)
3+module Main (main) where
4−import System.IO
4+
5+import Control.Applicative
5import Control.Exception (bracket)
6import Control.Exception (bracket)
7+import Control.Monad (forM_)
8+import Data.Attoparsec.ByteString
9+import RFC2616
6import System.Environment
10import System.Environment
11+import System.IO
7import qualified Data.ByteString.Char8 as B
12import qualified Data.ByteString.Char8 as B
8−import Data.Attoparsec
9
13
14+refill :: Handle -> IO B.ByteString
10−refill h = B.hGet h (4*1024)
15+refill h = B.hGet h (80*1024)
11
16
17+listy :: FilePath -> Handle -> IO ()
12listy file h = do
18listy file h = do
13 r <- parseWith (refill h) (many request) =<< refill h
19 r <- parseWith (refill h) (many request) =<< refill h
14 case r of
20 case r of
15 Fail _ _ msg -> hPutStrLn stderr $ file ++ ": " ++ msg
21 Fail _ _ msg -> hPutStrLn stderr $ file ++ ": " ++ msg
16 Done _ reqs -> print (length reqs)
22 Done _ reqs -> print (length reqs)
17−
23+
24+incrementy :: FilePath -> Handle -> IO ()
18−incrementy file h = go 0 =<< refill h
25+incrementy file h = go (0::Int) =<< refill h
19 where
26 where
20 go !n is = do
27 go !n is = do
21 r <- parseWith (refill h) request is
28 r <- parseWith (refill h) request is
@@ -28,7 +35,8 @@ incrementy file h = go 0 =<< refill h
28 then print (n+1)
35 then print (n+1)
29 else go (n+1) s
36 else go (n+1) s
30 | otherwise -> go (n+1) bs
37 | otherwise -> go (n+1) bs
31−
38+
39+main :: IO ()
32main = do
40main = do
33 args <- getArgs
41 args <- getArgs
34 forM_ args $ \arg ->
42 forM_ args $ \arg ->
examples/Makefile+6−2
@@ -1,4 +1,4 @@
1−CC := gcc
1+CC := cc
2CFLAGS := -O3 -Wall
2CFLAGS := -O3 -Wall
3# To make the code about 6% faster:
3# To make the code about 6% faster:
4# CFLAGS += -fwhole-program --combine
4# CFLAGS += -fwhole-program --combine
@@ -12,4 +12,8 @@ c-http: rfc2616.c http_parser.c
12clean:
12clean:
13 rm -f *.hi *.o c-http
13 rm -f *.hi *.o c-http
14
14
15−vpath %.c $(HOME)/hg/http-parser
15+http_parser.c: http_parser.h
16+ curl -O https://raw.githubusercontent.com/joyent/http-parser/master/http_parser.c
17+
18+http_parser.h:
19+ curl -O https://raw.githubusercontent.com/joyent/http-parser/master/http_parser.h
examples/Parsec_RFC2616.hs+9−6
@@ -23,15 +23,16 @@ token :: Stream s m Char => ParsecT s u m Char
23token = satisfy $ \c -> S.notMember (fromEnum c) set
23token = satisfy $ \c -> S.notMember (fromEnum c) set
24 where set = S.fromList . map fromEnum $ ['\0'..'\31'] ++ "()<>@,;:\\\"/[]?={} \t" ++ ['\128'..'\255']
24 where set = S.fromList . map fromEnum $ ['\0'..'\31'] ++ "()<>@,;:\\\"/[]?={} \t" ++ ['\128'..'\255']
25
25
26+isHorizontalSpace :: Char -> Bool
26isHorizontalSpace c = c == ' ' || c == '\t'
27isHorizontalSpace c = c == ' ' || c == '\t'
27
28
28skipHSpaces :: Stream s m Char => ParsecT s u m ()
29skipHSpaces :: Stream s m Char => ParsecT s u m ()
29skipHSpaces = skipMany1 (satisfy isHorizontalSpace)
30skipHSpaces = skipMany1 (satisfy isHorizontalSpace)
30
31
31data Request = Request {
32data Request = Request {
32− requestMethod :: String
33+ _requestMethod :: String
33− , requestUri :: String
34+ , _requestUri :: String
34− , requestProtocol :: String
35+ , _requestProtocol :: String
35 } deriving (Eq, Ord, Show)
36 } deriving (Eq, Ord, Show)
36
37
37requestLine :: Stream s m Char => ParsecT s u m Request
38requestLine :: Stream s m Char => ParsecT s u m Request
@@ -47,8 +48,8 @@ endOfLine :: Stream s m Char => ParsecT s u m ()
47endOfLine = (string "\r\n" *> pure ()) <|> (char '\n' *> pure ())
48endOfLine = (string "\r\n" *> pure ()) <|> (char '\n' *> pure ())
48
49
49data Header = Header {
50data Header = Header {
50− headerName :: String
51+ _headerName :: String
51− , headerValue :: [String]
52+ , _headerValue :: [String]
52 } deriving (Eq, Ord, Show)
53 } deriving (Eq, Ord, Show)
53
54
54messageHeader :: Stream s m Char => ParsecT s u m Header
55messageHeader :: Stream s m Char => ParsecT s u m Header
@@ -61,14 +62,16 @@ messageHeader = do
61request :: Stream s m Char => ParsecT s u m (Request, [Header])
62request :: Stream s m Char => ParsecT s u m (Request, [Header])
62request = (,) <$> requestLine <*> many messageHeader <* endOfLine
63request = (,) <$> requestLine <*> many messageHeader <* endOfLine
63
64
65+listy :: FilePath -> IO ()
64listy arg = do
66listy arg = do
65 r <- parseFromFile (many request) arg
67 r <- parseFromFile (many request) arg
66 case r of
68 case r of
67 Left err -> putStrLn $ arg ++ ": " ++ show err
69 Left err -> putStrLn $ arg ++ ": " ++ show err
68 Right rs -> print (length rs)
70 Right rs -> print (length rs)
69
71
72+chunky :: FilePath -> IO ()
70chunky arg = bracket (openFile arg ReadMode) hClose $ \h ->
73chunky arg = bracket (openFile arg ReadMode) hClose $ \h ->
71− loop 0 =<< B.hGetContents h
74+ loop (0::Int) =<< B.hGetContents h
72 where
75 where
73 loop !n bs
76 loop !n bs
74 | B.null bs = print n
77 | B.null bs = print n
examples/RFC2616.hs+24−49
@@ -5,86 +5,61 @@ module RFC2616
5 Header(..)
5 Header(..)
6 , Request(..)
6 , Request(..)
7 , Response(..)
7 , Response(..)
8− , isToken
9− , messageHeader
10 , request
8 , request
11− , requestLine
12 , response
9 , response
13− , responseLine
14− , lowerHeader
15− , lookupHeader
16 ) where
10 ) where
17
11
18import Control.Applicative
12import Control.Applicative
19−import Data.Attoparsec as P
13+import Data.Attoparsec.ByteString as P
20−import qualified Data.Attoparsec.Char8 as P8
21−import Data.Attoparsec.Char8 (char8, endOfLine, isDigit_w8)
14+import Data.Attoparsec.ByteString.Char8 (char8, endOfLine, isDigit_w8)
15+import Data.ByteString (ByteString)
22import Data.Word (Word8)
16import Data.Word (Word8)
23−import qualified Data.ByteString.Char8 as B hiding (map)
17+import Data.Attoparsec.ByteString.Char8 (isEndOfLine, isHorizontalSpace)
24−import qualified Data.ByteString as B (map)
25
18
26isToken :: Word8 -> Bool
19isToken :: Word8 -> Bool
27isToken w = w <= 127 && notInClass "\0-\31()<>@,;:\\\"/[]?={} \t" w
20isToken w = w <= 127 && notInClass "\0-\31()<>@,;:\\\"/[]?={} \t" w
28
21
29skipSpaces :: Parser ()
22skipSpaces :: Parser ()
30−skipSpaces = satisfy P8.isHorizontalSpace *> skipWhile P8.isHorizontalSpace
23+skipSpaces = satisfy isHorizontalSpace *> skipWhile isHorizontalSpace
31
24
32data Request = Request {
25data Request = Request {
33− requestMethod :: !B.ByteString
26+ requestMethod :: ByteString
34− , requestUri :: !B.ByteString
27+ , requestUri :: ByteString
35− , requestVersion :: !B.ByteString
28+ , requestVersion :: ByteString
36 } deriving (Eq, Ord, Show)
29 } deriving (Eq, Ord, Show)
37
30
38−httpVersion :: Parser B.ByteString
31+httpVersion :: Parser ByteString
39httpVersion = "HTTP/" *> P.takeWhile (\c -> isDigit_w8 c || c == 46)
32httpVersion = "HTTP/" *> P.takeWhile (\c -> isDigit_w8 c || c == 46)
40
33
41requestLine :: Parser Request
34requestLine :: Parser Request
42−requestLine = do
43− method <- P.takeWhile1 isToken <* char8 ' '
35+requestLine = Request <$> (takeWhile1 isToken <* char8 ' ')
44− uri <- P.takeWhile1 (/=32) <* char8 ' '
36+ <*> (takeWhile1 (/=32) <* char8 ' ')
45− version <- httpVersion <* endOfLine
37+ <*> (httpVersion <* endOfLine)
46− return $! Request method uri version
47
38
48data Header = Header {
39data Header = Header {
49− headerName :: !B.ByteString
40+ headerName :: ByteString
50− , headerValue :: [B.ByteString]
41+ , headerValue :: [ByteString]
51 } deriving (Eq, Ord, Show)
42 } deriving (Eq, Ord, Show)
52
43
53messageHeader :: Parser Header
44messageHeader :: Parser Header
54−messageHeader = do
45+messageHeader = Header
55− header <- P.takeWhile isToken <* char8 ':' <* skipWhile P8.isHorizontalSpace
46+ <$> (P.takeWhile isToken <* char8 ':' <* skipWhile isHorizontalSpace)
56− body <- takeTill P8.isEndOfLine <* endOfLine
47+ <*> ((:) <$> (takeTill isEndOfLine <* endOfLine)
57− bodies <- many $ skipSpaces *> takeTill P8.isEndOfLine <* endOfLine
48+ <*> (many $ skipSpaces *> takeTill isEndOfLine <* endOfLine))
58− return $! Header header (body:bodies)
59
49
60request :: Parser (Request, [Header])
50request :: Parser (Request, [Header])
61request = (,) <$> requestLine <*> many messageHeader <* endOfLine
51request = (,) <$> requestLine <*> many messageHeader <* endOfLine
62
52
63data Response = Response {
53data Response = Response {
64− responseVersion :: !B.ByteString
54+ responseVersion :: ByteString
65− , responseCode :: !B.ByteString
55+ , responseCode :: ByteString
66− , responseMsg :: !B.ByteString
56+ , responseMsg :: ByteString
67 } deriving (Eq, Ord, Show)
57 } deriving (Eq, Ord, Show)
68
58
69responseLine :: Parser Response
59responseLine :: Parser Response
70−responseLine = do
71− version <- httpVersion <* char8 ' '
60+responseLine = Response <$> (httpVersion <* char8 ' ')
72− code <- P.takeWhile isDigit_w8 <* char8 ' '
61+ <*> (P.takeWhile isDigit_w8 <* char8 ' ')
73− msg <- P.takeTill P8.isEndOfLine <* endOfLine
62+ <*> (takeTill isEndOfLine <* endOfLine)
74− return $! Response version code msg
75
63
76response :: Parser (Response, [Header])
64response :: Parser (Response, [Header])
77response = (,) <$> responseLine <*> many messageHeader <* endOfLine
65response = (,) <$> responseLine <*> many messageHeader <* endOfLine
78−
79−lowerHeader :: Header -> Header
80−lowerHeader (Header n v) = Header (B.map toLower n) (map (B.map toLower) v)
81− where toLower w | w >= 65 && w <= 90 = w + 32
82− | otherwise = w
83−
84−lookupHeader :: B.ByteString -> [Header] -> [B.ByteString]
85−lookupHeader k = go
86− where
87− go (Header n v:hs)
88− | k == n = v
89− | otherwise = go hs
90− go _ = []
examples/attoparsec-examples.cabalnew file+31
@@ -0,0 +1,31 @@
1+-- These examples are not intended to be installed.
2+-- So don't install 'em.
3+
4+name: attoparsec-examples
5+version: 0
6+cabal-version: >=1.6
7+build-type: Simple
8+
9+executable atto-rfc2616
10+ main-is: Atto_RFC2616.hs
11+ hs-source-dirs: .. .
12+ ghc-options: -O2 -Wall -rtsopts
13+ build-depends:
14+ array,
15+ base == 4.*,
16+ bytestring >= 0.10.4.0,
17+ deepseq >= 1.1,
18+ scientific >= 0.3.1,
19+ text >= 1.1.1.0
20+
21+executable parsec-rfc2616
22+ main-is: Parsec_RFC2616.hs
23+ hs-source-dirs: .
24+ ghc-options: -O2 -Wall -rtsopts
25+ build-depends:
26+ array,
27+ base == 4.*,
28+ containers,
29+ bytestring >= 0.10.4.0,
30+ parsec,
31+ text >= 1.1.1.0
examples/http-requests.txtnew file+494
@@ -0,0 +1,494 @@
1+GET / HTTP/1.1
2+Host: www.reddit.com
3+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
4+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
5+Accept-Language: en-us,en;q=0.5
6+Accept-Encoding: gzip, deflate
7+Connection: keep-alive
8+
9+GET /reddit.v_EZwRzV-Ns.css HTTP/1.1
10+Host: www.redditstatic.com
11+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
12+Accept: text/css,*/*;q=0.1
13+Accept-Language: en-us,en;q=0.5
14+Accept-Encoding: gzip, deflate
15+Connection: keep-alive
16+Referer: http://www.reddit.com/
17+
18+GET /reddit-init.en-us.O1zuMqOOQvY.js HTTP/1.1
19+Host: www.redditstatic.com
20+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
21+Accept: */*
22+Accept-Language: en-us,en;q=0.5
23+Accept-Encoding: gzip, deflate
24+Connection: keep-alive
25+Referer: http://www.reddit.com/
26+
27+GET /reddit.en-us.31yAfSoTsfo.js HTTP/1.1
28+Host: www.redditstatic.com
29+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
30+Accept: */*
31+Accept-Language: en-us,en;q=0.5
32+Accept-Encoding: gzip, deflate
33+Connection: keep-alive
34+Referer: http://www.reddit.com/
35+
36+GET /kill.png HTTP/1.1
37+Host: www.redditstatic.com
38+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
39+Accept: image/png,image/*;q=0.8,*/*;q=0.5
40+Accept-Language: en-us,en;q=0.5
41+Accept-Encoding: gzip, deflate
42+Connection: keep-alive
43+Referer: http://www.reddit.com/
44+
45+GET /icon.png HTTP/1.1
46+Host: www.redditstatic.com
47+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
48+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
49+Accept-Language: en-us,en;q=0.5
50+Accept-Encoding: gzip, deflate
51+Connection: keep-alive
52+
53+GET /favicon.ico HTTP/1.1
54+Host: www.redditstatic.com
55+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
56+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
57+Accept-Language: en-us,en;q=0.5
58+Accept-Encoding: gzip, deflate
59+Connection: keep-alive
60+
61+GET /AMZM4CWd6zstSC8y.jpg HTTP/1.1
62+Host: b.thumbs.redditmedia.com
63+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
64+Accept: image/png,image/*;q=0.8,*/*;q=0.5
65+Accept-Language: en-us,en;q=0.5
66+Accept-Encoding: gzip, deflate
67+Connection: keep-alive
68+Referer: http://www.reddit.com/
69+
70+GET /jz1d5Nm0w97-YyNm.jpg HTTP/1.1
71+Host: b.thumbs.redditmedia.com
72+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
73+Accept: image/png,image/*;q=0.8,*/*;q=0.5
74+Accept-Language: en-us,en;q=0.5
75+Accept-Encoding: gzip, deflate
76+Connection: keep-alive
77+Referer: http://www.reddit.com/
78+
79+GET /aWGO99I6yOcNUKXB.jpg HTTP/1.1
80+Host: a.thumbs.redditmedia.com
81+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
82+Accept: image/png,image/*;q=0.8,*/*;q=0.5
83+Accept-Language: en-us,en;q=0.5
84+Accept-Encoding: gzip, deflate
85+Connection: keep-alive
86+Referer: http://www.reddit.com/
87+
88+GET /rZ_rD5TjrJM0E9Aj.css HTTP/1.1
89+Host: e.thumbs.redditmedia.com
90+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
91+Accept: text/css,*/*;q=0.1
92+Accept-Language: en-us,en;q=0.5
93+Accept-Encoding: gzip, deflate
94+Connection: keep-alive
95+Referer: http://www.reddit.com/
96+
97+GET /tmsPwagFzyTvrGRx.jpg HTTP/1.1
98+Host: a.thumbs.redditmedia.com
99+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
100+Accept: image/png,image/*;q=0.8,*/*;q=0.5
101+Accept-Language: en-us,en;q=0.5
102+Accept-Encoding: gzip, deflate
103+Connection: keep-alive
104+Referer: http://www.reddit.com/
105+
106+GET /KYgUaLvXCK3TCEJx.jpg HTTP/1.1
107+Host: a.thumbs.redditmedia.com
108+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
109+Accept: image/png,image/*;q=0.8,*/*;q=0.5
110+Accept-Language: en-us,en;q=0.5
111+Accept-Encoding: gzip, deflate
112+Connection: keep-alive
113+Referer: http://www.reddit.com/
114+
115+GET /81pzxT5x2ozuEaxX.jpg HTTP/1.1
116+Host: e.thumbs.redditmedia.com
117+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
118+Accept: image/png,image/*;q=0.8,*/*;q=0.5
119+Accept-Language: en-us,en;q=0.5
120+Accept-Encoding: gzip, deflate
121+Connection: keep-alive
122+Referer: http://www.reddit.com/
123+
124+GET /MFqCUiUVPO5V8t6x.jpg HTTP/1.1
125+Host: a.thumbs.redditmedia.com
126+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
127+Accept: image/png,image/*;q=0.8,*/*;q=0.5
128+Accept-Language: en-us,en;q=0.5
129+Accept-Encoding: gzip, deflate
130+Connection: keep-alive
131+Referer: http://www.reddit.com/
132+
133+GET /TFpYTiAO5aEowokv.jpg HTTP/1.1
134+Host: e.thumbs.redditmedia.com
135+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
136+Accept: image/png,image/*;q=0.8,*/*;q=0.5
137+Accept-Language: en-us,en;q=0.5
138+Accept-Encoding: gzip, deflate
139+Connection: keep-alive
140+Referer: http://www.reddit.com/
141+
142+GET /eMWMpmm9APNeNqcF.jpg HTTP/1.1
143+Host: e.thumbs.redditmedia.com
144+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
145+Accept: image/png,image/*;q=0.8,*/*;q=0.5
146+Accept-Language: en-us,en;q=0.5
147+Accept-Encoding: gzip, deflate
148+Connection: keep-alive
149+Referer: http://www.reddit.com/
150+
151+GET /S-IpsJrOKuaK9GZ8.jpg HTTP/1.1
152+Host: c.thumbs.redditmedia.com
153+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
154+Accept: image/png,image/*;q=0.8,*/*;q=0.5
155+Accept-Language: en-us,en;q=0.5
156+Accept-Encoding: gzip, deflate
157+Connection: keep-alive
158+Referer: http://www.reddit.com/
159+
160+GET /3V6dj9PDsNnheDXn.jpg HTTP/1.1
161+Host: c.thumbs.redditmedia.com
162+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
163+Accept: image/png,image/*;q=0.8,*/*;q=0.5
164+Accept-Language: en-us,en;q=0.5
165+Accept-Encoding: gzip, deflate
166+Connection: keep-alive
167+Referer: http://www.reddit.com/
168+
169+GET /wQ3-VmNXhv8sg4SJ.jpg HTTP/1.1
170+Host: c.thumbs.redditmedia.com
171+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
172+Accept: image/png,image/*;q=0.8,*/*;q=0.5
173+Accept-Language: en-us,en;q=0.5
174+Accept-Encoding: gzip, deflate
175+Connection: keep-alive
176+Referer: http://www.reddit.com/
177+
178+GET /ixd1C1njpczEWC22.jpg HTTP/1.1
179+Host: c.thumbs.redditmedia.com
180+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
181+Accept: image/png,image/*;q=0.8,*/*;q=0.5
182+Accept-Language: en-us,en;q=0.5
183+Accept-Encoding: gzip, deflate
184+Connection: keep-alive
185+Referer: http://www.reddit.com/
186+
187+GET /nGsQj15VyOHMwmq8.jpg HTTP/1.1
188+Host: c.thumbs.redditmedia.com
189+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
190+Accept: image/png,image/*;q=0.8,*/*;q=0.5
191+Accept-Language: en-us,en;q=0.5
192+Accept-Encoding: gzip, deflate
193+Connection: keep-alive
194+Referer: http://www.reddit.com/
195+
196+GET /zT4yQmDxQLbIxK1b.jpg HTTP/1.1
197+Host: c.thumbs.redditmedia.com
198+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
199+Accept: image/png,image/*;q=0.8,*/*;q=0.5
200+Accept-Language: en-us,en;q=0.5
201+Accept-Encoding: gzip, deflate
202+Connection: keep-alive
203+Referer: http://www.reddit.com/
204+
205+GET /L5e1HcZLv1iu4nrG.jpg HTTP/1.1
206+Host: f.thumbs.redditmedia.com
207+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
208+Accept: image/png,image/*;q=0.8,*/*;q=0.5
209+Accept-Language: en-us,en;q=0.5
210+Accept-Encoding: gzip, deflate
211+Connection: keep-alive
212+Referer: http://www.reddit.com/
213+
214+GET /WJFFPxD8X4JO_lIG.jpg HTTP/1.1
215+Host: f.thumbs.redditmedia.com
216+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
217+Accept: image/png,image/*;q=0.8,*/*;q=0.5
218+Accept-Language: en-us,en;q=0.5
219+Accept-Encoding: gzip, deflate
220+Connection: keep-alive
221+Referer: http://www.reddit.com/
222+
223+GET /hVMVTDdjuY3bQox5.jpg HTTP/1.1
224+Host: f.thumbs.redditmedia.com
225+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
226+Accept: image/png,image/*;q=0.8,*/*;q=0.5
227+Accept-Language: en-us,en;q=0.5
228+Accept-Encoding: gzip, deflate
229+Connection: keep-alive
230+Referer: http://www.reddit.com/
231+
232+GET /rnWf8CjBcyPQs5y_.jpg HTTP/1.1
233+Host: f.thumbs.redditmedia.com
234+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
235+Accept: image/png,image/*;q=0.8,*/*;q=0.5
236+Accept-Language: en-us,en;q=0.5
237+Accept-Encoding: gzip, deflate
238+Connection: keep-alive
239+Referer: http://www.reddit.com/
240+
241+GET /gZJL1jNylKbGV4d-.jpg HTTP/1.1
242+Host: d.thumbs.redditmedia.com
243+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
244+Accept: image/png,image/*;q=0.8,*/*;q=0.5
245+Accept-Language: en-us,en;q=0.5
246+Accept-Encoding: gzip, deflate
247+Connection: keep-alive
248+Referer: http://www.reddit.com/
249+
250+GET /aNd2zNRLXiMnKUFh.jpg HTTP/1.1
251+Host: c.thumbs.redditmedia.com
252+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
253+Accept: image/png,image/*;q=0.8,*/*;q=0.5
254+Accept-Language: en-us,en;q=0.5
255+Accept-Encoding: gzip, deflate
256+Connection: keep-alive
257+Referer: http://www.reddit.com/
258+
259+GET /droparrowgray.gif HTTP/1.1
260+Host: www.redditstatic.com
261+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
262+Accept: image/png,image/*;q=0.8,*/*;q=0.5
263+Accept-Language: en-us,en;q=0.5
264+Accept-Encoding: gzip, deflate
265+Connection: keep-alive
266+Referer: http://www.redditstatic.com/reddit.v_EZwRzV-Ns.css
267+
268+GET /sprite-reddit.an0Lnf61Ap4.png HTTP/1.1
269+Host: www.redditstatic.com
270+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
271+Accept: image/png,image/*;q=0.8,*/*;q=0.5
272+Accept-Language: en-us,en;q=0.5
273+Accept-Encoding: gzip, deflate
274+Connection: keep-alive
275+Referer: http://www.redditstatic.com/reddit.v_EZwRzV-Ns.css
276+
277+GET /ga.js HTTP/1.1
278+Host: www.google-analytics.com
279+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
280+Accept: */*
281+Accept-Language: en-us,en;q=0.5
282+Accept-Encoding: gzip, deflate
283+Connection: keep-alive
284+Referer: http://www.reddit.com/
285+If-Modified-Since: Tue, 29 Oct 2013 19:33:51 GMT
286+
287+GET /reddit/ads.html?sr=-reddit.com&bust2 HTTP/1.1
288+Host: static.adzerk.net
289+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
290+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
291+Accept-Language: en-us,en;q=0.5
292+Accept-Encoding: gzip, deflate
293+Connection: keep-alive
294+Referer: http://www.reddit.com/
295+
296+GET /pixel/of_destiny.png?v=hOlmDALJCWWdjzfBV4ZxJPmrdCLWB%2Ftq7Z%2Ffp4Q%2FxXbVPPREuMJMVGzKraTuhhNWxCCwi6yFEZg%3D&r=783333388 HTTP/1.1
297+Host: pixel.redditmedia.com
298+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
299+Accept: image/png,image/*;q=0.8,*/*;q=0.5
300+Accept-Language: en-us,en;q=0.5
301+Accept-Encoding: gzip, deflate
302+Connection: keep-alive
303+Referer: http://www.reddit.com/
304+
305+GET /UNcO-h_QcS9PD-Gn.jpg HTTP/1.1
306+Host: c.thumbs.redditmedia.com
307+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
308+Accept: image/png,image/*;q=0.8,*/*;q=0.5
309+Accept-Language: en-us,en;q=0.5
310+Accept-Encoding: gzip, deflate
311+Connection: keep-alive
312+Referer: http://e.thumbs.redditmedia.com/rZ_rD5TjrJM0E9Aj.css
313+
314+GET /welcome-lines.png HTTP/1.1
315+Host: www.redditstatic.com
316+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
317+Accept: image/png,image/*;q=0.8,*/*;q=0.5
318+Accept-Language: en-us,en;q=0.5
319+Accept-Encoding: gzip, deflate
320+Connection: keep-alive
321+Referer: http://www.redditstatic.com/reddit.v_EZwRzV-Ns.css
322+
323+GET /welcome-upvote.png HTTP/1.1
324+Host: www.redditstatic.com
325+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
326+Accept: image/png,image/*;q=0.8,*/*;q=0.5
327+Accept-Language: en-us,en;q=0.5
328+Accept-Encoding: gzip, deflate
329+Connection: keep-alive
330+Referer: http://www.redditstatic.com/reddit.v_EZwRzV-Ns.css
331+
332+GET /__utm.gif?utmwv=5.5.1&utms=1&utmn=720496082&utmhn=www.reddit.com&utme=8(site*srpath*usertype*uitype)9(%20reddit.com*%20reddit.com-GET_listing*guest*web)11(3!2)&utmcs=UTF-8&utmsr=2560x1600&utmvp=1288x792&utmsc=24-bit&utmul=en-us&utmje=1&utmfl=13.0%20r0&utmdt=reddit%3A%20the%20front%20page%20of%20the%20internet&utmhid=2129416330&utmr=-&utmp=%2F&utmht=1400862512705&utmac=UA-12131688-1&utmcc=__utma%3D55650728.585571751.1400862513.1400862513.1400862513.1%3B%2B__utmz%3D55650728.1400862513.1.1.utmcsr%3D(direct)%7Cutmccn%3D(direct)%7Cutmcmd%3D(none)%3B&utmu=qR~ HTTP/1.1
333+Host: www.google-analytics.com
334+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
335+Accept: image/png,image/*;q=0.8,*/*;q=0.5
336+Accept-Language: en-us,en;q=0.5
337+Accept-Encoding: gzip, deflate
338+Connection: keep-alive
339+Referer: http://www.reddit.com/
340+
341+GET /ImnpOQhbXUPkwceN.png HTTP/1.1
342+Host: a.thumbs.redditmedia.com
343+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
344+Accept: image/png,image/*;q=0.8,*/*;q=0.5
345+Accept-Language: en-us,en;q=0.5
346+Accept-Encoding: gzip, deflate
347+Connection: keep-alive
348+Referer: http://www.reddit.com/
349+
350+GET /ajax/libs/jquery/1.7.1/jquery.min.js HTTP/1.1
351+Host: ajax.googleapis.com
352+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
353+Accept: */*
354+Accept-Language: en-us,en;q=0.5
355+Accept-Encoding: gzip, deflate
356+Connection: keep-alive
357+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
358+
359+GET /__utm.gif?utmwv=5.5.1&utms=2&utmn=1493472678&utmhn=www.reddit.com&utmt=event&utme=5(AdBlock*enabled*false)(0)8(site*srpath*usertype*uitype)9(%20reddit.com*%20reddit.com-GET_listing*guest*web)11(3!2)&utmcs=UTF-8&utmsr=2560x1600&utmvp=1288x792&utmsc=24-bit&utmul=en-us&utmje=1&utmfl=13.0%20r0&utmdt=reddit%3A%20the%20front%20page%20of%20the%20internet&utmhid=2129416330&utmr=-&utmp=%2F&utmht=1400862512708&utmac=UA-12131688-1&utmni=1&utmcc=__utma%3D55650728.585571751.1400862513.1400862513.1400862513.1%3B%2B__utmz%3D55650728.1400862513.1.1.utmcsr%3D(direct)%7Cutmccn%3D(direct)%7Cutmcmd%3D(none)%3B&utmu=6R~ HTTP/1.1
360+Host: www.google-analytics.com
361+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
362+Accept: image/png,image/*;q=0.8,*/*;q=0.5
363+Accept-Language: en-us,en;q=0.5
364+Accept-Encoding: gzip, deflate
365+Connection: keep-alive
366+Referer: http://www.reddit.com/
367+
368+GET /ados.js?q=43 HTTP/1.1
369+Host: secure.adzerk.net
370+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
371+Accept: */*
372+Accept-Language: en-us,en;q=0.5
373+Accept-Encoding: gzip, deflate
374+Connection: keep-alive
375+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
376+
377+GET /fetch-trackers?callback=jQuery111005268222517967478_1400862512407&ids%5B%5D=t3_25jzeq-t8_k2ii&_=1400862512408 HTTP/1.1
378+Host: tracker.redditmedia.com
379+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
380+Accept: */*
381+Accept-Language: en-us,en;q=0.5
382+Accept-Encoding: gzip, deflate
383+Connection: keep-alive
384+Referer: http://www.reddit.com/
385+
386+GET /ados?t=1400862512892&request={%22Placements%22:[{%22A%22:5146,%22S%22:24950,%22D%22:%22main%22,%22AT%22:5},{%22A%22:5146,%22S%22:24950,%22D%22:%22sponsorship%22,%22AT%22:8}],%22Keywords%22:%22-reddit.com%22,%22Referrer%22:%22http%3A%2F%2Fwww.reddit.com%2F%22,%22IsAsync%22:true,%22WriteResults%22:true} HTTP/1.1
387+Host: engine.adzerk.net
388+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
389+Accept: */*
390+Accept-Language: en-us,en;q=0.5
391+Accept-Encoding: gzip, deflate
392+Connection: keep-alive
393+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
394+
395+GET /pixel/of_doom.png?id=t3_25jzeq-t8_k2ii&hash=da31d967485cdbd459ce1e9a5dde279fef7fc381&r=1738649500 HTTP/1.1
396+Host: pixel.redditmedia.com
397+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
398+Accept: image/png,image/*;q=0.8,*/*;q=0.5
399+Accept-Language: en-us,en;q=0.5
400+Accept-Encoding: gzip, deflate
401+Connection: keep-alive
402+Referer: http://www.reddit.com/
403+
404+GET /Extensions/adFeedback.js HTTP/1.1
405+Host: static.adzrk.net
406+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
407+Accept: */*
408+Accept-Language: en-us,en;q=0.5
409+Accept-Encoding: gzip, deflate
410+Connection: keep-alive
411+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
412+
413+GET /Extensions/adFeedback.css HTTP/1.1
414+Host: static.adzrk.net
415+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
416+Accept: text/css,*/*;q=0.1
417+Accept-Language: en-us,en;q=0.5
418+Accept-Encoding: gzip, deflate
419+Connection: keep-alive
420+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
421+
422+GET /reddit/ads-load.html?bust2 HTTP/1.1
423+Host: static.adzerk.net
424+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
425+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
426+Accept-Language: en-us,en;q=0.5
427+Accept-Encoding: gzip, deflate
428+Connection: keep-alive
429+Referer: http://www.reddit.com/
430+
431+GET /Advertisers/a774d7d6148046efa89403a8db635a81.jpg HTTP/1.1
432+Host: static.adzerk.net
433+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
434+Accept: image/png,image/*;q=0.8,*/*;q=0.5
435+Accept-Language: en-us,en;q=0.5
436+Accept-Encoding: gzip, deflate
437+Connection: keep-alive
438+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
439+
440+GET /i.gif?e=eyJhdiI6NjIzNTcsImF0Ijo1LCJjbSI6MTE2MzUxLCJjaCI6Nzk4NCwiY3IiOjMzNzAxNSwiZGkiOiI4NmI2Y2UzYWM5NDM0MjhkOTk2ZTg4MjYwZDE5ZTE1YyIsImRtIjoxLCJmYyI6NDE2MTI4LCJmbCI6MjEwNDY0LCJrdyI6Ii1yZWRkaXQuY29tIiwibWsiOiItcmVkZGl0LmNvbSIsIm53Ijo1MTQ2LCJwYyI6MCwicHIiOjIwMzYyLCJydCI6MSwicmYiOiJodHRwOi8vd3d3LnJlZGRpdC5jb20vIiwic3QiOjI0OTUwLCJ1ayI6InVlMS01ZWIwOGFlZWQ5YTc0MDFjOTE5NWNiOTMzZWI3Yzk2NiIsInRzIjoxNDAwODYyNTkzNjQ1fQ&s=lwlbFf2Uywt7zVBFRj_qXXu7msY HTTP/1.1
441+Host: engine.adzerk.net
442+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
443+Accept: image/png,image/*;q=0.8,*/*;q=0.5
444+Accept-Language: en-us,en;q=0.5
445+Accept-Encoding: gzip, deflate
446+Connection: keep-alive
447+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
448+Cookie: azk=ue1-5eb08aeed9a7401c9195cb933eb7c966
449+
450+GET /BurstingPipe/adServer.bs?cn=tf&c=19&mc=imp&pli=9994987&PluID=0&ord=1400862593644&rtu=-1 HTTP/1.1
451+Host: bs.serving-sys.com
452+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
453+Accept: image/png,image/*;q=0.8,*/*;q=0.5
454+Accept-Language: en-us,en;q=0.5
455+Accept-Encoding: gzip, deflate
456+Connection: keep-alive
457+Referer: http://static.adzerk.net/reddit/ads.html?sr=-reddit.com&bust2
458+
459+GET /Advertisers/63cfd0044ffd49c0a71a6626f7a1d8f0.jpg HTTP/1.1
460+Host: static.adzerk.net
461+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
462+Accept: image/png,image/*;q=0.8,*/*;q=0.5
463+Accept-Language: en-us,en;q=0.5
464+Accept-Encoding: gzip, deflate
465+Connection: keep-alive
466+Referer: http://static.adzerk.net/reddit/ads-load.html?bust2
467+
468+GET /BurstingPipe/adServer.bs?cn=tf&c=19&mc=imp&pli=9962555&PluID=0&ord=1400862593645&rtu=-1 HTTP/1.1
469+Host: bs.serving-sys.com
470+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
471+Accept: image/png,image/*;q=0.8,*/*;q=0.5
472+Accept-Language: en-us,en;q=0.5
473+Accept-Encoding: gzip, deflate
474+Connection: keep-alive
475+Referer: http://static.adzerk.net/reddit/ads-load.html?bust2
476+Cookie: S_9994987=6754579095859875029; A4=01fmFvgRnI09SF00000; u2=d1263d39-874b-4a89-86cd-a2ab0860ed4e3Zl040
477+
478+GET /i.gif?e=eyJhdiI6NjIzNTcsImF0Ijo4LCJjbSI6MTE2MzUxLCJjaCI6Nzk4NCwiY3IiOjMzNzAxOCwiZGkiOiI3OTdlZjU3OWQ5NjE0ODdiODYyMGMyMGJkOTE4YzNiMSIsImRtIjoxLCJmYyI6NDE2MTMxLCJmbCI6MjEwNDY0LCJrdyI6Ii1yZWRkaXQuY29tIiwibWsiOiItcmVkZGl0LmNvbSIsIm53Ijo1MTQ2LCJwYyI6MCwicHIiOjIwMzYyLCJydCI6MSwicmYiOiJodHRwOi8vd3d3LnJlZGRpdC5jb20vIiwic3QiOjI0OTUwLCJ1ayI6InVlMS01ZWIwOGFlZWQ5YTc0MDFjOTE5NWNiOTMzZWI3Yzk2NiIsInRzIjoxNDAwODYyNTkzNjQ2fQ&s=OjzxzXAgQksbdQOHNm-bjZcnZPA HTTP/1.1
479+Host: engine.adzerk.net
480+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1
481+Accept: image/png,image/*;q=0.8,*/*;q=0.5
482+Accept-Language: en-us,en;q=0.5
483+Accept-Encoding: gzip, deflate
484+Connection: keep-alive
485+Referer: http://static.adzerk.net/reddit/ads-load.html?bust2
486+Cookie: azk=ue1-5eb08aeed9a7401c9195cb933eb7c966
487+
488+GET /subscribe?host_int=1042356184&ns_map=571794054_374233948806,464381511_13349283399&user_id=245722467&nid=1399334269710011966&ts=1400862514 HTTP/1.1
489+Host: notify8.dropbox.com
490+Accept-Encoding: identity
491+Connection: keep-alive
492+X-Dropbox-Locale: en_US
493+User-Agent: DropboxDesktopClient/2.7.54 (Macintosh; 10.8; ('i32',); en_US)
494+
examples/rfc2616.c+27−21
@@ -1,21 +1,22 @@
1/*
1/*
2− * This is a simple driver for Ryan Dahl's hand-bummed C http-parser
2+ * This is a simple driver for the http-parser package. It is
3− * package. It is intended to read one HTTP request after another
3+ * intended to read one HTTP request after another from a file,
4− * from a file, nothing more.
4+ * nothing more.
5 *
5 *
6 * For "feature parity" with the Haskell code in RFC2616.hs, we
6 * For "feature parity" with the Haskell code in RFC2616.hs, we
7 * allocate and populate a simple structure describing each request,
7 * allocate and populate a simple structure describing each request,
8 * since that's the sort of thing that many real applications would
8 * since that's the sort of thing that many real applications would
9 * themselves do and the library doesn't do this for us.
9 * themselves do and the library doesn't do this for us.
10 *
10 *
11− * For the http-parser source, see http://github.com/ry/http-parser/
11+ * For the http-parser source, see
12+ * https://github.com/joyent/http-parser/blob/master/http_parser.c
12 */
13 */
13
14
14/*
15/*
15 * Turn off this preprocessor symbol to have the callbacks do nothing
16 * Turn off this preprocessor symbol to have the callbacks do nothing
16 * at all, which "improves performance" by about 50%.
17 * at all, which "improves performance" by about 50%.
17 */
18 */
18−#define LOOK_BUSY
19+/*#define LOOK_BUSY*/
19
20
20#include <assert.h>
21#include <assert.h>
21#include <fcntl.h>
22#include <fcntl.h>
@@ -27,18 +28,18 @@
27#include <unistd.h>
28#include <unistd.h>
28
29
29#include "http_parser.h"
30#include "http_parser.h"
30−
31+
31struct http_string {
32struct http_string {
32 size_t len;
33 size_t len;
33 char value[0];
34 char value[0];
34};
35};
35−
36+
36struct http_header {
37struct http_header {
37 struct http_string *name;
38 struct http_string *name;
38 struct http_string *value;
39 struct http_string *value;
39 struct http_header *next;
40 struct http_header *next;
40};
41};
41−
42+
42struct http_request {
43struct http_request {
43 struct http_string *method;
44 struct http_string *method;
44 struct http_string *uri;
45 struct http_string *uri;
@@ -49,7 +50,7 @@ struct data {
49 size_t count;
50 size_t count;
50 struct http_request req;
51 struct http_request req;
51};
52};
52−
53+
53static void *xmalloc(size_t size)
54static void *xmalloc(size_t size)
54{
55{
55 void *ptr;
56 void *ptr;
@@ -78,7 +79,7 @@ static void xstrcat(struct http_string **dst, const char *src, size_t len)
78 *dst = xstrdup(src, len, 0);
79 *dst = xstrdup(src, len, 0);
79 return;
80 return;
80 }
81 }
81−
82+
82 p = xstrdup((*dst)->value, (*dst)->len, len);
83 p = xstrdup((*dst)->value, (*dst)->len, len);
83 memcpy(p->value + (*dst)->len, src, len);
84 memcpy(p->value + (*dst)->len, src, len);
84 p->len += len;
85 p->len += len;
@@ -98,7 +99,7 @@ static int begin(http_parser *p)
98static int url(http_parser *p, const char *at, size_t len)
99static int url(http_parser *p, const char *at, size_t len)
99{
100{
100#ifdef LOOK_BUSY
101#ifdef LOOK_BUSY
101− struct data *data = p->data;
102+ struct data *data = p->data;
102
103
103 xstrcat(&data->req.uri, at, len);
104 xstrcat(&data->req.uri, at, len);
104#endif
105#endif
@@ -119,7 +120,7 @@ static int header_field(http_parser *p, const char *at, size_t len)
119 hdr->name = xstrdup(at, len, 0);
120 hdr->name = xstrdup(at, len, 0);
120 hdr->value = NULL;
121 hdr->value = NULL;
121 hdr->next = NULL;
122 hdr->next = NULL;
122−
123+
123 if (data->req.last != NULL)
124 if (data->req.last != NULL)
124 data->req.last->next = hdr;
125 data->req.last->next = hdr;
125 data->req.last = hdr;
126 data->req.last = hdr;
@@ -150,7 +151,7 @@ static int complete(http_parser *p)
150
151
151 free(data->req.method);
152 free(data->req.method);
152 free(data->req.uri);
153 free(data->req.uri);
153−
154+
154 for (hdr = data->req.headers; hdr != NULL; hdr = next) {
155 for (hdr = data->req.headers; hdr != NULL; hdr = next) {
155 next = hdr->next;
156 next = hdr->next;
156 free(hdr->name);
157 free(hdr->name);
@@ -164,7 +165,7 @@ static int complete(http_parser *p)
164 data->req.headers = NULL;
165 data->req.headers = NULL;
165 data->req.last = NULL;
166 data->req.last = NULL;
166#endif
167#endif
167−
168+
168 /* Bludgeon http_parser into understanding that we really want to
169 /* Bludgeon http_parser into understanding that we really want to
169 * keep parsing after a request that in principle ought to close
170 * keep parsing after a request that in principle ought to close
170 * the "connection". */
171 * the "connection". */
@@ -180,16 +181,21 @@ static int complete(http_parser *p)
180static void parse(const char *path, int fd)
181static void parse(const char *path, int fd)
181{
182{
182 struct data data;
183 struct data data;
184+ http_parser_settings s;
183 http_parser p;
185 http_parser p;
184 ssize_t nread;
186 ssize_t nread;
185
187
186− http_parser_init(&p, HTTP_REQUEST);
188+ memset(&s, 0, sizeof(s));
187− p.on_message_begin = begin;
189+ s.on_message_begin = begin;
188− p.on_url = url;
190+ s.on_url = url;
189− p.on_header_field = header_field;
191+ s.on_header_field = header_field;
190− p.on_header_value = header_value;
192+ s.on_header_value = header_value;
191− p.on_message_complete = complete;
193+ s.on_message_complete = complete;
194+
192 p.data = &data;
195 p.data = &data;
196+
197+ http_parser_init(&p, HTTP_REQUEST);
198+
193 data.count = 0;
199 data.count = 0;
194 data.req.method = NULL;
200 data.req.method = NULL;
195 data.req.uri = NULL;
201 data.req.uri = NULL;
@@ -202,7 +208,7 @@ static void parse(const char *path, int fd)
202
208
203 nread = read(fd, buf, sizeof(buf));
209 nread = read(fd, buf, sizeof(buf));
204
210
205− np = http_parser_execute(&p, buf, nread);
211+ np = http_parser_execute(&p, &s, buf, nread);
206 if (np != nread) {
212 if (np != nread) {
207 fprintf(stderr, "%s: parse failed\n", path);
213 fprintf(stderr, "%s: parse failed\n", path);
208 break;
214 break;
tests/Makefiledeleted−14
@@ -1,14 +0,0 @@
1−all: TestFastSet.out qc.out
2−
3−%.out: %.exe
4− ./$< | tee $<.tmp
5− mv $<.tmp $@
6−
7−qc.exe: QC.hs
8− ghc -O -fno-warn-orphans --make -o $@ $<
9−
10−%.exe: %.hs
11− ghc -O -fno-warn-orphans --make -o $@ $<
12−
13−clean:
14− -rm -f *.hi *.o *.exe *.out
tests/QC.hs+6−2
@@ -1,15 +1,19 @@
1{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
1{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
2module Main (main) where
2module Main (main) where
3
3
4+import qualified QC.Buffer as Buffer
4import qualified QC.ByteString as ByteString
5import qualified QC.ByteString as ByteString
5import qualified QC.Combinator as Combinator
6import qualified QC.Combinator as Combinator
7+import qualified QC.Simple as Simple
6import qualified QC.Text as Text
8import qualified QC.Text as Text
7−import Test.Framework (defaultMain, testGroup)
9+import Test.Tasty (defaultMain, testGroup)
8
10
9main = defaultMain tests
11main = defaultMain tests
10
12
11−tests = [
13+tests = testGroup "tests" [
12 testGroup "bs" ByteString.tests
14 testGroup "bs" ByteString.tests
15+ , testGroup "buf" Buffer.tests
13 , testGroup "combinator" Combinator.tests
16 , testGroup "combinator" Combinator.tests
17+ , testGroup "simple" Simple.tests
14 , testGroup "text" Text.tests
18 , testGroup "text" Text.tests
15 ]
19 ]
tests/QC/Buffer.hsnew file+96
@@ -0,0 +1,96 @@
1+{-# LANGUAGE BangPatterns, CPP, FlexibleInstances, OverloadedStrings,
2+ TypeSynonymInstances #-}
3+
4+module QC.Buffer (tests) where
5+
6+#if !MIN_VERSION_base(4,8,0)
7+import Control.Applicative ((<$>))
8+import Data.Monoid (Monoid(mconcat))
9+#endif
10+import QC.Common ()
11+import Test.Tasty (TestTree)
12+import Test.Tasty.QuickCheck (testProperty)
13+import Test.QuickCheck
14+import qualified Data.Attoparsec.ByteString.Buffer as BB
15+import qualified Data.Attoparsec.Text.Buffer as BT
16+import qualified Data.ByteString as B
17+import qualified Data.ByteString.Unsafe as B
18+import qualified Data.Text as T
19+import qualified Data.Text.Unsafe as T
20+
21+data BP t b = BP [t] !t !b
22+ deriving (Eq, Show)
23+
24+type BPB = BP B.ByteString BB.Buffer
25+type BPT = BP T.Text BT.Buffer
26+
27+instance Arbitrary BPB where
28+ arbitrary = do
29+ bss <- arbitrary
30+ return $! toBP BB.buffer bss
31+
32+ shrink (BP bss _ _) = toBP BB.buffer <$> shrink bss
33+
34+instance Arbitrary BPT where
35+ arbitrary = do
36+ bss <- arbitrary
37+ return $! toBP BT.buffer bss
38+
39+ shrink (BP bss _ _) = toBP BT.buffer <$> shrink bss
40+
41+toBP :: (Monoid a, Monoid b) => (a -> b) -> [a] -> BP a b
42+toBP buf bss = BP bss (mconcat bss) (mconcat (map buf bss))
43+
44+b_unbuffer :: BPB -> Property
45+b_unbuffer (BP _ts t buf) = t === BB.unbuffer buf
46+
47+t_unbuffer :: BPT -> Property
48+t_unbuffer (BP _ts t buf) = t === BT.unbuffer buf
49+
50+b_length :: BPB -> Property
51+b_length (BP _ts t buf) = B.length t === BB.length buf
52+
53+t_length :: BPT -> Property
54+t_length (BP _ts t buf) = T.lengthWord16 t === BT.length buf
55+
56+b_unsafeIndex :: BPB -> Gen Property
57+b_unsafeIndex (BP _ts t buf) = do
58+ let l = B.length t
59+ i <- choose (0,l-1)
60+ return $ l === 0 .||. B.unsafeIndex t i === BB.unsafeIndex buf i
61+
62+t_iter :: BPT -> Gen Property
63+t_iter (BP _ts t buf) = do
64+ let l = T.lengthWord16 t
65+ i <- choose (0,l-1)
66+ let it (T.Iter c q) = (c,q)
67+ return $ l === 0 .||. it (T.iter t i) === it (BT.iter buf i)
68+
69+t_iter_ :: BPT -> Gen Property
70+t_iter_ (BP _ts t buf) = do
71+ let l = T.lengthWord16 t
72+ i <- choose (0,l-1)
73+ return $ l === 0 .||. T.iter_ t i === BT.iter_ buf i
74+
75+b_unsafeDrop :: BPB -> Gen Property
76+b_unsafeDrop (BP _ts t buf) = do
77+ i <- choose (0, B.length t)
78+ return $ B.unsafeDrop i t === BB.unsafeDrop i buf
79+
80+t_dropWord16 :: BPT -> Gen Property
81+t_dropWord16 (BP _ts t buf) = do
82+ i <- choose (0, T.lengthWord16 t)
83+ return $ T.dropWord16 i t === BT.dropWord16 i buf
84+
85+tests :: [TestTree]
86+tests = [
87+ testProperty "b_unbuffer" b_unbuffer
88+ , testProperty "t_unbuffer" t_unbuffer
89+ , testProperty "b_length" b_length
90+ , testProperty "t_length" t_length
91+ , testProperty "b_unsafeIndex" b_unsafeIndex
92+ , testProperty "t_iter" t_iter
93+ , testProperty "t_iter_" t_iter_
94+ , testProperty "b_unsafeDrop" b_unsafeDrop
95+ , testProperty "t_dropWord16" t_dropWord16
96+ ]
tests/QC/ByteString.hs+99−65
@@ -1,133 +1,163 @@
1−{-# LANGUAGE BangPatterns, OverloadedStrings #-}
1+{-# LANGUAGE BangPatterns, CPP, OverloadedStrings #-}
2module QC.ByteString (tests) where
2module QC.ByteString (tests) where
3
3
4+#if !MIN_VERSION_base(4,8,0)
4−import Control.Applicative ((<$>), (<*>))
5+import Control.Applicative ((<*>), (<$>))
6+#endif
5−import Data.Char (chr, ord)
7+import Data.Char (chr, ord, toUpper)
6import Data.Int (Int64)
8import Data.Int (Int64)
7import Data.Word (Word8)
9import Data.Word (Word8)
8import Prelude hiding (take, takeWhile)
10import Prelude hiding (take, takeWhile)
9−import QC.Common (parseBS, toStrictBS)
11+import QC.Common (ASCII(..), liftOp, parseBS, toStrictBS)
10−import Test.Framework (Test)
12+import Test.Tasty (TestTree)
11−import Test.Framework.Providers.QuickCheck2 (testProperty)
13+import Test.Tasty.QuickCheck (testProperty)
12import Test.QuickCheck
14import Test.QuickCheck
13import qualified Data.Attoparsec.ByteString as P
15import qualified Data.Attoparsec.ByteString as P
14import qualified Data.Attoparsec.ByteString.Char8 as P8
16import qualified Data.Attoparsec.ByteString.Char8 as P8
17+import qualified Data.Attoparsec.ByteString.FastSet as S
15import qualified Data.Attoparsec.ByteString.Lazy as PL
18import qualified Data.Attoparsec.ByteString.Lazy as PL
16import qualified Data.ByteString as B
19import qualified Data.ByteString as B
20+import qualified Data.ByteString.Char8 as B8
17import qualified Data.ByteString.Lazy as L
21import qualified Data.ByteString.Lazy as L
18import qualified Data.ByteString.Lazy.Char8 as L8
22import qualified Data.ByteString.Lazy.Char8 as L8
19
23
20-- Basic byte-level combinators.
24-- Basic byte-level combinators.
21
25
22−satisfy :: Word8 -> L.ByteString -> Bool
26+satisfy :: Word8 -> L.ByteString -> Property
23−satisfy w s = parseBS (P.satisfy (<=w)) (L.cons w s) == Just w
27+satisfy w s = parseBS (P.satisfy (<=w)) (L.cons w s) === Just w
24
28
25−satisfyWith :: Char -> L.ByteString -> Bool
29+satisfyWith :: Word8 -> L.ByteString -> Property
26−satisfyWith c s = parseBS (P.satisfyWith (chr . fromIntegral) (<=c))
30+satisfyWith w s = parseBS (P.satisfyWith (chr . fromIntegral) (<=c))
27− (L.cons (fromIntegral (ord c)) s) == Just c
31+ (L.cons (fromIntegral (ord c)) s) === Just c
32+ where
33+ c = chr (fromIntegral w)
28
34
29−word8 :: Word8 -> L.ByteString -> Bool
35+word8 :: Word8 -> L.ByteString -> Property
30−word8 w s = parseBS (P.word8 w) (L.cons w s) == Just w
36+word8 w s = parseBS (P.word8 w) (L.cons w s) === Just w
31
37
32−skip :: Word8 -> L.ByteString -> Bool
38+skip :: Word8 -> L.ByteString -> Property
33skip w s =
39skip w s =
34 case (parseBS (P.skip (<w)) s, L.uncons s) of
40 case (parseBS (P.skip (<w)) s, L.uncons s) of
35− (Nothing, mcs) -> maybe True (not . it) mcs
41+ (Nothing, mcs) -> maybe (property True) (expectFailure . it) mcs
36− (Just _, mcs) -> maybe False it mcs
42+ (Just _, mcs) -> maybe (property False) it mcs
37− where it cs = fst cs < w
43+ where it cs = liftOp "<" (<) (fst cs) w
38
44
39−anyWord8 :: L.ByteString -> Bool
45+anyWord8 :: L.ByteString -> Property
40anyWord8 s
46anyWord8 s
41− | L.null s = p == Nothing
47+ | L.null s = p === Nothing
42− | otherwise = p == Just (L.head s)
48+ | otherwise = p === Just (L.head s)
43 where p = parseBS P.anyWord8 s
49 where p = parseBS P.anyWord8 s
44
50
45−notWord8 :: Word8 -> NonEmptyList Word8 -> Bool
51+notWord8 :: Word8 -> NonEmptyList Word8 -> Property
46−notWord8 w (NonEmpty s) = parseBS (P.notWord8 w) bs == if v == w
52+notWord8 w (NonEmpty s) = parseBS (P.notWord8 w) bs === if v == w
47− then Nothing
53+ then Nothing
48− else Just v
54+ else Just v
49 where v = L.head bs
55 where v = L.head bs
50 bs = L.pack s
56 bs = L.pack s
51
57
52−peekWord8 :: L.ByteString -> Bool
58+peekWord8 :: L.ByteString -> Property
53peekWord8 s
59peekWord8 s
54− | L.null s = p == Just (Nothing, s)
60+ | L.null s = p === Just (Nothing, s)
55− | otherwise = p == Just (Just (L.head s), s)
61+ | otherwise = p === Just (Just (L.head s), s)
56 where p = parseBS ((,) <$> P.peekWord8 <*> P.takeLazyByteString) s
62 where p = parseBS ((,) <$> P.peekWord8 <*> P.takeLazyByteString) s
57
63
58−peekWord8' :: L.ByteString -> Bool
64+peekWord8' :: L.ByteString -> Property
59−peekWord8' s = parseBS P.peekWord8' s == (fst <$> L.uncons s)
65+peekWord8' s = parseBS P.peekWord8' s === (fst <$> L.uncons s)
60
66
61−string :: L.ByteString -> L.ByteString -> Bool
67+string :: L.ByteString -> L.ByteString -> Property
62−string s t = parseBS (P.string s') (s `L.append` t) == Just s'
68+string s t = parseBS (P.string s') (s `L.append` t) === Just s'
63 where s' = toStrictBS s
69 where s' = toStrictBS s
64
70
71+stringCI :: ASCII L.ByteString -> ASCII L.ByteString -> Property
72+stringCI (ASCII s) (ASCII t) =
73+ parseBS (P8.stringCI up) (s `L.append` t) === Just s'
74+ where s' = toStrictBS s
75+ up = B8.map toUpper s'
76+
77+strings :: L.ByteString -> L.ByteString -> L.ByteString -> Property
78+strings s t u =
79+ parseBS (P.string (toStrictBS s) >> P.string t') (L.concat [s,t,u])
80+ === Just t'
81+ where t' = toStrictBS t
82+
65−skipWhile :: Word8 -> L.ByteString -> Bool
83+skipWhile :: Word8 -> L.ByteString -> Property
66skipWhile w s =
84skipWhile w s =
67 let t = L.dropWhile (<=w) s
85 let t = L.dropWhile (<=w) s
68 in case PL.parse (P.skipWhile (<=w)) s of
86 in case PL.parse (P.skipWhile (<=w)) s of
69− PL.Done t' () -> t == t'
87+ PL.Done t' () -> t === t'
70− _ -> False
88+ _ -> property False
71
89
72−takeCount :: Positive Int -> L.ByteString -> Bool
90+takeCount :: Positive Int -> L.ByteString -> Property
73takeCount (Positive k) s =
91takeCount (Positive k) s =
74 case parseBS (P.take k) s of
92 case parseBS (P.take k) s of
75− Nothing -> fromIntegral k > L.length s
93+ Nothing -> liftOp ">" (>) (fromIntegral k) (L.length s)
76− Just _s -> fromIntegral k <= L.length s
94+ Just _s -> liftOp "<=" (<=) (fromIntegral k) (L.length s)
77
95
78−takeWhile :: Word8 -> L.ByteString -> Bool
96+takeWhile :: Word8 -> L.ByteString -> Property
79takeWhile w s =
97takeWhile w s =
80 let (h,t) = L.span (==w) s
98 let (h,t) = L.span (==w) s
81 in case PL.parse (P.takeWhile (==w)) s of
99 in case PL.parse (P.takeWhile (==w)) s of
82− PL.Done t' h' -> t == t' && toStrictBS h == h'
100+ PL.Done t' h' -> t === t' .&&. toStrictBS h === h'
83− _ -> False
101+ _ -> property False
84
102
85−take :: Int -> L.ByteString -> Bool
103+take :: Int -> L.ByteString -> Property
86−take n s = maybe (L.length s < fromIntegral n) (== B.take n (toStrictBS s)) $
104+take n s = maybe (property $ L.length s < fromIntegral n)
105+ (=== B.take n (toStrictBS s)) $
87 parseBS (P.take n) s
106 parseBS (P.take n) s
88
107
89−takeByteString :: L.ByteString -> Bool
108+takeByteString :: L.ByteString -> Property
90−takeByteString s = maybe False (== toStrictBS s) . parseBS P.takeByteString $ s
109+takeByteString s = maybe (property False) (=== toStrictBS s) .
110+ parseBS P.takeByteString $ s
91
111
92−takeLazyByteString :: L.ByteString -> Bool
112+takeLazyByteString :: L.ByteString -> Property
113+takeLazyByteString s = maybe (property False) (=== s) .
93−takeLazyByteString s = maybe False (== s) . parseBS P.takeLazyByteString $ s
114+ parseBS P.takeLazyByteString $ s
94
115
95−takeWhile1 :: Word8 -> L.ByteString -> Bool
116+takeWhile1 :: Word8 -> L.ByteString -> Property
96takeWhile1 w s =
117takeWhile1 w s =
97 let s' = L.cons w s
118 let s' = L.cons w s
98 (h,t) = L.span (<=w) s'
119 (h,t) = L.span (<=w) s'
99 in case PL.parse (P.takeWhile1 (<=w)) s' of
120 in case PL.parse (P.takeWhile1 (<=w)) s' of
100− PL.Done t' h' -> t == t' && toStrictBS h == h'
121+ PL.Done t' h' -> t === t' .&&. toStrictBS h === h'
101− _ -> False
122+ _ -> property False
102
123
103−takeTill :: Word8 -> L.ByteString -> Bool
124+takeTill :: Word8 -> L.ByteString -> Property
104takeTill w s =
125takeTill w s =
105 let (h,t) = L.break (==w) s
126 let (h,t) = L.break (==w) s
106 in case PL.parse (P.takeTill (==w)) s of
127 in case PL.parse (P.takeTill (==w)) s of
107− PL.Done t' h' -> t == t' && toStrictBS h == h'
128+ PL.Done t' h' -> t === t' .&&. toStrictBS h === h'
108− _ -> False
129+ _ -> property False
109
130
110−takeWhile1_empty :: Bool
131+takeWhile1_empty :: Property
111−takeWhile1_empty = parseBS (P.takeWhile1 undefined) L.empty == Nothing
132+takeWhile1_empty = parseBS (P.takeWhile1 undefined) L.empty === Nothing
112
133
113−endOfInput :: L.ByteString -> Bool
134+endOfInput :: L.ByteString -> Property
114−endOfInput s = parseBS P.endOfInput s == if L.null s
135+endOfInput s = parseBS P.endOfInput s === if L.null s
115− then Just ()
136+ then Just ()
116− else Nothing
137+ else Nothing
117
138
118−endOfLine :: L.ByteString -> Bool
139+endOfLine :: L.ByteString -> Property
119endOfLine s =
140endOfLine s =
120 case (parseBS P8.endOfLine s, L8.uncons s) of
141 case (parseBS P8.endOfLine s, L8.uncons s) of
121− (Nothing, mcs) -> maybe True (not . eol) mcs
142+ (Nothing, mcs) -> maybe (property True) (expectFailure . eol) mcs
122− (Just _, mcs) -> maybe False eol mcs
143+ (Just _, mcs) -> maybe (property False) eol mcs
144+ where eol (c,s') = c === '\n' .||.
123− where eol (c,s') = c == '\n' || (c, fst <$> L8.uncons s') == ('\r', Just '\n')
145+ (c, fst <$> L8.uncons s') === ('\r', Just '\n')
124
146
125−scan :: L.ByteString -> Positive Int64 -> Bool
147+scan :: L.ByteString -> Positive Int64 -> Property
126−scan s (Positive k) = parseBS p s == (Just $ toStrictBS $ L.take k s)
148+scan s (Positive k) = parseBS p s === Just (toStrictBS $ L.take k s)
127 where p = P.scan k $ \ n _ ->
149 where p = P.scan k $ \ n _ ->
128 if n > 0 then let !n' = n - 1 in Just n' else Nothing
150 if n > 0 then let !n' = n - 1 in Just n' else Nothing
129
151
152+members :: [Word8] -> Property
153+members s = property $ all (`S.memberWord8` set) s
154+ where set = S.fromList s
155+
156+nonmembers :: [Word8] -> [Word8] -> Property
157+nonmembers s s' = property . not . any (`S.memberWord8` set) $ filter (not . (`elem` s)) s'
158+ where set = S.fromList s
159+
130−tests :: [Test]
160+tests :: [TestTree]
131tests = [
161tests = [
132 testProperty "anyWord8" anyWord8
162 testProperty "anyWord8" anyWord8
133 , testProperty "endOfInput" endOfInput
163 , testProperty "endOfInput" endOfInput
@@ -141,6 +171,8 @@ tests = [
141 , testProperty "skip" skip
171 , testProperty "skip" skip
142 , testProperty "skipWhile" skipWhile
172 , testProperty "skipWhile" skipWhile
143 , testProperty "string" string
173 , testProperty "string" string
174+ , testProperty "stringCI" stringCI
175+ , testProperty "strings" strings
144 , testProperty "take" take
176 , testProperty "take" take
145 , testProperty "takeByteString" takeByteString
177 , testProperty "takeByteString" takeByteString
146 , testProperty "takeCount" takeCount
178 , testProperty "takeCount" takeCount
@@ -150,4 +182,6 @@ tests = [
150 , testProperty "takeWhile1" takeWhile1
182 , testProperty "takeWhile1" takeWhile1
151 , testProperty "takeWhile1_empty" takeWhile1_empty
183 , testProperty "takeWhile1_empty" takeWhile1_empty
152 , testProperty "word8" word8
184 , testProperty "word8" word8
185+ , testProperty "members" members
186+ , testProperty "nonmembers" nonmembers
153 ]
187 ]
tests/QC/Combinator.hs+18−5
@@ -1,10 +1,15 @@
1+{-# LANGUAGE CPP, OverloadedStrings #-}
2+
1module QC.Combinator where
3module QC.Combinator where
2
4
5+#if !MIN_VERSION_base(4,8,0)
3−import Control.Applicative
6+import Control.Applicative ((<*>), (<$>), (<*), (*>))
7+#endif
8+import Data.Maybe (fromJust, isJust)
4import Data.Word (Word8)
9import Data.Word (Word8)
5−import QC.Common (Repack, parseBS, repackBS)
10+import QC.Common (Repack, parseBS, repackBS, toLazyBS)
6−import Test.Framework (Test)
11+import Test.Tasty (TestTree)
7−import Test.Framework.Providers.QuickCheck2 (testProperty)
12+import Test.Tasty.QuickCheck (testProperty)
8import Test.QuickCheck
13import Test.QuickCheck
9import qualified Data.Attoparsec.ByteString.Char8 as P
14import qualified Data.Attoparsec.ByteString.Char8 as P
10import qualified Data.Attoparsec.Combinator as C
15import qualified Data.Attoparsec.Combinator as C
@@ -22,6 +27,13 @@ count (Positive (Small n)) rs s =
22 (length <$> parseBS (C.count n (P.string s)) input) == Just n
27 (length <$> parseBS (C.count n (P.string s)) input) == Just n
23 where input = repackBS rs (B.concat (replicate (n+1) s))
28 where input = repackBS rs (B.concat (replicate (n+1) s))
24
29
30+lookAhead :: NonEmptyList Word8 -> Bool
31+lookAhead (NonEmpty xs) =
32+ let ys = B.pack xs
33+ withLookAheadThenConsume = (\x y -> (x, y)) <$> C.lookAhead (P.string ys) <*> P.string ys
34+ mr = parseBS withLookAheadThenConsume $ toLazyBS ys
35+ in isJust mr && fst (fromJust mr) == snd (fromJust mr)
36+
25match :: Int -> NonNegative Int -> NonNegative Int -> Repack -> Bool
37match :: Int -> NonNegative Int -> NonNegative Int -> Repack -> Bool
26match n (NonNegative x) (NonNegative y) rs =
38match n (NonNegative x) (NonNegative y) rs =
27 parseBS (P.match parser) (repackBS rs input) == Just (input, n)
39 parseBS (P.match parser) (repackBS rs input) == Just (input, n)
@@ -31,9 +43,10 @@ match n (NonNegative x) (NonNegative y) rs =
31 B8.replicate x 'x', B8.pack (show n), B8.replicate y 'y'
43 B8.replicate x 'x', B8.pack (show n), B8.replicate y 'y'
32 ]
44 ]
33
45
34−tests :: [Test]
46+tests :: [TestTree]
35tests = [
47tests = [
36 testProperty "choice" choice
48 testProperty "choice" choice
37 , testProperty "count" count
49 , testProperty "count" count
50+ , testProperty "lookAhead" lookAhead
38 , testProperty "match" match
51 , testProperty "match" match
39 ]
52 ]
tests/QC/Common.hs+45−4
@@ -1,7 +1,9 @@
1+{-# LANGUAGE CPP, FlexibleInstances #-}
1{-# OPTIONS_GHC -fno-warn-orphans #-}
2{-# OPTIONS_GHC -fno-warn-orphans #-}
2module QC.Common
3module QC.Common
3 (
4 (
5+ ASCII(..)
4− parseBS
6+ , parseBS
5 , parseT
7 , parseT
6 , toLazyBS
8 , toLazyBS
7 , toStrictBS
9 , toStrictBS
@@ -10,10 +12,15 @@ module QC.Common
10 , repackBS_
12 , repackBS_
11 , repackT
13 , repackT
12 , repackT_
14 , repackT_
15+ , liftOp
13 ) where
16 ) where
14
17
18+#if !MIN_VERSION_base(4,8,0)
15−import Control.Applicative ((<$>), (<*>))
19+import Control.Applicative ((<*>), (<$>))
20+#endif
21+import Data.Char (isAlpha)
16import Test.QuickCheck
22import Test.QuickCheck
23+import Test.QuickCheck.Unicode (shrinkChar, string)
17import qualified Data.ByteString as B
24import qualified Data.ByteString as B
18import qualified Data.ByteString.Lazy as BL
25import qualified Data.ByteString.Lazy as BL
19import qualified Data.Text as T
26import qualified Data.Text as T
@@ -21,6 +28,16 @@ import qualified Data.Text.Lazy as TL
21import qualified Data.Attoparsec.ByteString.Lazy as BL
28import qualified Data.Attoparsec.ByteString.Lazy as BL
22import qualified Data.Attoparsec.Text.Lazy as TL
29import qualified Data.Attoparsec.Text.Lazy as TL
23
30
31+#if !MIN_VERSION_base(4,4,0)
32+-- This should really be a dependency on the random package :-(
33+instance Random Word8 where
34+ randomR = integralRandomR
35+ random = randomR (minBound,maxBound)
36+
37+instance Arbitrary Word8 where
38+ arbitrary = choose (minBound, maxBound)
39+#endif
40+
24parseBS :: BL.Parser r -> BL.ByteString -> Maybe r
41parseBS :: BL.Parser r -> BL.ByteString -> Maybe r
25parseBS p = BL.maybeResult . BL.parse p
42parseBS p = BL.maybeResult . BL.parse p
26
43
@@ -35,9 +52,22 @@ toLazyBS = BL.fromChunks . (:[])
35
52
36instance Arbitrary B.ByteString where
53instance Arbitrary B.ByteString where
37 arbitrary = B.pack <$> arbitrary
54 arbitrary = B.pack <$> arbitrary
55+ shrink = map B.pack . shrink . B.unpack
38
56
39instance Arbitrary BL.ByteString where
57instance Arbitrary BL.ByteString where
40 arbitrary = repackBS <$> arbitrary <*> arbitrary
58 arbitrary = repackBS <$> arbitrary <*> arbitrary
59+ shrink = map BL.pack . shrink . BL.unpack
60+
61+newtype ASCII a = ASCII { fromASCII :: a }
62+ deriving (Eq, Ord, Show)
63+
64+instance Arbitrary (ASCII B.ByteString) where
65+ arbitrary = (ASCII . B.pack) <$> listOf (choose (0,127))
66+ shrink = map (ASCII . B.pack) . shrink . B.unpack . fromASCII
67+
68+instance Arbitrary (ASCII BL.ByteString) where
69+ arbitrary = ASCII <$> (repackBS <$> arbitrary <*> (fromASCII <$> arbitrary))
70+ shrink = map (ASCII . BL.pack) . shrink . BL.unpack . fromASCII
41
71
42type Repack = NonEmptyList (Positive (Small Int))
72type Repack = NonEmptyList (Positive (Small Int))
43
73
@@ -54,10 +84,12 @@ repackBS_ = go . cycle
54 go _ _ = error "unpossible"
84 go _ _ = error "unpossible"
55
85
56instance Arbitrary T.Text where
86instance Arbitrary T.Text where
57− arbitrary = T.pack <$> arbitrary
87+ arbitrary = T.pack <$> string
88+ shrink = map T.pack . shrinkList shrinkChar . T.unpack
58
89
59instance Arbitrary TL.Text where
90instance Arbitrary TL.Text where
60− arbitrary = repackT <$> arbitrary <*> arbitrary
91+ arbitrary = TL.pack <$> string
92+ shrink = map TL.pack . shrinkList shrinkChar . TL.unpack
61
93
62repackT :: Repack -> T.Text -> TL.Text
94repackT :: Repack -> T.Text -> TL.Text
63repackT (NonEmpty bs) =
95repackT (NonEmpty bs) =
@@ -70,3 +102,12 @@ repackT_ = go . cycle
70 | otherwise = let (h,t) = T.splitAt b s
102 | otherwise = let (h,t) = T.splitAt b s
71 in h : go bs t
103 in h : go bs t
72 go _ _ = error "unpossible"
104 go _ _ = error "unpossible"
105+
106+liftOp :: (Show a, Testable prop) =>
107+ String -> (a -> a -> prop) -> a -> a -> Property
108+liftOp name f x y = counterexample desc (f x y)
109+ where op = case name of
110+ (c:_) | isAlpha c -> " `" ++ name ++ "` "
111+ | otherwise -> " " ++ name ++ " "
112+ _ -> " ??? "
113+ desc = "not (" ++ show x ++ op ++ show y ++ ")"
tests/QC/IPv6/Internal.hsnew file+316
@@ -0,0 +1,316 @@
1+-- -----------------------------------------------------------------------------
2+
3+-- |
4+-- Module : Text.IPv6Addr
5+-- Copyright : Copyright © Michel Boucey 2011-2015
6+-- License : BSD-Style
7+-- Maintainer : michel.boucey@gmail.com
8+--
9+-- Dealing with IPv6 address text representations, canonization and manipulations.
10+--
11+
12+-- -----------------------------------------------------------------------------
13+
14+{-# LANGUAGE OverloadedStrings #-}
15+
16+module QC.IPv6.Internal where {-
17+ ( expandTokens
18+ , macAddr
19+ , maybeIPv6AddrTokens
20+ , ipv4AddrToIPv6AddrTokens
21+ , ipv6TokensToText
22+ , ipv6TokensToIPv6Addr
23+ , isIPv6Addr
24+ , maybeTokIPv6Addr
25+ , maybeTokPureIPv6Addr
26+ , fromDoubleColon
27+ , fromIPv6Addr
28+ , toDoubleColon
29+ ) where -}
30+
31+import Debug.Trace
32+import Control.Monad (replicateM)
33+import Data.Attoparsec.Text
34+import Data.Char (isDigit,isHexDigit,toLower)
35+import Data.Monoid ((<>))
36+import Control.Applicative ((<|>),(<*))
37+import Data.List (group,isSuffixOf,elemIndex,elemIndices,intersperse)
38+import Data.Word (Word32)
39+import Numeric (showHex)
40+import qualified Data.Text as T
41+import qualified Data.Text.Read as R (decimal)
42+import Data.Maybe (fromJust)
43+
44+import QC.IPv6.Types
45+
46+tok0 = "0"
47+
48+-- | Returns the 'T.Text' of an IPv6 address.
49+fromIPv6Addr :: IPv6Addr -> T.Text
50+fromIPv6Addr (IPv6Addr t) = t
51+
52+-- | Given an arbitrary list of 'IPv6AddrToken', returns the corresponding 'T.Text'.
53+ipv6TokensToText :: [IPv6AddrToken] -> T.Text
54+ipv6TokensToText l = T.concat $ map ipv6TokenToText l
55+
56+-- | Returns the corresponding 'T.Text' of an IPv6 address token.
57+ipv6TokenToText :: IPv6AddrToken -> T.Text
58+ipv6TokenToText (SixteenBit s) = s
59+ipv6TokenToText Colon = ":"
60+ipv6TokenToText DoubleColon = "::"
61+ipv6TokenToText AllZeros = tok0 -- "A single 16-bit 0000 field MUST be represented as 0" (RFC 5952, 4.1)
62+ipv6TokenToText (IPv4Addr a) = a
63+
64+-- | Returns 'True' if a list of 'IPv6AddrToken' constitutes a valid IPv6 Address.
65+isIPv6Addr :: [IPv6AddrToken] -> Bool
66+isIPv6Addr [] = False
67+isIPv6Addr [DoubleColon] = True
68+isIPv6Addr [DoubleColon,SixteenBit tok1] = True
69+isIPv6Addr tks =
70+ diffNext tks && (do
71+ let cdctks = countDoubleColon tks
72+ let lentks = length tks
73+ let lasttk = last tks
74+ let lenconst = (lentks == 15 && cdctks == 0) || (lentks < 15 && cdctks == 1)
75+ firstValidToken tks &&
76+ (case countIPv4Addr tks of
77+ 0 -> case lasttk of
78+ SixteenBit _ -> lenconst
79+ DoubleColon -> lenconst
80+ AllZeros -> lenconst
81+ _ -> False
82+ 1 -> case lasttk of
83+ IPv4Addr _ -> (lentks == 13 && cdctks == 0) || (lentks < 12 && cdctks == 1)
84+ _ -> False
85+ otherwise -> False))
86+ where diffNext [] = False
87+ diffNext [_] = True
88+ diffNext (t:ts) = do
89+ let h = head ts
90+ case t of
91+ SixteenBit _ -> case h of
92+ SixteenBit _ -> False
93+ AllZeros -> False
94+ _ -> diffNext ts
95+ AllZeros -> case h of
96+ SixteenBit _ -> False
97+ AllZeros -> False
98+ _ -> diffNext ts
99+ _ -> diffNext ts
100+ firstValidToken l =
101+ case head l of
102+ SixteenBit _ -> True
103+ DoubleColon -> True
104+ AllZeros -> True
105+ _ -> False
106+ countDoubleColon l = length $ elemIndices DoubleColon l
107+ tok1 = "1"
108+
109+countIPv4Addr = foldr oneMoreIPv4Addr 0
110+ where
111+ oneMoreIPv4Addr t c = case t of
112+ IPv4Addr _ -> c + 1
113+ otherwise -> c
114+
115+-- | This is the main function which returns 'Just' the list of a tokenized IPv6
116+-- address text representation validated against RFC 4291 and canonized
117+-- in conformation with RFC 5952, or 'Nothing'.
118+maybeTokIPv6Addr :: T.Text -> Maybe [IPv6AddrToken]
119+maybeTokIPv6Addr t =
120+ case maybeIPv6AddrTokens t of
121+ Just ltks -> if isIPv6Addr ltks
122+ then Just $ (ipv4AddrReplacement . toDoubleColon . fromDoubleColon) ltks
123+ else Nothing
124+ Nothing -> Nothing
125+ where
126+ ipv4AddrReplacement ltks =
127+ if ipv4AddrRewrite ltks
128+ then init ltks ++ ipv4AddrToIPv6AddrTokens (last ltks)
129+ else ltks
130+
131+-- | Returns 'Just' the list of tokenized pure IPv6 address, always rewriting an
132+-- embedded IPv4 address if present.
133+maybeTokPureIPv6Addr :: T.Text -> Maybe [IPv6AddrToken]
134+maybeTokPureIPv6Addr t = do
135+ ltks <- maybeIPv6AddrTokens t
136+ if isIPv6Addr ltks
137+ then Just $ (toDoubleColon . ipv4AddrReplacement . fromDoubleColon) ltks
138+ else Nothing
139+ where
140+ ipv4AddrReplacement ltks' = init ltks' ++ ipv4AddrToIPv6AddrTokens (last ltks')
141+
142+-- | Tokenize a 'T.Text' into 'Just' a list of 'IPv6AddrToken', or 'Nothing'.
143+maybeIPv6AddrTokens :: T.Text -> Maybe [IPv6AddrToken]
144+maybeIPv6AddrTokens s =
145+ case readText s of
146+ Done r l -> traceShow (r,l) $ if r==T.empty then Just l else Nothing
147+ Fail {} -> Nothing
148+ where
149+ readText s = feed (parse (many1 $ ipv4Addr <|> sixteenBit <|> doubleColon <|> colon) s) T.empty
150+
151+-- | An embedded IPv4 address have to be rewritten to output a pure IPv6 Address
152+-- text representation in hexadecimal digits. But some well-known prefixed IPv6
153+-- addresses have to keep visible in their text representation the fact that
154+-- they deals with IPv4 to IPv6 transition process (RFC 5952 Section 5):
155+--
156+-- IPv4-compatible IPv6 address like "::1.2.3.4"
157+--
158+-- IPv4-mapped IPv6 address like "::ffff:1.2.3.4"
159+--
160+-- IPv4-translated address like "::ffff:0:1.2.3.4"
161+--
162+-- IPv4-translatable address like "64:ff9b::1.2.3.4"
163+--
164+-- ISATAP address like "fe80::5efe:1.2.3.4"
165+--
166+ipv4AddrRewrite :: [IPv6AddrToken] -> Bool
167+ipv4AddrRewrite tks =
168+ case last tks of
169+ IPv4Addr _ -> do
170+ let itks = init tks
171+ not (itks == [DoubleColon]
172+ || itks == [DoubleColon,SixteenBit tokffff,Colon]
173+ || itks == [DoubleColon,SixteenBit tokffff,Colon,AllZeros,Colon]
174+ || itks == [SixteenBit "64",Colon,SixteenBit "ff9b",DoubleColon]
175+ || [SixteenBit "200",Colon,SixteenBit tok5efe,Colon] `isSuffixOf` itks
176+ || [AllZeros,Colon,SixteenBit tok5efe,Colon] `isSuffixOf` itks
177+ || [DoubleColon,SixteenBit tok5efe,Colon] `isSuffixOf` itks)
178+ _ -> False
179+ where
180+ tokffff = "ffff"
181+ tok5efe = "5efe"
182+
183+-- | Rewrites an embedded 'IPv4Addr' into the corresponding list of pure 'IPv6Addr' tokens.
184+--
185+-- > ipv4AddrToIPv6AddrTokens (IPv4Addr "127.0.0.1") == [SixteenBits "7f0",Colon,SixteenBits "1"]
186+--
187+ipv4AddrToIPv6AddrTokens :: IPv6AddrToken -> [IPv6AddrToken]
188+ipv4AddrToIPv6AddrTokens t =
189+ case t of
190+ IPv4Addr a -> do
191+ let m = toHex a
192+ [ SixteenBit ((!!) m 0 <> addZero ((!!) m 1))
193+ , Colon
194+ , SixteenBit ((!!) m 2 <> addZero ((!!) m 3)) ]
195+ _ -> [t]
196+ where
197+ toHex a = map (\x -> T.pack $ showHex (read (T.unpack x)::Int) "") $ T.split (=='.') a
198+ addZero d = if T.length d == 1 then tok0 <> d else d
199+
200+expandTokens :: [IPv6AddrToken] -> [IPv6AddrToken]
201+expandTokens = map expandToken
202+ where expandToken (SixteenBit s) = SixteenBit $ T.justifyRight 4 '0' s
203+ expandToken AllZeros = SixteenBit "0000"
204+ expandToken t = t
205+
206+fromDoubleColon :: [IPv6AddrToken] -> [IPv6AddrToken]
207+fromDoubleColon tks =
208+ if DoubleColon `notElem` tks
209+ then tks
210+ else do let s = splitAt (fromJust $ elemIndex DoubleColon tks) tks
211+ let fsts = fst s
212+ let snds = if not (null (snd s)) then tail(snd s) else []
213+ let fste = if null fsts then [] else fsts ++ [Colon]
214+ let snde = if null snds then [] else Colon : snds
215+ fste ++ allZerosTokensReplacement(quantityOfAllZerosTokenToReplace tks) ++ snde
216+ where
217+ allZerosTokensReplacement x = intersperse Colon (replicate x AllZeros)
218+ quantityOfAllZerosTokenToReplace x =
219+ ntks tks - foldl (\c x -> if (x /= DoubleColon) && (x /= Colon) then c+1 else c) 0 x
220+ where
221+ ntks tks = if countIPv4Addr tks == 1 then 7 else 8
222+
223+toDoubleColon :: [IPv6AddrToken] -> [IPv6AddrToken]
224+toDoubleColon tks =
225+ zerosToDoubleColon tks (zerosRunToReplace $ zerosRunsList tks)
226+ where
227+ zerosToDoubleColon :: [IPv6AddrToken] -> (Int,Int) -> [IPv6AddrToken]
228+ -- No all zeros token, so no double colon replacement...
229+ zerosToDoubleColon ls (_,0) = ls
230+ -- "The symbol '::' MUST NOT be used to shorten just one 16-bit 0 field" (RFC 5952 4.2.2)
231+ zerosToDoubleColon ls (_,1) = ls
232+ zerosToDoubleColon ls (i,l) =
233+ let ls' = filter (/= Colon) ls
234+ in intersperse Colon (Prelude.take i ls') ++ [DoubleColon] ++ intersperse Colon (drop (i+l) ls')
235+ zerosRunToReplace t =
236+ let l = longestLengthZerosRun t
237+ in (firstLongestZerosRunIndex t l,l)
238+ where
239+ firstLongestZerosRunIndex x y = sum . snd . unzip $ Prelude.takeWhile (/=(True,y)) x
240+ longestLengthZerosRun x =
241+ maximum $ map longest x
242+ where longest t = case t of
243+ (True,i) -> i
244+ _ -> 0
245+ zerosRunsList x = map helper $ groupZerosRuns x
246+ where
247+ helper h = (head h == AllZeros, lh) where lh = length h
248+ groupZerosRuns = group . filter (/= Colon)
249+
250+ipv6TokensToIPv6Addr :: [IPv6AddrToken] -> Maybe IPv6Addr
251+ipv6TokensToIPv6Addr l = Just $ IPv6Addr $ ipv6TokensToText l
252+
253+fullSixteenBit :: T.Text -> Maybe IPv6AddrToken
254+fullSixteenBit t =
255+ case parse ipv6AddrFullChunk t of
256+ Done a b -> if a==T.empty then Just $ SixteenBit $ T.pack b else Nothing
257+ _ -> Nothing
258+
259+macAddr :: Parser (Maybe [IPv6AddrToken])
260+macAddr = do
261+ n1 <- count 2 hexaChar <* ":"
262+ n2 <- count 2 hexaChar <* ":"
263+ n3 <- count 2 hexaChar <* ":"
264+ n4 <- count 2 hexaChar <* ":"
265+ n5 <- count 2 hexaChar <* ":"
266+ n6 <- count 2 hexaChar
267+ return $ maybeIPv6AddrTokens $ T.pack $ concat [n1,n2,n3,n4,n5,n6]
268+
269+sixteenBit :: Parser IPv6AddrToken
270+sixteenBit = do
271+ r <- ipv6AddrFullChunk <|> count 3 hexaChar <|> count 2 hexaChar <|> count 1 hexaChar
272+ -- "Leading zeros MUST be suppressed" (RFC 5952, 4.1)
273+ let r' = T.dropWhile (=='0') $ T.pack r
274+ return $ if T.null r'
275+ then AllZeros
276+ -- Hexadecimal digits MUST be in lowercase (RFC 5952 4.3)
277+ else SixteenBit $ T.toLower r'
278+
279+ipv4Addr :: Parser IPv6AddrToken
280+ipv4Addr = do
281+ n1 <- manyDigits <* "."
282+ if n1 /= T.empty
283+ then do n2 <- manyDigits <* "."
284+ if n2 /= T.empty
285+ then do n3 <- manyDigits <* "."
286+ if n3 /= T.empty
287+ then do n4 <- manyDigits
288+ if n4 /= T.empty
289+ then return $ IPv4Addr $ T.intercalate "." [n1,n2,n3,n4]
290+ else parserFailure
291+ else parserFailure
292+ else parserFailure
293+ else parserFailure
294+ where
295+ parserFailure = fail "ipv4Addr parsing failure"
296+ manyDigits = do
297+ ds <- takeWhile1 isDigit
298+ case R.decimal ds of
299+ Right (n,_) -> return (if n < 256 then T.pack $ show n else T.empty)
300+ Left _ -> return T.empty
301+
302+doubleColon :: Parser IPv6AddrToken
303+doubleColon = do
304+ string "::"
305+ return DoubleColon
306+
307+colon :: Parser IPv6AddrToken
308+colon = do
309+ string ":"
310+ return Colon
311+
312+ipv6AddrFullChunk :: Parser String
313+ipv6AddrFullChunk = count 4 hexaChar
314+
315+hexaChar :: Parser Char
316+hexaChar = satisfy (inClass "0-9a-fA-F")
tests/QC/IPv6/Types.hsnew file+29
@@ -0,0 +1,29 @@
1+-- -----------------------------------------------------------------------------
2+
3+-- |
4+-- Module : Text.IPv6Addr
5+-- Copyright : Copyright © Michel Boucey 2011-2015
6+-- License : BSD-Style
7+-- Maintainer : michel.boucey@gmail.com
8+--
9+-- Dealing with IPv6 address text representations, canonization and manipulations.
10+--
11+
12+-- -----------------------------------------------------------------------------
13+
14+module QC.IPv6.Types where
15+
16+import qualified Data.Text as T
17+
18+data IPv6Addr = IPv6Addr T.Text
19+
20+instance Show IPv6Addr where
21+ show (IPv6Addr addr) = T.unpack addr
22+
23+data IPv6AddrToken
24+ = SixteenBit T.Text -- ^ A four hexadecimal digits group representing a 16-Bit chunk
25+ | AllZeros -- ^ An all zeros 16-Bit chunk
26+ | Colon -- ^ A separator between 16-Bit chunks
27+ | DoubleColon -- ^ A double-colon stands for a unique compression of many consecutive 16-Bit chunks
28+ | IPv4Addr T.Text -- ^ An embedded IPv4 address as representation of the last 32-Bit
29+ deriving (Eq,Show)
tests/QC/Rechunked.hsnew file+56
@@ -0,0 +1,56 @@
1+{-# LANGUAGE BangPatterns #-}
2+
3+module QC.Rechunked (
4+ rechunkBS
5+ , rechunkT
6+ ) where
7+
8+import Control.Monad (forM, forM_)
9+import Control.Monad.ST (ST, runST)
10+import Data.List (unfoldr)
11+import Test.QuickCheck (Gen, choose)
12+import qualified Data.ByteString as B
13+import qualified Data.Text as T
14+import qualified Data.Vector as V
15+import qualified Data.Vector.Generic as G
16+import qualified Data.Vector.Generic.Mutable as M
17+
18+rechunkBS :: B.ByteString -> Gen [B.ByteString]
19+rechunkBS = fmap (map B.copy) . rechunk_ B.splitAt B.length
20+
21+rechunkT :: T.Text -> Gen [T.Text]
22+rechunkT = fmap (map T.copy) . rechunk_ T.splitAt T.length
23+
24+rechunk_ :: (Int -> a -> (a,a)) -> (a -> Int) -> a -> Gen [a]
25+rechunk_ split len xs = (unfoldr go . (,) xs) `fmap` rechunkSizes (len xs)
26+ where go (b,r:rs) = Just (h, (t,rs))
27+ where (h,t) = split r b
28+ go (_,_) = Nothing
29+
30+rechunkSizes :: Int -> Gen [Int]
31+rechunkSizes n0 = shuffle =<< loop [] (0:repeat 1) n0
32+ where loop _ [] _ = error "it's 2015, where's my Stream type?"
33+ loop acc (lb:lbs) n
34+ | n <= 0 = shuffle (reverse acc)
35+ | otherwise = do
36+ !i <- choose (lb,n)
37+ loop (i:acc) lbs (n-i)
38+
39+shuffle :: [Int] -> Gen [Int]
40+shuffle (0:xs) = (0:) `fmap` shuffle xs
41+shuffle xs = fisherYates xs
42+
43+fisherYates :: [a] -> Gen [a]
44+fisherYates xs = (V.toList . V.backpermute v) `fmap` swapIndices (G.length v)
45+ where
46+ v = V.fromList xs
47+ swapIndices n0 = do
48+ swaps <- forM [0..n] $ \i -> ((,) i) `fmap` choose (i,n)
49+ return (runST (swapAll swaps))
50+ where
51+ n = n0 - 1
52+ swapAll :: [(Int,Int)] -> ST s (V.Vector Int)
53+ swapAll ijs = do
54+ mv <- G.unsafeThaw (G.enumFromTo 0 n :: V.Vector Int)
55+ forM_ ijs $ uncurry (M.swap mv)
56+ G.unsafeFreeze mv
tests/QC/Simple.hsnew file+37
@@ -0,0 +1,37 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
3+
4+module QC.Simple (
5+ tests
6+ ) where
7+
8+import Control.Applicative ((<|>))
9+import Data.ByteString (ByteString)
10+import Data.List (foldl')
11+import Data.Maybe (fromMaybe)
12+import QC.Rechunked (rechunkBS)
13+import Test.Tasty (TestTree)
14+import Test.Tasty.QuickCheck (testProperty)
15+import Test.QuickCheck (Property, counterexample, forAll)
16+import qualified Data.Attoparsec.ByteString.Char8 as A
17+
18+t_issue75 = expect issue75 "ab" (A.Done "" "b")
19+
20+issue75 :: A.Parser ByteString
21+issue75 = "a" >> ("b" <|> "")
22+
23+expect :: (Show r, Eq r) => A.Parser r -> ByteString -> A.Result r -> Property
24+expect p input wanted =
25+ forAll (rechunkBS input) $ \in' ->
26+ let result = parse p in'
27+ in counterexample (show result ++ " /= " ++ show wanted) $
28+ fromMaybe False (A.compareResults result wanted)
29+
30+parse :: A.Parser r -> [ByteString] -> A.Result r
31+parse p (x:xs) = foldl' A.feed (A.parse p x) xs
32+parse p [] = A.parse p ""
33+
34+tests :: [TestTree]
35+tests = [
36+ testProperty "issue75" t_issue75
37+ ]
tests/QC/Text.hs+100−66
@@ -1,67 +1,84 @@
1−{-# LANGUAGE BangPatterns, OverloadedStrings #-}
1+{-# LANGUAGE BangPatterns, CPP, OverloadedStrings #-}
2{-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
2{-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
3module QC.Text (tests) where
3module QC.Text (tests) where
4
4
5+#if !MIN_VERSION_base(4,8,0)
5−import Control.Applicative ((<$>), (<*>))
6+import Control.Applicative ((<*>), (<$>))
7+#endif
6import Data.Int (Int64)
8import Data.Int (Int64)
9+import Data.Word (Word8)
7import Prelude hiding (take, takeWhile)
10import Prelude hiding (take, takeWhile)
8−import QC.Common (parseT)
11+import QC.Common (liftOp, parseT)
9−import Test.Framework (Test)
12+import qualified QC.Text.FastSet as FastSet
13+import qualified QC.Text.Regressions as Regressions
14+import Test.Tasty (TestTree, testGroup)
10−import Test.Framework.Providers.QuickCheck2 (testProperty)
15+import Test.Tasty.QuickCheck (testProperty)
11import Test.QuickCheck
16import Test.QuickCheck
12import qualified Data.Attoparsec.Text as P
17import qualified Data.Attoparsec.Text as P
13import qualified Data.Attoparsec.Text.Lazy as PL
18import qualified Data.Attoparsec.Text.Lazy as PL
19+import qualified Data.Attoparsec.Text.FastSet as S
20+import qualified Data.ByteString as BS
14import qualified Data.Char as Char
21import qualified Data.Char as Char
15import qualified Data.Text as T
22import qualified Data.Text as T
23+import qualified Data.Text.Encoding as TE
16import qualified Data.Text.Lazy as L
24import qualified Data.Text.Lazy as L
17
25
18-- Basic byte-level combinators.
26-- Basic byte-level combinators.
19
27
20−satisfy :: Char -> L.Text -> Bool
28+satisfy :: Char -> L.Text -> Property
21−satisfy w s = parseT (P.satisfy (<=w)) (L.cons w s) == Just w
29+satisfy w s = parseT (P.satisfy (<=w)) (L.cons w s) === Just w
22
30
23−satisfyWith :: Char -> L.Text -> Bool
31+satisfyWith :: Char -> L.Text -> Property
24−satisfyWith c s = parseT (P.satisfyWith id (<=c)) (L.cons c s) == Just c
32+satisfyWith c s = parseT (P.satisfyWith id (<=c)) (L.cons c s) === Just c
25
33
26−char :: Char -> L.Text -> Bool
34+char :: Char -> L.Text -> Property
27−char w s = parseT (P.char w) (L.cons w s) == Just w
35+char w s = parseT (P.char w) (L.cons w s) === Just w
28
36
29−skip :: Char -> L.Text -> Bool
37+skip :: Char -> L.Text -> Property
30skip w s =
38skip w s =
31 case (parseT (P.skip (<w)) s, L.uncons s) of
39 case (parseT (P.skip (<w)) s, L.uncons s) of
32− (Nothing, mcs) -> maybe True (not . it) mcs
40+ (Nothing, mcs) -> maybe (property True) (expectFailure . it) mcs
33− (Just _, mcs) -> maybe False it mcs
41+ (Just _, mcs) -> maybe (property False) it mcs
34− where it cs = fst cs < w
42+ where it cs = liftOp "<" (<) (fst cs) w
35
43
36−anyChar :: L.Text -> Bool
44+anyChar :: L.Text -> Property
37anyChar s
45anyChar s
38− | L.null s = p == Nothing
46+ | L.null s = p === Nothing
39− | otherwise = p == Just (L.head s)
47+ | otherwise = p === Just (L.head s)
40 where p = parseT P.anyChar s
48 where p = parseT P.anyChar s
41
49
42−notChar :: Char -> NonEmptyList Char -> Bool
50+notChar :: Char -> NonEmptyList Char -> Property
43−notChar w (NonEmpty s) = parseT (P.notChar w) bs == if v == w
51+notChar w (NonEmpty s) = parseT (P.notChar w) bs === if v == w
44 then Nothing
52 then Nothing
45 else Just v
53 else Just v
46 where v = L.head bs
54 where v = L.head bs
47 bs = L.pack s
55 bs = L.pack s
48
56
49−peekChar :: L.Text -> Bool
57+peekChar :: L.Text -> Property
50peekChar s
58peekChar s
51− | L.null s = p == Just (Nothing, s)
59+ | L.null s = p === Just (Nothing, s)
52− | otherwise = p == Just (Just (L.head s), s)
60+ | otherwise = p === Just (Just (L.head s), s)
53 where p = parseT ((,) <$> P.peekChar <*> P.takeLazyText) s
61 where p = parseT ((,) <$> P.peekChar <*> P.takeLazyText) s
54
62
55−peekChar' :: L.Text -> Bool
63+peekChar' :: L.Text -> Property
56−peekChar' s = parseT P.peekChar' s == (fst <$> L.uncons s)
64+peekChar' s = parseT P.peekChar' s === (fst <$> L.uncons s)
57
65
58−string :: L.Text -> L.Text -> Bool
66+string :: L.Text -> L.Text -> Property
59−string s t = parseT (P.string s') (s `L.append` t) == Just s'
67+string s t = parseT (P.string s') (s `L.append` t) === Just s'
60 where s' = toStrict s
68 where s' = toStrict s
61
69
62−stringCI :: T.Text -> Bool
70+strings :: L.Text -> L.Text -> L.Text -> Property
71+strings s t u =
72+ parseT (P.string (toStrict s) >> P.string t') (L.concat [s,t,u])
73+ === Just t'
74+ where t' = toStrict t
75+
76+-- | Note: "simple, and efficient" works for well formed input...
77+-- i.e. e.g. Latin1 texts
78+stringCI :: [Word8] -> Property
63−stringCI s = P.parseOnly (P.stringCI fs) s == Right s
79+stringCI ws = P.parseOnly (P.stringCI fs) s === Right s
64 where fs = T.toCaseFold s
80 where fs = T.toCaseFold s
81+ s = TE.decodeLatin1 (BS.pack ws)
65
82
66asciiCI :: T.Text -> Gen Bool
83asciiCI :: T.Text -> Gen Bool
67asciiCI x =
84asciiCI x =
@@ -76,72 +93,84 @@ asciiCI x =
76toStrict :: L.Text -> T.Text
93toStrict :: L.Text -> T.Text
77toStrict = T.concat . L.toChunks
94toStrict = T.concat . L.toChunks
78
95
79−skipWhile :: Char -> L.Text -> Bool
96+skipWhile :: Char -> L.Text -> Property
80skipWhile w s =
97skipWhile w s =
81 let t = L.dropWhile (<=w) s
98 let t = L.dropWhile (<=w) s
82 in case PL.parse (P.skipWhile (<=w)) s of
99 in case PL.parse (P.skipWhile (<=w)) s of
83− PL.Done t' () -> t == t'
100+ PL.Done t' () -> t === t'
84− _ -> False
101+ _ -> property False
85
102
86−take :: Int -> L.Text -> Bool
103+take :: Int -> L.Text -> Property
87−take n s = maybe (L.length s < fromIntegral n) (== T.take n (toStrict s)) $
104+take n s = maybe (liftOp "<" (<) (L.length s) (fromIntegral n))
105+ (=== T.take n (toStrict s)) $
88 parseT (P.take n) s
106 parseT (P.take n) s
89
107
90−takeText :: L.Text -> Bool
108+takeText :: L.Text -> Property
91−takeText s = maybe False (== toStrict s) . parseT P.takeText $ s
109+takeText s = maybe (property False) (=== toStrict s) . parseT P.takeText $ s
92
110
93−takeLazyText :: L.Text -> Bool
111+takeLazyText :: L.Text -> Property
94−takeLazyText s = maybe False (== s) . parseT P.takeLazyText $ s
112+takeLazyText s = maybe (property False) (=== s) . parseT P.takeLazyText $ s
95
113
96−takeCount :: Positive Int -> L.Text -> Bool
114+takeCount :: Positive Int -> L.Text -> Property
97takeCount (Positive k) s =
115takeCount (Positive k) s =
98 case parseT (P.take k) s of
116 case parseT (P.take k) s of
99− Nothing -> fromIntegral k > L.length s
117+ Nothing -> liftOp ">" (>) (fromIntegral k) (L.length s)
100− Just _s -> fromIntegral k <= L.length s
118+ Just _s -> liftOp "<=" (<=) (fromIntegral k) (L.length s)
101
119
102−takeWhile :: Char -> L.Text -> Bool
120+takeWhile :: Char -> L.Text -> Property
103takeWhile w s =
121takeWhile w s =
104 let (h,t) = L.span (==w) s
122 let (h,t) = L.span (==w) s
105 in case PL.parse (P.takeWhile (==w)) s of
123 in case PL.parse (P.takeWhile (==w)) s of
106− PL.Done t' h' -> t == t' && toStrict h == h'
124+ PL.Done t' h' -> t === t' .&&. toStrict h === h'
107− _ -> False
125+ _ -> property False
108
126
109−takeWhile1 :: Char -> L.Text -> Bool
127+takeWhile1 :: Char -> L.Text -> Property
110takeWhile1 w s =
128takeWhile1 w s =
111− let s' = L.cons w s
129+ let s' = L.cons w s
112− (h,t) = L.span (<=w) s'
130+ (h,t) = L.span (<=w) s'
113 in case PL.parse (P.takeWhile1 (<=w)) s' of
131 in case PL.parse (P.takeWhile1 (<=w)) s' of
114− PL.Done t' h' -> t == t' && toStrict h == h'
132+ PL.Done t' h' -> t === t' .&&. toStrict h === h'
115− _ -> False
133+ _ -> property False
116
134
117−takeTill :: Char -> L.Text -> Bool
135+takeTill :: Char -> L.Text -> Property
118takeTill w s =
136takeTill w s =
119 let (h,t) = L.break (==w) s
137 let (h,t) = L.break (==w) s
120 in case PL.parse (P.takeTill (==w)) s of
138 in case PL.parse (P.takeTill (==w)) s of
121− PL.Done t' h' -> t == t' && toStrict h == h'
139+ PL.Done t' h' -> t === t' .&&. toStrict h === h'
122− _ -> False
140+ _ -> property False
123
141
124−takeWhile1_empty :: Bool
142+takeWhile1_empty :: Property
125−takeWhile1_empty = parseT (P.takeWhile1 undefined) L.empty == Nothing
143+takeWhile1_empty = parseT (P.takeWhile1 undefined) L.empty === Nothing
126
144
127−endOfInput :: L.Text -> Bool
145+endOfInput :: L.Text -> Property
128−endOfInput s = parseT P.endOfInput s == if L.null s
146+endOfInput s = parseT P.endOfInput s === if L.null s
129− then Just ()
147+ then Just ()
130− else Nothing
148+ else Nothing
131
149
132−endOfLine :: L.Text -> Bool
150+endOfLine :: L.Text -> Property
133endOfLine s =
151endOfLine s =
134 case (parseT P.endOfLine s, L.uncons s) of
152 case (parseT P.endOfLine s, L.uncons s) of
135− (Nothing, mcs) -> maybe True (not . eol) mcs
153+ (Nothing, mcs) -> maybe (property True) (expectFailure . eol) mcs
136− (Just _, mcs) -> maybe False eol mcs
154+ (Just _, mcs) -> maybe (property False) eol mcs
155+ where eol (c,s') = c === '\n' .||.
137− where eol (c,s') = c == '\n' || (c, fst <$> L.uncons s') == ('\r', Just '\n')
156+ (c, fst <$> L.uncons s') === ('\r', Just '\n')
138−
157+
139−scan :: L.Text -> Positive Int64 -> Bool
158+scan :: L.Text -> Positive Int64 -> Property
159+-- for some reason, if counterexample is removed, this test fails?
160+scan s (Positive k) = counterexample (show s)
140−scan s (Positive k) = parseT p s == (Just $ toStrict $ L.take k s)
161+ $ parseT p s === Just (toStrict $ L.take k s)
141 where p = P.scan k $ \ n _ ->
162 where p = P.scan k $ \ n _ ->
142 if n > 0 then let !n' = n - 1 in Just n' else Nothing
163 if n > 0 then let !n' = n - 1 in Just n' else Nothing
143
164
165+members :: String -> Property
166+members s = property $ all (`S.member` set) s
167+ where set = S.fromList s
168+
169+nonmembers :: String -> String -> Property
170+nonmembers s s' = property . not . any (`S.member` set) $ filter (not . (`elem` s)) s'
171+ where set = S.fromList s
172+
144−tests :: [Test]
173+tests :: [TestTree]
145tests = [
174tests = [
146 testProperty "anyChar" anyChar
175 testProperty "anyChar" anyChar
147 , testProperty "asciiCI" asciiCI
176 , testProperty "asciiCI" asciiCI
@@ -157,6 +186,7 @@ tests = [
157 , testProperty "skip" skip
186 , testProperty "skip" skip
158 , testProperty "skipWhile" skipWhile
187 , testProperty "skipWhile" skipWhile
159 , testProperty "string" string
188 , testProperty "string" string
189+ , testProperty "strings" strings
160 , testProperty "stringCI" stringCI
190 , testProperty "stringCI" stringCI
161 , testProperty "take" take
191 , testProperty "take" take
162 , testProperty "takeText" takeText
192 , testProperty "takeText" takeText
@@ -166,4 +196,8 @@ tests = [
166 , testProperty "takeWhile" takeWhile
196 , testProperty "takeWhile" takeWhile
167 , testProperty "takeWhile1" takeWhile1
197 , testProperty "takeWhile1" takeWhile1
168 , testProperty "takeWhile1_empty" takeWhile1_empty
198 , testProperty "takeWhile1_empty" takeWhile1_empty
199+ , testProperty "members" members
200+ , testProperty "nonmembers" nonmembers
201+ , testGroup "FastSet" FastSet.tests
202+ , testGroup "Regressions" Regressions.tests
169 ]
203 ]
tests/QC/Text/FastSet.hsnew file+15
@@ -0,0 +1,15 @@
1+module QC.Text.FastSet where
2+
3+import Test.Tasty (TestTree)
4+import Test.Tasty.QuickCheck (testProperty)
5+import Test.QuickCheck
6+import qualified Data.Attoparsec.Text.FastSet as FastSet
7+
8+membershipCorrect :: String -> String -> Property
9+membershipCorrect members others =
10+ let fs = FastSet.fromList members
11+ correct c = (c `FastSet.member` fs) == (c `elem` members)
12+ in property $ all correct (members ++ others)
13+
14+tests :: [TestTree]
15+tests = [ testProperty "membership is correct" membershipCorrect ]
tests/QC/Text/Regressions.hsnew file+54
@@ -0,0 +1,54 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+module QC.Text.Regressions (
4+ tests
5+ ) where
6+
7+import Data.Text (Text)
8+import qualified Data.Text as T
9+import Data.List (foldl')
10+import Data.Maybe (fromMaybe)
11+import Data.Char (isLower)
12+import Data.Monoid ((<>))
13+import QC.Rechunked (rechunkT)
14+import Test.Tasty (TestTree)
15+import Test.Tasty.QuickCheck (testProperty)
16+import Test.QuickCheck (Property, counterexample, forAll)
17+import qualified Data.Attoparsec.Text as A
18+
19+
20+--------------------------------------------------------------------------------
21+-- 105 was about runScanner not always returning the final state. The result
22+-- did depend on how the data was fed to the parser.
23+
24+t_issue105 :: Property
25+t_issue105 = expect issue105 "lowER" (A.Done "ER" "low")
26+
27+issue105 :: A.Parser Text
28+issue105 = do
29+ (_, firstFourLowercaseLetters) <- A.runScanner "" f
30+ return $ firstFourLowercaseLetters
31+
32+ where
33+ f :: Text -> Char -> Maybe Text
34+ f acc c = if T.length acc < 4 && isLower c
35+ then Just $ acc <> T.singleton c
36+ else Nothing
37+
38+
39+expect :: (Show r, Eq r) => A.Parser r -> Text -> A.Result r -> Property
40+expect p input wanted =
41+ forAll (rechunkT input) $ \in' ->
42+ let result = parse p in'
43+ in counterexample (show result ++ " /= " ++ show wanted) $
44+ fromMaybe False (A.compareResults result wanted)
45+
46+parse :: A.Parser r -> [Text] -> A.Result r
47+parse p (x:xs) = foldl' A.feed (A.parse p x) xs
48+parse p [] = A.parse p ""
49+
50+
51+tests :: [TestTree]
52+tests = [
53+ testProperty "issue105" t_issue105
54+ ]
tests/TestFastSet.hsdeleted−25
@@ -1,25 +0,0 @@
1−module Main (main) where
2−
3−import Data.Word (Word8)
4−import qualified Data.Attoparsec.FastSet as F
5−import System.Random (Random(..), RandomGen)
6−import Test.QuickCheck
7−
8−integralRandomR :: (Integral a, RandomGen g) => (a, a) -> g -> (a, g)
9−integralRandomR (a,b) g = case randomR (fromIntegral a :: Int,
10− fromIntegral b :: Int) g
11− of (x,g') -> (fromIntegral x, g')
12−
13−instance Random Word8 where
14− randomR = integralRandomR
15− random = randomR (minBound,maxBound)
16−
17−instance Arbitrary Word8 where
18− arbitrary = choose (minBound, maxBound)
19−
20−prop_AllMembers s =
21− let set = F.fromList s
22− in all (`F.memberWord8` set) s
23−
24−main = do
25− quickCheck prop_AllMembers