git.iko.soy/typeable/attoparsec
on master help sign in
51 files changed+1462−293
since splitsnapshots
unifiedsplit
.gitignore+1 .hgtags+5 .travis.yml+161 Data/Attoparsec.hs+1−1 Data/Attoparsec/ByteString.hs+7−4 Data/Attoparsec/ByteString/Buffer.hs+11−9 Data/Attoparsec/ByteString/Char8.hs+6−17 Data/Attoparsec/ByteString/FastSet.hs+1−1 Data/Attoparsec/ByteString/Internal.hs+84−33 Data/Attoparsec/ByteString/Lazy.hs+5−3 Data/Attoparsec/Char8.hs+1−1 Data/Attoparsec/Combinator.hs+12−5 Data/Attoparsec/Internal.hs+31−5 Data/Attoparsec/Internal/Types.hs+46−33 Data/Attoparsec/Lazy.hs+1−1 Data/Attoparsec/Number.hs+1−1 Data/Attoparsec/Text.hs+9−6 Data/Attoparsec/Text/Buffer.hs+19−9 Data/Attoparsec/Text/FastSet.hs+4−2 Data/Attoparsec/Text/Internal.hs+77−35 Data/Attoparsec/Text/Lazy.hs+6−8 Data/Attoparsec/Types.hs+1−1 Data/Attoparsec/Zepto.hs+82−44 README.markdown−6 attoparsec.cabal+58−13 benchmarks/Aeson.hs+6−2 benchmarks/Benchmarks.hs+8 benchmarks/Genome.hs+75 benchmarks/HeadersByteString.hs+10−1 benchmarks/HeadersByteString/Atto.hs+9−1 benchmarks/Links.hs+2−2 benchmarks/Numbers.hs+1 benchmarks/Sets.hs+21 benchmarks/TextFastSet.hs+121 benchmarks/attoparsec-benchmarks.cabal+4 benchmarks/http-request.txt+3−8 benchmarks/http-response.txt+2−4 benchmarks/warp-3.0.1.1/Network/Wai/Handler/Warp/RequestHeader.hs+1−1 changelog.md+49 tests/QC.hs+2−2 tests/QC/Buffer.hs+4−4 tests/QC/ByteString.hs+25−9 tests/QC/Combinator.hs+18−5 tests/QC/Common.hs+27−4 tests/QC/IPv6/Internal.hs+316 tests/QC/IPv6/Types.hs+29 tests/QC/Rechunked.hs+1−1 tests/QC/Simple.hs+3−3 tests/QC/Text.hs+26−8 tests/QC/Text/FastSet.hs+15 tests/QC/Text/Regressions.hs+54
.gitignore+1
@@ -3,3 +3,4 @@
33/dist/
44.cabal-sandbox
55cabal.sandbox.config
6+*.sw[op]
.hgtags+5
@@ -56,3 +56,8 @@ c9c2b05707b2fffcd749d5817e3e0d04e6ad6d0d 0.12.1.0
56569157c2b4c35aa5f683b4ac51e6ff52c696bc22e2 0.12.1.1
5757682ba515f7ba62500ad79371b37b90932e0c0e3d 0.12.1.2
58582066a72b79b478f59a9c7f1839c522fc59ba9af7 0.12.1.3
59+ef875ad3bebef2b6fb1bd7063de6a46a06d0d4b4 0.12.1.4
60+dc1fcff28d13a508f677c7b06af6420877842493 0.12.1.5
61+013e04e4526bfd5e9e4a611b42e6f03c413bea91 0.12.1.6
62+0ddbce995b77b8db391b4d6519f13feddf4b5bf3 0.13.0.0
63+46c16463ec20ad46c9cd0bfb1050c4268f34b749 0.13.0.1
.travis.ymlnew file+161
@@ -0,0 +1,161 @@
1+# This Travis job script has been generated by a script via
2+#
3+# haskell-ci '--tests-jobs=>=7.4' '--no-benchmarks' 'attoparsec.cabal'
4+#
5+# For more information, see https://github.com/haskell-CI/haskell-ci
6+#
7+# version: 0.3.20190815
8+#
9+language: c
10+dist: xenial
11+git:
12+ # whether to recursively clone submodules
13+ submodules: false
14+cache:
15+ directories:
16+ - $HOME/.cabal/packages
17+ - $HOME/.cabal/store
18+before_cache:
19+ - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log
20+ # remove files that are regenerated by 'cabal update'
21+ - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.*
22+ - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json
23+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache
24+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar
25+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx
26+ - rm -rfv $CABALHOME/packages/head.hackage
27+matrix:
28+ include:
29+ - compiler: ghc-8.8.1
30+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.8.1","cabal-install-3.0"]}}
31+ - compiler: ghc-8.6.5
32+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.6.5","cabal-install-2.4"]}}
33+ - compiler: ghc-8.4.4
34+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.4.4","cabal-install-2.4"]}}
35+ - compiler: ghc-8.2.2
36+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.2.2","cabal-install-2.4"]}}
37+ - compiler: ghc-8.0.2
38+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-8.0.2","cabal-install-2.4"]}}
39+ - compiler: ghc-7.10.3
40+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.10.3","cabal-install-2.4"]}}
41+ - compiler: ghc-7.8.4
42+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.8.4","cabal-install-2.4"]}}
43+ - compiler: ghc-7.6.3
44+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.6.3","cabal-install-2.4"]}}
45+ - compiler: ghc-7.4.2
46+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.4.2","cabal-install-2.4"]}}
47+ - compiler: ghc-7.2.2
48+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.2.2","cabal-install-2.4"]}}
49+ - compiler: ghc-7.0.4
50+ addons: {"apt":{"sources":["hvr-ghc"],"packages":["ghc-7.0.4","cabal-install-2.4"]}}
51+before_install:
52+ - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//')
53+ - WITHCOMPILER="-w $HC"
54+ - HCPKG="$HC-pkg"
55+ - unset CC
56+ - CABAL=/opt/ghc/bin/cabal
57+ - CABALHOME=$HOME/.cabal
58+ - export PATH="$CABALHOME/bin:$PATH"
59+ - TOP=$(pwd)
60+ - "HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\\d+)\\.(\\d+)\\.(\\d+)(\\.(\\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))')"
61+ - echo $HCNUMVER
62+ - CABAL="$CABAL -vnormal+nowrap+markoutput"
63+ - set -o pipefail
64+ - |
65+ echo 'function blue(s) { printf "\033[0;34m" s "\033[0m " }' >> .colorful.awk
66+ echo 'BEGIN { state = "output"; }' >> .colorful.awk
67+ echo '/^-----BEGIN CABAL OUTPUT-----$/ { state = "cabal" }' >> .colorful.awk
68+ echo '/^-----END CABAL OUTPUT-----$/ { state = "output" }' >> .colorful.awk
69+ echo '!/^(-----BEGIN CABAL OUTPUT-----|-----END CABAL OUTPUT-----)/ {' >> .colorful.awk
70+ echo ' if (state == "cabal") {' >> .colorful.awk
71+ echo ' print blue($0)' >> .colorful.awk
72+ echo ' } else {' >> .colorful.awk
73+ echo ' print $0' >> .colorful.awk
74+ echo ' }' >> .colorful.awk
75+ echo '}' >> .colorful.awk
76+ - cat .colorful.awk
77+ - |
78+ color_cabal_output () {
79+ awk -f $TOP/.colorful.awk
80+ }
81+ - echo text | color_cabal_output
82+install:
83+ - ${CABAL} --version
84+ - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"
85+ - TEST=--enable-tests
86+ - if [ $HCNUMVER -lt 70400 ] ; then TEST=--disable-tests ; fi
87+ - BENCH=--enable-benchmarks
88+ - BENCH=--disable-benchmarks
89+ - HEADHACKAGE=false
90+ - rm -f $CABALHOME/config
91+ - |
92+ echo "verbose: normal +nowrap +markoutput" >> $CABALHOME/config
93+ echo "remote-build-reporting: anonymous" >> $CABALHOME/config
94+ echo "write-ghc-environment-files: always" >> $CABALHOME/config
95+ echo "remote-repo-cache: $CABALHOME/packages" >> $CABALHOME/config
96+ echo "logs-dir: $CABALHOME/logs" >> $CABALHOME/config
97+ echo "world-file: $CABALHOME/world" >> $CABALHOME/config
98+ echo "extra-prog-path: $CABALHOME/bin" >> $CABALHOME/config
99+ echo "symlink-bindir: $CABALHOME/bin" >> $CABALHOME/config
100+ echo "installdir: $CABALHOME/bin" >> $CABALHOME/config
101+ echo "build-summary: $CABALHOME/logs/build.log" >> $CABALHOME/config
102+ echo "store-dir: $CABALHOME/store" >> $CABALHOME/config
103+ echo "install-dirs user" >> $CABALHOME/config
104+ echo " prefix: $CABALHOME" >> $CABALHOME/config
105+ echo "repository hackage.haskell.org" >> $CABALHOME/config
106+ echo " url: http://hackage.haskell.org/" >> $CABALHOME/config
107+ - cat $CABALHOME/config
108+ - rm -fv cabal.project cabal.project.local cabal.project.freeze
109+ - travis_retry ${CABAL} v2-update -v
110+ # Generate cabal.project
111+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
112+ - touch cabal.project
113+ - |
114+ echo "packages: ." >> cabal.project
115+ - |
116+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(attoparsec)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
117+ - cat cabal.project || true
118+ - cat cabal.project.local || true
119+ - if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi
120+ - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH} | color_cabal_output
121+ - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
122+ - rm cabal.project.freeze
123+ - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} --dep -j2 all | color_cabal_output
124+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --dep -j2 all | color_cabal_output
125+script:
126+ - DISTDIR=$(mktemp -d /tmp/dist-test.XXXX)
127+ # Packaging...
128+ - ${CABAL} v2-sdist all | color_cabal_output
129+ # Unpacking...
130+ - mv dist-newstyle/sdist/*.tar.gz ${DISTDIR}/
131+ - cd ${DISTDIR} || false
132+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec tar -xvf '{}' \;
133+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \;
134+ - PKGDIR_attoparsec="$(find . -maxdepth 1 -type d -regex '.*/attoparsec-[0-9.]*')"
135+ # Generate cabal.project
136+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
137+ - touch cabal.project
138+ - |
139+ echo "packages: ${PKGDIR_attoparsec}" >> cabal.project
140+ - |
141+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(attoparsec)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
142+ - cat cabal.project || true
143+ - cat cabal.project.local || true
144+ # Building...
145+ # this builds all libraries and executables (without tests/benchmarks)
146+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all | color_cabal_output
147+ # Building with tests and benchmarks...
148+ # build & run tests, build benchmarks
149+ - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} all | color_cabal_output
150+ # Testing...
151+ - if [ $HCNUMVER -ge 70400 ] ; then ${CABAL} v2-test $WITHCOMPILER ${TEST} ${BENCH} all | color_cabal_output ; fi
152+ # cabal check...
153+ - (cd ${PKGDIR_attoparsec} && ${CABAL} -vnormal check)
154+ # haddock...
155+ - ${CABAL} v2-haddock $WITHCOMPILER ${TEST} ${BENCH} all | color_cabal_output
156+ # Building without installed constraints for packages in global-db...
157+ - rm -f cabal.project.local
158+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all | color_cabal_output
159+
160+# REGENDATA ["--tests-jobs=>=7.4","--no-benchmarks","attoparsec.cabal"]
161+# EOF
Data/Attoparsec.hs+1−1
@@ -1,6 +1,6 @@
11-- |
22-- Module : Data.Attoparsec
3−-- Copyright : Bryan O'Sullivan 2007-2014
3+-- Copyright : Bryan O'Sullivan 2007-2015
44-- License : BSD3
55--
66-- Maintainer : bos@serpentine.com
Data/Attoparsec/ByteString.hs+7−4
@@ -4,7 +4,7 @@
44#endif
55-- |
66-- Module : Data.Attoparsec.ByteString
7−-- Copyright : Bryan O'Sullivan 2007-2014
7+-- Copyright : Bryan O'Sullivan 2007-2015
88-- License : BSD3
99--
1010-- Maintainer : bos@serpentine.com
@@ -63,6 +63,7 @@ module Data.Attoparsec.ByteString
6363 , I.skipWhile
6464 , I.take
6565 , I.scan
66+ , I.runScanner
6667 , I.takeWhile
6768 , I.takeWhile1
6869 , I.takeTill
@@ -96,6 +97,7 @@ module Data.Attoparsec.ByteString
9697 ) where
9798
9899import Data.Attoparsec.Combinator
100+import Data.List (intercalate)
99101import qualified Data.Attoparsec.ByteString.Internal as I
100102import qualified Data.Attoparsec.Internal as I
101103import qualified Data.ByteString as B
@@ -222,6 +224,7 @@ maybeResult _ = Nothing
222224-- | Convert a 'Result' value to an 'Either' value. A 'T.Partial'
223225-- result is treated as failure.
224226eitherResult :: Result r -> Either String r
225−eitherResult (T.Done _ r) = Right r
226−eitherResult (T.Fail _ _ msg) = Left msg
227−eitherResult _ = Left "Result: incomplete input"
227+eitherResult (T.Done _ r) = Right r
228+eitherResult (T.Fail _ [] msg) = Left msg
229+eitherResult (T.Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
230+eitherResult _ = Left "Result: incomplete input"
Data/Attoparsec/ByteString/Buffer.hs+11−9
@@ -1,7 +1,7 @@
1−{-# LANGUAGE BangPatterns, CPP #-}
1+{-# LANGUAGE BangPatterns #-}
22-- |
33-- Module : Data.Attoparsec.ByteString.Buffer
4−-- Copyright : Bryan O'Sullivan 2007-2014
4+-- Copyright : Bryan O'Sullivan 2007-2015
55-- License : BSD3
66--
77-- Maintainer : bos@serpentine.com
@@ -57,9 +57,8 @@ import Control.Exception (assert)
5757import Data.ByteString.Internal (ByteString(..), memcpy, nullForeignPtr)
5858import Data.Attoparsec.Internal.Fhthagn (inlinePerformIO)
5959import Data.List (foldl1')
60−#if !MIN_VERSION_base(4,8,0)
61−import Data.Monoid (Monoid(..))
62−#endif
60+import Data.Monoid as Mon (Monoid(..))
61+import Data.Semigroup (Semigroup(..))
6362import Data.Word (Word8)
6463import Foreign.ForeignPtr (ForeignPtr, withForeignPtr)
6564import Foreign.Ptr (castPtr, plusPtr)
@@ -88,14 +87,17 @@ buffer (PS fp off len) = Buf fp off len len 0
8887unbuffer :: Buffer -> ByteString
8988unbuffer (Buf fp off len _ _) = PS fp off len
9089
90+instance Semigroup Buffer where
91+ (Buf _ _ _ 0 _) <> b = b
92+ a <> (Buf _ _ _ 0 _) = a
93+ buf <> (Buf fp off len _ _) = append buf fp off len
94+
9195instance Monoid Buffer where
9296 mempty = Buf nullForeignPtr 0 0 0 0
9397
94− mappend (Buf _ _ _ 0 _) b = b
95− mappend a (Buf _ _ _ 0 _) = a
96− mappend buf (Buf fp off len _ _) = append buf fp off len
98+ mappend = (<>)
9799
98− mconcat [] = mempty
100+ mconcat [] = Mon.mempty
99101 mconcat xs = foldl1' mappend xs
100102
101103pappend :: Buffer -> ByteString -> Buffer
Data/Attoparsec/ByteString/Char8.hs+6−17
@@ -7,7 +7,7 @@
77
88-- |
99-- Module : Data.Attoparsec.ByteString.Char8
10−-- Copyright : Bryan O'Sullivan 2007-2014
10+-- Copyright : Bryan O'Sullivan 2007-2015
1111-- License : BSD3
1212--
1313-- Maintainer : bos@serpentine.com
@@ -70,7 +70,7 @@ module Data.Attoparsec.ByteString.Char8
7070
7171 -- * Efficient string handling
7272 , I.string
73− , stringCI
73+ , I.stringCI
7474 , skipSpace
7575 , skipWhile
7676 , I.take
@@ -128,10 +128,10 @@ module Data.Attoparsec.ByteString.Char8
128128 ) where
129129
130130#if !MIN_VERSION_base(4,8,0)
131−import Control.Applicative (pure, (*>), (<*))
131+import Control.Applicative (pure, (*>), (<*), (<$>))
132132import Data.Word (Word)
133133#endif
134−import Control.Applicative ((<$>), (<|>))
134+import Control.Applicative ((<|>))
135135import Control.Monad (void, when)
136136import Data.Attoparsec.ByteString.FastSet (charClass, memberChar)
137137import Data.Attoparsec.ByteString.Internal (Parser)
@@ -169,16 +169,6 @@ instance (a ~ B.ByteString) => IsString (Parser a) where
169169-- currency sign in ISO-8859-1). Haskell 'Char' values above U+00FF
170170-- are truncated, so e.g. U+1D6B7 is truncated to the byte @0xB7@.
171171
172−-- ASCII-specific but fast, oh yes.
173−toLower :: Word8 -> Word8
174−toLower w | w >= 65 && w <= 90 = w + 32
175− | otherwise = w
176−
177−-- | Satisfy a literal string, ignoring case.
178−stringCI :: B.ByteString -> Parser B.ByteString
179−stringCI = I.stringTransform (B8.map toLower)
180−{-# INLINE stringCI #-}
181−
182172-- | Consume input as long as the predicate returns 'True', and return
183173-- the consumed input.
184174--
@@ -454,9 +444,8 @@ hexadecimal = B8.foldl' step 0 `fmap` I.takeWhile1 isHexDigit
454444
455445-- | Parse and decode an unsigned decimal number.
456446decimal :: Integral a => Parser a
457−decimal = B8.foldl' step 0 `fmap` I.takeWhile1 isDig
458− where isDig w = w >= 48 && w <= 57
459− step a w = a * 10 + fromIntegral (w - 48)
447+decimal = B8.foldl' step 0 `fmap` I.takeWhile1 isDigit_w8
448+ where step a w = a * 10 + fromIntegral (w - 48)
460449{-# SPECIALISE decimal :: Parser Int #-}
461450{-# SPECIALISE decimal :: Parser Int8 #-}
462451{-# SPECIALISE decimal :: Parser Int16 #-}
Data/Attoparsec/ByteString/FastSet.hs+1−1
@@ -3,7 +3,7 @@
33-----------------------------------------------------------------------------
44-- |
55-- Module : Data.Attoparsec.ByteString.FastSet
6−-- Copyright : Bryan O'Sullivan 2007-2014
6+-- Copyright : Bryan O'Sullivan 2007-2015
77-- License : BSD3
88--
99-- Maintainer : bos@serpentine.com
Data/Attoparsec/ByteString/Internal.hs+84−33
@@ -1,7 +1,8 @@
1−{-# LANGUAGE BangPatterns, GADTs, OverloadedStrings, RecordWildCards #-}
1+{-# LANGUAGE BangPatterns, CPP, GADTs, OverloadedStrings, RankNTypes,
2+ RecordWildCards #-}
23-- |
34-- Module : Data.Attoparsec.ByteString.Internal
4−-- Copyright : Bryan O'Sullivan 2007-2014
5+-- Copyright : Bryan O'Sullivan 2007-2015
56-- License : BSD3
67--
78-- Maintainer : bos@serpentine.com
@@ -46,7 +47,7 @@ module Data.Attoparsec.ByteString.Internal
4647 -- * Efficient string handling
4748 , skipWhile
4849 , string
49− , stringTransform
50+ , stringCI
5051 , take
5152 , scan
5253 , runScanner
@@ -65,7 +66,10 @@ module Data.Attoparsec.ByteString.Internal
6566 , atEnd
6667 ) where
6768
68−import Control.Applicative ((<|>), (<$>))
69+#if !MIN_VERSION_base(4,8,0)
70+import Control.Applicative ((<$>))
71+#endif
72+import Control.Applicative ((<|>))
6973import Control.Monad (when)
7074import Data.Attoparsec.ByteString.Buffer (Buffer, buffer)
7175import Data.Attoparsec.ByteString.FastSet (charClass, memberWord8)
@@ -74,6 +78,7 @@ import Data.Attoparsec.Internal
7478import Data.Attoparsec.Internal.Fhthagn (inlinePerformIO)
7579import Data.Attoparsec.Internal.Types hiding (Parser, Failure, Success)
7680import Data.ByteString (ByteString)
81+import Data.List (intercalate)
7782import Data.Word (Word8)
7883import Foreign.ForeignPtr (withForeignPtr)
7984import Foreign.Ptr (castPtr, minusPtr, plusPtr)
@@ -139,19 +144,12 @@ storable = hack undefined
139144 return . inlinePerformIO . withForeignPtr fp $ \p ->
140145 peek (castPtr $ p `plusPtr` o)
141146
142−-- | Consume @n@ bytes of input, but succeed only if the predicate
143−-- returns 'True'.
144−takeWith :: Int -> (ByteString -> Bool) -> Parser ByteString
145−takeWith n0 p = do
146− let n = max n0 0
147− s <- ensure n
148− if p s
149− then advance n >> return s
150− else fail "takeWith"
151−
152147-- | Consume exactly @n@ bytes of input.
153148take :: Int -> Parser ByteString
154−take n = takeWith n (const True)
149+take n0 = do
150+ let n = max n0 0
151+ s <- ensure n
152+ advance n >> return s
155153{-# INLINE take #-}
156154
157155-- | @string s@ parses a sequence of bytes that identically match
@@ -170,13 +168,59 @@ take n = takeWith n (const True)
170168-- before failing. In attoparsec, the above parser will /succeed/ on
171169-- that input, because the failed first branch will consume nothing.
172170string :: ByteString -> Parser ByteString
173−string s = takeWith (B.length s) (==s)
171+string s = string_ (stringSuspended id) id s
174172{-# INLINE string #-}
175173
176−stringTransform :: (ByteString -> ByteString) -> ByteString
177− -> Parser ByteString
178−stringTransform f s = takeWith (B.length s) ((==f s) . f)
179−{-# INLINE stringTransform #-}
174+-- ASCII-specific but fast, oh yes.
175+toLower :: Word8 -> Word8
176+toLower w | w >= 65 && w <= 90 = w + 32
177+ | otherwise = w
178+
179+-- | Satisfy a literal string, ignoring case.
180+stringCI :: ByteString -> Parser ByteString
181+stringCI s = string_ (stringSuspended lower) lower s
182+ where lower = B8.map toLower
183+{-# INLINE stringCI #-}
184+
185+string_ :: (forall r. ByteString -> ByteString -> Buffer -> Pos -> More
186+ -> Failure r -> Success ByteString r -> Result r)
187+ -> (ByteString -> ByteString)
188+ -> ByteString -> Parser ByteString
189+string_ suspended f s0 = T.Parser $ \t pos more lose succ ->
190+ let n = B.length s
191+ s = f s0
192+ in if lengthAtLeast pos n t
193+ then let t' = substring pos (Pos n) t
194+ in if s == f t'
195+ then succ t (pos + Pos n) more t'
196+ else lose t pos more [] "string"
197+ else let t' = Buf.unsafeDrop (fromPos pos) t
198+ in if f t' `B.isPrefixOf` s
199+ then suspended s (B.drop (B.length t') s) t pos more lose succ
200+ else lose t pos more [] "string"
201+{-# INLINE string_ #-}
202+
203+stringSuspended :: (ByteString -> ByteString)
204+ -> ByteString -> ByteString -> Buffer -> Pos -> More
205+ -> Failure r
206+ -> Success ByteString r
207+ -> Result r
208+stringSuspended f s0 s t pos more lose succ =
209+ runParser (demandInput_ >>= go) t pos more lose succ
210+ where go s'0 = T.Parser $ \t' pos' more' lose' succ' ->
211+ let m = B.length s
212+ s' = f s'0
213+ n = B.length s'
214+ in if n >= m
215+ then if B.unsafeTake m s' == s
216+ then let o = Pos (B.length s0)
217+ in succ' t' (pos' + o) more'
218+ (substring pos' o t')
219+ else lose' t' pos' more' [] "string"
220+ else if s' == B.unsafeTake n s
221+ then stringSuspended f s0 (B.unsafeDrop n s)
222+ t' pos' more' lose' succ'
223+ else lose' t' pos' more' [] "string"
180224
181225-- | Skip past input for as long as the predicate returns 'True'.
182226skipWhile :: (Word8 -> Bool) -> Parser ()
@@ -213,15 +257,24 @@ takeTill p = takeWhile (not . p)
213257-- parsers loop until a failure occurs. Careless use will thus result
214258-- in an infinite loop.
215259takeWhile :: (Word8 -> Bool) -> Parser ByteString
216−takeWhile p = (B.concat . reverse) `fmap` go []
260+takeWhile p = do
261+ s <- B8.takeWhile p <$> get
262+ continue <- inputSpansChunks (B.length s)
263+ if continue
264+ then takeWhileAcc p [s]
265+ else return s
266+{-# INLINE takeWhile #-}
267+
268+takeWhileAcc :: (Word8 -> Bool) -> [ByteString] -> Parser ByteString
269+takeWhileAcc p = go
217270 where
218271 go acc = do
219272 s <- B8.takeWhile p <$> get
220273 continue <- inputSpansChunks (B.length s)
221274 if continue
222275 then go (s:acc)
223− else return (s:acc)
224−{-# INLINE takeWhile #-}
276+ else return $ concatReverse (s:acc)
277+{-# INLINE takeWhileAcc #-}
225278
226279takeRest :: Parser [ByteString]
227280takeRest = go []
@@ -285,16 +338,13 @@ scan_ f s0 p = go [] s0
285338-- parsers loop until a failure occurs. Careless use will thus result
286339-- in an infinite loop.
287340scan :: s -> (s -> Word8 -> Maybe s) -> Parser ByteString
288−scan = scan_ $ \_ chunks ->
289− case chunks of
290− [x] -> return x
291− xs -> return $! B.concat $ reverse xs
341+scan = scan_ $ \_ chunks -> return $! concatReverse chunks
292342{-# INLINE scan #-}
293343
294344-- | Like 'scan', but generalized to return the final state of the
295345-- scanner.
296346runScanner :: s -> (s -> Word8 -> Maybe s) -> Parser (ByteString, s)
297−runScanner = scan_ $ \s xs -> return (B.concat (reverse xs), s)
347+runScanner = scan_ $ \s xs -> let !sx = concatReverse xs in return (sx, s)
298348{-# INLINE runScanner #-}
299349
300350-- | Consume input as long as the predicate returns 'True', and return
@@ -314,8 +364,9 @@ takeWhile1 p = do
314364 advance len
315365 eoc <- endOfChunk
316366 if eoc
317− then (s<>) `fmap` takeWhile p
367+ then takeWhileAcc p [s]
318368 else return s
369+{-# INLINE takeWhile1 #-}
319370
320371-- | Match any byte in a set.
321372--
@@ -416,9 +467,10 @@ parse m s = T.runParser m (buffer s) (Pos 0) Incomplete failK successK
416467-- @
417468parseOnly :: Parser a -> ByteString -> Either String a
418469parseOnly m s = case T.runParser m (buffer s) (Pos 0) Complete failK successK of
419− Fail _ _ err -> Left err
420− Done _ a -> Right a
421− _ -> error "parseOnly: impossible error!"
470+ Fail _ [] err -> Left err
471+ Fail _ ctxs err -> Left (intercalate " > " ctxs ++ ": " ++ err)
472+ Done _ a -> Right a
473+ _ -> error "parseOnly: impossible error!"
422474{-# INLINE parseOnly #-}
423475
424476get :: Parser ByteString
@@ -465,7 +517,6 @@ ensure n = T.Parser $ \t pos more lose succ ->
465517 then succ t pos more (substring pos (Pos n) t)
466518 -- The uncommon case is kept out-of-line to reduce code size:
467519 else ensureSuspended n t pos more lose succ
468−-- Non-recursive so the bounds check can be inlined:
469520{-# INLINE ensure #-}
470521
471522-- | Return both the result of a parse and the portion of the input
Data/Attoparsec/ByteString/Lazy.hs+5−3
@@ -5,7 +5,7 @@
55
66-- |
77-- Module : Data.Attoparsec.ByteString.Lazy
8−-- Copyright : Bryan O'Sullivan 2007-2014
8+-- Copyright : Bryan O'Sullivan 2007-2015
99-- License : BSD3
1010--
1111-- Maintainer : bos@serpentine.com
@@ -41,6 +41,7 @@ module Data.Attoparsec.ByteString.Lazy
4141
4242import Control.DeepSeq (NFData(rnf))
4343import Data.ByteString.Lazy.Internal (ByteString(..), chunk)
44+import Data.List (intercalate)
4445import qualified Data.ByteString as B
4546import qualified Data.Attoparsec.ByteString as A
4647import qualified Data.Attoparsec.Internal.Types as T
@@ -104,5 +105,6 @@ maybeResult _ = Nothing
104105
105106-- | Convert a 'Result' value to an 'Either' value.
106107eitherResult :: Result r -> Either String r
107−eitherResult (Done _ r) = Right r
108−eitherResult (Fail _ _ msg) = Left msg
108+eitherResult (Done _ r) = Right r
109+eitherResult (Fail _ [] msg) = Left msg
110+eitherResult (Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
Data/Attoparsec/Char8.hs+1−1
@@ -1,6 +1,6 @@
11-- |
22-- Module : Data.Attoparsec.Char8
3−-- Copyright : Bryan O'Sullivan 2007-2014
3+-- Copyright : Bryan O'Sullivan 2007-2015
44-- License : BSD3
55--
66-- Maintainer : bos@serpentine.com
Data/Attoparsec/Combinator.hs+12−5
@@ -4,7 +4,7 @@
44#endif
55-- |
66-- Module : Data.Attoparsec.Combinator
7−-- Copyright : Daan Leijen 1999-2001, Bryan O'Sullivan 2007-2014
7+-- Copyright : Daan Leijen 1999-2001, Bryan O'Sullivan 2007-2015
88-- License : BSD3
99--
1010-- Maintainer : bos@serpentine.com
@@ -36,14 +36,14 @@ module Data.Attoparsec.Combinator
3636 , satisfyElem
3737 , endOfInput
3838 , atEnd
39+ , lookAhead
3940 ) where
4041
4142#if !MIN_VERSION_base(4,8,0)
42−import Control.Applicative (Applicative(..))
43+import Control.Applicative (Applicative(..), (<$>))
4344import Data.Monoid (Monoid(mappend))
4445#endif
45−import Control.Applicative (Alternative(..), empty, liftA2, many, (<|>),
46− (<$>))
46+import Control.Applicative (Alternative(..), empty, liftA2, many, (<|>))
4747import Control.Monad (MonadPlus(..))
4848import Data.Attoparsec.Internal.Types (Parser(..), IResult(..))
4949import Data.Attoparsec.Internal (endOfInput, atEnd, satisfyElem)
@@ -242,7 +242,14 @@ eitherP a b = (Left <$> a) <|> (Right <$> b)
242242-- | If a parser has returned a 'T.Partial' result, supply it with more
243243-- input.
244244feed :: Monoid i => IResult i r -> i -> IResult i r
245−feed f@(Fail _ _ _) _ = f
245+feed (Fail t ctxs msg) d = Fail (mappend t d) ctxs msg
246246feed (Partial k) d = k d
247247feed (Done t r) d = Done (mappend t d) r
248248{-# INLINE feed #-}
249+
250+-- | Apply a parser without consuming any input.
251+lookAhead :: Parser i a -> Parser i a
252+lookAhead p = Parser $ \t pos more lose succ ->
253+ let succ' t' _pos' more' = succ t' pos more'
254+ in runParser p t pos more lose succ'
255+{-# INLINE lookAhead #-}
Data/Attoparsec/Internal.hs+31−5
@@ -1,7 +1,7 @@
1−{-# LANGUAGE BangPatterns, ScopedTypeVariables #-}
1+{-# LANGUAGE BangPatterns, CPP, ScopedTypeVariables #-}
22-- |
33-- Module : Data.Attoparsec.Internal
4−-- Copyright : Bryan O'Sullivan 2007-2014
4+-- Copyright : Bryan O'Sullivan 2007-2015
55-- License : BSD3
66--
77-- Maintainer : bos@serpentine.com
@@ -15,13 +15,18 @@ module Data.Attoparsec.Internal
1515 ( compareResults
1616 , prompt
1717 , demandInput
18+ , demandInput_
1819 , wantInput
1920 , endOfInput
2021 , atEnd
2122 , satisfyElem
23+ , concatReverse
2224 ) where
2325
26+#if !MIN_VERSION_base(4,8,0)
2427import Control.Applicative ((<$>))
28+import Data.Monoid (Monoid, mconcat)
29+#endif
2530import Data.Attoparsec.Internal.Types
2631import Data.ByteString (ByteString)
2732import Data.Text (Text)
@@ -40,8 +45,8 @@ compareResults (Done t0 r0) (Done t1 r1) =
4045compareResults (Partial _) (Partial _) = Nothing
4146compareResults _ _ = Just False
4247
43−-- | Ask for input. If we receive any, pass it to a success
44−-- continuation, otherwise to a failure continuation.
48+-- | Ask for input. If we receive any, pass the augmented input to a
49+-- success continuation, otherwise to a failure continuation.
4550prompt :: Chunk t
4651 => State t -> Pos -> More
4752 -> (State t -> Pos -> More -> IResult t r)
@@ -68,12 +73,25 @@ demandInput :: Chunk t => Parser t ()
6873demandInput = Parser $ \t pos more lose succ ->
6974 case more of
7075 Complete -> lose t pos more [] "not enough input"
71− _ -> let lose' t' pos' more' = lose t' pos' more' [] "not enough input"
76+ _ -> let lose' _ pos' more' = lose t pos' more' [] "not enough input"
7277 succ' t' pos' more' = succ t' pos' more' ()
7378 in prompt t pos more lose' succ'
7479{-# SPECIALIZE demandInput :: Parser ByteString () #-}
7580{-# SPECIALIZE demandInput :: Parser Text () #-}
7681
82+-- | Immediately demand more input via a 'Partial' continuation
83+-- result. Return the new input.
84+demandInput_ :: Chunk t => Parser t t
85+demandInput_ = Parser $ \t pos more lose succ ->
86+ case more of
87+ Complete -> lose t pos more [] "not enough input"
88+ _ -> Partial $ \s ->
89+ if nullChunk s
90+ then lose t pos Complete [] "not enough input"
91+ else succ (pappendChunk t s) pos more s
92+{-# SPECIALIZE demandInput_ :: Parser ByteString ByteString #-}
93+{-# SPECIALIZE demandInput_ :: Parser Text Text #-}
94+
7795-- | This parser always succeeds. It returns 'True' if any input is
7896-- available either immediately or on demand, and 'False' if the end
7997-- of all input has been reached.
@@ -143,3 +161,11 @@ satisfyElem p = Parser $ \t pos more lose succ ->
143161 | otherwise -> lose t pos more [] "satisfyElem"
144162 Nothing -> satisfySuspended p t pos more lose succ
145163{-# INLINE satisfyElem #-}
164+
165+-- | Concatenate a monoid after reversing its elements. Used to
166+-- glue together a series of textual chunks that have been accumulated
167+-- \"backwards\".
168+concatReverse :: Monoid m => [m] -> m
169+concatReverse [x] = x
170+concatReverse xs = mconcat (reverse xs)
171+{-# INLINE concatReverse #-}
Data/Attoparsec/Internal/Types.hs+46−33
@@ -1,8 +1,8 @@
1−{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, OverloadedStrings,
1+{-# LANGUAGE CPP, BangPatterns, GeneralizedNewtypeDeriving, OverloadedStrings,
22 Rank2Types, RecordWildCards, TypeFamilies #-}
33-- |
44-- Module : Data.Attoparsec.Internal.Types
5−-- Copyright : Bryan O'Sullivan 2007-2014
5+-- Copyright : Bryan O'Sullivan 2007-2015
66-- License : BSD3
77--
88-- Maintainer : bos@serpentine.com
@@ -25,13 +25,13 @@ module Data.Attoparsec.Internal.Types
2525 , Chunk(..)
2626 ) where
2727
28−#if !MIN_VERSION_base(4,8,0)
29−import Control.Applicative (Applicative(..))
30−import Data.Monoid (Monoid(..))
31−#endif
32−import Control.Applicative (Alternative(..), (<$>))
28+import Control.Applicative as App (Applicative(..), (<$>))
29+import Control.Applicative (Alternative(..))
3330import Control.DeepSeq (NFData(rnf))
3431import Control.Monad (MonadPlus(..))
32+import qualified Control.Monad.Fail as Fail (MonadFail(..))
33+import Data.Monoid as Mon (Monoid(..))
34+import Data.Semigroup (Semigroup(..))
3535import Data.Word (Word8)
3636import Data.ByteString (ByteString)
3737import qualified Data.ByteString as BS
@@ -39,7 +39,7 @@ import Data.ByteString.Internal (w2c)
3939import Data.Text (Text)
4040import qualified Data.Text as Text
4141import Data.Text.Unsafe (Iter(..))
42−import Prelude hiding (getChar, succ)
42+import Prelude hiding (succ)
4343import qualified Data.Attoparsec.ByteString.Buffer as B
4444import qualified Data.Attoparsec.Text.Buffer as T
4545
@@ -70,10 +70,13 @@ data IResult i r =
7070 -- not yet been consumed (if any) when the parse succeeded.
7171
7272instance (Show i, Show r) => Show (IResult i r) where
73− show (Fail t stk msg) =
74− unwords [ "Fail", show t, show stk, show msg]
75− show (Partial _) = "Partial _"
76− show (Done t r) = unwords ["Done", show t, show r]
73+ showsPrec d ir = showParen (d > 10) $
74+ case ir of
75+ (Fail t stk msg) -> showString "Fail" . f t . f stk . f msg
76+ (Partial _) -> showString "Partial _"
77+ (Done t r) -> showString "Done" . f t . f r
78+ where f :: Show a => a -> ShowS
79+ f x = showChar ' ' . showsPrec 11 x
7780
7881instance (NFData i, NFData r) => NFData (IResult i r) where
7982 rnf (Fail t stk msg) = rnf t `seq` rnf stk `seq` rnf msg
@@ -124,17 +127,21 @@ type Success i t a r = t -> Pos -> More -> a -> IResult i r
124127data More = Complete | Incomplete
125128 deriving (Eq, Show)
126129
127−instance Monoid More where
128− mappend c@Complete _ = c
129− mappend _ m = m
130− mempty = Incomplete
130+instance Semigroup More where
131+ c@Complete <> _ = c
132+ _ <> m = m
133+
134+instance Mon.Monoid More where
135+ mappend = (<>)
136+ mempty = Incomplete
131137
132138instance Monad (Parser i) where
133− fail err = Parser $ \t pos more lose _succ -> lose t pos more [] msg
134− where msg = "Failed reading: " ++ err
139+#if !(MIN_VERSION_base(4,13,0))
140+ fail = Fail.fail
135141 {-# INLINE fail #-}
142+#endif
136143
137− return v = Parser $ \t pos more _lose succ -> succ t pos more v
144+ return = App.pure
138145 {-# INLINE return #-}
139146
140147 m >>= k = Parser $ \t !pos more lose succ ->
@@ -142,6 +149,15 @@ instance Monad (Parser i) where
142149 in runParser m t pos more lose succ'
143150 {-# INLINE (>>=) #-}
144151
152+ (>>) = (*>)
153+ {-# INLINE (>>) #-}
154+
155+
156+instance Fail.MonadFail (Parser i) where
157+ fail err = Parser $ \t pos more lose _succ -> lose t pos more [] msg
158+ where msg = "Failed reading: " ++ err
159+ {-# INLINE fail #-}
160+
145161plus :: Parser i a -> Parser i a -> Parser i a
146162plus f g = Parser $ \t pos more lose succ ->
147163 let lose' t' _pos' more' _ctx _msg = runParser g t' pos more' lose succ
@@ -166,23 +182,23 @@ apP d e = do
166182{-# INLINE apP #-}
167183
168184instance Applicative (Parser i) where
169− pure = return
185+ pure v = Parser $ \t !pos more _lose succ -> succ t pos more v
170186 {-# INLINE pure #-}
171187 (<*>) = apP
172188 {-# INLINE (<*>) #-}
173−
174− -- These definitions are equal to the defaults, but this
175− -- way the optimizer doesn't have to work so hard to figure
176− -- that out.
177− (*>) = (>>)
189+ m *> k = m >>= \_ -> k
178190 {-# INLINE (*>) #-}
179− x <* y = x >>= \a -> y >> return a
191+ x <* y = x >>= \a -> y >> pure a
180192 {-# INLINE (<*) #-}
181193
194+instance Semigroup (Parser i a) where
195+ (<>) = plus
196+ {-# INLINE (<>) #-}
197+
182198instance Monoid (Parser i a) where
183199 mempty = fail "mempty"
184200 {-# INLINE mempty #-}
185− mappend = plus
201+ mappend = (<>)
186202 {-# INLINE mappend #-}
187203
188204instance Alternative (Parser i) where
@@ -193,8 +209,9 @@ instance Alternative (Parser i) where
193209 {-# INLINE (<|>) #-}
194210
195211 many v = many_v
196− where many_v = some_v <|> pure []
197− some_v = (:) <$> v <*> many_v
212+ where
213+ many_v = some_v <|> pure []
214+ some_v = (:) <$> v <*> many_v
198215 {-# INLINE many #-}
199216
200217 some v = some_v
@@ -203,10 +220,6 @@ instance Alternative (Parser i) where
203220 some_v = (:) <$> v <*> many_v
204221 {-# INLINE some #-}
205222
206−(<>) :: (Monoid m) => m -> m -> m
207−(<>) = mappend
208−{-# INLINE (<>) #-}
209−
210223-- | A common interface for input chunks.
211224class Monoid c => Chunk c where
212225 type ChunkElem c
Data/Attoparsec/Lazy.hs+1−1
@@ -1,6 +1,6 @@
11-- |
22-- Module : Data.Attoparsec.Lazy
3−-- Copyright : Bryan O'Sullivan 2007-2014
3+-- Copyright : Bryan O'Sullivan 2007-2015
44-- License : BSD3
55--
66-- Maintainer : bos@serpentine.com
Data/Attoparsec/Number.hs+1−1
@@ -1,7 +1,7 @@
11{-# LANGUAGE DeriveDataTypeable #-}
22-- |
33-- Module : Data.Attoparsec.Number
4−-- Copyright : Bryan O'Sullivan 2007-2014
4+-- Copyright : Bryan O'Sullivan 2007-2015
55-- License : BSD3
66--
77-- Maintainer : bos@serpentine.com
Data/Attoparsec/Text.hs+9−6
@@ -6,7 +6,7 @@
66
77-- |
88-- Module : Data.Attoparsec.Text
9−-- Copyright : Bryan O'Sullivan 2007-2014
9+-- Copyright : Bryan O'Sullivan 2007-2015
1010-- License : BSD3
1111--
1212-- Maintainer : bos@serpentine.com
@@ -72,6 +72,7 @@ module Data.Attoparsec.Text
7272 , skipSpace
7373 , I.skipWhile
7474 , I.scan
75+ , I.runScanner
7576 , I.take
7677 , I.takeWhile
7778 , I.takeWhile1
@@ -126,10 +127,10 @@ module Data.Attoparsec.Text
126127 ) where
127128
128129#if !MIN_VERSION_base(4,8,0)
129−import Control.Applicative (pure, (*>), (<*))
130+import Control.Applicative (pure, (*>), (<*), (<$>))
130131import Data.Word (Word)
131132#endif
132−import Control.Applicative ((<$>), (<|>))
133+import Control.Applicative ((<|>))
133134import Data.Attoparsec.Combinator
134135import Data.Attoparsec.Number (Number(..))
135136import Data.Scientific (Scientific)
@@ -138,6 +139,7 @@ import Data.Attoparsec.Text.Internal (Parser, Result, parse, takeWhile1)
138139import Data.Bits (Bits, (.|.), shiftL)
139140import Data.Char (isAlpha, isDigit, isSpace, ord)
140141import Data.Int (Int8, Int16, Int32, Int64)
142+import Data.List (intercalate)
141143import Data.Text (Text)
142144import Data.Word (Word8, Word16, Word32, Word64)
143145import qualified Data.Attoparsec.Internal as I
@@ -264,9 +266,10 @@ maybeResult _ = Nothing
264266-- | Convert a 'Result' value to an 'Either' value. A 'Partial' result
265267-- is treated as failure.
266268eitherResult :: Result r -> Either String r
267−eitherResult (T.Done _ r) = Right r
268−eitherResult (T.Fail _ _ msg) = Left msg
269−eitherResult _ = Left "Result: incomplete input"
269+eitherResult (T.Done _ r) = Right r
270+eitherResult (T.Fail _ [] msg) = Left msg
271+eitherResult (T.Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
272+eitherResult _ = Left "Result: incomplete input"
270273
271274-- | A predicate that matches either a carriage return @\'\\r\'@ or
272275-- newline @\'\\n\'@ character.
Data/Attoparsec/Text/Buffer.hs+19−9
@@ -3,7 +3,7 @@
33
44-- |
55-- Module : Data.Attoparsec.Text.Buffer
6−-- Copyright : Bryan O'Sullivan 2007-2014
6+-- Copyright : Bryan O'Sullivan 2007-2015
77-- License : BSD3
88--
99-- Maintainer : bos@serpentine.com
@@ -28,6 +28,7 @@ module Data.Attoparsec.Text.Buffer
2828 Buffer
2929 , buffer
3030 , unbuffer
31+ , unbufferAt
3132 , length
3233 , pappend
3334 , iter
@@ -39,9 +40,8 @@ module Data.Attoparsec.Text.Buffer
3940import Control.Exception (assert)
4041import Data.Bits (shiftR)
4142import Data.List (foldl1')
42−#if !MIN_VERSION_base(4,8,0)
43−import Data.Monoid (Monoid(..))
44−#endif
43+import Data.Monoid as Mon (Monoid(..))
44+import Data.Semigroup (Semigroup(..))
4545import Data.Text ()
4646import Data.Text.Internal (Text(..))
4747import Data.Text.Internal.Encoding.Utf16 (chr2)
@@ -74,15 +74,25 @@ buffer (Text arr off len) = Buf arr off len len 0
7474unbuffer :: Buffer -> Text
7575unbuffer (Buf arr off len _ _) = Text arr off len
7676
77+unbufferAt :: Int -> Buffer -> Text
78+unbufferAt s (Buf arr off len _ _) =
79+ assert (s >= 0 && s <= len) $
80+ Text arr (off+s) (len-s)
81+
82+instance Semigroup Buffer where
83+ (Buf _ _ _ 0 _) <> b = b
84+ a <> (Buf _ _ _ 0 _) = a
85+ buf <> (Buf arr off len _ _) = append buf arr off len
86+ {-# INLINE (<>) #-}
87+
7788instance Monoid Buffer where
7889 mempty = Buf A.empty 0 0 0 0
90+ {-# INLINE mempty #-}
7991
80− mappend (Buf _ _ _ 0 _) b = b
81− mappend a (Buf _ _ _ 0 _) = a
82− mappend buf (Buf arr off len _ _) = append buf arr off len
92+ mappend = (<>)
8393
84− mconcat [] = mempty
85− mconcat xs = foldl1' mappend xs
94+ mconcat [] = Mon.mempty
95+ mconcat xs = foldl1' (<>) xs
8696
8797pappend :: Buffer -> Text -> Buffer
8898pappend (Buf _ _ _ 0 _) t = buffer t
Data/Attoparsec/Text/FastSet.hs+4−2
@@ -3,7 +3,7 @@
33------------------------------------------------------------------------------
44-- |
55-- Module : Data.Attoparsec.FastSet
6−-- Copyright : Felipe Lessa 2010, Bryan O'Sullivan 2007-2014
6+-- Copyright : Felipe Lessa 2010, Bryan O'Sullivan 2007-2015
77-- License : BSD3
88--
99-- Maintainer : felipe.lessa@gmail.com
@@ -62,7 +62,9 @@ resolveCollisions (a:b:entries) = a' : resolveCollisions (b' : entries)
6262
6363pad :: Int -> [Entry] -> [Entry]
6464pad = go 0
65− where go !_ !m [] = replicate (max 1 m) empty
65+ where -- ensure that we pad enough so that lookups beyond the
66+ -- last hash in the table fall within the array
67+ go !_ !m [] = replicate (max 1 m + 1) empty
6668 go k m (e:entries) = map (const empty) [k..i - 1] ++ e :
6769 go (i + 1) (m + i - k - 1) entries
6870 where i = index e
Data/Attoparsec/Text/Internal.hs+77−35
@@ -3,7 +3,7 @@
33{-# OPTIONS_GHC -fno-warn-orphans #-}
44-- |
55-- Module : Data.Attoparsec.Text.Internal
6−-- Copyright : Bryan O'Sullivan 2007-2014
6+-- Copyright : Bryan O'Sullivan 2007-2015
77-- License : BSD3
88--
99-- Maintainer : bos@serpentine.com
@@ -65,14 +65,15 @@ module Data.Attoparsec.Text.Internal
6565 , atEnd
6666 ) where
6767
68−import Control.Applicative ((<|>), (<$>))
68+import Control.Applicative ((<|>), (<$>), pure, (*>))
6969import Control.Monad (when)
7070import Data.Attoparsec.Combinator ((<?>))
7171import Data.Attoparsec.Internal
7272import Data.Attoparsec.Internal.Types hiding (Parser, Failure, Success)
7373import qualified Data.Attoparsec.Text.Buffer as Buf
7474import Data.Attoparsec.Text.Buffer (Buffer, buffer)
75−import Data.Char (chr, ord)
75+import Data.Char (isAsciiUpper, isAsciiLower, toUpper, toLower)
76+import Data.List (intercalate)
7677import Data.String (IsString(..))
7778import Data.Text.Internal (Text(..))
7879import Prelude hiding (getChar, succ, take, takeWhile)
@@ -159,9 +160,46 @@ take n = takeWith (max n 0) (const True)
159160-- before failing. In attoparsec, the above parser will /succeed/ on
160161-- that input, because the failed first branch will consume nothing.
161162string :: Text -> Parser Text
162−string s = takeWith (T.length s) (==s)
163+string s = string_ (stringSuspended id) id s
163164{-# INLINE string #-}
164165
166+string_ :: (forall r. Text -> Text -> Buffer -> Pos -> More
167+ -> Failure r -> Success Text r -> Result r)
168+ -> (Text -> Text)
169+ -> Text -> Parser Text
170+string_ suspended f s0 = T.Parser $ \t pos more lose succ ->
171+ let s = f s0
172+ ft = f (Buf.unbufferAt (fromPos pos) t)
173+ in case T.commonPrefixes s ft of
174+ Nothing
175+ | T.null s -> succ t pos more T.empty
176+ | T.null ft -> suspended s s t pos more lose succ
177+ | otherwise -> lose t pos more [] "string"
178+ Just (pfx,ssfx,tsfx)
179+ | T.null ssfx -> let l = Pos (T.lengthWord16 pfx)
180+ in succ t (pos + l) more (substring pos l t)
181+ | not (T.null tsfx) -> lose t pos more [] "string"
182+ | otherwise -> suspended s ssfx t pos more lose succ
183+{-# INLINE string_ #-}
184+
185+stringSuspended :: (Text -> Text)
186+ -> Text -> Text -> Buffer -> Pos -> More
187+ -> Failure r
188+ -> Success Text r
189+ -> Result r
190+stringSuspended f s000 s0 t0 pos0 more0 lose0 succ0 =
191+ runParser (demandInput_ >>= go) t0 pos0 more0 lose0 succ0
192+ where
193+ go s' = T.Parser $ \t pos more lose succ ->
194+ let s = f s'
195+ in case T.commonPrefixes s0 s of
196+ Nothing -> lose t pos more [] "string"
197+ Just (_pfx,ssfx,tsfx)
198+ | T.null ssfx -> let l = Pos (T.lengthWord16 s000)
199+ in succ t (pos + l) more (substring pos l t)
200+ | T.null tsfx -> stringSuspended f s000 ssfx t pos more lose succ
201+ | otherwise -> lose t pos more [] "string"
202+
165203-- | Satisfy a literal string, ignoring case.
166204--
167205-- Note: this function is currently quite inefficient. Unicode case
@@ -184,23 +222,16 @@ stringCI s = go 0
184222
185223-- | Satisfy a literal string, ignoring case for characters in the ASCII range.
186224asciiCI :: Text -> Parser Text
187−asciiCI input = do
188− (k,t) <- ensure n
189− if asciiToLower t == s
190− then advance k >> return t
191− else fail "asciiCI"
192− where
193− n = T.length input
194− s = asciiToLower input
195−
196− -- convert letters in the ASCII range to lower-case
197− asciiToLower = T.map f
198− where
199− offset = ord 'a' - ord 'A'
200− f c | 'A' <= c && c <= 'Z' = chr (ord c + offset)
201− | otherwise = c
225+asciiCI s = fmap fst $ match $ T.foldr ((*>) . asciiCharCI) (pure ()) s
202226{-# INLINE asciiCI #-}
203227
228+asciiCharCI :: Char -> Parser Char
229+asciiCharCI c
230+ | isAsciiUpper c = char c <|> char (toLower c)
231+ | isAsciiLower c = char c <|> char (toUpper c)
232+ | otherwise = char c
233+{-# INLINE asciiCharCI #-}
234+
204235-- | Skip past input for as long as the predicate returns 'True'.
205236skipWhile :: (Char -> Bool) -> Parser ()
206237skipWhile p = go
@@ -236,14 +267,25 @@ takeTill p = takeWhile (not . p)
236267-- parsers loop until a failure occurs. Careless use will thus result
237268-- in an infinite loop.
238269takeWhile :: (Char -> Bool) -> Parser Text
239−takeWhile p = (T.concat . reverse) `fmap` go []
270+takeWhile p = do
271+ h <- T.takeWhile p <$> get
272+ continue <- inputSpansChunks (size h)
273+ -- only use slow concat path if necessary
274+ if continue
275+ then takeWhileAcc p [h]
276+ else return h
277+{-# INLINE takeWhile #-}
278+
279+takeWhileAcc :: (Char -> Bool) -> [Text] -> Parser Text
280+takeWhileAcc p = go
240281 where
241282 go acc = do
242283 h <- T.takeWhile p <$> get
243284 continue <- inputSpansChunks (size h)
244285 if continue
245286 then go (h:acc)
246− else return (h:acc)
287+ else return $ concatReverse (h:acc)
288+{-# INLINE takeWhileAcc #-}
247289
248290takeRest :: Parser [Text]
249291takeRest = go []
@@ -266,7 +308,7 @@ takeLazyText :: Parser L.Text
266308takeLazyText = L.fromChunks `fmap` takeRest
267309
268310data Scan s = Continue s
269− | Finished {-# UNPACK #-} !Int Text
311+ | Finished s {-# UNPACK #-} !Int Text
270312
271313scan_ :: (s -> [Text] -> Parser r) -> s -> (s -> Char -> Maybe s) -> Parser r
272314scan_ f s0 p = go [] s0
@@ -275,7 +317,7 @@ scan_ f s0 p = go [] s0
275317 case T.uncons t of
276318 Just (c,t') -> case p s c of
277319 Just s' -> scanner s' (n+1) t'
278− Nothing -> Finished n t
320+ Nothing -> Finished s n t
279321 Nothing -> Continue s
280322 go acc s = do
281323 input <- get
@@ -284,8 +326,8 @@ scan_ f s0 p = go [] s0
284326 if continue
285327 then go (input : acc) s'
286328 else f s' (input : acc)
287− Finished n t -> do advance (size input - size t)
288− f s (T.take n input : acc)
329+ Finished s' n t -> do advance (size input - size t)
330+ f s' (T.take n input : acc)
289331{-# INLINE scan_ #-}
290332
291333-- | A stateful scanner. The predicate consumes and transforms a
@@ -301,16 +343,13 @@ scan_ f s0 p = go [] s0
301343-- parsers loop until a failure occurs. Careless use will thus result
302344-- in an infinite loop.
303345scan :: s -> (s -> Char -> Maybe s) -> Parser Text
304−scan = scan_ $ \_ chunks ->
305− case chunks of
306− [x] -> return x
307− xs -> return . T.concat . reverse $ xs
346+scan = scan_ $ \_ chunks -> return $! concatReverse chunks
308347{-# INLINE scan #-}
309348
310349-- | Like 'scan', but generalized to return the final state of the
311350-- scanner.
312351runScanner :: s -> (s -> Char -> Maybe s) -> Parser (Text, s)
313−runScanner = scan_ $ \s xs -> return (T.concat (reverse xs), s)
352+runScanner = scan_ $ \s xs -> let !sx = concatReverse xs in return (sx, s)
314353{-# INLINE runScanner #-}
315354
316355-- | Consume input as long as the predicate returns 'True', and return
@@ -328,8 +367,9 @@ takeWhile1 p = do
328367 advance size'
329368 eoc <- endOfChunk
330369 if eoc
331− then (h<>) `fmap` takeWhile p
370+ then takeWhileAcc p [h]
332371 else return h
372+{-# INLINE takeWhile1 #-}
333373
334374-- | Match any character in a set.
335375--
@@ -429,9 +469,10 @@ parse m s = runParser m (buffer s) 0 Incomplete failK successK
429469-- @
430470parseOnly :: Parser a -> Text -> Either String a
431471parseOnly m s = case runParser m (buffer s) 0 Complete failK successK of
432− Fail _ _ err -> Left err
433− Done _ a -> Right a
434− _ -> error "parseOnly: impossible error!"
472+ Fail _ [] err -> Left err
473+ Fail _ ctxs err -> Left (intercalate " > " ctxs ++ ": " ++ err)
474+ Done _ a -> Right a
475+ _ -> error "parseOnly: impossible error!"
435476{-# INLINE parseOnly #-}
436477
437478get :: Parser Text
@@ -476,7 +517,6 @@ ensure n = T.Parser $ \t pos more lose succ ->
476517 Just n' -> succ t pos more (n', substring pos n' t)
477518 -- The uncommon case is kept out-of-line to reduce code size:
478519 Nothing -> ensureSuspended n t pos more lose succ
479−-- Non-recursive so the bounds check can be inlined:
480520{-# INLINE ensure #-}
481521
482522-- | Return both the result of a parse and the portion of the input
@@ -487,6 +527,8 @@ match p = T.Parser $ \t pos more lose succ ->
487527 (substring pos (pos'-pos) t', a)
488528 in runParser p t pos more lose succ'
489529
530+-- | Ensure that at least @n@ code points of input are available.
531+-- Returns the number of words consumed while traversing.
490532lengthAtLeast :: Pos -> Int -> Buffer -> Maybe Pos
491533lengthAtLeast pos n t = go 0 (fromPos pos)
492534 where go i !p
Data/Attoparsec/Text/Lazy.hs+6−8
@@ -6,7 +6,7 @@
66{-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
77-- |
88-- Module : Data.Attoparsec.Text.Lazy
9−-- Copyright : Bryan O'Sullivan 2007-2014
9+-- Copyright : Bryan O'Sullivan 2007-2015
1010-- License : BSD3
1111--
1212-- Maintainer : bos@serpentine.com
@@ -41,6 +41,7 @@ module Data.Attoparsec.Text.Lazy
4141 ) where
4242
4343import Control.DeepSeq (NFData(rnf))
44+import Data.List (intercalate)
4445import Data.Text.Lazy.Internal (Text(..), chunk)
4546import qualified Data.Attoparsec.Internal.Types as T
4647import qualified Data.Attoparsec.Text as A
@@ -59,11 +60,7 @@ data Result r = Fail Text [String] String
5960 -- ^ The parse succeeded. The 'Text' is the
6061 -- input that had not yet been consumed (if any) when
6162 -- the parse succeeded.
62−
63−instance Show r => Show (Result r) where
64− show (Fail bs stk msg) =
65− "Fail " ++ show bs ++ " " ++ show stk ++ " " ++ show msg
66− show (Done bs r) = "Done " ++ show bs ++ " " ++ show r
63+ deriving (Show)
6764
6865instance NFData r => NFData (Result r) where
6966 rnf (Fail bs ctxs msg) = rnf bs `seq` rnf ctxs `seq` rnf msg
@@ -99,5 +96,6 @@ maybeResult _ = Nothing
9996
10097-- | Convert a 'Result' value to an 'Either' value.
10198eitherResult :: Result r -> Either String r
102−eitherResult (Done _ r) = Right r
103−eitherResult (Fail _ _ msg) = Left msg
99+eitherResult (Done _ r) = Right r
100+eitherResult (Fail _ [] msg) = Left msg
101+eitherResult (Fail _ ctxs msg) = Left (intercalate " > " ctxs ++ ": " ++ msg)
Data/Attoparsec/Types.hs+1−1
@@ -1,6 +1,6 @@
11-- |
22-- Module : Data.Attoparsec.Types
3−-- Copyright : Bryan O'Sullivan 2007-2014
3+-- Copyright : Bryan O'Sullivan 2007-2015
44-- License : BSD3
55--
66-- Maintainer : bos@serpentine.com
Data/Attoparsec/Zepto.hs+82−44
@@ -2,11 +2,11 @@
22#if __GLASGOW_HASKELL__ >= 702
33{-# LANGUAGE Trustworthy #-} -- Data.ByteString.Unsafe
44#endif
5−{-# LANGUAGE BangPatterns, MagicHash, UnboxedTuples #-}
5+{-# LANGUAGE BangPatterns #-}
66
77-- |
88-- Module : Data.Attoparsec.Zepto
9−-- Copyright : Bryan O'Sullivan 2007-2014
9+-- Copyright : Bryan O'Sullivan 2007-2015
1010-- License : BSD3
1111--
1212-- Maintainer : bos@serpentine.com
@@ -27,101 +27,139 @@
2727module Data.Attoparsec.Zepto
2828 (
2929 Parser
30+ , ZeptoT
3031 , parse
32+ , parseT
3133 , atEnd
3234 , string
3335 , take
3436 , takeWhile
3537 ) where
3638
37−import Data.Word (Word8)
3839import Control.Applicative
39−import Control.Monad
40−import Data.Monoid
41−import qualified Data.ByteString as B
42−import qualified Data.ByteString.Unsafe as B
40+import Control.Monad (MonadPlus(..), ap)
41+import qualified Control.Monad.Fail as Fail
42+import Control.Monad.IO.Class (MonadIO(..))
4343import Data.ByteString (ByteString)
44+import Data.Functor.Identity (Identity(runIdentity))
45+import Data.Monoid as Mon (Monoid(..))
46+import Data.Semigroup (Semigroup(..))
47+import Data.Word (Word8)
4448import Prelude hiding (take, takeWhile)
49+import qualified Data.ByteString as B
50+import qualified Data.ByteString.Unsafe as B
4551
4652newtype S = S {
4753 input :: ByteString
4854 }
4955
5056data Result a = Fail String
51− | OK !a
57+ | OK !a S
5258
5359-- | A simple parser.
5460--
5561-- This monad is strict in its state, and the monadic bind operator
5662-- ('>>=') evaluates each result to weak head normal form before
5763-- passing it along.
58−newtype Parser a = Parser {
59− runParser :: S -> (# Result a, S #)
64+newtype ZeptoT m a = Parser {
65+ runParser :: S -> m (Result a)
6066 }
6167
62−instance Functor Parser where
63− fmap f m = Parser $ \s -> case runParser m s of
64− (# OK a, s' #) -> (# OK (f a), s' #)
65− (# Fail err, s' #) -> (# Fail err, s' #)
68+type Parser a = ZeptoT Identity a
69+
70+instance Monad m => Functor (ZeptoT m) where
71+ fmap f m = Parser $ \s -> do
72+ result <- runParser m s
73+ case result of
74+ OK a s' -> return (OK (f a) s')
75+ Fail err -> return (Fail err)
6676 {-# INLINE fmap #-}
6777
68−instance Monad Parser where
69− return a = Parser $ \s -> (# OK a, s #)
78+instance MonadIO m => MonadIO (ZeptoT m) where
79+ liftIO act = Parser $ \s -> do
80+ result <- liftIO act
81+ return (OK result s)
82+ {-# INLINE liftIO #-}
83+
84+instance Monad m => Monad (ZeptoT m) where
85+ return = pure
7086 {-# INLINE return #-}
7187
72− m >>= k = Parser $ \s -> case runParser m s of
73− (# OK a, s' #) -> runParser (k a) s'
74− (# Fail err, s' #) -> (# Fail err, s' #)
88+ m >>= k = Parser $ \s -> do
89+ result <- runParser m s
90+ case result of
91+ OK a s' -> runParser (k a) s'
92+ Fail err -> return (Fail err)
7593 {-# INLINE (>>=) #-}
7694
77− fail msg = Parser $ \s -> (# Fail msg, s #)
95+#if !(MIN_VERSION_base(4,13,0))
96+ fail = Fail.fail
97+ {-# INLINE fail #-}
98+#endif
7899
79−instance MonadPlus Parser where
100+instance Monad m => Fail.MonadFail (ZeptoT m) where
101+ fail msg = Parser $ \_ -> return (Fail msg)
102+ {-# INLINE fail #-}
103+
104+instance Monad m => MonadPlus (ZeptoT m) where
80105 mzero = fail "mzero"
81106 {-# INLINE mzero #-}
82107
83− mplus a b = Parser $ \s ->
84− case runParser a s of
85− (# ok@(OK _), s' #) -> (# ok, s' #)
86− (# _, _ #) -> case runParser b s of
87− (# ok@(OK _), s'' #) -> (# ok, s'' #)
88− (# err, s'' #) -> (# err, s'' #)
108+ mplus a b = Parser $ \s -> do
109+ result <- runParser a s
110+ case result of
111+ ok@(OK _ _) -> return ok
112+ _ -> runParser b s
89113 {-# INLINE mplus #-}
90114
91−instance Applicative Parser where
92− pure = return
115+instance (Monad m) => Applicative (ZeptoT m) where
116+ pure a = Parser $ \s -> return (OK a s)
93117 {-# INLINE pure #-}
94118 (<*>) = ap
95119 {-# INLINE (<*>) #-}
96120
97−gets :: (S -> a) -> Parser a
98−gets f = Parser $ \s -> (# OK (f s), s #)
121+gets :: Monad m => (S -> a) -> ZeptoT m a
122+gets f = Parser $ \s -> return (OK (f s) s)
99123{-# INLINE gets #-}
100124
101−put :: S -> Parser ()
102−put s = Parser $ \_ -> (# OK (), s #)
125+put :: Monad m => S -> ZeptoT m ()
126+put s = Parser $ \_ -> return (OK () s)
103127{-# INLINE put #-}
104128
105129-- | Run a parser.
106130parse :: Parser a -> ByteString -> Either String a
107−parse p bs = case runParser p (S bs) of
108− (# OK a, _ #) -> Right a
109− (# Fail err, _ #) -> Left err
110−
111−instance Monoid (Parser a) where
131+parse p bs = case runIdentity (runParser p (S bs)) of
132+ (OK a _) -> Right a
133+ (Fail err) -> Left err
134+{-# INLINE parse #-}
135+
136+-- | Run a parser on top of the given base monad.
137+parseT :: Monad m => ZeptoT m a -> ByteString -> m (Either String a)
138+parseT p bs = do
139+ result <- runParser p (S bs)
140+ case result of
141+ OK a _ -> return (Right a)
142+ Fail err -> return (Left err)
143+{-# INLINE parseT #-}
144+
145+instance Monad m => Semigroup (ZeptoT m a) where
146+ (<>) = mplus
147+ {-# INLINE (<>) #-}
148+
149+instance Monad m => Mon.Monoid (ZeptoT m a) where
112150 mempty = fail "mempty"
113151 {-# INLINE mempty #-}
114− mappend = mplus
152+ mappend = (<>)
115153 {-# INLINE mappend #-}
116154
117−instance Alternative Parser where
155+instance Monad m => Alternative (ZeptoT m) where
118156 empty = fail "empty"
119157 {-# INLINE empty #-}
120158 (<|>) = mplus
121159 {-# INLINE (<|>) #-}
122160
123161-- | Consume input while the predicate returns 'True'.
124−takeWhile :: (Word8 -> Bool) -> Parser ByteString
162+takeWhile :: Monad m => (Word8 -> Bool) -> ZeptoT m ByteString
125163takeWhile p = do
126164 (h,t) <- gets (B.span p . input)
127165 put (S t)
@@ -129,7 +167,7 @@ takeWhile p = do
129167{-# INLINE takeWhile #-}
130168
131169-- | Consume @n@ bytes of input.
132−take :: Int -> Parser ByteString
170+take :: Monad m => Int -> ZeptoT m ByteString
133171take !n = do
134172 s <- gets input
135173 if B.length s >= n
@@ -138,7 +176,7 @@ take !n = do
138176{-# INLINE take #-}
139177
140178-- | Match a string exactly.
141−string :: ByteString -> Parser ()
179+string :: Monad m => ByteString -> ZeptoT m ()
142180string s = do
143181 i <- gets input
144182 if s `B.isPrefixOf` i
@@ -147,7 +185,7 @@ string s = do
147185{-# INLINE string #-}
148186
149187-- | Indicate whether the end of the input has been reached.
150−atEnd :: Parser Bool
188+atEnd :: Monad m => ZeptoT m Bool
151189atEnd = do
152190 i <- gets input
153191 return $! B.null i
README.markdown−6
@@ -16,12 +16,6 @@ Master [git repository](https://github.com/bos/attoparsec):
1616
1717* `git clone git://github.com/bos/attoparsec.git`
1818
19−There's also a [Mercurial mirror](https://bitbucket.org/bos/attoparsec):
20−
21−* `hg clone https://bitbucket.org/bos/attoparsec`
22−
23−(You can create and contribute changes using either Mercurial or git.)
24−
2519Authors
2620-------
2721
attoparsec.cabal+58−13
@@ -1,12 +1,13 @@
11name: attoparsec
2−version: 0.12.1.3
2+version: 0.13.2.2
3+x-revision: 3
34license: BSD3
45license-file: LICENSE
56category: Text, Parsing
67author: Bryan O'Sullivan <bos@serpentine.com>
7−maintainer: Bryan O'Sullivan <bos@serpentine.com>
8+maintainer: Bryan O'Sullivan <bos@serpentine.com>, Ben Gamari <ben@smart-cactus.org>
89stability: experimental
9−tested-with: GHC == 7.0, GHC == 7.2, GHC == 7.4, GHC == 7.6, GHC == 7.8
10+tested-with: GHC == 7.0.4, GHC == 7.2.2, GHC == 7.4.2, GHC ==7.6.3, GHC ==7.8.4, GHC ==7.10.3, GHC ==8.0.2, GHC ==8.2.2, GHC==8.4.4, GHC==8.6.5, GHC==8.8.1
1011synopsis: Fast combinator parsing for bytestrings and text
1112cabal-version: >= 1.8
1213homepage: https://github.com/bos/attoparsec
@@ -21,6 +22,7 @@ extra-source-files:
2122 benchmarks/*.cabal
2223 benchmarks/*.hs
2324 benchmarks/*.txt
25+ benchmarks/json-data/*.json
2426 benchmarks/Makefile
2527 benchmarks/med.txt.bz2
2628 changelog.md
@@ -29,6 +31,7 @@ extra-source-files:
2931 examples/Makefile
3032 tests/*.hs
3133 tests/QC/*.hs
34+ tests/QC/IPv6/*.hs
3235
3336Flag developer
3437 Description: Whether to build the library in development mode
@@ -37,22 +40,30 @@ Flag developer
3740
3841library
3942 build-depends: array,
40− base >= 4.2 && < 5,
43+ base >= 4.3 && < 5,
4144 bytestring,
4245 containers,
4346 deepseq,
4447 scientific >= 0.3.1 && < 0.4,
48+ transformers >= 0.2 && (< 0.4 || >= 0.4.1.0) && < 0.6,
4549 text >= 1.1.1.3
4650 if impl(ghc < 7.4)
4751 build-depends:
4852 bytestring < 0.10.4.0
4953
54+ if !impl(ghc >= 8.0)
55+ -- Data.Semigroup && Control.Monad.Fail are available in base-4.9+
56+ build-depends: fail == 4.9.*,
57+ semigroups >=0.16.1 && <0.20
58+
5059 exposed-modules: Data.Attoparsec
5160 Data.Attoparsec.ByteString
5261 Data.Attoparsec.ByteString.Char8
5362 Data.Attoparsec.ByteString.Lazy
5463 Data.Attoparsec.Char8
5564 Data.Attoparsec.Combinator
65+ Data.Attoparsec.Internal
66+ Data.Attoparsec.Internal.Types
5667 Data.Attoparsec.Lazy
5768 Data.Attoparsec.Number
5869 Data.Attoparsec.Text
@@ -62,9 +73,7 @@ library
6273 other-modules: Data.Attoparsec.ByteString.Buffer
6374 Data.Attoparsec.ByteString.FastSet
6475 Data.Attoparsec.ByteString.Internal
65− Data.Attoparsec.Internal
6676 Data.Attoparsec.Internal.Fhthagn
67− Data.Attoparsec.Internal.Types
6877 Data.Attoparsec.Text.Buffer
6978 Data.Attoparsec.Text.FastSet
7079 Data.Attoparsec.Text.Internal
@@ -82,9 +91,31 @@ test-suite tests
8291 QC.ByteString
8392 QC.Combinator
8493 QC.Common
94+ QC.IPv6.Internal
95+ QC.IPv6.Types
8596 QC.Rechunked
8697 QC.Simple
8798 QC.Text
99+ QC.Text.FastSet
100+ QC.Text.Regressions
101+
102+ other-modules: Data.Attoparsec.ByteString
103+ Data.Attoparsec.ByteString.Buffer
104+ Data.Attoparsec.ByteString.Char8
105+ Data.Attoparsec.ByteString.FastSet
106+ Data.Attoparsec.ByteString.Internal
107+ Data.Attoparsec.ByteString.Lazy
108+ Data.Attoparsec.Combinator
109+ Data.Attoparsec.Internal
110+ Data.Attoparsec.Internal.Fhthagn
111+ Data.Attoparsec.Internal.Types
112+ Data.Attoparsec.Number
113+ Data.Attoparsec.Text
114+ Data.Attoparsec.Text.Buffer
115+ Data.Attoparsec.Text.FastSet
116+ Data.Attoparsec.Text.Internal
117+ Data.Attoparsec.Text.Lazy
118+ Data.Attoparsec.Zepto
88119
89120 ghc-options:
90121 -Wall -threaded -rtsopts
@@ -94,17 +125,23 @@ test-suite tests
94125
95126 build-depends:
96127 array,
97− base >= 4 && < 5,
128+ base,
98129 bytestring,
99130 deepseq >= 1.1,
100− QuickCheck >= 2.7,
131+ QuickCheck >= 2.13.2 && < 2.14,
101132 quickcheck-unicode,
102133 scientific,
103− test-framework >= 0.8.0.2,
104− test-framework-quickcheck2 >= 0.3.0.3,
134+ tasty >= 0.11,
135+ tasty-quickcheck >= 0.8,
105136 text,
137+ transformers >= 0.2 && (< 0.4 || >= 0.4.1.0) && < 0.6,
106138 vector
107139
140+ if !impl(ghc >= 8.0)
141+ -- Data.Semigroup && Control.Monad.Fail are available in base-4.9+
142+ build-depends: fail == 4.9.*,
143+ semigroups >=0.16.1 && <0.20
144+
108145benchmark benchmarks
109146 type: exitcode-stdio-1.0
110147 hs-source-dirs: benchmarks benchmarks/warp-3.0.1.1 .
@@ -116,7 +153,11 @@ benchmark benchmarks
116153 HeadersByteString.Atto
117154 HeadersText
118155 Links
156+ Network.Wai.Handler.Warp.ReadInt
157+ Network.Wai.Handler.Warp.RequestHeader
119158 Numbers
159+ Sets
160+ TextFastSet
120161 Warp
121162 ghc-options: -O2 -Wall
122163
@@ -128,6 +169,7 @@ benchmark benchmarks
128169 base == 4.*,
129170 bytestring >= 0.10.4.0,
130171 case-insensitive,
172+ containers,
131173 criterion >= 1.0,
132174 deepseq >= 1.1,
133175 directory,
@@ -137,13 +179,16 @@ benchmark benchmarks
137179 parsec >= 3.1.2,
138180 scientific,
139181 text >= 1.1.1.0,
182+ transformers >= 0.2 && (< 0.4 || >= 0.4.1.0) && < 0.6,
140183 unordered-containers,
141184 vector
142185
186+ if !impl(ghc >= 8.0)
187+ -- Data.Semigroup && Control.Monad.Fail are available in base-4.9+
188+ build-depends: fail == 4.9.*,
189+ semigroups >=0.16.1 && <0.19
190+
143191source-repository head
144192 type: git
145193 location: https://github.com/bos/attoparsec
146194
147−source-repository head
148− type: mercurial
149− location: https://bitbucket.org/bos/attoparsec
benchmarks/Aeson.hs+6−2
@@ -10,8 +10,13 @@ module Aeson
1010import Data.ByteString.Builder
1111 (Builder, byteString, toLazyByteString, charUtf8, word8)
1212
13+#if !MIN_VERSION_base(4,8,0)
14+import Control.Applicative ((*>), (<$>), (<*), pure)
15+import Data.Monoid (mappend, mempty)
16+#endif
17+
1318import Common (pathTo)
14−import Control.Applicative ((*>), (<$>), (<*), liftA2, pure)
19+import Control.Applicative (liftA2)
1520import Control.DeepSeq (NFData(..))
1621import Control.Monad (forM)
1722import Data.Attoparsec.ByteString.Char8 (Parser, char, endOfInput, scientific,
@@ -20,7 +25,6 @@ import Data.Bits ((.|.), shiftL)
2025import Data.ByteString (ByteString)
2126import Data.Char (chr)
2227import Data.List (sort)
23−import Data.Monoid (mappend, mempty)
2428import Data.Scientific (Scientific)
2529import Data.Text (Text)
2630import Data.Text.Encoding (decodeUtf8')
benchmarks/Benchmarks.hs+8
@@ -13,6 +13,7 @@ import Text.Parsec.Text ()
1313import Text.Parsec.Text.Lazy ()
1414import qualified Warp
1515import qualified Aeson
16+import qualified Genome
1617import qualified Data.Attoparsec.ByteString as AB
1718import qualified Data.Attoparsec.ByteString.Char8 as AC
1819import qualified Data.Attoparsec.ByteString.Lazy as ABL
@@ -27,6 +28,7 @@ import qualified HeadersByteString
2728import qualified HeadersText
2829import qualified Links
2930import qualified Text.Parsec as P
31+import qualified Sets
3032
3133main :: IO ()
3234main = do
@@ -69,10 +71,14 @@ main = do
6971 , bench "isAlpha_ascii" $ nf (ABL.parse (AC.takeWhile AC.isAlpha_ascii)) bl
7072 , bench "isAlpha_iso8859_15" $
7173 nf (ABL.parse (AC.takeWhile AC.isAlpha_iso8859_15)) bl
74+ , bench "T isAlpha" $ nf (AT.parse (AT.takeWhile isAlpha)) t
75+ , bench "TL isAlpha" $ nf (ATL.parse (AT.takeWhile isAlpha)) tl
7276 ]
7377 , bgroup "takeWhile1" [
7478 bench "isAlpha" $ nf (ABL.parse (AC.takeWhile1 isAlpha)) bl
7579 , bench "isAlpha_ascii" $ nf (ABL.parse (AC.takeWhile1 AC.isAlpha_ascii)) bl
80+ , bench "T isAlpha" $ nf (AT.parse (AT.takeWhile1 isAlpha)) t
81+ , bench "TL isAlpha" $ nf (ATL.parse (AT.takeWhile1 isAlpha)) tl
7682 ]
7783 , bench "word32LE" $ nf (AB.parse word32LE) b
7884 , bgroup "scan" [
@@ -80,10 +86,12 @@ main = do
8086 , bench "long" $ nf (AB.parse quotedString) b
8187 ]
8288 , aeson
89+ , Genome.genome
8390 , headersBS
8491 , headersT
8592 , Links.links
8693 , numbers
94+ , Sets.benchmarks
8795 , Warp.benchmarks
8896 ]
8997
benchmarks/Genome.hsnew file+75
@@ -0,0 +1,75 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+module Genome
4+ (
5+ genome
6+ ) where
7+
8+import Control.Applicative
9+import Criterion.Main
10+import Data.ByteString (ByteString)
11+import qualified Data.ByteString.Char8 as B8
12+import qualified Data.ByteString.Lazy.Char8 as L8
13+import Data.Attoparsec.ByteString.Char8 as B
14+import qualified Data.Attoparsec.ByteString.Lazy as BL
15+import Data.Text (Text)
16+import qualified Data.Text as T
17+import qualified Data.Text.Lazy as TL
18+import Data.Attoparsec.Text as T
19+import qualified Data.Attoparsec.Text.Lazy as TL
20+import Common (rechunkBS, rechunkT)
21+
22+genome :: Benchmark
23+genome = bgroup "genome" [
24+ bgroup "bytestring" [
25+ bench "s" $ nf (map (B.parse searchBS)) (B8.tails geneB)
26+ , bench "l" $ nf (map (BL.parse searchBS)) (L8.tails geneBL)
27+ , bgroup "CI" [
28+ bench "s" $ nf (map (B.parse searchBSCI)) (B8.tails geneB)
29+ , bench "l" $ nf (map (BL.parse searchBSCI)) (L8.tails geneBL)
30+ ]
31+ ]
32+ , bgroup "text" [
33+ bench "s" $ nf (map (T.parse searchT)) (T.tails geneT)
34+ , bench "l" $ nf (map (TL.parse searchT)) (TL.tails geneTL)
35+ , bgroup "CI" [
36+ bench "s" $ nf (map (T.parse searchTCI)) (T.tails geneT)
37+ , bench "l" $ nf (map (TL.parse searchTCI)) (TL.tails geneTL)
38+ ]
39+ ]
40+ ]
41+ where geneB = B8.pack gene
42+ geneBL = rechunkBS 4 geneB
43+ geneT = T.pack gene
44+ geneTL = rechunkT 4 geneT
45+
46+searchBS :: B.Parser ByteString
47+searchBS = "caac" *> ("aaca" <|> "aact")
48+
49+searchBSCI :: B.Parser ByteString
50+searchBSCI = B.stringCI "CAAC" *> (B.stringCI "AACA" <|> B.stringCI "AACT")
51+
52+searchT :: T.Parser Text
53+searchT = "caac" *> ("aaca" <|> "aact")
54+
55+searchTCI :: T.Parser Text
56+searchTCI = T.asciiCI "CAAC" *> (T.asciiCI "AACA" <|> T.asciiCI "AACT")
57+
58+-- Dictyostelium discoideum developmental protein DG1094 (gacT) gene,
59+-- partial cds. http://www.ncbi.nlm.nih.gov/nuccore/AF081586.1
60+
61+gene :: String
62+gene = "atcgatttagaaagatacaaagatagaaccatcaataataaacaagagaagagagcaagt\
63+ \agagatattaataaagagattgaaagagagattgaaaagaagagattatcaccaagagaa\
64+ \agattaaatttatttggtctttcttcctcatcttcatcagtgaattcaacattaacaaga\
65+ \tctacagcaaatattatctctacaatagacggtagtggaggtagtaatcgtaatagtaaa\
66+ \aattatggtaatggctcatcctcctcctcaaatagaagatatagtaatactattaatcaa\
67+ \caattacaaatgcaattacaacaacttcaaatccaacaacaacaatatcaacaaactcaa\
68+ \caatctcaaataccattacaatatcaacaacaacaacagcaacaacaacaacaaaccact\
69+ \acaactacaactacatcaagtggtagtaatagattctcttcaaatagatataaaccagtt\
70+ \gatcttacacaatcatcttcaaactttcgttattcacgtgaaatttatgatgatgattat\
71+ \tattcaaataataatttaatgatgtttggtaatgagcaaccaaatcaaacaccaatttct\
72+ \gtatcatcttcatctgcattcacacgtcaaagatctcaaagttgctttgaaccagagaat\
73+ \cttgtattgctacaacaacaatatcaacaatatcaacaacaacaacaacaacaacaacaa\
74+ \attccattccaagcaaatccacaatatagtaatgctgttattgaacaaaaattggatcaa\
75+ \attagagataccattaataatttacatagagataaccgagtctctaga"
benchmarks/HeadersByteString.hs+10−1
@@ -9,6 +9,15 @@ import qualified Data.Attoparsec.ByteString.Char8 as B
99import qualified Data.Attoparsec.ByteString.Lazy as BL
1010import qualified Data.ByteString.Char8 as B
1111
12+-- Note: In the benchmarks for parsing an http request
13+-- from a strict bytestring, we consider warp's implementation,
14+-- which has highly optimized code for handling the first
15+-- line in particular. It's treatment of the headers
16+-- is more relaxed from the treatment they are given by the
17+-- attoparsec parser benchmarked here. Consequently, it
18+-- is should not be possible to match its performance since
19+-- it accepts header names with disallowed characters.
20+
1221headers :: IO Benchmark
1322headers = do
1423 req <- B.readFile =<< pathTo "http-request.txt"
@@ -18,7 +27,7 @@ headers = do
1827 return $ bgroup "headers" [
1928 bgroup "B" [
2029 bench "request" $ nf (B.parseOnly request) req
21− , bench "warp" $ nfIO (parseHeaderLines [req])
30+ , bench "warp" $ nfIO (parseHeaderLines (B.split '\n' req))
2231 , bench "response" $ nf (B.parseOnly response) resp
2332 ]
2433 , bgroup "BL" [
benchmarks/HeadersByteString/Atto.hs+9−1
@@ -15,8 +15,16 @@ import qualified Data.ByteString.Char8 as B
1515instance NFData HttpVersion where
1616 rnf !_ = ()
1717
18+isHeaderChar :: Char -> Bool
19+isHeaderChar c =
20+ (c >= 'a' && c <= 'z') ||
21+ (c >= 'A' && c <= 'Z') ||
22+ (c >= '0' && c <= '9') ||
23+ (c == '_') ||
24+ (c == '-')
25+
1826header = do
19− name <- B.takeWhile1 (B.inClass "a-zA-Z0-9_-") <* B.char ':' <* B.skipSpace
27+ name <- B.takeWhile1 isHeaderChar <* B.char ':' <* B.skipSpace
2028 body <- bodyLine
2129 return (name, body)
2230
benchmarks/Links.hs+2−2
@@ -5,8 +5,8 @@ module Links (links) where
55import Control.Applicative
66import Control.DeepSeq (NFData(..))
77import Criterion.Main (Benchmark, bench, nf)
8−import Data.Attoparsec as A
9−import Data.Attoparsec.Char8 as A8
8+import Data.Attoparsec.ByteString as A
9+import Data.Attoparsec.ByteString.Char8 as A8
1010import Data.ByteString.Char8 as B8
1111
1212data Link = Link {
benchmarks/Numbers.hs+1
@@ -1,4 +1,5 @@
11{-# LANGUAGE BangPatterns #-}
2+{-# OPTIONS_GHC -fno-warn-deprecations #-}
23
34module Numbers (numbers) where
45
benchmarks/Sets.hsnew file+21
@@ -0,0 +1,21 @@
1+module Sets (benchmarks) where
2+
3+import Criterion
4+import Data.Char (ord)
5+import qualified Data.Attoparsec.Text.FastSet as FastSet
6+import qualified TextFastSet
7+import qualified Data.HashSet as HashSet
8+import qualified Data.IntSet as IntSet
9+
10+smallSet :: String
11+smallSet = ['a'..'z'] ++ ['A'..'Z'] ++ ['0'..'9']
12+
13+benchmarks :: Benchmark
14+benchmarks = bgroup "sets" [
15+ bench "Fast" $ whnf (FastSet.member '*') (FastSet.fromList smallSet)
16+ , bench "Hash" $ whnf (HashSet.member '*') (HashSet.fromList smallSet)
17+ , bench "Int" $ whnf (IntSet.member (ord '*'))
18+ (IntSet.fromList (map ord smallSet))
19+ , bench "TextFast" $ whnf (TextFastSet.member '*')
20+ (TextFastSet.fromList smallSet)
21+ ]
benchmarks/TextFastSet.hsnew file+121
@@ -0,0 +1,121 @@
1+{-# LANGUAGE BangPatterns #-}
2+
3+------------------------------------------------------------------------------
4+-- |
5+-- Module : Data.Attoparsec.FastSet
6+-- Copyright : Felipe Lessa 2010, Bryan O'Sullivan 2007-2015
7+-- License : BSD3
8+--
9+-- Maintainer : felipe.lessa@gmail.com
10+-- Stability : experimental
11+-- Portability : unknown
12+--
13+-- Fast set membership tests for 'Char' values. We test for
14+-- membership using a hashtable implemented with Robin Hood
15+-- collision resolution. The set representation is unboxed,
16+-- and the characters and hashes interleaved, for efficiency.
17+--
18+--
19+-----------------------------------------------------------------------------
20+module TextFastSet
21+ (
22+ -- * Data type
23+ FastSet
24+ -- * Construction
25+ , fromList
26+ , set
27+ -- * Lookup
28+ , member
29+ -- * Handy interface
30+ , charClass
31+ ) where
32+
33+import Data.Bits ((.|.), (.&.), shiftR)
34+import Data.Function (on)
35+import Data.List (sort, sortBy)
36+import qualified Data.Array.Base as AB
37+import qualified Data.Array.Unboxed as A
38+import qualified Data.Text as T
39+
40+data FastSet = FastSet {
41+ table :: {-# UNPACK #-} !(A.UArray Int Int)
42+ , mask :: {-# UNPACK #-} !Int
43+ }
44+
45+data Entry = Entry {
46+ key :: {-# UNPACK #-} !Char
47+ , initialIndex :: {-# UNPACK #-} !Int
48+ , index :: {-# UNPACK #-} !Int
49+ }
50+
51+offset :: Entry -> Int
52+offset e = index e - initialIndex e
53+
54+resolveCollisions :: [Entry] -> [Entry]
55+resolveCollisions [] = []
56+resolveCollisions [e] = [e]
57+resolveCollisions (a:b:entries) = a' : resolveCollisions (b' : entries)
58+ where (a', b')
59+ | index a < index b = (a, b)
60+ | offset a < offset b = (b { index=index a }, a { index=index a + 1 })
61+ | otherwise = (a, b { index=index a + 1 })
62+
63+pad :: Int -> [Entry] -> [Entry]
64+pad = go 0
65+ where -- ensure that we pad enough so that lookups beyond the
66+ -- last hash in the table fall within the array
67+ go !_ !m [] = replicate (max 1 m + 1) empty
68+ go k m (e:entries) = map (const empty) [k..i - 1] ++ e :
69+ go (i + 1) (m + i - k - 1) entries
70+ where i = index e
71+ empty = Entry '\0' maxBound 0
72+
73+nextPowerOf2 :: Int -> Int
74+nextPowerOf2 0 = 1
75+nextPowerOf2 x = go (x - 1) 1
76+ where go y 32 = y + 1
77+ go y k = go (y .|. (y `shiftR` k)) $ k * 2
78+
79+fastHash :: Char -> Int
80+fastHash = fromEnum
81+
82+fromList :: String -> FastSet
83+fromList s = FastSet (AB.listArray (0, length interleaved - 1) interleaved)
84+ mask'
85+ where s' = ordNub (sort s)
86+ l = length s'
87+ mask' = nextPowerOf2 ((5 * l) `div` 4) - 1
88+ entries = pad mask' .
89+ resolveCollisions .
90+ sortBy (compare `on` initialIndex) .
91+ zipWith (\c i -> Entry c i i) s' .
92+ map ((.&. mask') . fastHash) $ s'
93+ interleaved = concatMap (\e -> [fromEnum $ key e, initialIndex e])
94+ entries
95+
96+ordNub :: Eq a => [a] -> [a]
97+ordNub [] = []
98+ordNub (y:ys) = go y ys
99+ where go x (z:zs)
100+ | x == z = go x zs
101+ | otherwise = x : go z zs
102+ go x [] = [x]
103+
104+set :: T.Text -> FastSet
105+set = fromList . T.unpack
106+
107+-- | Check the set for membership.
108+member :: Char -> FastSet -> Bool
109+member c a = go (2 * i)
110+ where i = fastHash c .&. mask a
111+ lookupAt j b = (i' <= i) && (c == c' || b)
112+ where c' = toEnum $ AB.unsafeAt (table a) j
113+ i' = AB.unsafeAt (table a) $ j + 1
114+ go j = lookupAt j . lookupAt (j + 2) . lookupAt (j + 4) .
115+ lookupAt (j + 6) . go $ j + 8
116+
117+charClass :: String -> FastSet
118+charClass = fromList . go
119+ where go (a:'-':b:xs) = [a..b] ++ go xs
120+ go (x:xs) = x : go xs
121+ go _ = ""
benchmarks/attoparsec-benchmarks.cabal+4
@@ -16,6 +16,8 @@ executable attoparsec-benchmarks
1616 Links
1717 Numbers
1818 Network.Wai.Handler.Warp.ReadInt
19+ Sets
20+ TextFastSet
1921 Warp
2022 hs-source-dirs: .. . warp-3.0.1.1
2123 ghc-options: -O2 -Wall -rtsopts
@@ -24,6 +26,7 @@ executable attoparsec-benchmarks
2426 base == 4.*,
2527 bytestring >= 0.10.4.0,
2628 case-insensitive,
29+ containers,
2730 criterion >= 1.0,
2831 deepseq >= 1.1,
2932 directory,
@@ -33,5 +36,6 @@ executable attoparsec-benchmarks
3336 parsec >= 3.1.2,
3437 scientific >= 0.3.1,
3538 text >= 1.1.1.0,
39+ transformers,
3640 unordered-containers,
3741 vector
benchmarks/http-request.txt+3−8
@@ -1,14 +1,9 @@
11GET / HTTP/1.1
22Host: twitter.com
3−Accept: text/html, application/xhtml+xml, application/xml; q=0.9,
4− image/webp, */*; q=0.8
3+Accept: text/html, application/xhtml+xml, application/xml; q=0.9, image/webp, */*; q=0.8
54Accept-Encoding: gzip,deflate,sdch
65Accept-Language: en-GB,en-US;q=0.8,en;q=0.6
76Cache-Control: max-age=0
8−Cookie: guest_id=v1%3A139; _twitter_sess=BAh7CSIKZmxhc2hJQz-e1e1;
9− __utma=43838368.452555194.1399611824.1; __utmb=43838368;
10− __utmc=43838368; __utmz=1399611824.1.1.utmcsr=(direct)|utmcmd=(none)
11−User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5)
12− AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.86
13− Safari/537.36
7+Cookie: guest_id=v1%3A139; _twitter_sess=BAh7CSIKZmxhc2hJQz-e1e1; __utma=43838368.452555194.1399611824.1; __utmb=43838368; __utmc=43838368; __utmz=1399611824.1.1.utmcsr=(direct)|utmcmd=(none)
8+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.86 Safari/537.36
149
benchmarks/http-response.txt+2−4
@@ -3,10 +3,8 @@ Date: Fri, 09 May 2014 04:24:57 GMT
33Expires: -1
44Cache-Control: private, max-age=0
55Content-Type: text/html; charset=ISO-8859-1
6−Set-Cookie: PREF=ID=1a871afc4240db5e:FF:LM=1399609497:S=e5MZ0itEekjVwNcU;
7− expires=Sun, 08-May-2016 04:24:57 GMT; path=/; domain=.google.com
8−Set-Cookie: NID=67=-WlfaDG6VSEPk7abAjrK98HBSoCD2ID6JKkUR95tEumzDmg7Fc8pSQ8;
9− expires=Sat, 08-Nov-2014 04:24:57 GMT; path=/; domain=.google.com; HttpOnly
6+Set-Cookie: PREF=ID=1a871afc4240db5e:FF:LM=1399609497:S=e5MZ0itEekjVwNcU; expires=Sun, 08-May-2016 04:24:57 GMT; path=/; domain=.google.com
7+Set-Cookie: NID=67=-WlfaDG6VSEPk7abAjrK98HBSoCD2ID6JKkUR95tEumzDmg7Fc8pSQ8; expires=Sat, 08-Nov-2014 04:24:57 GMT; path=/; domain=.google.com; HttpOnly
108P3P: CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info."
119Server: gws
1210X-XSS-Protection: 1; mode=block
benchmarks/warp-3.0.1.1/Network/Wai/Handler/Warp/RequestHeader.hs+1−1
@@ -139,7 +139,7 @@ parseRequestLine requestLine@(PS fptr off len) = withForeignPtr fptr $ \ptr -> d
139139
140140parseHeader :: ByteString -> H.Header
141141parseHeader s =
142− let (k, rest) = S.breakByte 58 s -- ':'
142+ let (k, rest) = S.break (== 58) s -- ':'
143143 rest' = S.dropWhile (\c -> c == 32 || c == 9) $ S.drop 1 rest
144144 in (CI.mk k, rest')
145145
changelog.md+49
@@ -1,3 +1,52 @@
1+0.13.2.1
2+
3+* Improved performance of `Data.Attoparsec.Text.asciiCI`
4+
5+0.13.2.0
6+
7+* `pure` is now strict in `Position`
8+
9+0.13.1.0
10+
11+* `runScanner` now correctly returns the final state
12+ (https://github.com/bos/attoparsec/issues/105).
13+* `Parser`, `ZeptoT`, `Buffer`, and `More` now expose `Semigroup` instances.
14+* `Parser`, and `ZeptoT` now expose `MonadFail` instances.
15+
16+0.13.0.2
17+
18+* Restore the fast specialised character set implementation for Text
19+* Move testsuite from test-framework to tasty
20+* Performance optimization of takeWhile and takeWhile1
21+
22+0.13.0.1
23+
24+* Fixed a bug in the implementations of inClass and notInClass for
25+ Text (https://github.com/bos/attoparsec/issues/103)
26+
27+0.13.0.0
28+
29+* Made the parser type in the Zepto module a monad transformer
30+ (needed by aeson's string unescaping parser).
31+
32+0.12.1.6
33+
34+* Fixed a case folding bug in the ByteString version of stringCI.
35+
36+0.12.1.5
37+
38+* Fixed an indexing bug in the new Text implementation of string,
39+ reported by Michel Boucey.
40+
41+0.12.1.4
42+
43+* Fixed a case where the string parser would consume an unnecessary
44+ amount of input before failing a match, when it could bail much
45+ earlier (https://github.com/bos/attoparsec/issues/97)
46+
47+* Added more context to error messages
48+ (https://github.com/bos/attoparsec/pull/79)
49+
1500.12.1.3
251
352* Fixed incorrect tracking of Text lengths
tests/QC.hs+2−2
@@ -6,11 +6,11 @@ import qualified QC.ByteString as ByteString
66import qualified QC.Combinator as Combinator
77import qualified QC.Simple as Simple
88import qualified QC.Text as Text
9−import Test.Framework (defaultMain, testGroup)
9+import Test.Tasty (defaultMain, testGroup)
1010
1111main = defaultMain tests
1212
13−tests = [
13+tests = testGroup "tests" [
1414 testGroup "bs" ByteString.tests
1515 , testGroup "buf" Buffer.tests
1616 , testGroup "combinator" Combinator.tests
tests/QC/Buffer.hs+4−4
@@ -3,13 +3,13 @@
33
44module QC.Buffer (tests) where
55
6−import Control.Applicative ((<$>))
76#if !MIN_VERSION_base(4,8,0)
7+import Control.Applicative ((<$>))
88import Data.Monoid (Monoid(mconcat))
99#endif
1010import QC.Common ()
11−import Test.Framework (Test)
12−import Test.Framework.Providers.QuickCheck2 (testProperty)
11+import Test.Tasty (TestTree)
12+import Test.Tasty.QuickCheck (testProperty)
1313import Test.QuickCheck
1414import qualified Data.Attoparsec.ByteString.Buffer as BB
1515import qualified Data.Attoparsec.Text.Buffer as BT
@@ -82,7 +82,7 @@ t_dropWord16 (BP _ts t buf) = do
8282 i <- choose (0, T.lengthWord16 t)
8383 return $ T.dropWord16 i t === BT.dropWord16 i buf
8484
85−tests :: [Test]
85+tests :: [TestTree]
8686tests = [
8787 testProperty "b_unbuffer" b_unbuffer
8888 , testProperty "t_unbuffer" t_unbuffer
tests/QC/ByteString.hs+25−9
@@ -2,22 +2,22 @@
22module QC.ByteString (tests) where
33
44#if !MIN_VERSION_base(4,8,0)
5−import Control.Applicative ((<*>))
5+import Control.Applicative ((<*>), (<$>))
66#endif
7−import Control.Applicative ((<$>))
8−import Data.Char (chr, ord)
7+import Data.Char (chr, ord, toUpper)
98import Data.Int (Int64)
109import Data.Word (Word8)
1110import Prelude hiding (take, takeWhile)
12−import QC.Common (liftOp, parseBS, toStrictBS)
13−import Test.Framework (Test)
14−import Test.Framework.Providers.QuickCheck2 (testProperty)
11+import QC.Common (ASCII(..), liftOp, parseBS, toStrictBS)
12+import Test.Tasty (TestTree)
13+import Test.Tasty.QuickCheck (testProperty)
1514import Test.QuickCheck
1615import qualified Data.Attoparsec.ByteString as P
1716import qualified Data.Attoparsec.ByteString.Char8 as P8
1817import qualified Data.Attoparsec.ByteString.FastSet as S
1918import qualified Data.Attoparsec.ByteString.Lazy as PL
2019import qualified Data.ByteString as B
20+import qualified Data.ByteString.Char8 as B8
2121import qualified Data.ByteString.Lazy as L
2222import qualified Data.ByteString.Lazy.Char8 as L8
2323
@@ -26,9 +26,11 @@ import qualified Data.ByteString.Lazy.Char8 as L8
2626satisfy :: Word8 -> L.ByteString -> Property
2727satisfy w s = parseBS (P.satisfy (<=w)) (L.cons w s) === Just w
2828
29−satisfyWith :: Char -> L.ByteString -> Property
30−satisfyWith c s = parseBS (P.satisfyWith (chr . fromIntegral) (<=c))
29+satisfyWith :: Word8 -> L.ByteString -> Property
30+satisfyWith w s = parseBS (P.satisfyWith (chr . fromIntegral) (<=c))
3131 (L.cons (fromIntegral (ord c)) s) === Just c
32+ where
33+ c = chr (fromIntegral w)
3234
3335word8 :: Word8 -> L.ByteString -> Property
3436word8 w s = parseBS (P.word8 w) (L.cons w s) === Just w
@@ -66,6 +68,18 @@ string :: L.ByteString -> L.ByteString -> Property
6668string s t = parseBS (P.string s') (s `L.append` t) === Just s'
6769 where s' = toStrictBS s
6870
71+stringCI :: ASCII L.ByteString -> ASCII L.ByteString -> Property
72+stringCI (ASCII s) (ASCII t) =
73+ parseBS (P8.stringCI up) (s `L.append` t) === Just s'
74+ where s' = toStrictBS s
75+ up = B8.map toUpper s'
76+
77+strings :: L.ByteString -> L.ByteString -> L.ByteString -> Property
78+strings s t u =
79+ parseBS (P.string (toStrictBS s) >> P.string t') (L.concat [s,t,u])
80+ === Just t'
81+ where t' = toStrictBS t
82+
6983skipWhile :: Word8 -> L.ByteString -> Property
7084skipWhile w s =
7185 let t = L.dropWhile (<=w) s
@@ -143,7 +157,7 @@ nonmembers :: [Word8] -> [Word8] -> Property
143157nonmembers s s' = property . not . any (`S.memberWord8` set) $ filter (not . (`elem` s)) s'
144158 where set = S.fromList s
145159
146−tests :: [Test]
160+tests :: [TestTree]
147161tests = [
148162 testProperty "anyWord8" anyWord8
149163 , testProperty "endOfInput" endOfInput
@@ -157,6 +171,8 @@ tests = [
157171 , testProperty "skip" skip
158172 , testProperty "skipWhile" skipWhile
159173 , testProperty "string" string
174+ , testProperty "stringCI" stringCI
175+ , testProperty "strings" strings
160176 , testProperty "take" take
161177 , testProperty "takeByteString" takeByteString
162178 , testProperty "takeCount" takeCount
tests/QC/Combinator.hs+18−5
@@ -1,10 +1,15 @@
1+{-# LANGUAGE CPP, OverloadedStrings #-}
2+
13module QC.Combinator where
24
3−import Control.Applicative
5+#if !MIN_VERSION_base(4,8,0)
6+import Control.Applicative ((<*>), (<$>), (<*), (*>))
7+#endif
8+import Data.Maybe (fromJust, isJust)
49import Data.Word (Word8)
5−import QC.Common (Repack, parseBS, repackBS)
6−import Test.Framework (Test)
7−import Test.Framework.Providers.QuickCheck2 (testProperty)
10+import QC.Common (Repack, parseBS, repackBS, toLazyBS)
11+import Test.Tasty (TestTree)
12+import Test.Tasty.QuickCheck (testProperty)
813import Test.QuickCheck
914import qualified Data.Attoparsec.ByteString.Char8 as P
1015import qualified Data.Attoparsec.Combinator as C
@@ -22,6 +27,13 @@ count (Positive (Small n)) rs s =
2227 (length <$> parseBS (C.count n (P.string s)) input) == Just n
2328 where input = repackBS rs (B.concat (replicate (n+1) s))
2429
30+lookAhead :: NonEmptyList Word8 -> Bool
31+lookAhead (NonEmpty xs) =
32+ let ys = B.pack xs
33+ withLookAheadThenConsume = (\x y -> (x, y)) <$> C.lookAhead (P.string ys) <*> P.string ys
34+ mr = parseBS withLookAheadThenConsume $ toLazyBS ys
35+ in isJust mr && fst (fromJust mr) == snd (fromJust mr)
36+
2537match :: Int -> NonNegative Int -> NonNegative Int -> Repack -> Bool
2638match n (NonNegative x) (NonNegative y) rs =
2739 parseBS (P.match parser) (repackBS rs input) == Just (input, n)
@@ -31,9 +43,10 @@ match n (NonNegative x) (NonNegative y) rs =
3143 B8.replicate x 'x', B8.pack (show n), B8.replicate y 'y'
3244 ]
3345
34−tests :: [Test]
46+tests :: [TestTree]
3547tests = [
3648 testProperty "choice" choice
3749 , testProperty "count" count
50+ , testProperty "lookAhead" lookAhead
3851 , testProperty "match" match
3952 ]
tests/QC/Common.hs+27−4
@@ -1,8 +1,9 @@
1−{-# LANGUAGE CPP #-}
1+{-# LANGUAGE CPP, FlexibleInstances #-}
22{-# OPTIONS_GHC -fno-warn-orphans #-}
33module QC.Common
44 (
5− parseBS
5+ ASCII(..)
6+ , parseBS
67 , parseT
78 , toLazyBS
89 , toStrictBS
@@ -15,9 +16,8 @@ module QC.Common
1516 ) where
1617
1718#if !MIN_VERSION_base(4,8,0)
18−import Control.Applicative ((<*>))
19+import Control.Applicative ((<*>), (<$>))
1920#endif
20−import Control.Applicative ((<$>))
2121import Data.Char (isAlpha)
2222import Test.QuickCheck
2323import Test.QuickCheck.Unicode (shrinkChar, string)
@@ -28,6 +28,16 @@ import qualified Data.Text.Lazy as TL
2828import qualified Data.Attoparsec.ByteString.Lazy as BL
2929import qualified Data.Attoparsec.Text.Lazy as TL
3030
31+#if !MIN_VERSION_base(4,4,0)
32+-- This should really be a dependency on the random package :-(
33+instance Random Word8 where
34+ randomR = integralRandomR
35+ random = randomR (minBound,maxBound)
36+
37+instance Arbitrary Word8 where
38+ arbitrary = choose (minBound, maxBound)
39+#endif
40+
3141parseBS :: BL.Parser r -> BL.ByteString -> Maybe r
3242parseBS p = BL.maybeResult . BL.parse p
3343
@@ -42,9 +52,22 @@ toLazyBS = BL.fromChunks . (:[])
4252
4353instance Arbitrary B.ByteString where
4454 arbitrary = B.pack <$> arbitrary
55+ shrink = map B.pack . shrink . B.unpack
4556
4657instance Arbitrary BL.ByteString where
4758 arbitrary = repackBS <$> arbitrary <*> arbitrary
59+ shrink = map BL.pack . shrink . BL.unpack
60+
61+newtype ASCII a = ASCII { fromASCII :: a }
62+ deriving (Eq, Ord, Show)
63+
64+instance Arbitrary (ASCII B.ByteString) where
65+ arbitrary = (ASCII . B.pack) <$> listOf (choose (0,127))
66+ shrink = map (ASCII . B.pack) . shrink . B.unpack . fromASCII
67+
68+instance Arbitrary (ASCII BL.ByteString) where
69+ arbitrary = ASCII <$> (repackBS <$> arbitrary <*> (fromASCII <$> arbitrary))
70+ shrink = map (ASCII . BL.pack) . shrink . BL.unpack . fromASCII
4871
4972type Repack = NonEmptyList (Positive (Small Int))
5073
tests/QC/IPv6/Internal.hsnew file+316
@@ -0,0 +1,316 @@
1+-- -----------------------------------------------------------------------------
2+
3+-- |
4+-- Module : Text.IPv6Addr
5+-- Copyright : Copyright © Michel Boucey 2011-2015
6+-- License : BSD-Style
7+-- Maintainer : michel.boucey@gmail.com
8+--
9+-- Dealing with IPv6 address text representations, canonization and manipulations.
10+--
11+
12+-- -----------------------------------------------------------------------------
13+
14+{-# LANGUAGE OverloadedStrings #-}
15+
16+module QC.IPv6.Internal where {-
17+ ( expandTokens
18+ , macAddr
19+ , maybeIPv6AddrTokens
20+ , ipv4AddrToIPv6AddrTokens
21+ , ipv6TokensToText
22+ , ipv6TokensToIPv6Addr
23+ , isIPv6Addr
24+ , maybeTokIPv6Addr
25+ , maybeTokPureIPv6Addr
26+ , fromDoubleColon
27+ , fromIPv6Addr
28+ , toDoubleColon
29+ ) where -}
30+
31+import Debug.Trace
32+import Control.Monad (replicateM)
33+import Data.Attoparsec.Text
34+import Data.Char (isDigit,isHexDigit,toLower)
35+import Data.Monoid ((<>))
36+import Control.Applicative ((<|>),(<*))
37+import Data.List (group,isSuffixOf,elemIndex,elemIndices,intersperse)
38+import Data.Word (Word32)
39+import Numeric (showHex)
40+import qualified Data.Text as T
41+import qualified Data.Text.Read as R (decimal)
42+import Data.Maybe (fromJust)
43+
44+import QC.IPv6.Types
45+
46+tok0 = "0"
47+
48+-- | Returns the 'T.Text' of an IPv6 address.
49+fromIPv6Addr :: IPv6Addr -> T.Text
50+fromIPv6Addr (IPv6Addr t) = t
51+
52+-- | Given an arbitrary list of 'IPv6AddrToken', returns the corresponding 'T.Text'.
53+ipv6TokensToText :: [IPv6AddrToken] -> T.Text
54+ipv6TokensToText l = T.concat $ map ipv6TokenToText l
55+
56+-- | Returns the corresponding 'T.Text' of an IPv6 address token.
57+ipv6TokenToText :: IPv6AddrToken -> T.Text
58+ipv6TokenToText (SixteenBit s) = s
59+ipv6TokenToText Colon = ":"
60+ipv6TokenToText DoubleColon = "::"
61+ipv6TokenToText AllZeros = tok0 -- "A single 16-bit 0000 field MUST be represented as 0" (RFC 5952, 4.1)
62+ipv6TokenToText (IPv4Addr a) = a
63+
64+-- | Returns 'True' if a list of 'IPv6AddrToken' constitutes a valid IPv6 Address.
65+isIPv6Addr :: [IPv6AddrToken] -> Bool
66+isIPv6Addr [] = False
67+isIPv6Addr [DoubleColon] = True
68+isIPv6Addr [DoubleColon,SixteenBit tok1] = True
69+isIPv6Addr tks =
70+ diffNext tks && (do
71+ let cdctks = countDoubleColon tks
72+ let lentks = length tks
73+ let lasttk = last tks
74+ let lenconst = (lentks == 15 && cdctks == 0) || (lentks < 15 && cdctks == 1)
75+ firstValidToken tks &&
76+ (case countIPv4Addr tks of
77+ 0 -> case lasttk of
78+ SixteenBit _ -> lenconst
79+ DoubleColon -> lenconst
80+ AllZeros -> lenconst
81+ _ -> False
82+ 1 -> case lasttk of
83+ IPv4Addr _ -> (lentks == 13 && cdctks == 0) || (lentks < 12 && cdctks == 1)
84+ _ -> False
85+ otherwise -> False))
86+ where diffNext [] = False
87+ diffNext [_] = True
88+ diffNext (t:ts) = do
89+ let h = head ts
90+ case t of
91+ SixteenBit _ -> case h of
92+ SixteenBit _ -> False
93+ AllZeros -> False
94+ _ -> diffNext ts
95+ AllZeros -> case h of
96+ SixteenBit _ -> False
97+ AllZeros -> False
98+ _ -> diffNext ts
99+ _ -> diffNext ts
100+ firstValidToken l =
101+ case head l of
102+ SixteenBit _ -> True
103+ DoubleColon -> True
104+ AllZeros -> True
105+ _ -> False
106+ countDoubleColon l = length $ elemIndices DoubleColon l
107+ tok1 = "1"
108+
109+countIPv4Addr = foldr oneMoreIPv4Addr 0
110+ where
111+ oneMoreIPv4Addr t c = case t of
112+ IPv4Addr _ -> c + 1
113+ otherwise -> c
114+
115+-- | This is the main function which returns 'Just' the list of a tokenized IPv6
116+-- address text representation validated against RFC 4291 and canonized
117+-- in conformation with RFC 5952, or 'Nothing'.
118+maybeTokIPv6Addr :: T.Text -> Maybe [IPv6AddrToken]
119+maybeTokIPv6Addr t =
120+ case maybeIPv6AddrTokens t of
121+ Just ltks -> if isIPv6Addr ltks
122+ then Just $ (ipv4AddrReplacement . toDoubleColon . fromDoubleColon) ltks
123+ else Nothing
124+ Nothing -> Nothing
125+ where
126+ ipv4AddrReplacement ltks =
127+ if ipv4AddrRewrite ltks
128+ then init ltks ++ ipv4AddrToIPv6AddrTokens (last ltks)
129+ else ltks
130+
131+-- | Returns 'Just' the list of tokenized pure IPv6 address, always rewriting an
132+-- embedded IPv4 address if present.
133+maybeTokPureIPv6Addr :: T.Text -> Maybe [IPv6AddrToken]
134+maybeTokPureIPv6Addr t = do
135+ ltks <- maybeIPv6AddrTokens t
136+ if isIPv6Addr ltks
137+ then Just $ (toDoubleColon . ipv4AddrReplacement . fromDoubleColon) ltks
138+ else Nothing
139+ where
140+ ipv4AddrReplacement ltks' = init ltks' ++ ipv4AddrToIPv6AddrTokens (last ltks')
141+
142+-- | Tokenize a 'T.Text' into 'Just' a list of 'IPv6AddrToken', or 'Nothing'.
143+maybeIPv6AddrTokens :: T.Text -> Maybe [IPv6AddrToken]
144+maybeIPv6AddrTokens s =
145+ case readText s of
146+ Done r l -> traceShow (r,l) $ if r==T.empty then Just l else Nothing
147+ Fail {} -> Nothing
148+ where
149+ readText s = feed (parse (many1 $ ipv4Addr <|> sixteenBit <|> doubleColon <|> colon) s) T.empty
150+
151+-- | An embedded IPv4 address have to be rewritten to output a pure IPv6 Address
152+-- text representation in hexadecimal digits. But some well-known prefixed IPv6
153+-- addresses have to keep visible in their text representation the fact that
154+-- they deals with IPv4 to IPv6 transition process (RFC 5952 Section 5):
155+--
156+-- IPv4-compatible IPv6 address like "::1.2.3.4"
157+--
158+-- IPv4-mapped IPv6 address like "::ffff:1.2.3.4"
159+--
160+-- IPv4-translated address like "::ffff:0:1.2.3.4"
161+--
162+-- IPv4-translatable address like "64:ff9b::1.2.3.4"
163+--
164+-- ISATAP address like "fe80::5efe:1.2.3.4"
165+--
166+ipv4AddrRewrite :: [IPv6AddrToken] -> Bool
167+ipv4AddrRewrite tks =
168+ case last tks of
169+ IPv4Addr _ -> do
170+ let itks = init tks
171+ not (itks == [DoubleColon]
172+ || itks == [DoubleColon,SixteenBit tokffff,Colon]
173+ || itks == [DoubleColon,SixteenBit tokffff,Colon,AllZeros,Colon]
174+ || itks == [SixteenBit "64",Colon,SixteenBit "ff9b",DoubleColon]
175+ || [SixteenBit "200",Colon,SixteenBit tok5efe,Colon] `isSuffixOf` itks
176+ || [AllZeros,Colon,SixteenBit tok5efe,Colon] `isSuffixOf` itks
177+ || [DoubleColon,SixteenBit tok5efe,Colon] `isSuffixOf` itks)
178+ _ -> False
179+ where
180+ tokffff = "ffff"
181+ tok5efe = "5efe"
182+
183+-- | Rewrites an embedded 'IPv4Addr' into the corresponding list of pure 'IPv6Addr' tokens.
184+--
185+-- > ipv4AddrToIPv6AddrTokens (IPv4Addr "127.0.0.1") == [SixteenBits "7f0",Colon,SixteenBits "1"]
186+--
187+ipv4AddrToIPv6AddrTokens :: IPv6AddrToken -> [IPv6AddrToken]
188+ipv4AddrToIPv6AddrTokens t =
189+ case t of
190+ IPv4Addr a -> do
191+ let m = toHex a
192+ [ SixteenBit ((!!) m 0 <> addZero ((!!) m 1))
193+ , Colon
194+ , SixteenBit ((!!) m 2 <> addZero ((!!) m 3)) ]
195+ _ -> [t]
196+ where
197+ toHex a = map (\x -> T.pack $ showHex (read (T.unpack x)::Int) "") $ T.split (=='.') a
198+ addZero d = if T.length d == 1 then tok0 <> d else d
199+
200+expandTokens :: [IPv6AddrToken] -> [IPv6AddrToken]
201+expandTokens = map expandToken
202+ where expandToken (SixteenBit s) = SixteenBit $ T.justifyRight 4 '0' s
203+ expandToken AllZeros = SixteenBit "0000"
204+ expandToken t = t
205+
206+fromDoubleColon :: [IPv6AddrToken] -> [IPv6AddrToken]
207+fromDoubleColon tks =
208+ if DoubleColon `notElem` tks
209+ then tks
210+ else do let s = splitAt (fromJust $ elemIndex DoubleColon tks) tks
211+ let fsts = fst s
212+ let snds = if not (null (snd s)) then tail(snd s) else []
213+ let fste = if null fsts then [] else fsts ++ [Colon]
214+ let snde = if null snds then [] else Colon : snds
215+ fste ++ allZerosTokensReplacement(quantityOfAllZerosTokenToReplace tks) ++ snde
216+ where
217+ allZerosTokensReplacement x = intersperse Colon (replicate x AllZeros)
218+ quantityOfAllZerosTokenToReplace x =
219+ ntks tks - foldl (\c x -> if (x /= DoubleColon) && (x /= Colon) then c+1 else c) 0 x
220+ where
221+ ntks tks = if countIPv4Addr tks == 1 then 7 else 8
222+
223+toDoubleColon :: [IPv6AddrToken] -> [IPv6AddrToken]
224+toDoubleColon tks =
225+ zerosToDoubleColon tks (zerosRunToReplace $ zerosRunsList tks)
226+ where
227+ zerosToDoubleColon :: [IPv6AddrToken] -> (Int,Int) -> [IPv6AddrToken]
228+ -- No all zeros token, so no double colon replacement...
229+ zerosToDoubleColon ls (_,0) = ls
230+ -- "The symbol '::' MUST NOT be used to shorten just one 16-bit 0 field" (RFC 5952 4.2.2)
231+ zerosToDoubleColon ls (_,1) = ls
232+ zerosToDoubleColon ls (i,l) =
233+ let ls' = filter (/= Colon) ls
234+ in intersperse Colon (Prelude.take i ls') ++ [DoubleColon] ++ intersperse Colon (drop (i+l) ls')
235+ zerosRunToReplace t =
236+ let l = longestLengthZerosRun t
237+ in (firstLongestZerosRunIndex t l,l)
238+ where
239+ firstLongestZerosRunIndex x y = sum . snd . unzip $ Prelude.takeWhile (/=(True,y)) x
240+ longestLengthZerosRun x =
241+ maximum $ map longest x
242+ where longest t = case t of
243+ (True,i) -> i
244+ _ -> 0
245+ zerosRunsList x = map helper $ groupZerosRuns x
246+ where
247+ helper h = (head h == AllZeros, lh) where lh = length h
248+ groupZerosRuns = group . filter (/= Colon)
249+
250+ipv6TokensToIPv6Addr :: [IPv6AddrToken] -> Maybe IPv6Addr
251+ipv6TokensToIPv6Addr l = Just $ IPv6Addr $ ipv6TokensToText l
252+
253+fullSixteenBit :: T.Text -> Maybe IPv6AddrToken
254+fullSixteenBit t =
255+ case parse ipv6AddrFullChunk t of
256+ Done a b -> if a==T.empty then Just $ SixteenBit $ T.pack b else Nothing
257+ _ -> Nothing
258+
259+macAddr :: Parser (Maybe [IPv6AddrToken])
260+macAddr = do
261+ n1 <- count 2 hexaChar <* ":"
262+ n2 <- count 2 hexaChar <* ":"
263+ n3 <- count 2 hexaChar <* ":"
264+ n4 <- count 2 hexaChar <* ":"
265+ n5 <- count 2 hexaChar <* ":"
266+ n6 <- count 2 hexaChar
267+ return $ maybeIPv6AddrTokens $ T.pack $ concat [n1,n2,n3,n4,n5,n6]
268+
269+sixteenBit :: Parser IPv6AddrToken
270+sixteenBit = do
271+ r <- ipv6AddrFullChunk <|> count 3 hexaChar <|> count 2 hexaChar <|> count 1 hexaChar
272+ -- "Leading zeros MUST be suppressed" (RFC 5952, 4.1)
273+ let r' = T.dropWhile (=='0') $ T.pack r
274+ return $ if T.null r'
275+ then AllZeros
276+ -- Hexadecimal digits MUST be in lowercase (RFC 5952 4.3)
277+ else SixteenBit $ T.toLower r'
278+
279+ipv4Addr :: Parser IPv6AddrToken
280+ipv4Addr = do
281+ n1 <- manyDigits <* "."
282+ if n1 /= T.empty
283+ then do n2 <- manyDigits <* "."
284+ if n2 /= T.empty
285+ then do n3 <- manyDigits <* "."
286+ if n3 /= T.empty
287+ then do n4 <- manyDigits
288+ if n4 /= T.empty
289+ then return $ IPv4Addr $ T.intercalate "." [n1,n2,n3,n4]
290+ else parserFailure
291+ else parserFailure
292+ else parserFailure
293+ else parserFailure
294+ where
295+ parserFailure = fail "ipv4Addr parsing failure"
296+ manyDigits = do
297+ ds <- takeWhile1 isDigit
298+ case R.decimal ds of
299+ Right (n,_) -> return (if n < 256 then T.pack $ show n else T.empty)
300+ Left _ -> return T.empty
301+
302+doubleColon :: Parser IPv6AddrToken
303+doubleColon = do
304+ string "::"
305+ return DoubleColon
306+
307+colon :: Parser IPv6AddrToken
308+colon = do
309+ string ":"
310+ return Colon
311+
312+ipv6AddrFullChunk :: Parser String
313+ipv6AddrFullChunk = count 4 hexaChar
314+
315+hexaChar :: Parser Char
316+hexaChar = satisfy (inClass "0-9a-fA-F")
tests/QC/IPv6/Types.hsnew file+29
@@ -0,0 +1,29 @@
1+-- -----------------------------------------------------------------------------
2+
3+-- |
4+-- Module : Text.IPv6Addr
5+-- Copyright : Copyright © Michel Boucey 2011-2015
6+-- License : BSD-Style
7+-- Maintainer : michel.boucey@gmail.com
8+--
9+-- Dealing with IPv6 address text representations, canonization and manipulations.
10+--
11+
12+-- -----------------------------------------------------------------------------
13+
14+module QC.IPv6.Types where
15+
16+import qualified Data.Text as T
17+
18+data IPv6Addr = IPv6Addr T.Text
19+
20+instance Show IPv6Addr where
21+ show (IPv6Addr addr) = T.unpack addr
22+
23+data IPv6AddrToken
24+ = SixteenBit T.Text -- ^ A four hexadecimal digits group representing a 16-Bit chunk
25+ | AllZeros -- ^ An all zeros 16-Bit chunk
26+ | Colon -- ^ A separator between 16-Bit chunks
27+ | DoubleColon -- ^ A double-colon stands for a unique compression of many consecutive 16-Bit chunks
28+ | IPv4Addr T.Text -- ^ An embedded IPv4 address as representation of the last 32-Bit
29+ deriving (Eq,Show)
tests/QC/Rechunked.hs+1−1
@@ -29,7 +29,7 @@ rechunk_ split len xs = (unfoldr go . (,) xs) `fmap` rechunkSizes (len xs)
2929
3030rechunkSizes :: Int -> Gen [Int]
3131rechunkSizes n0 = shuffle =<< loop [] (0:repeat 1) n0
32− where loop _ [] _ = error "it's 2014, where's my Stream type?"
32+ where loop _ [] _ = error "it's 2015, where's my Stream type?"
3333 loop acc (lb:lbs) n
3434 | n <= 0 = shuffle (reverse acc)
3535 | otherwise = do
tests/QC/Simple.hs+3−3
@@ -10,8 +10,8 @@ import Data.ByteString (ByteString)
1010import Data.List (foldl')
1111import Data.Maybe (fromMaybe)
1212import QC.Rechunked (rechunkBS)
13−import Test.Framework (Test)
14−import Test.Framework.Providers.QuickCheck2 (testProperty)
13+import Test.Tasty (TestTree)
14+import Test.Tasty.QuickCheck (testProperty)
1515import Test.QuickCheck (Property, counterexample, forAll)
1616import qualified Data.Attoparsec.ByteString.Char8 as A
1717
@@ -31,7 +31,7 @@ parse :: A.Parser r -> [ByteString] -> A.Result r
3131parse p (x:xs) = foldl' A.feed (A.parse p x) xs
3232parse p [] = A.parse p ""
3333
34−tests :: [Test]
34+tests :: [TestTree]
3535tests = [
3636 testProperty "issue75" t_issue75
3737 ]
tests/QC/Text.hs+26−8
@@ -3,20 +3,24 @@
33module QC.Text (tests) where
44
55#if !MIN_VERSION_base(4,8,0)
6−import Control.Applicative ((<*>))
6+import Control.Applicative ((<*>), (<$>))
77#endif
8−import Control.Applicative ((<$>))
98import Data.Int (Int64)
9+import Data.Word (Word8)
1010import Prelude hiding (take, takeWhile)
1111import QC.Common (liftOp, parseT)
12−import Test.Framework (Test)
13−import Test.Framework.Providers.QuickCheck2 (testProperty)
12+import qualified QC.Text.FastSet as FastSet
13+import qualified QC.Text.Regressions as Regressions
14+import Test.Tasty (TestTree, testGroup)
15+import Test.Tasty.QuickCheck (testProperty)
1416import Test.QuickCheck
1517import qualified Data.Attoparsec.Text as P
1618import qualified Data.Attoparsec.Text.Lazy as PL
1719import qualified Data.Attoparsec.Text.FastSet as S
20+import qualified Data.ByteString as BS
1821import qualified Data.Char as Char
1922import qualified Data.Text as T
23+import qualified Data.Text.Encoding as TE
2024import qualified Data.Text.Lazy as L
2125
2226-- Basic byte-level combinators.
@@ -63,9 +67,18 @@ string :: L.Text -> L.Text -> Property
6367string s t = parseT (P.string s') (s `L.append` t) === Just s'
6468 where s' = toStrict s
6569
66−stringCI :: T.Text -> Property
67−stringCI s = P.parseOnly (P.stringCI fs) s === Right s
70+strings :: L.Text -> L.Text -> L.Text -> Property
71+strings s t u =
72+ parseT (P.string (toStrict s) >> P.string t') (L.concat [s,t,u])
73+ === Just t'
74+ where t' = toStrict t
75+
76+-- | Note: "simple, and efficient" works for well formed input...
77+-- i.e. e.g. Latin1 texts
78+stringCI :: [Word8] -> Property
79+stringCI ws = P.parseOnly (P.stringCI fs) s === Right s
6880 where fs = T.toCaseFold s
81+ s = TE.decodeLatin1 (BS.pack ws)
6982
7083asciiCI :: T.Text -> Gen Bool
7184asciiCI x =
@@ -143,7 +156,9 @@ endOfLine s =
143156 (c, fst <$> L.uncons s') === ('\r', Just '\n')
144157
145158scan :: L.Text -> Positive Int64 -> Property
146−scan s (Positive k) = parseT p s === Just (toStrict $ L.take k s)
159+-- for some reason, if counterexample is removed, this test fails?
160+scan s (Positive k) = counterexample (show s)
161+ $ parseT p s === Just (toStrict $ L.take k s)
147162 where p = P.scan k $ \ n _ ->
148163 if n > 0 then let !n' = n - 1 in Just n' else Nothing
149164
@@ -155,7 +170,7 @@ nonmembers :: String -> String -> Property
155170nonmembers s s' = property . not . any (`S.member` set) $ filter (not . (`elem` s)) s'
156171 where set = S.fromList s
157172
158−tests :: [Test]
173+tests :: [TestTree]
159174tests = [
160175 testProperty "anyChar" anyChar
161176 , testProperty "asciiCI" asciiCI
@@ -171,6 +186,7 @@ tests = [
171186 , testProperty "skip" skip
172187 , testProperty "skipWhile" skipWhile
173188 , testProperty "string" string
189+ , testProperty "strings" strings
174190 , testProperty "stringCI" stringCI
175191 , testProperty "take" take
176192 , testProperty "takeText" takeText
@@ -182,4 +198,6 @@ tests = [
182198 , testProperty "takeWhile1_empty" takeWhile1_empty
183199 , testProperty "members" members
184200 , testProperty "nonmembers" nonmembers
201+ , testGroup "FastSet" FastSet.tests
202+ , testGroup "Regressions" Regressions.tests
185203 ]
tests/QC/Text/FastSet.hsnew file+15
@@ -0,0 +1,15 @@
1+module QC.Text.FastSet where
2+
3+import Test.Tasty (TestTree)
4+import Test.Tasty.QuickCheck (testProperty)
5+import Test.QuickCheck
6+import qualified Data.Attoparsec.Text.FastSet as FastSet
7+
8+membershipCorrect :: String -> String -> Property
9+membershipCorrect members others =
10+ let fs = FastSet.fromList members
11+ correct c = (c `FastSet.member` fs) == (c `elem` members)
12+ in property $ all correct (members ++ others)
13+
14+tests :: [TestTree]
15+tests = [ testProperty "membership is correct" membershipCorrect ]
tests/QC/Text/Regressions.hsnew file+54
@@ -0,0 +1,54 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+module QC.Text.Regressions (
4+ tests
5+ ) where
6+
7+import Data.Text (Text)
8+import qualified Data.Text as T
9+import Data.List (foldl')
10+import Data.Maybe (fromMaybe)
11+import Data.Char (isLower)
12+import Data.Monoid ((<>))
13+import QC.Rechunked (rechunkT)
14+import Test.Tasty (TestTree)
15+import Test.Tasty.QuickCheck (testProperty)
16+import Test.QuickCheck (Property, counterexample, forAll)
17+import qualified Data.Attoparsec.Text as A
18+
19+
20+--------------------------------------------------------------------------------
21+-- 105 was about runScanner not always returning the final state. The result
22+-- did depend on how the data was fed to the parser.
23+
24+t_issue105 :: Property
25+t_issue105 = expect issue105 "lowER" (A.Done "ER" "low")
26+
27+issue105 :: A.Parser Text
28+issue105 = do
29+ (_, firstFourLowercaseLetters) <- A.runScanner "" f
30+ return $ firstFourLowercaseLetters
31+
32+ where
33+ f :: Text -> Char -> Maybe Text
34+ f acc c = if T.length acc < 4 && isLower c
35+ then Just $ acc <> T.singleton c
36+ else Nothing
37+
38+
39+expect :: (Show r, Eq r) => A.Parser r -> Text -> A.Result r -> Property
40+expect p input wanted =
41+ forAll (rechunkT input) $ \in' ->
42+ let result = parse p in'
43+ in counterexample (show result ++ " /= " ++ show wanted) $
44+ fromMaybe False (A.compareResults result wanted)
45+
46+parse :: A.Parser r -> [Text] -> A.Result r
47+parse p (x:xs) = foldl' A.feed (A.parse p x) xs
48+parse p [] = A.parse p ""
49+
50+
51+tests :: [TestTree]
52+tests = [
53+ testProperty "issue105" t_issue105
54+ ]