6c4cc7f · text · git.iko.soy

git.iko.soy/text

on master help sign in

Merge pull request #155 from RyanGlScott/master

6c4cc7fBryan O'Sullivan10 years ago

Data/Text/Encoding.hs+7−18

compare with master

unifiedsplit

@@ -55,13 +55,9 @@ module Data.Text.Encoding

55 , encodeUtf32LE

56 , encodeUtf32BE

58−#if MIN_VERSION_bytestring(0,10,4)

59 -- * Encoding Text using ByteString Builders

58 -- * Encoding Text using ByteString Builders

60− -- | /Note/ that these functions are only available if built against

61− -- @bytestring >= 0.10.4.0@.

62 , encodeUtf8Builder

59 , encodeUtf8Builder

63 , encodeUtf8BuilderEscaped

60 , encodeUtf8BuilderEscaped

64−#endif

65 ) where

61 ) where

67#if __GLASGOW_HASKELL__ >= 702

63#if __GLASGOW_HASKELL__ >= 702

@@ -70,24 +66,15 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

70import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

66import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

71#endif

67#endif

73−#if MIN_VERSION_bytestring(0,10,4)

74−import Data.Bits ((.&.))

75−import Data.Text.Internal.Unsafe.Char (ord)

76−import qualified Data.ByteString.Builder as B

77−import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)

78−import qualified Data.ByteString.Builder.Prim as BP

79−import qualified Data.ByteString.Builder.Prim.Internal as BP

80−import qualified Data.Text.Internal.Encoding.Utf16 as U16

81−#endif

82−

83import Control.Exception (evaluate, try)

69import Control.Exception (evaluate, try)

84import Control.Monad.ST (runST)

70import Control.Monad.ST (runST)

71+import Data.Bits ((.&.))

85import Data.ByteString as B

72import Data.ByteString as B

86import Data.ByteString.Internal as B hiding (c2w)

73import Data.ByteString.Internal as B hiding (c2w)

87import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

74import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

88import Data.Text.Internal (Text(..), safe, text)

75import Data.Text.Internal (Text(..), safe, text)

89import Data.Text.Internal.Private (runText)

76import Data.Text.Internal.Private (runText)

90−import Data.Text.Internal.Unsafe.Char (unsafeWrite)

77+import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)

91import Data.Text.Internal.Unsafe.Shift (shiftR)

78import Data.Text.Internal.Unsafe.Shift (shiftR)

92import Data.Text.Show ()

79import Data.Text.Show ()

93import Data.Text.Unsafe (unsafeDupablePerformIO)

80import Data.Text.Unsafe (unsafeDupablePerformIO)

@@ -98,8 +85,13 @@ import Foreign.Marshal.Utils (with)

98import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

85import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

99import Foreign.Storable (Storable, peek, poke)

86import Foreign.Storable (Storable, peek, poke)

100import GHC.Base (ByteArray#, MutableByteArray#)

87import GHC.Base (ByteArray#, MutableByteArray#)

88+import qualified Data.ByteString.Builder as B

89+import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)

90+import qualified Data.ByteString.Builder.Prim as BP

91+import qualified Data.ByteString.Builder.Prim.Internal as BP

101import qualified Data.Text.Array as A

92import qualified Data.Text.Array as A

102import qualified Data.Text.Internal.Encoding.Fusion as E

93import qualified Data.Text.Internal.Encoding.Fusion as E

94+import qualified Data.Text.Internal.Encoding.Utf16 as U16

103import qualified Data.Text.Internal.Fusion as F

95import qualified Data.Text.Internal.Fusion as F

104

105#include "text_cbits.h"

97#include "text_cbits.h"

@@ -315,8 +307,6 @@ decodeUtf8' :: ByteString -> Either UnicodeException Text

315decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode

307decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode

316{-# INLINE decodeUtf8' #-}

308{-# INLINE decodeUtf8' #-}

317

309

318−#if MIN_VERSION_bytestring(0,10,4)

319−

320-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

310-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

321encodeUtf8Builder :: Text -> B.Builder

311encodeUtf8Builder :: Text -> B.Builder

322encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

312encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

@@ -377,7 +367,6 @@ encodeUtf8BuilderEscaped be =

377 outerLoop i (B.BufferRange op ope)

367 outerLoop i (B.BufferRange op ope)

378 where

368 where

379 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)

369 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)

380−#endif

381

370

382-- | Encode text using UTF-8 encoding.

371-- | Encode text using UTF-8 encoding.

383encodeUtf8 :: Text -> ByteString

372encodeUtf8 :: Text -> ByteString

Data/Text/Lazy/Encoding.hs+5−15

@@ -46,27 +46,23 @@ module Data.Text.Lazy.Encoding

46 , encodeUtf32LE

47 , encodeUtf32BE

49−#if MIN_VERSION_bytestring(0,10,4)

50 -- * Encoding Text using ByteString Builders

49 -- * Encoding Text using ByteString Builders

51 , encodeUtf8Builder

50 , encodeUtf8Builder

52 , encodeUtf8BuilderEscaped

51 , encodeUtf8BuilderEscaped

53−#endif

54 ) where

52 ) where

56import Control.Exception (evaluate, try)

54import Control.Exception (evaluate, try)

55+import Data.Monoid (Monoid(..))

57import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

56import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

58import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)

57import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)

59−import qualified Data.ByteString as S

60−import qualified Data.ByteString.Lazy as B

61−import qualified Data.ByteString.Lazy.Internal as B

62−import qualified Data.ByteString.Unsafe as B

63−#if MIN_VERSION_bytestring(0,10,4)

64import Data.Word (Word8)

58import Data.Word (Word8)

65−import Data.Monoid (Monoid(..))

59+import qualified Data.ByteString as S

66import qualified Data.ByteString.Builder as B

60import qualified Data.ByteString.Builder as B

67import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)

61import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)

68import qualified Data.ByteString.Builder.Prim as BP

62import qualified Data.ByteString.Builder.Prim as BP

69−#endif

63+import qualified Data.ByteString.Lazy as B

64+import qualified Data.ByteString.Lazy.Internal as B

65+import qualified Data.ByteString.Unsafe as B

70import qualified Data.Text as T

66import qualified Data.Text as T

71import qualified Data.Text.Encoding as TE

67import qualified Data.Text.Encoding as TE

72import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E

68import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E

@@ -146,7 +142,6 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

146{-# INLINE decodeUtf8' #-}

142{-# INLINE decodeUtf8' #-}

147

143

148encodeUtf8 :: Text -> B.ByteString

144encodeUtf8 :: Text -> B.ByteString

149−#if MIN_VERSION_bytestring(0,10,4)

150encodeUtf8 Empty = B.empty

145encodeUtf8 Empty = B.empty

151encodeUtf8 lt@(Chunk t _) =

146encodeUtf8 lt@(Chunk t _) =

152 B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt

147 B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt

@@ -168,11 +163,6 @@ encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

168encodeUtf8BuilderEscaped prim =

163encodeUtf8BuilderEscaped prim =

169 foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty

164 foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty

170

165

171−#else

172−encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)

173−encodeUtf8 Empty = B.Empty

174−#endif

175−

176-- | Decode text from little endian UTF-16 encoding.

166-- | Decode text from little endian UTF-16 encoding.

177decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text

167decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text

178decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

168decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

benchmarks/text-benchmarks.cabal+11−1

@@ -15,6 +15,11 @@ build-type: Simple

16cabal-version: >=1.2

18+flag bytestring-builder

19+ description: Depend on the bytestring-builder package for backwards compatibility.

20+ default: False

21+ manual: False

22+

18flag llvm

23flag llvm

19 description: use LLVM

24 description: use LLVM

20 default: False

25 default: False

@@ -33,7 +38,6 @@ executable text-benchmarks

33 build-depends: base == 4.*,

38 build-depends: base == 4.*,

34 binary,

39 binary,

35 blaze-builder,

40 blaze-builder,

36− bytestring,

37 bytestring-lexing >= 0.5.0,

41 bytestring-lexing >= 0.5.0,

38 containers,

42 containers,

39 criterion >= 0.10.0.0,

43 criterion >= 0.10.0.0,

@@ -46,6 +50,12 @@ executable text-benchmarks

46 utf8-string,

50 utf8-string,

47 vector

51 vector

53+ if flag(bytestring-builder)

54+ build-depends: bytestring >= 0.9 && < 0.10.4,

55+ bytestring-builder >= 0.10.4

56+ else

57+ build-depends: bytestring >= 0.10.4

58+

49executable text-multilang

59executable text-multilang

50 hs-source-dirs: haskell

60 hs-source-dirs: haskell

51 main-is: Multilang.hs

61 main-is: Multilang.hs

tests/literal-rule-test.sh+1−1

@@ -6,7 +6,7 @@ function check_firings() {

6 rule=$1

7 expected=$2

8 build="ghc -O -ddump-rule-firings LiteralRuleTest.hs"

9− build="$build -i.. -I../include -DMIN_VERSION_bytestring(a,b,c)=1"

9+ build="$build -i.. -I../include"

10 touch LiteralRuleTest.hs

11 echo -n "Want to see $expected firings of rule $rule... " >&2

12 firings=$($build 2>&1 | grep "Rule fired: $rule\$" | wc -l)

text.cabal+15−4

@@ -64,6 +64,11 @@ extra-source-files:

64 tests/scripts/*.sh

65 tests/text-tests.cabal

67+flag bytestring-builder

68+ description: Depend on the bytestring-builder package for backwards compatibility.

69+ default: False

70+ manual: False

71+

67flag developer

72flag developer

68 description: operate in developer mode

73 description: operate in developer mode

69 default: False

74 default: False

@@ -133,10 +138,11 @@ library

133 deepseq >= 1.1.0.0,

138 deepseq >= 1.1.0.0,

134 ghc-prim >= 0.2

139 ghc-prim >= 0.2

135

140

136− if impl(ghc >= 7.7)

141+ if flag(bytestring-builder)

137− build-depends: bytestring >= 0.10.4.0

142+ build-depends: bytestring >= 0.9 && < 0.10.4,

143+ bytestring-builder >= 0.10.4

138 else

144 else

139− build-depends: bytestring >= 0.9

145+ build-depends: bytestring >= 0.10.4

140

146

141 cpp-options: -DHAVE_DEEPSEQ

147 cpp-options: -DHAVE_DEEPSEQ

142 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2

148 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2

@@ -171,7 +177,6 @@ test-suite tests

171 array,

177 array,

172 base,

178 base,

173 binary,

179 binary,

174− bytestring,

175 deepseq,

180 deepseq,

176 directory,

181 directory,

177 ghc-prim,

182 ghc-prim,

@@ -181,6 +186,12 @@ test-suite tests

181 test-framework-hunit >= 0.2,

186 test-framework-hunit >= 0.2,

182 test-framework-quickcheck2 >= 0.2

187 test-framework-quickcheck2 >= 0.2

183

188

189+ if flag(bytestring-builder)

190+ build-depends: bytestring >= 0.9 && < 0.10.4,

191+ bytestring-builder >= 0.10.4

192+ else

193+ build-depends: bytestring >= 0.10.4

194+

184 if flag(integer-simple)

195 if flag(integer-simple)

185 cpp-options: -DINTEGER_SIMPLE

196 cpp-options: -DINTEGER_SIMPLE

186 build-depends: integer-simple >= 0.1 && < 0.5

197 build-depends: integer-simple >= 0.1 && < 0.5