08b39b5 · text · git.iko.soy

git.iko.soy/text

on master help sign in

Merge the new bytestring builder code

08b39b5Bryan O'Sullivan13 years ago

.gitignorenew file+2

compare with master

unifiedsplit

@@ -0,0 +1,2 @@

1+dist

2+cabal-dev

Data/Text/Encoding.hs+104

@@ -54,6 +54,14 @@ module Data.Text.Encoding

54 , encodeUtf16BE

55 , encodeUtf32LE

56 , encodeUtf32BE

57+

58+#if MIN_VERSION_bytestring(0,10,4)

59+ -- * Generic encoding of Text

60+ -- , encodeStreamWithB

61+ -- , encodeTextWithB

62+ -- , encodeUtf8Builder

63+ , encodeUtf8Escaped

64+#endif

57 ) where

65 ) where

59import Control.Exception (evaluate, try)

67import Control.Exception (evaluate, try)

@@ -66,6 +74,13 @@ import Control.Monad.ST (runST)

66import Data.Bits ((.&.))

74import Data.Bits ((.&.))

67import Data.ByteString as B

75import Data.ByteString as B

68import Data.ByteString.Internal as B

76import Data.ByteString.Internal as B

77+#if MIN_VERSION_bytestring(0,10,4)

78+import qualified Data.ByteString.Builder as B

79+import qualified Data.ByteString.Builder.Internal as B

80+import qualified Data.ByteString.Builder.Prim as BP

81+import qualified Data.ByteString.Builder.Prim.Internal as BP

82+import qualified Data.ByteString.Lazy as BL

83+#endif

69import Data.Text ()

84import Data.Text ()

70import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

85import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

71import Data.Text.Internal (Text(..), safe, textP)

86import Data.Text.Internal (Text(..), safe, textP)

@@ -291,6 +306,61 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDec

291

306

292-- | Encode text using UTF-8 encoding.

307-- | Encode text using UTF-8 encoding.

293encodeUtf8 :: Text -> ByteString

308encodeUtf8 :: Text -> ByteString

309+#if MIN_VERSION_bytestring(0,10,4)

310+

311+encodeUtf8 =

312+ BL.toStrict . B.toLazyByteString

313+ . encodeUtf8Escaped (BP.liftFixedToBounded BP.word8)

314+

315+-- | Encode text using UTF-8 encoding and escape the ASCII characters using

316+-- a 'BP.PrimBounded'.

317+encodeUtf8Escaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

318+encodeUtf8Escaped be (Text arr off len) =

319+ B.builder step

320+ where

321+ bound = max 4 $ BP.sizeBound be

322+ iend = off + len

323+ step !k =

324+ outerLoop off

325+ where

326+ outerLoop !i0 !br@(B.BufferRange op0 ope)

327+ | i0 >= iend = k br

328+ | op0 `plusPtr` bound < ope =

329+ goPartial (i0 + min outRemaining inpRemaining)

330+ | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)

331+ where

332+ outRemaining = (ope `minusPtr` op0) `div` bound

333+ inpRemaining = iend - i0

334+

335+ goPartial !iendTmp = go i0 op0

336+ where

337+ go !i !op

338+ | i < iendTmp = case A.unsafeIndex arr i of

339+ w | w <= 0x7F -> do

340+ BP.runB be (fromIntegral w) op >>= go (i + 1)

341+ | w <= 0x7FF -> do

342+ poke8 0 $ (w `shiftR` 6) + 0xC0

343+ poke8 1 $ (w .&. 0x3f) + 0x80

344+ go (i + 1) (op `plusPtr` 2)

345+ | 0xD800 <= w && w <= 0xDBFF -> do

346+ let c = ord $ U16.chr2 w (A.unsafeIndex arr (i+1))

347+ poke8 0 $ (c `shiftR` 18) + 0xF0

348+ poke8 1 $ ((c `shiftR` 12) .&. 0x3F) + 0x80

349+ poke8 2 $ ((c `shiftR` 6) .&. 0x3F) + 0x80

350+ poke8 3 $ (c .&. 0x3F) + 0x80

351+ go (i + 2) (op `plusPtr` 4)

352+ | otherwise -> do

353+ poke8 0 $ (w `shiftR` 12) + 0xE0

354+ poke8 1 $ ((w `shiftR` 6) .&. 0x3F) + 0x80

355+ poke8 2 $ (w .&. 0x3F) + 0x80

356+ go (i + 1) (op `plusPtr` 3)

357+ | otherwise =

358+ outerLoop i (B.BufferRange op ope)

359+ where

360+ poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)

361+

362+#else

363+

294encodeUtf8 (Text arr off len) = unsafeDupablePerformIO $ do

364encodeUtf8 (Text arr off len) = unsafeDupablePerformIO $ do

295 let size0 = max len 4

365 let size0 = max len 4

296 mallocByteString size0 >>= start size0 off 0

366 mallocByteString size0 >>= start size0 off 0

@@ -344,6 +414,7 @@ encodeUtf8 (Text arr off len) = unsafeDupablePerformIO $ do

344 poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80

414 poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80

345 poke8 (m+2) $ (w .&. 0x3F) + 0x80

415 poke8 (m+2) $ (w .&. 0x3F) + 0x80

346 go (n+1) (m+3)

416 go (n+1) (m+3)

417+#endif

347

418

348-- | Decode text from little endian UTF-16 encoding.

419-- | Decode text from little endian UTF-16 encoding.

349decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

420decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

@@ -432,3 +503,36 @@ foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_stat

432

503

433foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

504foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

434 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

505 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

506+

507+

508+{-

509+-- | Encode all elements of a 'F.Stream' using a 'B.BoundedEncoding'.

510+{-# INLINE encodeStreamWithB #-}

511+encodeStreamWithB :: B.BoundedEncoding a -> F.Stream a -> B.Builder

512+encodeStreamWithB be =

513+ \(F.Stream next s0 _) -> B.builder $ step next s0

514+ where

515+ bound = B.sizeBound be

516+ step next s0 k (B.BufferRange op0 ope0) =

517+ go s0 op0

518+ where

519+ go s !op = case next s of

520+ F.Done -> k (B.BufferRange op ope0)

521+ F.Skip s' -> go s' op

522+ F.Yield x s'

523+ | op `plusPtr` bound <= ope0 -> B.runB be x op >>= go s'

524+ | otherwise ->

525+ return $ B.bufferFull bound op (step next s k)

526+

527+

528+-- |

529+-- | /Subject to fusion./

530+-- Encode all 'Char's of a 'T.Text' using a 'B.BoundedEncoding'.

531+{-# INLINE encodeTextWithB #-}

532+encodeTextWithB :: B.BoundedEncoding Char -> Text -> B.Builder

533+encodeTextWithB be = encodeStreamWithB be . F.stream

534+

535+-- | Encode text using UTF-8 encoding.

536+encodeUtf8Builder :: Text -> B.Builder

537+encodeUtf8Builder = encodeUtf8Escaped (B.fromF B.word8)

538+-}

Data/Text/Lazy/Encoding.hs+14

@@ -54,6 +54,11 @@ import qualified Data.ByteString as S

54import qualified Data.ByteString.Lazy as B

55import qualified Data.ByteString.Lazy.Internal as B

56import qualified Data.ByteString.Unsafe as B

57+#if MIN_VERSION_bytestring(0,10,4)

58+import Data.Monoid (mempty, (<>))

59+import qualified Data.ByteString.Builder as B

60+import qualified Data.ByteString.Builder.Prim as BP

61+#endif

57import qualified Data.Text.Encoding as TE

62import qualified Data.Text.Encoding as TE

58import qualified Data.Text.Lazy as L

63import qualified Data.Text.Lazy as L

59import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E

64import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E

@@ -135,8 +140,17 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

135{-# INLINE decodeUtf8' #-}

140{-# INLINE decodeUtf8' #-}

136

141

137encodeUtf8 :: Text -> B.ByteString

142encodeUtf8 :: Text -> B.ByteString

143+#if MIN_VERSION_bytestring(0,10,4)

144+encodeUtf8 =

145+ B.toLazyByteString . go

146+ where

147+ go Empty = mempty

148+ go (Chunk c cs) =

149+ TE.encodeUtf8Escaped (BP.liftFixedToBounded BP.word8) c <> go cs

150+#else

138encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)

151encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)

139encodeUtf8 Empty = B.Empty

152encodeUtf8 Empty = B.Empty

153+#endif

140

154

141-- | Decode text from little endian UTF-16 encoding.

155-- | Decode text from little endian UTF-16 encoding.

142decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text

156decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text

text.cabal+5−1

@@ -121,10 +121,14 @@ library

121 build-depends:

122 array >= 0.3,

123 base >= 4.2 && < 5,

124− bytestring >= 0.9,

125 deepseq >= 1.1.0.0,

124 deepseq >= 1.1.0.0,

126 ghc-prim >= 0.2

125 ghc-prim >= 0.2

127

126

127+ if impl(ghc >= 7.7)

128+ build-depends: bytestring >= 0.10.4.0

129+ else

130+ build-depends: bytestring >= 0.9

131+

128 cpp-options: -DHAVE_DEEPSEQ

132 cpp-options: -DHAVE_DEEPSEQ

129 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2

133 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2

130 if flag(developer)

134 if flag(developer)