git.iko.soy/text
on master help sign in
Rewrite encodeUtf8 for speed
8dbe7adBryan O'Sullivan
Data/Text/Encoding.hs+60−5
compare with master
unifiedsplit
@@ -1,3 +1,4 @@
1+{-# LANGUAGE BangPatterns #-}
1-- |
2-- |
2-- Module : Data.Text.Encoding
3-- Module : Data.Text.Encoding
3-- Copyright : (c) Tom Harper 2008-2009,
4-- Copyright : (c) Tom Harper 2008-2009,
@@ -34,17 +35,29 @@ module Data.Text.Encoding
34
35
35 -- * Encoding Text to ByteStrings
36 -- * Encoding Text to ByteStrings
36 , encodeUtf8
37 , encodeUtf8
38+ , encodeUtf8'
37 , encodeUtf16LE
39 , encodeUtf16LE
38 , encodeUtf16BE
40 , encodeUtf16BE
39 , encodeUtf32LE
41 , encodeUtf32LE
40 , encodeUtf32BE
42 , encodeUtf32BE
41 ) where
43 ) where
42
44
43−import Data.ByteString (ByteString)
45+import Data.Bits ((.&.))
44−import qualified Data.Text.Fusion as F
46+import Data.ByteString as B
47+import Data.ByteString.Internal as B
45import Data.Text.Encoding.Error (OnDecodeError, strictDecode)
48import Data.Text.Encoding.Error (OnDecodeError, strictDecode)
49+import Data.Text.Encoding.Utf16 (chr2)
50+import Data.Text.Internal (Text(..))
51+import Data.Text.UnsafeChar (ord)
52+import Data.Text.UnsafeShift (shiftL, shiftR)
53+import Data.Word (Word8)
54+import Foreign.ForeignPtr (withForeignPtr)
55+import Foreign.Ptr (plusPtr)
56+import Foreign.Storable (poke)
57+import System.IO.Unsafe (unsafePerformIO)
58+import qualified Data.Text.Array as A
46import qualified Data.Text.Encoding.Fusion as E
59import qualified Data.Text.Encoding.Fusion as E
47−import Data.Text.Internal (Text)
60+import qualified Data.Text.Fusion as F
48
61
49-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
62-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
50decodeASCII :: ByteString -> Text
63decodeASCII :: ByteString -> Text
@@ -61,10 +74,52 @@ decodeUtf8 :: ByteString -> Text
61decodeUtf8 = decodeUtf8With strictDecode
74decodeUtf8 = decodeUtf8With strictDecode
62{-# INLINE decodeUtf8 #-}
75{-# INLINE decodeUtf8 #-}
63
76
77+-- | Encode text using UTF-8 encoding.
78+encodeUtf8' :: Text -> ByteString
79+encodeUtf8' txt = E.unstream (E.restreamUtf8 (F.stream txt))
80+{-# INLINE encodeUtf8' #-}
81+
64-- | Encode text using UTF-8 encoding.
82-- | Encode text using UTF-8 encoding.
65encodeUtf8 :: Text -> ByteString
83encodeUtf8 :: Text -> ByteString
66−encodeUtf8 txt = E.unstream (E.restreamUtf8 (F.stream txt))
84+encodeUtf8 (Text arr off len) = unsafePerformIO $ do
85+ let size0 = min len 4
86+ mallocByteString size0 >>= start size0 off 0
87+ where
88+ start size n0 m0 fp = withForeignPtr fp $ loop n0 m0
89+ where
90+ loop n1 m1 ptr = go n1 m1
91+ where
92+ go !n !m
93+ | n-off == len = return $! PS fp 0 m
94+ | size-m < 4 = {-# SCC "encodeUtf8/resize" #-} do
95+ let newSize = size `shiftL` 1
96+ fp' <- mallocByteString newSize
97+ withForeignPtr fp' $ \ptr' -> memcpy ptr' ptr (fromIntegral m)
98+ start newSize n m fp'
99+ | otherwise = do
100+ let poke8 k v = poke (ptr `plusPtr` k) (fromIntegral v :: Word8)
101+ w = A.unsafeIndex arr n
102+ case undefined of
103+ _| w <= 0x7F -> do
104+ poke8 m w
105+ go (n+1) (m+1)
106+ | w <= 0x7FF -> do
107+ poke8 m $ (w `shiftR` 6) + 0xC0
108+ poke8 (m+1) $ (w .&. 0x3f) + 0x80
109+ go (n+1) (m+2)
110+ | 0xD800 <= w && w <= 0xDBFF -> do
111+ let c = ord $ chr2 w (A.unsafeIndex arr (n+1))
112+ poke8 m $ (c `shiftR` 18) + 0xF0
113+ poke8 (m+1) $ ((c `shiftR` 12) .&. 0x3F) + 0x80
114+ poke8 (m+2) $ ((c `shiftR` 6) .&. 0x3F) + 0x80
115+ poke8 (m+3) $ (c .&. 0x3F) + 0x80
116+ go (n+2) (m+4)
117+ | otherwise -> do
118+ poke8 m $ (w `shiftR` 12) + 0xE0
119+ poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80
120+ poke8 (m+2) $ (w .&. 0x3F) + 0x80
121+ go (n+1) (m+3)
67−{-# INLINE encodeUtf8 #-}
122+{- INLINE encodeUtf8 #-}
68
123
69-- | Decode text from little endian UTF-16 encoding.
124-- | Decode text from little endian UTF-16 encoding.
70decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
125decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text