d90cb3a · text · git.iko.soy

git.iko.soy/text

on master help sign in

A valiant attempt at improving UTF-8 encoding performance.

d90cb3aBryan O'Sullivan15 years ago

Data/Text/Encoding.hs+15−45

compare with master

unifiedsplit

@@ -53,24 +53,22 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

53#else

54import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

55#endif

56−import Data.Bits ((.&.))

57import Data.ByteString as B

56import Data.ByteString as B

58import Data.ByteString.Internal as B

57import Data.ByteString.Internal as B

59import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

58import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

60import Data.Text.Internal (Text(..), textP)

59import Data.Text.Internal (Text(..), textP)

61−import Data.Text.UnsafeChar (ord, unsafeWrite)

60+import Data.Text.UnsafeChar (unsafeWrite)

62−import Data.Text.UnsafeShift (shiftL, shiftR)

63import Data.Word (Word8)

61import Data.Word (Word8)

64import Foreign.C.Types (CSize)

62import Foreign.C.Types (CSize)

65−import Foreign.ForeignPtr (withForeignPtr)

63+import Foreign.ForeignPtr (newForeignPtr, withForeignPtr)

66import Foreign.Marshal.Utils (with)

64import Foreign.Marshal.Utils (with)

67import Foreign.Ptr (Ptr, plusPtr)

65import Foreign.Ptr (Ptr, plusPtr)

68import Foreign.Storable (peek, poke)

66import Foreign.Storable (peek, poke)

69−import GHC.Base (MutableByteArray#)

67+import GHC.Base (ByteArray#, MutableByteArray#)

70import System.IO.Unsafe (unsafePerformIO)

68import System.IO.Unsafe (unsafePerformIO)

71import qualified Data.Text.Array as A

69import qualified Data.Text.Array as A

72import qualified Data.Text.Encoding.Fusion as E

70import qualified Data.Text.Encoding.Fusion as E

73−import qualified Data.Text.Encoding.Utf16 as U16

71+

74import qualified Data.Text.Fusion as F

72import qualified Data.Text.Fusion as F

76-- $strict

74-- $strict

@@ -145,45 +143,14 @@ decodeUtf8' = unsafePerformIO . try . evaluate . decodeUtf8With strictDecode

145

143

146-- | Encode text using UTF-8 encoding.

144-- | Encode text using UTF-8 encoding.

147encodeUtf8 :: Text -> ByteString

145encodeUtf8 :: Text -> ByteString

148−encodeUtf8 (Text arr off len) = unsafePerformIO $ do

146+encodeUtf8 (Text arr off len)

149− let size0 = max len 4

147+ | len == 0 = empty

150− mallocByteString size0 >>= start size0 off 0

148+ | otherwise = unsafePerformIO $ do

151− where

149+ with (fromIntegral len ::CSize) $ \lenPtr -> do

152− start size n0 m0 fp = withForeignPtr fp $ loop n0 m0

150+ dptr <- c_encode_utf8 lenPtr (A.aBA arr) (fromIntegral off)

153− where

151+ fp <- newForeignPtr c_free_finalizer dptr

154− loop n1 m1 ptr = go n1 m1

152+ dlen <- peek lenPtr

155− where

156− go !n !m

157− | n == off+len = return (PS fp 0 m)

153+ return (PS fp 0 (fromIntegral dlen))

158− | otherwise = do

159− let poke8 k v = poke (ptr `plusPtr` k) (fromIntegral v :: Word8)

160− ensure k act

161− | size-m >= k = act

162− | otherwise = {-# SCC "resizeUtf8/ensure" #-} do

163− let newSize = size `shiftL` 1

164− fp' <- mallocByteString newSize

165− withForeignPtr fp' $ \ptr' ->

166− memcpy ptr' ptr (fromIntegral m)

167− start newSize n m fp'

168− {-# INLINE ensure #-}

169− case A.unsafeIndex arr n of

170− w| w <= 0x7F -> poke8 m w >> go (n+1) (m+1)

171− | w <= 0x7FF -> ensure 2 $ do

172− poke8 m $ (w `shiftR` 6) + 0xC0

173− poke8 (m+1) $ (w .&. 0x3f) + 0x80

174− go (n+1) (m+2)

175− | 0xD800 <= w && w <= 0xDBFF -> ensure 4 $ do

176− let c = ord $ U16.chr2 w (A.unsafeIndex arr (n+1))

177− poke8 m $ (c `shiftR` 18) + 0xF0

178− poke8 (m+1) $ ((c `shiftR` 12) .&. 0x3F) + 0x80

179− poke8 (m+2) $ ((c `shiftR` 6) .&. 0x3F) + 0x80

180− poke8 (m+3) $ (c .&. 0x3F) + 0x80

181− go (n+2) (m+4)

182− | otherwise -> ensure 3 $ do

183− poke8 m $ (w `shiftR` 12) + 0xE0

184− poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80

185− poke8 (m+2) $ (w .&. 0x3F) + 0x80

186− go (n+1) (m+3)

187

154

188-- | Decode text from little endian UTF-16 encoding.

155-- | Decode text from little endian UTF-16 encoding.

189decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

156decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

@@ -264,3 +231,6 @@ encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))

264foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

231foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

265 :: MutableByteArray# s -> Ptr CSize

232 :: MutableByteArray# s -> Ptr CSize

266 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

233 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

234+

235+foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8

236+ :: Ptr CSize -> ByteArray# -> CSize -> IO (Ptr Word8)

cbits/cbits.c+70−1

@@ -6,9 +6,10 @@

6 * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.

7 */

9−#include <string.h>

10#include <stdint.h>

9#include <stdint.h>

11#include <stdio.h>

10#include <stdio.h>

11+#include <stdlib.h>

12+#include <string.h>

13void _hs_text_memcpy(void *dest, size_t doff, const void *src, size_t soff,

14void _hs_text_memcpy(void *dest, size_t doff, const void *src, size_t soff,

14 size_t n)

15 size_t n)

@@ -131,3 +132,71 @@ _hs_text_decode_utf8(uint16_t *dest, size_t *destoff,

131

132

132 return s;

133 return s;

133}

134}

135+

136+uint8_t *_hs_text_encode_utf8(size_t *plen,

137+ const uint16_t const *src, size_t srcoff)

138+{

139+ size_t srclen = *plen;

140+ size_t destlen = srclen > 4 ? srclen : 4;

141+ uint8_t *dest = malloc(destlen);

142+ uint8_t *d = dest;

143+ const uint8_t const *dend = dest + destlen;

144+ const uint16_t const *s = src + srcoff;

145+ const uint16_t const *srcend = s + srclen;

146+

147+ do {

148+ const uint16_t w = *s++;

149+ if (w < 0x80) {

150+ if (dend - d < 1) {

151+ destlen *= 2;

152+ uint8_t *newdest = realloc(dest, destlen);

153+ d = newdest + (d - dest);

154+ dest = newdest;

155+ dend = newdest + destlen;

156+ }

157+ *d++ = w;

158+ while (s < srcend && d < dend && *s < 0x80) {

159+ *d++ = *s++;

160+ }

161+ } else if (w < 0x800) {

162+ if (dend - d < 2) {

163+ destlen *= 2;

164+ uint8_t *newdest = realloc(dest, destlen);

165+ d = newdest + (d - dest);

166+ dest = newdest;

167+ dend = newdest + destlen;

168+ }

169+ *d++ = 0xc0 | (w >> 6);

170+ *d++ = 0x80 | (w & 0x3f);

171+ } else if (w >= 0xd800 && w < 0xdc00) {

172+ if (dend - d < 4) {

173+ destlen *= 2;

174+ uint8_t *newdest = realloc(dest, destlen);

175+ d = newdest + (d - dest);

176+ dest = newdest;

177+ dend = newdest + destlen;

178+ }

179+ const uint32_t c = (((((uint32_t) w) - 0xd800) << 10) |

180+ (((uint32_t) *s++) - 0xdc00)) + 0x10000;

181+ *d++ = 0xf0 | (c >> 18);

182+ *d++ = 0x80 | ((c >> 12) & 0x3f);

183+ *d++ = 0x80 | ((c >> 6) & 0x3f);

184+ *d++ = 0x80 | (c & 0x3f);

185+ } else {

186+ if (dend - d < 3) {

187+ destlen *= 2;

188+ uint8_t *newdest = realloc(dest, destlen);

189+ d = newdest + (d - dest);

190+ dest = newdest;

191+ dend = newdest + destlen;

192+ }

193+ *d++ = 0xe0 | (w >> 12);

194+ *d++ = 0x80 | ((w >> 6) & 0x3f);

195+ *d++ = 0x80 | (w & 0x3f);

196+ }

197+ } while (s < srcend);

198+

199+ *plen = d - dest;

200+

201+ return dest;

202+}

text.cabal+1

@@ -65,6 +65,7 @@ flag developer

66library

67 c-sources: cbits/cbits.c

68+ cc-options: -Wall

69 exposed-modules:

70 exposed-modules:

70 Data.Text

71 Data.Text