d90cb3a · text · git.iko.soy

git.iko.soy/text

A valiant attempt at improving UTF-8 encoding performance.

This didn't actually work - it slowed down aeson encoding by almost 2x!

commitd90cb3aab27e81cf786d2318b053024bcb6d1ca8 authorBryan O'Sullivan <bos@serpentine.com> dateFri 23 Dec 2011, 11:57 (15 years ago) parentf0158d3

3 files changed+86−46

compare with master

unifiedsplit

Data/Text/Encoding.hs+15−45 cbits/cbits.c+70−1 text.cabal+1

Data/Text/Encoding.hs+15−45

@@ -53,24 +53,22 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

5353#else

5454import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

5555#endif

56−import Data.Bits ((.&.))

5756import Data.ByteString as B

5857import Data.ByteString.Internal as B

5958import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

6059import Data.Text.Internal (Text(..), textP)

61−import Data.Text.UnsafeChar (ord, unsafeWrite)

62−import Data.Text.UnsafeShift (shiftL, shiftR)

60+import Data.Text.UnsafeChar (unsafeWrite)

6361import Data.Word (Word8)

6462import Foreign.C.Types (CSize)

65−import Foreign.ForeignPtr (withForeignPtr)

63+import Foreign.ForeignPtr (newForeignPtr, withForeignPtr)

6664import Foreign.Marshal.Utils (with)

6765import Foreign.Ptr (Ptr, plusPtr)

6866import Foreign.Storable (peek, poke)

69−import GHC.Base (MutableByteArray#)

67+import GHC.Base (ByteArray#, MutableByteArray#)

7068import System.IO.Unsafe (unsafePerformIO)

7169import qualified Data.Text.Array as A

7270import qualified Data.Text.Encoding.Fusion as E

73−import qualified Data.Text.Encoding.Utf16 as U16

71+

7472import qualified Data.Text.Fusion as F

7573

7674-- $strict

@@ -145,45 +143,14 @@ decodeUtf8' = unsafePerformIO . try . evaluate . decodeUtf8With strictDecode

145143

146144-- | Encode text using UTF-8 encoding.

147145encodeUtf8 :: Text -> ByteString

148−encodeUtf8 (Text arr off len) = unsafePerformIO $ do

149− let size0 = max len 4

150− mallocByteString size0 >>= start size0 off 0

151− where

152− start size n0 m0 fp = withForeignPtr fp $ loop n0 m0

153− where

154− loop n1 m1 ptr = go n1 m1

155− where

156− go !n !m

157− | n == off+len = return (PS fp 0 m)

158− | otherwise = do

159− let poke8 k v = poke (ptr `plusPtr` k) (fromIntegral v :: Word8)

160− ensure k act

161− | size-m >= k = act

162− | otherwise = {-# SCC "resizeUtf8/ensure" #-} do

163− let newSize = size `shiftL` 1

164− fp' <- mallocByteString newSize

165− withForeignPtr fp' $ \ptr' ->

166− memcpy ptr' ptr (fromIntegral m)

167− start newSize n m fp'

168− {-# INLINE ensure #-}

169− case A.unsafeIndex arr n of

170− w| w <= 0x7F -> poke8 m w >> go (n+1) (m+1)

171− | w <= 0x7FF -> ensure 2 $ do

172− poke8 m $ (w `shiftR` 6) + 0xC0

173− poke8 (m+1) $ (w .&. 0x3f) + 0x80

174− go (n+1) (m+2)

175− | 0xD800 <= w && w <= 0xDBFF -> ensure 4 $ do

176− let c = ord $ U16.chr2 w (A.unsafeIndex arr (n+1))

177− poke8 m $ (c `shiftR` 18) + 0xF0

178− poke8 (m+1) $ ((c `shiftR` 12) .&. 0x3F) + 0x80

179− poke8 (m+2) $ ((c `shiftR` 6) .&. 0x3F) + 0x80

180− poke8 (m+3) $ (c .&. 0x3F) + 0x80

181− go (n+2) (m+4)

182− | otherwise -> ensure 3 $ do

183− poke8 m $ (w `shiftR` 12) + 0xE0

184− poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80

185− poke8 (m+2) $ (w .&. 0x3F) + 0x80

186− go (n+1) (m+3)

146+encodeUtf8 (Text arr off len)

147+ | len == 0 = empty

148+ | otherwise = unsafePerformIO $ do

149+ with (fromIntegral len ::CSize) $ \lenPtr -> do

150+ dptr <- c_encode_utf8 lenPtr (A.aBA arr) (fromIntegral off)

151+ fp <- newForeignPtr c_free_finalizer dptr

152+ dlen <- peek lenPtr

153+ return (PS fp 0 (fromIntegral dlen))

187154

188155-- | Decode text from little endian UTF-16 encoding.

189156decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

@@ -264,3 +231,6 @@ encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))

264231foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

265232 :: MutableByteArray# s -> Ptr CSize

266233 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

234+

235+foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8

236+ :: Ptr CSize -> ByteArray# -> CSize -> IO (Ptr Word8)

cbits/cbits.c+70−1

@@ -6,9 +6,10 @@

66 * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.

77 */

9−#include <string.h>

109#include <stdint.h>

1110#include <stdio.h>

11+#include <stdlib.h>

12+#include <string.h>

1213

1314void _hs_text_memcpy(void *dest, size_t doff, const void *src, size_t soff,

1415 size_t n)

@@ -131,3 +132,71 @@ _hs_text_decode_utf8(uint16_t *dest, size_t *destoff,

131132

132133 return s;

133134}

135+

136+uint8_t *_hs_text_encode_utf8(size_t *plen,

137+ const uint16_t const *src, size_t srcoff)

138+{

139+ size_t srclen = *plen;

140+ size_t destlen = srclen > 4 ? srclen : 4;

141+ uint8_t *dest = malloc(destlen);

142+ uint8_t *d = dest;

143+ const uint8_t const *dend = dest + destlen;

144+ const uint16_t const *s = src + srcoff;

145+ const uint16_t const *srcend = s + srclen;

146+

147+ do {

148+ const uint16_t w = *s++;

149+ if (w < 0x80) {

150+ if (dend - d < 1) {

151+ destlen *= 2;

152+ uint8_t *newdest = realloc(dest, destlen);

153+ d = newdest + (d - dest);

154+ dest = newdest;

155+ dend = newdest + destlen;

156+ }

157+ *d++ = w;

158+ while (s < srcend && d < dend && *s < 0x80) {

159+ *d++ = *s++;

160+ }

161+ } else if (w < 0x800) {

162+ if (dend - d < 2) {

163+ destlen *= 2;

164+ uint8_t *newdest = realloc(dest, destlen);

165+ d = newdest + (d - dest);

166+ dest = newdest;

167+ dend = newdest + destlen;

168+ }

169+ *d++ = 0xc0 | (w >> 6);

170+ *d++ = 0x80 | (w & 0x3f);

171+ } else if (w >= 0xd800 && w < 0xdc00) {

172+ if (dend - d < 4) {

173+ destlen *= 2;

174+ uint8_t *newdest = realloc(dest, destlen);

175+ d = newdest + (d - dest);

176+ dest = newdest;

177+ dend = newdest + destlen;

178+ }

179+ const uint32_t c = (((((uint32_t) w) - 0xd800) << 10) |

180+ (((uint32_t) *s++) - 0xdc00)) + 0x10000;

181+ *d++ = 0xf0 | (c >> 18);

182+ *d++ = 0x80 | ((c >> 12) & 0x3f);

183+ *d++ = 0x80 | ((c >> 6) & 0x3f);

184+ *d++ = 0x80 | (c & 0x3f);

185+ } else {

186+ if (dend - d < 3) {

187+ destlen *= 2;

188+ uint8_t *newdest = realloc(dest, destlen);

189+ d = newdest + (d - dest);

190+ dest = newdest;

191+ dend = newdest + destlen;

192+ }

193+ *d++ = 0xe0 | (w >> 12);

194+ *d++ = 0x80 | ((w >> 6) & 0x3f);

195+ *d++ = 0x80 | (w & 0x3f);

196+ }

197+ } while (s < srcend);

198+

199+ *plen = d - dest;

200+

201+ return dest;

202+}

text.cabal+1

@@ -65,6 +65,7 @@ flag developer

6565

6666library

6767 c-sources: cbits/cbits.c

68+ cc-options: -Wall

6869

6970 exposed-modules:

7071 Data.Text