2b68d6b · text · git.iko.soy

git.iko.soy/text

encodeUtf8_2: a C-based encoding function

Not surprisingly, this is a lot faster than encodeUtf8_1 and the Builder-based rewrite under almost all circumstances. It's slower on tiny inputs (20%), but roughly twice as fast as encodeUtf8_1 on longer inputs. --HG-- extra : amend_source : 093410e1295572be039d87a9c21e97d250c5f9f9

commit2b68d6b8ee424ab4a24f2f0f6773b97b6b1910b5 authorBryan O'Sullivan <bos@serpentine.com> dateTue 7 Jan 2014, 11:19 (13 years ago) parent1c65333

3 files changed+55−2

compare with master

unifiedsplit

Data/Text/Encoding.hs+17−2 benchmarks/haskell/Benchmarks/Pure.hs+2 cbits/cbits.c+36

Data/Text/Encoding.hs+17−2

@@ -52,6 +52,7 @@ module Data.Text.Encoding

5252 , encodeUtf8

5353 , encodeUtf8_0

5454 , encodeUtf8_1

55+ , encodeUtf8_2

5556 , encodeUtf16LE

5657 , encodeUtf16BE

5758 , encodeUtf32LE

@@ -101,12 +102,12 @@ import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)

101102import Data.Text.Internal.Private (runText)

102103import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)

103104import Data.Word (Word8, Word32)

104−import Foreign.C.Types (CSize)

105+import Foreign.C.Types (CSize(..))

105106import Foreign.ForeignPtr (touchForeignPtr, withForeignPtr)

106107import Foreign.Marshal.Utils (with)

107108import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

108109import Foreign.Storable (Storable, peek, poke)

109−import GHC.Base (MutableByteArray#)

110+import GHC.Base (ByteArray#, MutableByteArray#)

110111import qualified Data.Text.Array as A

111112import qualified Data.Text.Internal.Encoding.Fusion as E

112113import qualified Data.Text.Internal.Encoding.Utf16 as U16

@@ -521,6 +522,17 @@ encodeUtf8_1 (Text arr off len)

521522 hot (n+2) (ptr `plusPtr` 4)

522523 hot n0 ptr0

523524

525+encodeUtf8_2 :: Text -> ByteString

526+encodeUtf8_2 (Text arr off len)

527+ | len == 0 = B.empty

528+ | otherwise = unsafeDupablePerformIO $ do

529+ fp <- mallocByteString (len*4)

530+ withForeignPtr fp $ \ptr ->

531+ with ptr $ \destPtr -> do

532+ c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

533+ newDest <- peek destPtr

534+ return (PS fp 0 (newDest `minusPtr` ptr))

535+

524536-- | Decode text from little endian UTF-16 encoding.

525537decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

526538decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

@@ -608,3 +620,6 @@ foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_stat

608620

609621foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

610622 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

623+

624+foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8

625+ :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO ()

benchmarks/haskell/Benchmarks/Pure.hs+2

@@ -113,6 +113,7 @@ benchmark kind fp = do

113113 , bgroup "encode"

114114 [ benchT $ nf T.encodeUtf8 ta

115115 , benchT1 $ nf T.encodeUtf8_1 ta

116+ , benchT2 $ nf T.encodeUtf8_2 ta

116117 , benchTL $ nf TL.encodeUtf8 tla

117118 , benchBS $ nf BS.pack sa

118119 , benchBSL $ nf BL.pack sa

@@ -412,6 +413,7 @@ benchmark kind fp = do

412413 benchS = bench ("String+" ++ kind)

413414 benchT = bench ("Text+" ++ kind)

414415 benchT1 = bench ("Text1+" ++ kind)

416+ benchT2 = bench ("Text2+" ++ kind)

415417 benchTL = bench ("LazyText+" ++ kind)

416418 benchBS = bench ("ByteString+" ++ kind)

417419 benchBSL = bench ("LazyByteString+" ++ kind)

cbits/cbits.c+36

@@ -229,3 +229,39 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

229229 ret -= 1;

230230 return ret;

231231}

232+

233+void

234+_hs_text_encode_utf8(uint8_t **destp, const uint16_t *src, size_t srcoff,

235+ size_t srclen)

236+{

237+ const uint16_t *srcend;

238+ uint8_t *dest = *destp;

239+

240+ src += srcoff;

241+ srcend = src + srclen;

242+

243+ while (src < srcend) {

244+ uint16_t w = *src++;

245+

246+ if (w <= 0x7F)

247+ *dest++ = w;

248+ else if (w <= 0x7FF) {

249+ *dest++ = (w >> 6) | 0xC0;

250+ *dest++ = (w & 0x3f) | 0x80;

251+ }

252+ else if (w < 0xD800 || w > 0xDBFF) {

253+ *dest++ = (w >> 12) | 0xE0;

254+ *dest++ = ((w >> 6) & 0x3F) | 0x80;

255+ *dest++ = (w & 0x3F) | 0x80;

256+ } else {

257+ uint32_t c = ((((uint32_t) w) - 0xD800) << 10) +

258+ (((uint32_t) *src++) - 0xDC00) + 0x10000;

259+ *dest++ = (c >> 18) | 0xE0;

260+ *dest++ = ((c >> 12) & 0x3F) | 0x80;

261+ *dest++ = ((c >> 6) & 0x3F) | 0x80;

262+ *dest++ = (c & 0x3F) | 0x80;

263+ }

264+ }

265+

266+ *destp = dest;

267+}