2b68d6b · text · git.iko.soy

git.iko.soy/text

on master help sign in

encodeUtf8_2: a C-based encoding function

2b68d6bBryan O'Sullivan13 years ago

Data/Text/Encoding.hs+17−2

compare with master

unifiedsplit

@@ -52,6 +52,7 @@ module Data.Text.Encoding

52 , encodeUtf8

53 , encodeUtf8_0

54 , encodeUtf8_1

55+ , encodeUtf8_2

55 , encodeUtf16LE

56 , encodeUtf16LE

56 , encodeUtf16BE

57 , encodeUtf16BE

57 , encodeUtf32LE

58 , encodeUtf32LE

@@ -101,12 +102,12 @@ import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)

101import Data.Text.Internal.Private (runText)

102import Data.Text.Internal.Private (runText)

102import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)

103import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)

103import Data.Word (Word8, Word32)

104import Data.Word (Word8, Word32)

104−import Foreign.C.Types (CSize)

105+import Foreign.C.Types (CSize(..))

105import Foreign.ForeignPtr (touchForeignPtr, withForeignPtr)

106import Foreign.ForeignPtr (touchForeignPtr, withForeignPtr)

106import Foreign.Marshal.Utils (with)

107import Foreign.Marshal.Utils (with)

107import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

108import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

108import Foreign.Storable (Storable, peek, poke)

109import Foreign.Storable (Storable, peek, poke)

109−import GHC.Base (MutableByteArray#)

110+import GHC.Base (ByteArray#, MutableByteArray#)

110import qualified Data.Text.Array as A

111import qualified Data.Text.Array as A

111import qualified Data.Text.Internal.Encoding.Fusion as E

112import qualified Data.Text.Internal.Encoding.Fusion as E

112import qualified Data.Text.Internal.Encoding.Utf16 as U16

113import qualified Data.Text.Internal.Encoding.Utf16 as U16

@@ -521,6 +522,17 @@ encodeUtf8_1 (Text arr off len)

521 hot (n+2) (ptr `plusPtr` 4)

522 hot (n+2) (ptr `plusPtr` 4)

522 hot n0 ptr0

523 hot n0 ptr0

523

524

525+encodeUtf8_2 :: Text -> ByteString

526+encodeUtf8_2 (Text arr off len)

527+ | len == 0 = B.empty

528+ | otherwise = unsafeDupablePerformIO $ do

529+ fp <- mallocByteString (len*4)

530+ withForeignPtr fp $ \ptr ->

531+ with ptr $ \destPtr -> do

532+ c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

533+ newDest <- peek destPtr

534+ return (PS fp 0 (newDest `minusPtr` ptr))

535+

524-- | Decode text from little endian UTF-16 encoding.

536-- | Decode text from little endian UTF-16 encoding.

525decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

537decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text

526decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

538decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

@@ -608,3 +620,6 @@ foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_stat

608

620

609foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

621foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

610 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

622 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

623+

624+foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8

625+ :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO ()

benchmarks/haskell/Benchmarks/Pure.hs+2

@@ -113,6 +113,7 @@ benchmark kind fp = do

113 , bgroup "encode"

114 [ benchT $ nf T.encodeUtf8 ta

115 , benchT1 $ nf T.encodeUtf8_1 ta

116+ , benchT2 $ nf T.encodeUtf8_2 ta

116 , benchTL $ nf TL.encodeUtf8 tla

117 , benchTL $ nf TL.encodeUtf8 tla

117 , benchBS $ nf BS.pack sa

118 , benchBS $ nf BS.pack sa

118 , benchBSL $ nf BL.pack sa

119 , benchBSL $ nf BL.pack sa

@@ -412,6 +413,7 @@ benchmark kind fp = do

412 benchS = bench ("String+" ++ kind)

413 benchS = bench ("String+" ++ kind)

413 benchT = bench ("Text+" ++ kind)

414 benchT = bench ("Text+" ++ kind)

414 benchT1 = bench ("Text1+" ++ kind)

415 benchT1 = bench ("Text1+" ++ kind)

416+ benchT2 = bench ("Text2+" ++ kind)

415 benchTL = bench ("LazyText+" ++ kind)

417 benchTL = bench ("LazyText+" ++ kind)

416 benchBS = bench ("ByteString+" ++ kind)

418 benchBS = bench ("ByteString+" ++ kind)

417 benchBSL = bench ("LazyByteString+" ++ kind)

419 benchBSL = bench ("LazyByteString+" ++ kind)

cbits/cbits.c+36

@@ -229,3 +229,39 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

229 ret -= 1;

230 return ret;

231}

232+

233+void

234+_hs_text_encode_utf8(uint8_t **destp, const uint16_t *src, size_t srcoff,

235+ size_t srclen)

236+{

237+ const uint16_t *srcend;

238+ uint8_t *dest = *destp;

239+

240+ src += srcoff;

241+ srcend = src + srclen;

242+

243+ while (src < srcend) {

244+ uint16_t w = *src++;

245+

246+ if (w <= 0x7F)

247+ *dest++ = w;

248+ else if (w <= 0x7FF) {

249+ *dest++ = (w >> 6) | 0xC0;

250+ *dest++ = (w & 0x3f) | 0x80;

251+ }

252+ else if (w < 0xD800 || w > 0xDBFF) {

253+ *dest++ = (w >> 12) | 0xE0;

254+ *dest++ = ((w >> 6) & 0x3F) | 0x80;

255+ *dest++ = (w & 0x3F) | 0x80;

256+ } else {

257+ uint32_t c = ((((uint32_t) w) - 0xD800) << 10) +

258+ (((uint32_t) *src++) - 0xDC00) + 0x10000;

259+ *dest++ = (c >> 18) | 0xE0;

260+ *dest++ = ((c >> 12) & 0x3F) | 0x80;

261+ *dest++ = ((c >> 6) & 0x3F) | 0x80;

262+ *dest++ = (c & 0x3F) | 0x80;

263+ }

264+ }

265+

266+ *destp = dest;

267+}