7c06306 · text · git.iko.soy

git.iko.soy/text

Add new `Data.Text.Encoding.decodeLatin1` ISO-8859-1 decoding function

This has about an order of magnitude lower runtime and/or call-overhead as compared to the more generic `text-icu` approach, e.g. according to criterion with GHC 7.4.1 on Linux/x86_64: * 12 times faster for empty input strings, * 6 times faster for 16-byte strings, and * 3 times faster for 1024-byte strings. `decodeLatin1` is also faster compared to using `decodeUtf8` for plain ASCII: * 2 times faster for 16-byte input strings, * ~40% faster for 1024-byte strings.

commit7c06306bd5b7382cb101f8632b5a1fc50697fe94 authorHerbert Valerio Riedel <hvr@gnu.org> dateSun 23 Sep 2012, 17:36 (14 years ago) parent24fd53e

3 files changed+39−2

compare with master

unifiedsplit

Data/Text/Encoding.hs+18−2 cbits/cbits.c+14 tests/Tests/Properties.hs+7

Data/Text/Encoding.hs+18−2

@@ -23,6 +23,7 @@ module Data.Text.Encoding

2323 -- * Decoding ByteStrings to Text

2424 -- $strict

2525 decodeASCII

26+ , decodeLatin1

2627 , decodeUtf8

2728 , decodeUtf16LE

2829 , decodeUtf16BE

@@ -57,7 +58,7 @@ import Data.Bits ((.&.))

5758import Data.ByteString as B

5859import Data.ByteString.Internal as B

5960import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

60−import Data.Text.Internal (Text(..))

61+import Data.Text.Internal (Text(..), textP)

6162import Data.Text.Private (runText)

6263import Data.Text.UnsafeChar (ord, unsafeWrite)

6364import Data.Text.UnsafeShift (shiftL, shiftR)

@@ -90,11 +91,23 @@ import qualified Data.Text.Fusion as F

9091-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

9192-- encoded text.

9293--

93−-- This function is deprecated. Use 'decodeUtf8' instead.

94+-- This function is deprecated. Use 'decodeLatin1' instead.

9495decodeASCII :: ByteString -> Text

9596decodeASCII = decodeUtf8

9697{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

9798

99+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.

100+--

101+-- 'decodeLatin1' is semantically equivalent to

102+-- @Data.Text.pack . Data.ByteString.Char8.unpack@

103+decodeLatin1 :: ByteString -> Text

104+decodeLatin1 (PS fp off len) = textP a 0 len

105+ where

106+ a = A.run (A.new len >>= unsafeIOToST . go)

107+ go dest = withForeignPtr fp $ \ptr -> do

108+ c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))

109+ return dest

110+

98111-- | Decode a 'ByteString' containing UTF-8 encoded text.

99112decodeUtf8With :: OnDecodeError -> ByteString -> Text

100113decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

@@ -279,3 +292,6 @@ encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))

279292foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

280293 :: MutableByteArray# s -> Ptr CSize

281294 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

295+

296+foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

297+ :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

cbits/cbits.c+14

@@ -61,6 +61,20 @@ decode(uint32_t *state, uint32_t* codep, uint32_t byte) {

6161 return *state = utf8d[256 + *state + type];

6262}

6363

64+/*

65+ * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode

66+ * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to

67+ * an UTF16 array

68+ */

69+void

70+_hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

71+ const uint8_t const *srcend)

72+{

73+ const uint8_t *p = src;

74+ while (p != srcend)

75+ *dest++ = *p++;

76+}

77+

6478/*

6579 * A best-effort decoder. Runs until it hits either end of input or

6680 * the start of an invalid byte sequence.

tests/Tests/Properties.hs+7

@@ -65,10 +65,16 @@ tl_unstreamChunks x = f 11 x == f 1000 x

6565tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

6666tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

6767

68+-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'

69+encodeL1 :: T.Text -> B.ByteString

70+encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

71+

6872t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

6973 where a = T.map (\c -> chr (ord c `mod` 128)) t

7074tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

7175 where a = TL.map (\c -> chr (ord c `mod` 128)) t

76+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

77+ where a = T.map (\c -> chr (ord c `mod` 256)) t

7278t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

7379t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

7480tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id

@@ -754,6 +760,7 @@ tests =

754760 testGroup "transcoding" [

755761 testProperty "t_ascii" t_ascii,

756762 testProperty "tl_ascii" tl_ascii,

763+ testProperty "t_latin1" t_latin1,

757764 testProperty "t_utf8" t_utf8,

758765 testProperty "t_utf8'" t_utf8',

759766 testProperty "tl_utf8" tl_utf8,