54fca94 · text · git.iko.soy

git.iko.soy/text

on master help sign in

Merge pull request #18 from hvr/pull-req-16

54fca94Bryan O'Sullivan14 years ago

Data/Text/Encoding.hs+18−2

compare with master

unifiedsplit

@@ -26,6 +26,7 @@ module Data.Text.Encoding

26 -- * Decoding ByteStrings to Text

27 -- $strict

28 decodeASCII

29+ , decodeLatin1

29 , decodeUtf8

30 , decodeUtf8

30 , decodeUtf16LE

31 , decodeUtf16LE

31 , decodeUtf16BE

32 , decodeUtf16BE

@@ -60,7 +61,7 @@ import Data.Bits ((.&.))

60import Data.ByteString as B

61import Data.ByteString as B

61import Data.ByteString.Internal as B

62import Data.ByteString.Internal as B

62import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

63import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

63−import Data.Text.Internal (Text(..))

64+import Data.Text.Internal (Text(..), textP)

64import Data.Text.Private (runText)

65import Data.Text.Private (runText)

65import Data.Text.UnsafeChar (ord, unsafeWrite)

66import Data.Text.UnsafeChar (ord, unsafeWrite)

66import Data.Text.UnsafeShift (shiftL, shiftR)

67import Data.Text.UnsafeShift (shiftL, shiftR)

@@ -93,11 +94,23 @@ import qualified Data.Text.Fusion as F

93-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

94-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

94-- encoded text.

95-- encoded text.

95--

96--

96−-- This function is deprecated. Use 'decodeUtf8' instead.

97+-- This function is deprecated. Use 'decodeLatin1' instead.

97decodeASCII :: ByteString -> Text

98decodeASCII :: ByteString -> Text

98decodeASCII = decodeUtf8

99decodeASCII = decodeUtf8

99{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

100{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

100

101

102+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.

103+--

104+-- 'decodeLatin1' is semantically equivalent to

105+-- @Data.Text.pack . Data.ByteString.Char8.unpack@

106+decodeLatin1 :: ByteString -> Text

107+decodeLatin1 (PS fp off len) = textP a 0 len

108+ where

109+ a = A.run (A.new len >>= unsafeIOToST . go)

110+ go dest = withForeignPtr fp $ \ptr -> do

111+ c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))

112+ return dest

113+

101-- | Decode a 'ByteString' containing UTF-8 encoded text.

114-- | Decode a 'ByteString' containing UTF-8 encoded text.

102decodeUtf8With :: OnDecodeError -> ByteString -> Text

115decodeUtf8With :: OnDecodeError -> ByteString -> Text

103decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

116decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

@@ -282,3 +295,6 @@ encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))

282foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

295foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

283 :: MutableByteArray# s -> Ptr CSize

296 :: MutableByteArray# s -> Ptr CSize

284 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

297 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

298+

299+foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

300+ :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

Data/Text/Lazy/Encoding.hs+6−1

@@ -23,6 +23,7 @@ module Data.Text.Lazy.Encoding

23 -- * Decoding ByteStrings to Text

24 -- $strict

25 decodeASCII

26+ , decodeLatin1

26 , decodeUtf8

27 , decodeUtf8

27 , decodeUtf16LE

28 , decodeUtf16LE

28 , decodeUtf16BE

29 , decodeUtf16BE

@@ -77,11 +78,15 @@ import qualified Data.Text.Lazy.Fusion as F

77-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

78-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

78-- encoded text.

79-- encoded text.

79--

80--

80−-- This function is deprecated. Use 'decodeUtf8' instead.

81+-- This function is deprecated. Use 'decodeLatin1' instead.

81decodeASCII :: B.ByteString -> Text

82decodeASCII :: B.ByteString -> Text

82decodeASCII = decodeUtf8

83decodeASCII = decodeUtf8

83{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

84{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

86+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.

87+decodeLatin1 :: B.ByteString -> Text

88+decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks

89+

85-- | Decode a 'ByteString' containing UTF-8 encoded text.

90-- | Decode a 'ByteString' containing UTF-8 encoded text.

86decodeUtf8With :: OnDecodeError -> B.ByteString -> Text

91decodeUtf8With :: OnDecodeError -> B.ByteString -> Text

87decodeUtf8With onErr bs0 = fast bs0

92decodeUtf8With onErr bs0 = fast bs0

cbits/cbits.c+38

@@ -61,6 +61,44 @@ decode(uint32_t *state, uint32_t* codep, uint32_t byte) {

61 return *state = utf8d[256 + *state + type];

62}

64+/*

65+ * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode

66+ * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to

67+ * an UTF16 array

68+ */

69+void

70+_hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

71+ const uint8_t const *srcend)

72+{

73+ const uint8_t *p = src;

74+

75+#if defined(__i386__) || defined(__x86_64__)

76+ /* This optimization works on a little-endian systems by using

77+ (aligned) 32-bit loads instead of 8-bit loads

78+ */

79+

80+ /* consume unaligned prefix */

81+ while (p != srcend && (uintptr_t)p & 0x3)

82+ *dest++ = *p++;

83+

84+ /* iterate over 32-bit aligned loads */

85+ while (p < srcend - 3) {

86+ const uint32_t w = *((const uint32_t *)p);

87+

88+ *dest++ = w & 0xff;

89+ *dest++ = (w >> 8) & 0xff;

90+ *dest++ = (w >> 16) & 0xff;

91+ *dest++ = (w >> 24) & 0xff;

92+

93+ p += 4;

94+ }

95+#endif

96+

97+ /* handle unaligned suffix */

98+ while (p != srcend)

99+ *dest++ = *p++;

100+}

101+

64/*

102/*

65 * A best-effort decoder. Runs until it hits either end of input or

103 * A best-effort decoder. Runs until it hits either end of input or

66 * the start of an invalid byte sequence.

104 * the start of an invalid byte sequence.

tests/Tests/Properties.hs+13

@@ -32,6 +32,7 @@ import Test.Framework (Test, testGroup)

32import Test.Framework.Providers.QuickCheck2 (testProperty)

33import qualified Data.Bits as Bits (shiftL, shiftR)

34import qualified Data.ByteString as B

35+import qualified Data.ByteString.Lazy as BL

35import qualified Data.List as L

36import qualified Data.List as L

36import qualified Data.Text as T

37import qualified Data.Text as T

37import qualified Data.Text.Encoding as E

38import qualified Data.Text.Encoding as E

@@ -65,10 +66,20 @@ tl_unstreamChunks x = f 11 x == f 1000 x

65tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

66tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

66tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

67tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

69+-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'

70+encodeL1 :: T.Text -> B.ByteString

71+encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

72+encodeLazyL1 :: TL.Text -> BL.ByteString

73+encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

74+

68t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

75t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

69 where a = T.map (\c -> chr (ord c `mod` 128)) t

76 where a = T.map (\c -> chr (ord c `mod` 128)) t

70tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

77tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

71 where a = TL.map (\c -> chr (ord c `mod` 128)) t

78 where a = TL.map (\c -> chr (ord c `mod` 128)) t

79+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

80+ where a = T.map (\c -> chr (ord c `mod` 256)) t

81+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

82+ where a = TL.map (\c -> chr (ord c `mod` 256)) t

72t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

83t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

73t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

84t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

74tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id

85tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id

@@ -754,6 +765,8 @@ tests =

754 testGroup "transcoding" [

765 testGroup "transcoding" [

755 testProperty "t_ascii" t_ascii,

766 testProperty "t_ascii" t_ascii,

756 testProperty "tl_ascii" tl_ascii,

767 testProperty "tl_ascii" tl_ascii,

768+ testProperty "t_latin1" t_latin1,

769+ testProperty "tl_latin1" tl_latin1,

757 testProperty "t_utf8" t_utf8,

770 testProperty "t_utf8" t_utf8,

758 testProperty "t_utf8'" t_utf8',

771 testProperty "t_utf8'" t_utf8',

759 testProperty "tl_utf8" tl_utf8,

772 testProperty "tl_utf8" tl_utf8,