2ceb1cb · text · git.iko.soy

Data/Text.hs+3

compare with master

unifiedsplit

@@ -1,5 +1,8 @@

1{-# LANGUAGE BangPatterns, CPP, MagicHash, Rank2Types, UnboxedTuples #-}

2{-# OPTIONS_GHC -fno-warn-orphans #-}

3+#if __GLASGOW_HASKELL__ >= 702

4+{-# LANGUAGE Trustworthy #-}

5+#endif

3

6

4-- |

7-- |

5-- Module : Data.Text

8-- Module : Data.Text

Data/Text/Encoding.hs+21−2

@@ -1,5 +1,8 @@

1{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, MagicHash,

2 UnliftedFFITypes #-}

3+#if __GLASGOW_HASKELL__ >= 702

4+{-# LANGUAGE Trustworthy #-}

5+#endif

3-- |

6-- |

4-- Module : Data.Text.Encoding

7-- Module : Data.Text.Encoding

@@ -23,6 +26,7 @@ module Data.Text.Encoding

23 -- * Decoding ByteStrings to Text

26 -- * Decoding ByteStrings to Text

24 -- $strict

27 -- $strict

25 decodeASCII

28 decodeASCII

29+ , decodeLatin1

26 , decodeUtf8

30 , decodeUtf8

27 , decodeUtf16LE

31 , decodeUtf16LE

28 , decodeUtf16BE

32 , decodeUtf16BE

@@ -57,7 +61,7 @@ import Data.Bits ((.&.))

57import Data.ByteString as B

61import Data.ByteString as B

58import Data.ByteString.Internal as B

62import Data.ByteString.Internal as B

59import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

63import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

60−import Data.Text.Internal (Text(..))

64+import Data.Text.Internal (Text(..), textP)

61import Data.Text.Private (runText)

65import Data.Text.Private (runText)

62import Data.Text.UnsafeChar (ord, unsafeWrite)

66import Data.Text.UnsafeChar (ord, unsafeWrite)

63import Data.Text.UnsafeShift (shiftL, shiftR)

67import Data.Text.UnsafeShift (shiftL, shiftR)

@@ -90,11 +94,23 @@ import qualified Data.Text.Fusion as F

90-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

94-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

91-- encoded text.

95-- encoded text.

92--

96--

93−-- This function is deprecated. Use 'decodeUtf8' instead.

97+-- This function is deprecated. Use 'decodeLatin1' instead.

94decodeASCII :: ByteString -> Text

98decodeASCII :: ByteString -> Text

95decodeASCII = decodeUtf8

99decodeASCII = decodeUtf8

96{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

100{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

97

101

102+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.

103+--

104+-- 'decodeLatin1' is semantically equivalent to

105+-- @Data.Text.pack . Data.ByteString.Char8.unpack@

106+decodeLatin1 :: ByteString -> Text

107+decodeLatin1 (PS fp off len) = textP a 0 len

108+ where

109+ a = A.run (A.new len >>= unsafeIOToST . go)

110+ go dest = withForeignPtr fp $ \ptr -> do

111+ c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))

112+ return dest

113+

98-- | Decode a 'ByteString' containing UTF-8 encoded text.

114-- | Decode a 'ByteString' containing UTF-8 encoded text.

99decodeUtf8With :: OnDecodeError -> ByteString -> Text

115decodeUtf8With :: OnDecodeError -> ByteString -> Text

100decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

116decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

@@ -279,3 +295,6 @@ encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))

279foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

295foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

280 :: MutableByteArray# s -> Ptr CSize

296 :: MutableByteArray# s -> Ptr CSize

281 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

297 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

298+

299+foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

300+ :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

Data/Text/Encoding/Error.hs+3

@@ -1,4 +1,7 @@

1{-# LANGUAGE CPP, DeriveDataTypeable #-}

2+#if __GLASGOW_HASKELL__ >= 702

3+{-# LANGUAGE Trustworthy #-}

4+#endif

2-- |

5-- |

3-- Module : Data.Text.Encoding.Error

6-- Module : Data.Text.Encoding.Error

4-- Copyright : (c) Bryan O'Sullivan 2009

7-- Copyright : (c) Bryan O'Sullivan 2009

Data/Text/IO.hs+3

@@ -1,4 +1,7 @@

1{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-}

2+#if __GLASGOW_HASKELL__ >= 702

3+{-# LANGUAGE Trustworthy #-}

4+#endif

2-- |

5-- |

3-- Module : Data.Text.IO

6-- Module : Data.Text.IO

Data/Text/Lazy.hs+3

@@ -1,5 +1,8 @@

1{-# OPTIONS_GHC -fno-warn-orphans #-}

2{-# LANGUAGE BangPatterns, MagicHash, CPP #-}

3+#if __GLASGOW_HASKELL__ >= 702

4+{-# LANGUAGE Trustworthy #-}

5+#endif

3-- |

6-- |

4-- Module : Data.Text.Lazy

7-- Module : Data.Text.Lazy

Data/Text/Lazy/Encoding.hs+10−2

@@ -1,4 +1,7 @@

1−{-# LANGUAGE BangPatterns #-}

1+{-# LANGUAGE BangPatterns,CPP #-}

2+#if __GLASGOW_HASKELL__ >= 702

3+{-# LANGUAGE Trustworthy #-}

4+#endif

2-- |

5-- |

3-- Module : Data.Text.Lazy.Encoding

6-- Module : Data.Text.Lazy.Encoding

@@ -20,6 +23,7 @@ module Data.Text.Lazy.Encoding

20 -- * Decoding ByteStrings to Text

23 -- * Decoding ByteStrings to Text

21 -- $strict

24 -- $strict

22 decodeASCII

25 decodeASCII

26+ , decodeLatin1

23 , decodeUtf8

27 , decodeUtf8

24 , decodeUtf16LE

28 , decodeUtf16LE

25 , decodeUtf16BE

29 , decodeUtf16BE

@@ -74,11 +78,15 @@ import qualified Data.Text.Lazy.Fusion as F

74-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

78-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

75-- encoded text.

79-- encoded text.

76--

80--

77−-- This function is deprecated. Use 'decodeUtf8' instead.

81+-- This function is deprecated. Use 'decodeLatin1' instead.

78decodeASCII :: B.ByteString -> Text

82decodeASCII :: B.ByteString -> Text

79decodeASCII = decodeUtf8

83decodeASCII = decodeUtf8

80{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

84{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

81

85

86+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.

87+decodeLatin1 :: B.ByteString -> Text

88+decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks

89+

82-- | Decode a 'ByteString' containing UTF-8 encoded text.

90-- | Decode a 'ByteString' containing UTF-8 encoded text.

83decodeUtf8With :: OnDecodeError -> B.ByteString -> Text

91decodeUtf8With :: OnDecodeError -> B.ByteString -> Text

84decodeUtf8With onErr bs0 = fast bs0

92decodeUtf8With onErr bs0 = fast bs0

Data/Text/Lazy/IO.hs+3

@@ -1,4 +1,7 @@

1{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-}

2+#if __GLASGOW_HASKELL__ >= 702

3+{-# LANGUAGE Trustworthy #-}

4+#endif

2-- |

5-- |

3-- Module : Data.Text.Lazy.IO

6-- Module : Data.Text.Lazy.IO

Data/Text/Lazy/Read.hs+4−1

@@ -1,4 +1,7 @@

1−{-# LANGUAGE OverloadedStrings #-}

1+{-# LANGUAGE OverloadedStrings, CPP #-}

2+#if __GLASGOW_HASKELL__ >= 702

3+{-# LANGUAGE Trustworthy #-}

4+#endif

2

5

3-- |

6-- |

4-- Module : Data.Text.Lazy.Read

7-- Module : Data.Text.Lazy.Read

Data/Text/Read.hs+4−1

@@ -1,4 +1,7 @@

1−{-# LANGUAGE OverloadedStrings, UnboxedTuples #-}

1+{-# LANGUAGE OverloadedStrings, UnboxedTuples, CPP #-}

2+#if __GLASGOW_HASKELL__ >= 702

3+{-# LANGUAGE Trustworthy #-}

4+#endif

2

5

3-- |

6-- |

4-- Module : Data.Text.Read

7-- Module : Data.Text.Read

cbits/cbits.c+38

@@ -61,6 +61,44 @@ decode(uint32_t *state, uint32_t* codep, uint32_t byte) {

61 return *state = utf8d[256 + *state + type];

62}

63

64+/*

65+ * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode

66+ * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to

67+ * an UTF16 array

68+ */

69+void

70+_hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

71+ const uint8_t const *srcend)

72+{

73+ const uint8_t *p = src;

74+

75+#if defined(__i386__) || defined(__x86_64__)

76+ /* This optimization works on a little-endian systems by using

77+ (aligned) 32-bit loads instead of 8-bit loads

78+ */

79+

80+ /* consume unaligned prefix */

81+ while (p != srcend && (uintptr_t)p & 0x3)

82+ *dest++ = *p++;

83+

84+ /* iterate over 32-bit aligned loads */

85+ while (p < srcend - 3) {

86+ const uint32_t w = *((const uint32_t *)p);

87+

88+ *dest++ = w & 0xff;

89+ *dest++ = (w >> 8) & 0xff;

90+ *dest++ = (w >> 16) & 0xff;

91+ *dest++ = (w >> 24) & 0xff;

92+

93+ p += 4;

94+ }

95+#endif

96+

97+ /* handle unaligned suffix */

98+ while (p != srcend)

99+ *dest++ = *p++;

100+}

101+

64/*

102/*

65 * A best-effort decoder. Runs until it hits either end of input or

103 * A best-effort decoder. Runs until it hits either end of input or

66 * the start of an invalid byte sequence.

104 * the start of an invalid byte sequence.

tests/Tests/Properties.hs+13

@@ -32,6 +32,7 @@ import Test.Framework (Test, testGroup)

32import Test.Framework.Providers.QuickCheck2 (testProperty)

33import qualified Data.Bits as Bits (shiftL, shiftR)

34import qualified Data.ByteString as B

35+import qualified Data.ByteString.Lazy as BL

35import qualified Data.List as L

36import qualified Data.List as L

36import qualified Data.Text as T

37import qualified Data.Text as T

37import qualified Data.Text.Encoding as E

38import qualified Data.Text.Encoding as E

@@ -65,10 +66,20 @@ tl_unstreamChunks x = f 11 x == f 1000 x

65tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

66tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

66tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

67tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

67

68

69+-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'

70+encodeL1 :: T.Text -> B.ByteString

71+encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

72+encodeLazyL1 :: TL.Text -> BL.ByteString

73+encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

74+

68t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

75t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

69 where a = T.map (\c -> chr (ord c `mod` 128)) t

76 where a = T.map (\c -> chr (ord c `mod` 128)) t

70tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

77tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

71 where a = TL.map (\c -> chr (ord c `mod` 128)) t

78 where a = TL.map (\c -> chr (ord c `mod` 128)) t

79+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

80+ where a = T.map (\c -> chr (ord c `mod` 256)) t

81+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

82+ where a = TL.map (\c -> chr (ord c `mod` 256)) t

72t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

83t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

73t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

84t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

74tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id

85tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id

@@ -754,6 +765,8 @@ tests =

754 testGroup "transcoding" [

765 testGroup "transcoding" [

755 testProperty "t_ascii" t_ascii,

766 testProperty "t_ascii" t_ascii,

756 testProperty "tl_ascii" tl_ascii,

767 testProperty "tl_ascii" tl_ascii,

768+ testProperty "t_latin1" t_latin1,

769+ testProperty "tl_latin1" tl_latin1,

757 testProperty "t_utf8" t_utf8,

770 testProperty "t_utf8" t_utf8,

758 testProperty "t_utf8'" t_utf8',

771 testProperty "t_utf8'" t_utf8',

759 testProperty "tl_utf8" tl_utf8,

772 testProperty "tl_utf8" tl_utf8,

text.cabal+1−1

@@ -131,7 +131,7 @@ library

131 cpp-options: -DHAVE_DEEPSEQ

132 else

133 build-depends: extensible-exceptions

134− extensions: PatternSignatures

134+ extensions: ScopedTypeVariables

135

136 ghc-options: -Wall -funbox-strict-fields -O2

137 if impl(ghc >= 6.8)