git.iko.soy/text
on master help sign in
Add new `Data.Text.Encoding.decodeLatin1` ISO-8859-1 decoding function
7c06306Herbert Valerio Riedel
Data/Text/Encoding.hs+18−2
compare with master
unifiedsplit
@@ -23,6 +23,7 @@ module Data.Text.Encoding
23 -- * Decoding ByteStrings to Text
23 -- * Decoding ByteStrings to Text
24 -- $strict
24 -- $strict
25 decodeASCII
25 decodeASCII
26+ , decodeLatin1
26 , decodeUtf8
27 , decodeUtf8
27 , decodeUtf16LE
28 , decodeUtf16LE
28 , decodeUtf16BE
29 , decodeUtf16BE
@@ -57,7 +58,7 @@ import Data.Bits ((.&.))
57import Data.ByteString as B
58import Data.ByteString as B
58import Data.ByteString.Internal as B
59import Data.ByteString.Internal as B
59import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
60import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
60−import Data.Text.Internal (Text(..))
61+import Data.Text.Internal (Text(..), textP)
61import Data.Text.Private (runText)
62import Data.Text.Private (runText)
62import Data.Text.UnsafeChar (ord, unsafeWrite)
63import Data.Text.UnsafeChar (ord, unsafeWrite)
63import Data.Text.UnsafeShift (shiftL, shiftR)
64import Data.Text.UnsafeShift (shiftL, shiftR)
@@ -90,11 +91,23 @@ import qualified Data.Text.Fusion as F
90-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
91-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
91-- encoded text.
92-- encoded text.
92--
93--
93−-- This function is deprecated. Use 'decodeUtf8' instead.
94+-- This function is deprecated. Use 'decodeLatin1' instead.
94decodeASCII :: ByteString -> Text
95decodeASCII :: ByteString -> Text
95decodeASCII = decodeUtf8
96decodeASCII = decodeUtf8
96{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
97{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
97
98
99+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
100+--
101+-- 'decodeLatin1' is semantically equivalent to
102+-- @Data.Text.pack . Data.ByteString.Char8.unpack@
103+decodeLatin1 :: ByteString -> Text
104+decodeLatin1 (PS fp off len) = textP a 0 len
105+ where
106+ a = A.run (A.new len >>= unsafeIOToST . go)
107+ go dest = withForeignPtr fp $ \ptr -> do
108+ c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))
109+ return dest
110+
98-- | Decode a 'ByteString' containing UTF-8 encoded text.
111-- | Decode a 'ByteString' containing UTF-8 encoded text.
99decodeUtf8With :: OnDecodeError -> ByteString -> Text
112decodeUtf8With :: OnDecodeError -> ByteString -> Text
100decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
113decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
@@ -279,3 +292,6 @@ encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
279foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
292foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
280 :: MutableByteArray# s -> Ptr CSize
293 :: MutableByteArray# s -> Ptr CSize
281 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
294 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
295+
296+foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
297+ :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
cbits/cbits.c+14
@@ -61,6 +61,20 @@ decode(uint32_t *state, uint32_t* codep, uint32_t byte) {
61 return *state = utf8d[256 + *state + type];
61 return *state = utf8d[256 + *state + type];
62}
62}
63
63
64+/*
65+ * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode
66+ * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to
67+ * an UTF16 array
68+ */
69+void
70+_hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,
71+ const uint8_t const *srcend)
72+{
73+ const uint8_t *p = src;
74+ while (p != srcend)
75+ *dest++ = *p++;
76+}
77+
64/*
78/*
65 * A best-effort decoder. Runs until it hits either end of input or
79 * A best-effort decoder. Runs until it hits either end of input or
66 * the start of an invalid byte sequence.
80 * the start of an invalid byte sequence.
tests/Tests/Properties.hs+7
@@ -65,10 +65,16 @@ tl_unstreamChunks x = f 11 x == f 1000 x
65tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id
65tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id
66tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id
66tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id
67
67
68+-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'
69+encodeL1 :: T.Text -> B.ByteString
70+encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack
71+
68t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
72t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
69 where a = T.map (\c -> chr (ord c `mod` 128)) t
73 where a = T.map (\c -> chr (ord c `mod` 128)) t
70tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a
74tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a
71 where a = TL.map (\c -> chr (ord c `mod` 128)) t
75 where a = TL.map (\c -> chr (ord c `mod` 128)) t
76+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a
77+ where a = T.map (\c -> chr (ord c `mod` 256)) t
72t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id
78t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id
73t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)
79t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)
74tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id
80tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id
@@ -754,6 +760,7 @@ tests =
754 testGroup "transcoding" [
760 testGroup "transcoding" [
755 testProperty "t_ascii" t_ascii,
761 testProperty "t_ascii" t_ascii,
756 testProperty "tl_ascii" tl_ascii,
762 testProperty "tl_ascii" tl_ascii,
763+ testProperty "t_latin1" t_latin1,
757 testProperty "t_utf8" t_utf8,
764 testProperty "t_utf8" t_utf8,
758 testProperty "t_utf8'" t_utf8',
765 testProperty "t_utf8'" t_utf8',
759 testProperty "tl_utf8" tl_utf8,
766 testProperty "tl_utf8" tl_utf8,