git.iko.soy/text
on master help sign in
Add `Data.Text.Lazy.Encoding.decodeLatin1` ISO-8859-1 decoding function
3cec9a2Herbert Valerio Riedel
Data/Text/Lazy/Encoding.hs+6−1
compare with master
unifiedsplit
@@ -20,6 +20,7 @@ module Data.Text.Lazy.Encoding
20 -- * Decoding ByteStrings to Text
20 -- * Decoding ByteStrings to Text
21 -- $strict
21 -- $strict
22 decodeASCII
22 decodeASCII
23+ , decodeLatin1
23 , decodeUtf8
24 , decodeUtf8
24 , decodeUtf16LE
25 , decodeUtf16LE
25 , decodeUtf16BE
26 , decodeUtf16BE
@@ -74,11 +75,15 @@ import qualified Data.Text.Lazy.Fusion as F
74-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
75-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
75-- encoded text.
76-- encoded text.
76--
77--
77−-- This function is deprecated. Use 'decodeUtf8' instead.
78+-- This function is deprecated. Use 'decodeLatin1' instead.
78decodeASCII :: B.ByteString -> Text
79decodeASCII :: B.ByteString -> Text
79decodeASCII = decodeUtf8
80decodeASCII = decodeUtf8
80{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
81{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
81
82
83+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
84+decodeLatin1 :: B.ByteString -> Text
85+decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks
86+
82-- | Decode a 'ByteString' containing UTF-8 encoded text.
87-- | Decode a 'ByteString' containing UTF-8 encoded text.
83decodeUtf8With :: OnDecodeError -> B.ByteString -> Text
88decodeUtf8With :: OnDecodeError -> B.ByteString -> Text
84decodeUtf8With onErr bs0 = fast bs0
89decodeUtf8With onErr bs0 = fast bs0
tests/Tests/Properties.hs+6
@@ -32,6 +32,7 @@ import Test.Framework (Test, testGroup)
32import Test.Framework.Providers.QuickCheck2 (testProperty)
32import Test.Framework.Providers.QuickCheck2 (testProperty)
33import qualified Data.Bits as Bits (shiftL, shiftR)
33import qualified Data.Bits as Bits (shiftL, shiftR)
34import qualified Data.ByteString as B
34import qualified Data.ByteString as B
35+import qualified Data.ByteString.Lazy as BL
35import qualified Data.List as L
36import qualified Data.List as L
36import qualified Data.Text as T
37import qualified Data.Text as T
37import qualified Data.Text.Encoding as E
38import qualified Data.Text.Encoding as E
@@ -68,6 +69,8 @@ tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id
68-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'
69-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'
69encodeL1 :: T.Text -> B.ByteString
70encodeL1 :: T.Text -> B.ByteString
70encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack
71encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack
72+encodeLazyL1 :: TL.Text -> BL.ByteString
73+encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks
71
74
72t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
75t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
73 where a = T.map (\c -> chr (ord c `mod` 128)) t
76 where a = T.map (\c -> chr (ord c `mod` 128)) t
@@ -75,6 +78,8 @@ tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a
75 where a = TL.map (\c -> chr (ord c `mod` 128)) t
78 where a = TL.map (\c -> chr (ord c `mod` 128)) t
76t_latin1 t = E.decodeLatin1 (encodeL1 a) == a
79t_latin1 t = E.decodeLatin1 (encodeL1 a) == a
77 where a = T.map (\c -> chr (ord c `mod` 256)) t
80 where a = T.map (\c -> chr (ord c `mod` 256)) t
81+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a
82+ where a = TL.map (\c -> chr (ord c `mod` 256)) t
78t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id
83t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id
79t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)
84t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)
80tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id
85tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id
@@ -761,6 +766,7 @@ tests =
761 testProperty "t_ascii" t_ascii,
766 testProperty "t_ascii" t_ascii,
762 testProperty "tl_ascii" tl_ascii,
767 testProperty "tl_ascii" tl_ascii,
763 testProperty "t_latin1" t_latin1,
768 testProperty "t_latin1" t_latin1,
769+ testProperty "tl_latin1" tl_latin1,
764 testProperty "t_utf8" t_utf8,
770 testProperty "t_utf8" t_utf8,
765 testProperty "t_utf8'" t_utf8',
771 testProperty "t_utf8'" t_utf8',
766 testProperty "tl_utf8" tl_utf8,
772 testProperty "tl_utf8" tl_utf8,