3cec9a2 · text · git.iko.soy

git.iko.soy/text

Add `Data.Text.Lazy.Encoding.decodeLatin1` ISO-8859-1 decoding function

See 7c06306bd5b7382cb101f8632b5a1fc50697fe94 for more information

commit3cec9a28a29d5468f15b614dd3db058773099494 authorHerbert Valerio Riedel <hvr@gnu.org> dateSun 23 Sep 2012, 17:37 (14 years ago) parent7c06306

2 files changed+12−1

compare with master

unifiedsplit

Data/Text/Lazy/Encoding.hs+6−1 tests/Tests/Properties.hs+6

Data/Text/Lazy/Encoding.hs+6−1

@@ -20,6 +20,7 @@ module Data.Text.Lazy.Encoding

2020 -- * Decoding ByteStrings to Text

2121 -- $strict

2222 decodeASCII

23+ , decodeLatin1

2324 , decodeUtf8

2425 , decodeUtf16LE

2526 , decodeUtf16BE

@@ -74,11 +75,15 @@ import qualified Data.Text.Lazy.Fusion as F

7475-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII

7576-- encoded text.

7677--

77−-- This function is deprecated. Use 'decodeUtf8' instead.

78+-- This function is deprecated. Use 'decodeLatin1' instead.

7879decodeASCII :: B.ByteString -> Text

7980decodeASCII = decodeUtf8

8081{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}

8182

83+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.

84+decodeLatin1 :: B.ByteString -> Text

85+decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks

86+

8287-- | Decode a 'ByteString' containing UTF-8 encoded text.

8388decodeUtf8With :: OnDecodeError -> B.ByteString -> Text

8489decodeUtf8With onErr bs0 = fast bs0

tests/Tests/Properties.hs+6

@@ -32,6 +32,7 @@ import Test.Framework (Test, testGroup)

3232import Test.Framework.Providers.QuickCheck2 (testProperty)

3333import qualified Data.Bits as Bits (shiftL, shiftR)

3434import qualified Data.ByteString as B

35+import qualified Data.ByteString.Lazy as BL

3536import qualified Data.List as L

3637import qualified Data.Text as T

3738import qualified Data.Text.Encoding as E

@@ -68,6 +69,8 @@ tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

6869-- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'

6970encodeL1 :: T.Text -> B.ByteString

7071encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

72+encodeLazyL1 :: TL.Text -> BL.ByteString

73+encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

7174

7275t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

7376 where a = T.map (\c -> chr (ord c `mod` 128)) t

@@ -75,6 +78,8 @@ tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

7578 where a = TL.map (\c -> chr (ord c `mod` 128)) t

7679t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

7780 where a = T.map (\c -> chr (ord c `mod` 256)) t

81+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

82+ where a = TL.map (\c -> chr (ord c `mod` 256)) t

7883t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

7984t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

8085tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id

@@ -761,6 +766,7 @@ tests =

761766 testProperty "t_ascii" t_ascii,

762767 testProperty "tl_ascii" tl_ascii,

763768 testProperty "t_latin1" t_latin1,

769+ testProperty "tl_latin1" tl_latin1,

764770 testProperty "t_utf8" t_utf8,

765771 testProperty "t_utf8'" t_utf8',

766772 testProperty "tl_utf8" tl_utf8,