git.iko.soy/text
on master help sign in
Test encode/decode functions against all of Unicode.
841909fJohn Millikin
tests/Properties.hs+7−7
compare with master
unifiedsplit
@@ -32,7 +32,7 @@ import Data.Text.Search (indices)
32import qualified Data.Text.Lazy.Search as S (indices)
32import qualified Data.Text.Lazy.Search as S (indices)
33import qualified SlowFunctions as Slow
33import qualified SlowFunctions as Slow
34
34
35−import QuickCheckUtils (NotEmpty(..), small)
35+import QuickCheckUtils (NotEmpty(..), small, genUnicode)
36
36
37-- Ensure that two potentially bottom values (in the sense of crashing
37-- Ensure that two potentially bottom values (in the sense of crashing
38-- for some inputs, not looping infinitely) either both crash, or both
38-- for some inputs, not looping infinitely) either both crash, or both
@@ -62,12 +62,12 @@ tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id
62
62
63t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
63t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
64 where a = T.map (\c -> chr (ord c `mod` 128)) t
64 where a = T.map (\c -> chr (ord c `mod` 128)) t
65−t_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id
65+t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id
66−tl_utf8 = (EL.decodeUtf8 . EL.encodeUtf8) `eq` id
66+tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id
67−t_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id
67+t_utf16LE = forAll genUnicode $ (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id
68−t_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id
68+t_utf16BE = forAll genUnicode $ (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id
69−t_utf32LE = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id
69+t_utf32LE = forAll genUnicode $ (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id
70−t_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
70+t_utf32BE = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
71
71
72class Stringy s where
72class Stringy s where
73 packS :: String -> s
73 packS :: String -> s
tests/QuickCheckUtils.hs+54−1
@@ -3,12 +3,15 @@
3module QuickCheckUtils where
3module QuickCheckUtils where
4
4
5import Control.Arrow (first)
5import Control.Arrow (first)
6+import Data.Char (chr)
7+import Data.Bits ((.&.))
6import Data.Int (Int64)
8import Data.Int (Int64)
7import Data.Word (Word8, Word16, Word32)
9import Data.Word (Word8, Word16, Word32)
10+import Data.String (IsString, fromString)
8import qualified Data.Text as T
11import qualified Data.Text as T
9import qualified Data.Text.Lazy as TL
12import qualified Data.Text.Lazy as TL
10import System.Random (Random(..), RandomGen)
13import System.Random (Random(..), RandomGen)
11−import Test.QuickCheck (Arbitrary(..), choose, sized, vector)
14+import Test.QuickCheck (Gen, Arbitrary(..), choose, sized, vector, oneof)
12import qualified Data.ByteString as B
15import qualified Data.ByteString as B
13
16
14instance Random Int64 where
17instance Random Int64 where
@@ -42,6 +45,56 @@ instance Random Word32 where
42instance Arbitrary Word32 where
45instance Arbitrary Word32 where
43 arbitrary = choose (minBound,maxBound)
46 arbitrary = choose (minBound,maxBound)
44
47
48+genUnicode :: IsString a => Gen a
49+genUnicode = fmap fromString string where
50+ string = sized $ \n ->
51+ do k <- choose (0,n)
52+ sequence [ char | _ <- [1..k] ]
53+
54+ excluding :: [a -> Bool] -> Gen a -> Gen a
55+ excluding bad gen = loop
56+ where
57+ loop = do
58+ x <- gen
59+ if or (map ($ x) bad)
60+ then loop
61+ else return x
62+
63+ reserved = [lowSurrogate, highSurrogate, noncharacter]
64+ lowSurrogate c = c >= 0xDC00 && c <= 0xDFFF
65+ highSurrogate c = c >= 0xD800 && c <= 0xDBFF
66+ noncharacter c = masked == 0xFFFE || masked == 0xFFFF
67+ where
68+ masked = c .&. 0xFFFF
69+
70+ ascii = choose (0,0x7F)
71+ plane0 = choose (0xF0, 0xFFFF)
72+ plane1 = oneof [ choose (0x10000, 0x10FFF)
73+ , choose (0x11000, 0x11FFF)
74+ , choose (0x12000, 0x12FFF)
75+ , choose (0x13000, 0x13FFF)
76+ , choose (0x1D000, 0x1DFFF)
77+ , choose (0x1F000, 0x1FFFF)
78+ ]
79+ plane2 = oneof [ choose (0x20000, 0x20FFF)
80+ , choose (0x21000, 0x21FFF)
81+ , choose (0x22000, 0x22FFF)
82+ , choose (0x23000, 0x23FFF)
83+ , choose (0x24000, 0x24FFF)
84+ , choose (0x25000, 0x25FFF)
85+ , choose (0x26000, 0x26FFF)
86+ , choose (0x27000, 0x27FFF)
87+ , choose (0x28000, 0x28FFF)
88+ , choose (0x29000, 0x29FFF)
89+ , choose (0x2A000, 0x2AFFF)
90+ , choose (0x2B000, 0x2BFFF)
91+ , choose (0x2F000, 0x2FFFF)
92+ ]
93+ plane14 = choose (0xE0000, 0xE0FFF)
94+ planes = [ascii, plane0, plane1, plane2, plane14]
95+
96+ char = chr `fmap` excluding reserved (oneof planes)
97+
45instance Arbitrary T.Text where
98instance Arbitrary T.Text where
46 arbitrary = T.pack `fmap` arbitrary
99 arbitrary = T.pack `fmap` arbitrary
47
100