on master help sign in
Merge fix for gh-61 into 1.0 branch
commitf91d74928efda2dd71496b9d498b6ef686ce1752 authorBryan O'Sullivan <bos@serpentine.com> dateMon 30 Dec 2013, 00:36 (13 years ago) parentba4ae052c3b395
3 files changed+84−28
cbits/cbits.c+31−10 tests/Tests/Properties.hs+44−11 tests/Tests/QuickCheckUtils.hs+9−7
cbits/cbits.c+31−10
@@ -127,11 +127,18 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,
127127 * state0 != UTF8_ACCEPT, UTF8_REJECT
128128 *
129129 */
130−const uint8_t *
131−_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
132− const uint8_t **const src,
133− const uint8_t *const srcend,
134− uint32_t *codepoint0, uint32_t *state0)
130+#if defined(__GNUC__) || defined(__clang__)
131+static inline uint8_t const *
132+_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff,
133+ const uint8_t const **src, const uint8_t const *srcend,
134+ uint32_t *codepoint0, uint32_t *state0)
135+ __attribute((always_inline));
136+#endif
137+
138+static inline uint8_t const *
139+_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff,
140+ const uint8_t const **src, const uint8_t const *srcend,
141+ uint32_t *codepoint0, uint32_t *state0)
135142{
136143 uint16_t *d = dest + *destoff;
137144 const uint8_t *s = *src, *last = *src;
@@ -185,10 +192,6 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
185192 last = s;
186193 }
187194
188− /* Invalid encoding, back up to the errant character */
189− if (state == UTF8_REJECT)
190− s -= 1;
191−
192195 *destoff = d - dest;
193196 *codepoint0 = codepoint;
194197 *state0 = state;
@@ -197,6 +200,19 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
197200 return s;
198201}
199202
203+uint8_t const *
204+_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
205+ const uint8_t const **src,
206+ const uint8_t const *srcend,
207+ uint32_t *codepoint0, uint32_t *state0)
208+{
209+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
210+ codepoint0, state0);
211+ if (*state0 == UTF8_REJECT)
212+ ret -=1;
213+ return ret;
214+}
215+
200216/*
201217 * Helper to decode buffer and discard final decoder state
202218 */
@@ -206,5 +222,10 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
206222{
207223 uint32_t codepoint;
208224 uint32_t state = UTF8_ACCEPT;
209− return _hs_text_decode_utf8_state(dest, destoff, &src, srcend, &codepoint, &state);
225+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
226+ &codepoint, &state);
227+ /* Back up if we have an incomplete or invalid encoding */
228+ if (state != UTF8_ACCEPT)
229+ ret -= 1;
230+ return ret;
210231}
tests/Tests/Properties.hs+44−11
@@ -13,6 +13,7 @@ import Test.QuickCheck
1313import Test.QuickCheck.Monadic
1414import Text.Show.Functions ()
1515
16+import Control.Applicative ((<$>), (<*>))
1617import Control.Arrow ((***), second)
1718import Control.Exception (catch)
1819import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)
@@ -110,23 +111,55 @@ t_utf8_incr = do
110111 E.Some t _ f' = f a
111112 in t : feedChunksOf n f' b
112113
113−-- This is a poor attempt to ensure that the error handling paths on
114−-- decode are exercised in some way. Proper testing would be rather
115−-- more involved.
116−t_utf8_err :: DecodeErr -> B.ByteString -> Property
117−t_utf8_err (DE _ de) bs = monadicIO $ do
118− l <- run $ let len = T.length (E.decodeUtf8With de bs)
119− in (len `seq` return (Right len)) `catch`
120− (\(e::UnicodeException) -> return (Left e))
121− case l of
122− Left err -> assert $ length (show err) >= 0
123− Right n -> assert $ n >= 0
114+data Badness = Solo | Leading | Trailing
115+ deriving (Eq, Show)
116+
117+instance Arbitrary Badness where
118+ arbitrary = elements [Solo, Leading, Trailing]
119+
120+t_utf8_err :: Badness -> DecodeErr -> Property
121+t_utf8_err bad de = do
122+ let gen = case bad of
123+ Solo -> genInvalidUTF8
124+ Leading -> B.append <$> genInvalidUTF8 <*> genUTF8
125+ Trailing -> B.append <$> genUTF8 <*> genInvalidUTF8
126+ genUTF8 = E.encodeUtf8 <$> genUnicode
127+ forAll gen $ \bs -> do
128+ onErr <- genDecodeErr de
129+ monadicIO $ do
130+ l <- run $ let len = T.length (E.decodeUtf8With onErr bs)
131+ in (len `seq` return (Right len)) `catch`
132+ (\(e::UnicodeException) -> return (Left e))
133+ assert $ case l of
134+ Left err -> length (show err) >= 0
135+ Right _ -> de /= Strict
124136
125137t_utf8_err' :: B.ByteString -> Property
126138t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of
127139 Left err -> length (show err) >= 0
128140 Right t -> T.length t >= 0
129141
142+genInvalidUTF8 :: Gen B.ByteString
143+genInvalidUTF8 = B.pack <$> oneof [
144+ -- invalid leading byte of a 2-byte sequence
145+ (:) <$> choose (0xC0, 0xC1) <*> upTo 1 contByte
146+ -- invalid leading byte of a 4-byte sequence
147+ , (:) <$> choose (0xF5, 0xFF) <*> upTo 3 contByte
148+ -- continuation bytes without a start byte
149+ , listOf1 contByte
150+ -- short 2-byte sequence
151+ , (:[]) <$> choose (0xC2, 0xDF)
152+ -- short 3-byte sequence
153+ , (:) <$> choose (0xE0, 0xEF) <*> upTo 1 contByte
154+ -- short 4-byte sequence
155+ , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte
156+ ]
157+ where
158+ contByte = (0x80 +) <$> choose (0, 0x3f)
159+ upTo n gen = do
160+ k <- choose (0,n)
161+ vectorOf k gen
162+
130163s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
131164 where _types = s :: String
132165sf_Eq p s =
tests/Tests/QuickCheckUtils.hs+9−7
@@ -18,6 +18,7 @@ module Tests.QuickCheckUtils
1818 , integralRandomR
1919
2020 , DecodeErr (..)
21+ , genDecodeErr
2122
2223 , Stringy (..)
2324 , eq
@@ -194,16 +195,17 @@ integralRandomR (a,b) g = case randomR (fromIntegral a :: Integer,
194195 fromIntegral b :: Integer) g of
195196 (x,h) -> (fromIntegral x, h)
196197
197−data DecodeErr = DE String T.OnDecodeError
198+data DecodeErr = Lenient | Ignore | Strict | Replace
199+ deriving (Show, Eq)
198200
199−instance Show DecodeErr where
200− show (DE d _) = "DE " ++ d
201+genDecodeErr :: DecodeErr -> Gen T.OnDecodeError
202+genDecodeErr Lenient = return T.lenientDecode
203+genDecodeErr Ignore = return T.ignore
204+genDecodeErr Strict = return T.strictDecode
205+genDecodeErr Replace = arbitrary
201206
202207instance Arbitrary DecodeErr where
203− arbitrary = oneof [ return $ DE "lenient" T.lenientDecode
204− , return $ DE "ignore" T.ignore
205− , return $ DE "strict" T.strictDecode
206− , DE "replace" `fmap` arbitrary ]
208+ arbitrary = elements [Lenient, Ignore, Strict, Replace]
207209
208210class Stringy s where
209211 packS :: String -> s