git.iko.soy/text
on master help sign in
Merge from 1.0 branch
7bceda0Bryan O'Sullivan
cbits/cbits.c+31−10
compare with master
unifiedsplit
@@ -127,11 +127,18 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,
127 * state0 != UTF8_ACCEPT, UTF8_REJECT
127 * state0 != UTF8_ACCEPT, UTF8_REJECT
128 *
128 *
129 */
129 */
130−const uint8_t *
130+#if defined(__GNUC__) || defined(__clang__)
131+static inline uint8_t const *
131−_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
132+_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff,
132− const uint8_t **const src,
133+ const uint8_t const **src, const uint8_t const *srcend,
133− const uint8_t *const srcend,
134+ uint32_t *codepoint0, uint32_t *state0)
135+ __attribute((always_inline));
136+#endif
137+
138+static inline uint8_t const *
139+_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff,
140+ const uint8_t const **src, const uint8_t const *srcend,
134− uint32_t *codepoint0, uint32_t *state0)
141+ uint32_t *codepoint0, uint32_t *state0)
135{
142{
136 uint16_t *d = dest + *destoff;
143 uint16_t *d = dest + *destoff;
137 const uint8_t *s = *src, *last = *src;
144 const uint8_t *s = *src, *last = *src;
@@ -185,10 +192,6 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
185 last = s;
192 last = s;
186 }
193 }
187
194
188− /* Invalid encoding, back up to the errant character */
189− if (state == UTF8_REJECT)
190− s -= 1;
191−
192 *destoff = d - dest;
195 *destoff = d - dest;
193 *codepoint0 = codepoint;
196 *codepoint0 = codepoint;
194 *state0 = state;
197 *state0 = state;
@@ -197,6 +200,19 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
197 return s;
200 return s;
198}
201}
199
202
203+uint8_t const *
204+_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
205+ const uint8_t const **src,
206+ const uint8_t const *srcend,
207+ uint32_t *codepoint0, uint32_t *state0)
208+{
209+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
210+ codepoint0, state0);
211+ if (*state0 == UTF8_REJECT)
212+ ret -=1;
213+ return ret;
214+}
215+
200/*
216/*
201 * Helper to decode buffer and discard final decoder state
217 * Helper to decode buffer and discard final decoder state
202 */
218 */
@@ -206,5 +222,10 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
206{
222{
207 uint32_t codepoint;
223 uint32_t codepoint;
208 uint32_t state = UTF8_ACCEPT;
224 uint32_t state = UTF8_ACCEPT;
209− return _hs_text_decode_utf8_state(dest, destoff, &src, srcend, &codepoint, &state);
225+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
226+ &codepoint, &state);
227+ /* Back up if we have an incomplete or invalid encoding */
228+ if (state != UTF8_ACCEPT)
229+ ret -= 1;
230+ return ret;
210}
231}
changelog+5
@@ -1,3 +1,8 @@
1+1.0.0.1
2+
3+ * decodeUtf8: Fixed a regression that caused us to incorrectly
4+ identify truncated UTF-8 as valid (gh-61)
5+
11.0.0.0
61.0.0.0
2
7
3 * Added support for Unicode 6.3.0 to case conversion functions
8 * Added support for Unicode 6.3.0 to case conversion functions
tests/Tests/Properties.hs+44−11
@@ -13,6 +13,7 @@ import Test.QuickCheck
13import Test.QuickCheck.Monadic
13import Test.QuickCheck.Monadic
14import Text.Show.Functions ()
14import Text.Show.Functions ()
15
15
16+import Control.Applicative ((<$>), (<*>))
16import Control.Arrow ((***), second)
17import Control.Arrow ((***), second)
17import Control.Exception (catch)
18import Control.Exception (catch)
18import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)
19import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)
@@ -110,23 +111,55 @@ t_utf8_incr = do
110 E.Some t _ f' = f a
111 E.Some t _ f' = f a
111 in t : feedChunksOf n f' b
112 in t : feedChunksOf n f' b
112
113
113−-- This is a poor attempt to ensure that the error handling paths on
114+data Badness = Solo | Leading | Trailing
114−-- decode are exercised in some way. Proper testing would be rather
115+ deriving (Eq, Show)
115−-- more involved.
116+
117+instance Arbitrary Badness where
118+ arbitrary = elements [Solo, Leading, Trailing]
119+
116−t_utf8_err :: DecodeErr -> B.ByteString -> Property
120+t_utf8_err :: Badness -> DecodeErr -> Property
121+t_utf8_err bad de = do
122+ let gen = case bad of
123+ Solo -> genInvalidUTF8
124+ Leading -> B.append <$> genInvalidUTF8 <*> genUTF8
125+ Trailing -> B.append <$> genUTF8 <*> genInvalidUTF8
126+ genUTF8 = E.encodeUtf8 <$> genUnicode
127+ forAll gen $ \bs -> do
128+ onErr <- genDecodeErr de
117−t_utf8_err (DE _ de) bs = monadicIO $ do
129+ monadicIO $ do
118− l <- run $ let len = T.length (E.decodeUtf8With de bs)
130+ l <- run $ let len = T.length (E.decodeUtf8With onErr bs)
119− in (len `seq` return (Right len)) `catch`
131+ in (len `seq` return (Right len)) `catch`
120− (\(e::UnicodeException) -> return (Left e))
132+ (\(e::UnicodeException) -> return (Left e))
121− case l of
133+ assert $ case l of
122− Left err -> assert $ length (show err) >= 0
134+ Left err -> length (show err) >= 0
123− Right n -> assert $ n >= 0
135+ Right _ -> de /= Strict
124
136
125t_utf8_err' :: B.ByteString -> Property
137t_utf8_err' :: B.ByteString -> Property
126t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of
138t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of
127 Left err -> length (show err) >= 0
139 Left err -> length (show err) >= 0
128 Right t -> T.length t >= 0
140 Right t -> T.length t >= 0
129
141
142+genInvalidUTF8 :: Gen B.ByteString
143+genInvalidUTF8 = B.pack <$> oneof [
144+ -- invalid leading byte of a 2-byte sequence
145+ (:) <$> choose (0xC0, 0xC1) <*> upTo 1 contByte
146+ -- invalid leading byte of a 4-byte sequence
147+ , (:) <$> choose (0xF5, 0xFF) <*> upTo 3 contByte
148+ -- continuation bytes without a start byte
149+ , listOf1 contByte
150+ -- short 2-byte sequence
151+ , (:[]) <$> choose (0xC2, 0xDF)
152+ -- short 3-byte sequence
153+ , (:) <$> choose (0xE0, 0xEF) <*> upTo 1 contByte
154+ -- short 4-byte sequence
155+ , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte
156+ ]
157+ where
158+ contByte = (0x80 +) <$> choose (0, 0x3f)
159+ upTo n gen = do
160+ k <- choose (0,n)
161+ vectorOf k gen
162+
130s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
163s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
131 where _types = s :: String
164 where _types = s :: String
132sf_Eq p s =
165sf_Eq p s =
tests/Tests/QuickCheckUtils.hs+9−7
@@ -18,6 +18,7 @@ module Tests.QuickCheckUtils
18 , integralRandomR
18 , integralRandomR
19
19
20 , DecodeErr (..)
20 , DecodeErr (..)
21+ , genDecodeErr
21
22
22 , Stringy (..)
23 , Stringy (..)
23 , eq
24 , eq
@@ -194,16 +195,17 @@ integralRandomR (a,b) g = case randomR (fromIntegral a :: Integer,
194 fromIntegral b :: Integer) g of
195 fromIntegral b :: Integer) g of
195 (x,h) -> (fromIntegral x, h)
196 (x,h) -> (fromIntegral x, h)
196
197
197−data DecodeErr = DE String T.OnDecodeError
198+data DecodeErr = Lenient | Ignore | Strict | Replace
199+ deriving (Show, Eq)
198
200
199−instance Show DecodeErr where
201+genDecodeErr :: DecodeErr -> Gen T.OnDecodeError
200− show (DE d _) = "DE " ++ d
202+genDecodeErr Lenient = return T.lenientDecode
203+genDecodeErr Ignore = return T.ignore
204+genDecodeErr Strict = return T.strictDecode
205+genDecodeErr Replace = arbitrary
201
206
202instance Arbitrary DecodeErr where
207instance Arbitrary DecodeErr where
203− arbitrary = oneof [ return $ DE "lenient" T.lenientDecode
208+ arbitrary = elements [Lenient, Ignore, Strict, Replace]
204− , return $ DE "ignore" T.ignore
205− , return $ DE "strict" T.strictDecode
206− , DE "replace" `fmap` arbitrary ]
207
209
208class Stringy s where
210class Stringy s where
209 packS :: String -> s
211 packS :: String -> s