git.iko.soy/text
on master help sign in
Merge pull request #182 from kuk0/decode
45f389bBryan O'Sullivan
cbits/cbits.c+5−10
compare with master
unifiedsplit
@@ -207,10 +207,8 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
207 uint32_t *codepoint0, uint32_t *state0)
207 uint32_t *codepoint0, uint32_t *state0)
208{
208{
209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
210− codepoint0, state0);
210+ codepoint0, state0);
211− if (*state0 == UTF8_REJECT)
211+ return *src;
212− ret -=1;
213− return ret;
214}
212}
215
213
216/*
214/*
@@ -222,12 +220,9 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
222{
220{
223 uint32_t codepoint;
221 uint32_t codepoint;
224 uint32_t state = UTF8_ACCEPT;
222 uint32_t state = UTF8_ACCEPT;
225− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
223+ _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
226− &codepoint, &state);
224+ &codepoint, &state);
227− /* Back up if we have an incomplete or invalid encoding */
228− if (state != UTF8_ACCEPT)
229− ret -= 1;
230− return ret;
225+ return src;
231}
226}
232
227
233void
228void
tests/Tests/Properties.hs+36
@@ -180,6 +180,33 @@ genInvalidUTF8 = B.pack <$> oneof [
180 k <- choose (0,n)
180 k <- choose (0,n)
181 vectorOf k gen
181 vectorOf k gen
182
182
183+-- See http://unicode.org/faq/utf_bom.html#gen8
184+-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...
185+-- When faced with this illegal byte sequence ... a UTF-8 conformant process
186+-- must treat the first byte 110xxxxx2 as an illegal termination error
187+-- (e.g. filter it out or replace by 0xFFFD) ...
188+-- ... and continue processing at the second byte 0xxxxxxx2
189+t_decode_with_error2 =
190+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"
191+t_decode_with_error3 =
192+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"
193+t_decode_with_error4 =
194+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
195+
196+t_decode_with_error2' =
197+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of
198+ E.Some x _ _ -> x === "xa"
199+t_decode_with_error3' =
200+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of
201+ E.Some x _ _ -> x === "xaa"
202+t_decode_with_error4' =
203+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of
204+ E.Some x _ _ -> x === "xaaa"
205+
206+t_infix_concat bs1 text bs2 rep =
207+ text `T.isInfixOf`
208+ E.decodeUtf8With (\_ _ -> rep) (B.concat [bs1, E.encodeUtf8 text, bs2])
209+
183s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
210s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
184 where _types = s :: String
211 where _types = s :: String
185sf_Eq p s =
212sf_Eq p s =
@@ -955,6 +982,15 @@ tests =
955 testGroup "errors" [
982 testGroup "errors" [
956 testProperty "t_utf8_err" t_utf8_err,
983 testProperty "t_utf8_err" t_utf8_err,
957 testProperty "t_utf8_err'" t_utf8_err'
984 testProperty "t_utf8_err'" t_utf8_err'
985+ ],
986+ testGroup "error recovery" [
987+ testProperty "t_decode_with_error2" t_decode_with_error2,
988+ testProperty "t_decode_with_error3" t_decode_with_error3,
989+ testProperty "t_decode_with_error4" t_decode_with_error4,
990+ testProperty "t_decode_with_error2'" t_decode_with_error2',
991+ testProperty "t_decode_with_error3'" t_decode_with_error3',
992+ testProperty "t_decode_with_error4'" t_decode_with_error4',
993+ testProperty "t_infix_concat" t_infix_concat
958 ]
994 ]
959 ],
995 ],
960
996