git.iko.soy/text
on master help sign in
fix utf8 error recovery
commita125908830a974870e835298fd35807b7a529574 authorKubo Kovac <kuko@fb.com> dateMon 22 May 2017, 16:27 (9 years ago) parent240e1a5
2 files changed+21−6
compare with master
unifiedsplit
cbits/cbits.c+3−6 tests/Tests/Properties.hs+18
cbits/cbits.c+3−6
@@ -222,12 +222,9 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
222222{
223223 uint32_t codepoint;
224224 uint32_t state = UTF8_ACCEPT;
225− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
226− &codepoint, &state);
227− /* Back up if we have an incomplete or invalid encoding */
228− if (state != UTF8_ACCEPT)
229− ret -= 1;
230− return ret;
225+ _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
226+ &codepoint, &state);
227+ return src;
231228}
232229
233230void
tests/Tests/Properties.hs+18
@@ -180,6 +180,19 @@ genInvalidUTF8 = B.pack <$> oneof [
180180 k <- choose (0,n)
181181 vectorOf k gen
182182
183+-- See http://unicode.org/faq/utf_bom.html#gen8
184+-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...
185+-- When faced with this illegal byte sequence ... a UTF-8 conformant process
186+-- must treat the first byte 110xxxxx2 as an illegal termination error
187+-- (e.g. filter it out or replace by 0xFFFD) ...
188+-- ... and continue processing at the second byte 0xxxxxxx2
189+t_decode_with_error2 =
190+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"
191+t_decode_with_error3 =
192+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"
193+t_decode_with_error4 =
194+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
195+
183196s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
184197 where _types = s :: String
185198sf_Eq p s =
@@ -955,6 +968,11 @@ tests =
955968 testGroup "errors" [
956969 testProperty "t_utf8_err" t_utf8_err,
957970 testProperty "t_utf8_err'" t_utf8_err'
971+ ],
972+ testGroup "error recovery" [
973+ testProperty "t_decode_with_error2" t_decode_with_error2,
974+ testProperty "t_decode_with_error3" t_decode_with_error3,
975+ testProperty "t_decode_with_error4" t_decode_with_error4
958976 ]
959977 ],
960978