git.iko.soy/text
on master help sign in
fix utf8 error recovery for stream decoding
commitc976329a5eaf1546390adff38625527e6b6f1d85 authorKubo Kovac <kuko@fb.com> dateMon 22 May 2017, 17:47 (9 years ago) parenta125908
2 files changed+16−5
compare with master
unifiedsplit
cbits/cbits.c+2−4 tests/Tests/Properties.hs+14−1
cbits/cbits.c+2−4
@@ -207,10 +207,8 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
207207 uint32_t *codepoint0, uint32_t *state0)
208208{
209209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
210− codepoint0, state0);
211− if (*state0 == UTF8_REJECT)
212− ret -=1;
213− return ret;
210+ codepoint0, state0);
211+ return *src;
214212}
215213
216214/*
tests/Tests/Properties.hs+14−1
@@ -193,6 +193,16 @@ t_decode_with_error3 =
193193t_decode_with_error4 =
194194 E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
195195
196+t_decode_with_error2' =
197+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of
198+ E.Some x _ _ -> x === "xa"
199+t_decode_with_error3' =
200+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of
201+ E.Some x _ _ -> x === "xaa"
202+t_decode_with_error4' =
203+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of
204+ E.Some x _ _ -> x === "xaaa"
205+
196206s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
197207 where _types = s :: String
198208sf_Eq p s =
@@ -972,7 +982,10 @@ tests =
972982 testGroup "error recovery" [
973983 testProperty "t_decode_with_error2" t_decode_with_error2,
974984 testProperty "t_decode_with_error3" t_decode_with_error3,
975− testProperty "t_decode_with_error4" t_decode_with_error4
985+ testProperty "t_decode_with_error4" t_decode_with_error4,
986+ testProperty "t_decode_with_error2'" t_decode_with_error2',
987+ testProperty "t_decode_with_error3'" t_decode_with_error3',
988+ testProperty "t_decode_with_error4'" t_decode_with_error4'
976989 ]
977990 ],
978991