git.iko.soy/text
on master help sign in
fix utf8 error recovery for stream decoding
c976329Kubo Kovac
cbits/cbits.c+2−4
compare with master
unifiedsplit
@@ -207,10 +207,8 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
207 uint32_t *codepoint0, uint32_t *state0)
207 uint32_t *codepoint0, uint32_t *state0)
208{
208{
209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
210− codepoint0, state0);
210+ codepoint0, state0);
211− if (*state0 == UTF8_REJECT)
211+ return *src;
212− ret -=1;
213− return ret;
214}
212}
215
213
216/*
214/*
tests/Tests/Properties.hs+14−1
@@ -193,6 +193,16 @@ t_decode_with_error3 =
193t_decode_with_error4 =
193t_decode_with_error4 =
194 E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
194 E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
195
195
196+t_decode_with_error2' =
197+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of
198+ E.Some x _ _ -> x === "xa"
199+t_decode_with_error3' =
200+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of
201+ E.Some x _ _ -> x === "xaa"
202+t_decode_with_error4' =
203+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of
204+ E.Some x _ _ -> x === "xaaa"
205+
196s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
206s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
197 where _types = s :: String
207 where _types = s :: String
198sf_Eq p s =
208sf_Eq p s =
@@ -972,7 +982,10 @@ tests =
972 testGroup "error recovery" [
982 testGroup "error recovery" [
973 testProperty "t_decode_with_error2" t_decode_with_error2,
983 testProperty "t_decode_with_error2" t_decode_with_error2,
974 testProperty "t_decode_with_error3" t_decode_with_error3,
984 testProperty "t_decode_with_error3" t_decode_with_error3,
975− testProperty "t_decode_with_error4" t_decode_with_error4
985+ testProperty "t_decode_with_error4" t_decode_with_error4,
986+ testProperty "t_decode_with_error2'" t_decode_with_error2',
987+ testProperty "t_decode_with_error3'" t_decode_with_error3',
988+ testProperty "t_decode_with_error4'" t_decode_with_error4'
976 ]
989 ]
977 ],
990 ],
978
991