git.iko.soy/text
on master help sign in
Merge pull request #182 from kuk0/decode

Fixing #181

commit45f389b51535100c12fd26e936ff571d04d1df22 authorBryan O'Sullivan <bos@serpentine.com> dateMon 7 Aug 2017, 19:26 (9 years ago) parent58962e5d42bafb
2 files changed+41−10
compare with master
unifiedsplit
cbits/cbits.c+5−10 tests/Tests/Properties.hs+36
cbits/cbits.c+5−10
@@ -207,10 +207,8 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
207207 uint32_t *codepoint0, uint32_t *state0)
208208{
209209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
210− codepoint0, state0);
211− if (*state0 == UTF8_REJECT)
212− ret -=1;
213− return ret;
210+ codepoint0, state0);
211+ return *src;
214212}
215213
216214/*
@@ -222,12 +220,9 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
222220{
223221 uint32_t codepoint;
224222 uint32_t state = UTF8_ACCEPT;
225− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
226− &codepoint, &state);
227− /* Back up if we have an incomplete or invalid encoding */
228− if (state != UTF8_ACCEPT)
229− ret -= 1;
230− return ret;
223+ _hs_text_decode_utf8_int(dest, destoff, &src, srcend,
224+ &codepoint, &state);
225+ return src;
231226}
232227
233228void
tests/Tests/Properties.hs+36
@@ -180,6 +180,33 @@ genInvalidUTF8 = B.pack <$> oneof [
180180 k <- choose (0,n)
181181 vectorOf k gen
182182
183+-- See http://unicode.org/faq/utf_bom.html#gen8
184+-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...
185+-- When faced with this illegal byte sequence ... a UTF-8 conformant process
186+-- must treat the first byte 110xxxxx2 as an illegal termination error
187+-- (e.g. filter it out or replace by 0xFFFD) ...
188+-- ... and continue processing at the second byte 0xxxxxxx2
189+t_decode_with_error2 =
190+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"
191+t_decode_with_error3 =
192+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"
193+t_decode_with_error4 =
194+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
195+
196+t_decode_with_error2' =
197+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of
198+ E.Some x _ _ -> x === "xa"
199+t_decode_with_error3' =
200+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of
201+ E.Some x _ _ -> x === "xaa"
202+t_decode_with_error4' =
203+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of
204+ E.Some x _ _ -> x === "xaaa"
205+
206+t_infix_concat bs1 text bs2 rep =
207+ text `T.isInfixOf`
208+ E.decodeUtf8With (\_ _ -> rep) (B.concat [bs1, E.encodeUtf8 text, bs2])
209+
183210s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)
184211 where _types = s :: String
185212sf_Eq p s =
@@ -955,6 +982,15 @@ tests =
955982 testGroup "errors" [
956983 testProperty "t_utf8_err" t_utf8_err,
957984 testProperty "t_utf8_err'" t_utf8_err'
985+ ],
986+ testGroup "error recovery" [
987+ testProperty "t_decode_with_error2" t_decode_with_error2,
988+ testProperty "t_decode_with_error3" t_decode_with_error3,
989+ testProperty "t_decode_with_error4" t_decode_with_error4,
990+ testProperty "t_decode_with_error2'" t_decode_with_error2',
991+ testProperty "t_decode_with_error3'" t_decode_with_error3',
992+ testProperty "t_decode_with_error4'" t_decode_with_error4',
993+ testProperty "t_infix_concat" t_infix_concat
958994 ]
959995 ],
960996