238949e · text · git.iko.soy

git.iko.soy/text

Present undecoded bytestring when streaming

commit238949ef5de204afd19ad20bfc133435969cbcf3 authorBryan O'Sullivan <bos@serpentine.com> dateMon 23 Sep 2013, 09:46 (13 years ago) parent67d52b0

3 files changed+33−19

compare with master

unifiedsplit

Data/Text/Encoding.hs+17−10 cbits/cbits.c+15−8 tests/Tests/Properties.hs+1−1

Data/Text/Encoding.hs+17−10

@@ -77,7 +77,7 @@ import Data.Word (Word8, Word32)

7777import Foreign.C.Types (CSize)

7878import Foreign.ForeignPtr (withForeignPtr)

7979import Foreign.Marshal.Utils (with)

80−import Foreign.Ptr (Ptr, minusPtr, plusPtr)

80+import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

8181import Foreign.Storable (Storable, peek, poke)

8282import GHC.Base (MutableByteArray#)

8383import qualified Data.Text.Array as A

@@ -196,12 +196,14 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

196196-- or continuation where it is encountered.

197197

198198-- | A stream oriented decoding result.

199−data Decoding = Some Text (ByteString -> Decoding)

199+data Decoding = Some Text ByteString (ByteString -> Decoding)

200200

201201instance Show Decoding where

202− showsPrec d (Some t _) = showParen (d > prec) $

203− showString "Some " . showsPrec (prec+1) t

204− where prec = 10

202+ showsPrec d (Some t bs _) = showParen (d > prec) $

203+ showString "Some " . showsPrec prec' t .

204+ showChar ' ' . showsPrec prec' bs .

205+ showString " _"

206+ where prec = 10; prec' = prec + 1

205207

206208newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)

207209newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)

@@ -224,7 +226,7 @@ streamDecodeUtf8With onErr = decodeChunk 0 0

224226 -- We create a slightly larger than necessary buffer to accommodate a

225227 -- potential surrogate pair started in the last buffer

226228 decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding

227− decodeChunk codepoint0 state0 (PS fp off len) =

229+ decodeChunk codepoint0 state0 bs@(PS fp off len) =

228230 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)

229231 where

230232 decodeChunkToBuffer :: A.MArray s -> IO Decoding

@@ -232,10 +234,12 @@ streamDecodeUtf8With onErr = decodeChunk 0 0

232234 with (0::CSize) $ \destOffPtr ->

233235 with codepoint0 $ \codepointPtr ->

234236 with state0 $ \statePtr ->

237+ with nullPtr $ \curPtrPtr ->

235238 let end = ptr `plusPtr` (off + len)

236239 loop curPtr = do

237− curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr curPtr end

238− codepointPtr statePtr

240+ poke curPtrPtr curPtr

241+ curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr

242+ curPtrPtr end codepointPtr statePtr

239243 state <- peek statePtr

240244 case state of

241245 UTF8_REJECT -> do

@@ -258,7 +262,10 @@ streamDecodeUtf8With onErr = decodeChunk 0 0

258262 chunkText <- unsafeSTToIO $ do

259263 arr <- A.unsafeFreeze dest

260264 return $! textP arr 0 (fromIntegral n)

261− return $ Some chunkText (decodeChunk codepoint state)

265+ lastPtr <- peek curPtrPtr

266+ let left = lastPtr `minusPtr` curPtr

267+ return $ Some chunkText (B.drop left bs)

268+ (decodeChunk codepoint state)

262269 in loop (ptr `plusPtr` off)

263270 desc = "Data.Text.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"

264271

@@ -421,7 +428,7 @@ foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

421428

422429foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state

423430 :: MutableByteArray# s -> Ptr CSize

424− -> Ptr Word8 -> Ptr Word8

431+ -> Ptr (Ptr Word8) -> Ptr Word8

425432 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)

426433

427434foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

cbits/cbits.c+15−8

@@ -104,10 +104,13 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

104104 * A best-effort decoder. Runs until it hits either end of input or

105105 * the start of an invalid byte sequence.

106106 *

107− * At exit, updates *destoff with the next offset to write to, and

108− * returns the next source offset to read from. Moreover, this function

109− * exposes the internal decoder state (state0 and codepoint0), allowing one

110− * to restart the decoder after it terminates (say, due to a partial codepoint).

107+ * At exit, we update *destoff with the next offset to write to, *src

108+ * with the next source location past the last one successfully

109+ * decoded, and return the next source location to read from.

110+ *

111+ * Moreover, we expose the internal decoder state (state0 and

112+ * codepoint0), allowing one to restart the decoder after it

113+ * terminates (say, due to a partial codepoint).

111114 *

112115 * In particular, there are a few possible outcomes,

113116 *

@@ -126,11 +129,12 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

126129 */

127130const uint8_t *

128131_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

129− const uint8_t *const src, const uint8_t *const srcend,

132+ const uint8_t **const src,

133+ const uint8_t *const srcend,

130134 uint32_t *codepoint0, uint32_t *state0)

131135{

132136 uint16_t *d = dest + *destoff;

133− const uint8_t *s = src;

137+ const uint8_t *s = *src, *last = *src;

134138 uint32_t state = *state0;

135139 uint32_t codepoint = *codepoint0;

136140

@@ -162,6 +166,7 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

162166 *d++ = (uint16_t) ((codepoint >> 16) & 0xff);

163167 *d++ = (uint16_t) ((codepoint >> 24) & 0xff);

164168 }

169+ last = s;

165170 }

166171#endif

167172

@@ -177,6 +182,7 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

177182 *d++ = (uint16_t) (0xD7C0 + (codepoint >> 10));

178183 *d++ = (uint16_t) (0xDC00 + (codepoint & 0x3FF));

179184 }

185+ last = s;

180186 }

181187

182188 /* Invalid encoding, back up to the errant character */

@@ -186,6 +192,7 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

186192 *destoff = d - dest;

187193 *codepoint0 = codepoint;

188194 *state0 = state;

195+ *src = last;

189196

190197 return s;

191198}

@@ -195,9 +202,9 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

195202 */

196203const uint8_t *

197204_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

198− const uint8_t *const src, const uint8_t *const srcend)

205+ const uint8_t *src, const uint8_t *const srcend)

199206{

200207 uint32_t codepoint;

201208 uint32_t state = UTF8_ACCEPT;

202− return _hs_text_decode_utf8_state(dest, destoff, src, srcend, &codepoint, &state);

209+ return _hs_text_decode_utf8_state(dest, destoff, &src, srcend, &codepoint, &state);

203210}

tests/Tests/Properties.hs+1−1

@@ -107,7 +107,7 @@ t_utf8_incr = do

107107 feedChunksOf n f bs

108108 | B.null bs = []

109109 | otherwise = let (a,b) = B.splitAt n bs

110− E.Some t f' = f a

110+ E.Some t _ f' = f a

111111 in t : feedChunksOf n f' b

112112

113113-- This is a poor attempt to ensure that the error handling paths on