238949e · text · git.iko.soy

git.iko.soy/text

on master help sign in

Present undecoded bytestring when streaming

238949eBryan O'Sullivan13 years ago

Data/Text/Encoding.hs+17−10

compare with master

unifiedsplit

@@ -77,7 +77,7 @@ import Data.Word (Word8, Word32)

77import Foreign.C.Types (CSize)

78import Foreign.ForeignPtr (withForeignPtr)

79import Foreign.Marshal.Utils (with)

80−import Foreign.Ptr (Ptr, minusPtr, plusPtr)

80+import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

81import Foreign.Storable (Storable, peek, poke)

82import GHC.Base (MutableByteArray#)

83import qualified Data.Text.Array as A

@@ -196,12 +196,14 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

196-- or continuation where it is encountered.

197

198-- | A stream oriented decoding result.

199−data Decoding = Some Text (ByteString -> Decoding)

199+data Decoding = Some Text ByteString (ByteString -> Decoding)

200

201instance Show Decoding where

202− showsPrec d (Some t _) = showParen (d > prec) $

202+ showsPrec d (Some t bs _) = showParen (d > prec) $

203− showString "Some " . showsPrec (prec+1) t

203+ showString "Some " . showsPrec prec' t .

204+ showChar ' ' . showsPrec prec' bs .

205+ showString " _"

204− where prec = 10

206+ where prec = 10; prec' = prec + 1

205

207

206newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)

208newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)

207newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)

209newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)

@@ -224,7 +226,7 @@ streamDecodeUtf8With onErr = decodeChunk 0 0

224 -- We create a slightly larger than necessary buffer to accommodate a

226 -- We create a slightly larger than necessary buffer to accommodate a

225 -- potential surrogate pair started in the last buffer

227 -- potential surrogate pair started in the last buffer

226 decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding

228 decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding

227− decodeChunk codepoint0 state0 (PS fp off len) =

229+ decodeChunk codepoint0 state0 bs@(PS fp off len) =

228 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)

230 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)

229 where

231 where

230 decodeChunkToBuffer :: A.MArray s -> IO Decoding

232 decodeChunkToBuffer :: A.MArray s -> IO Decoding

@@ -232,10 +234,12 @@ streamDecodeUtf8With onErr = decodeChunk 0 0

232 with (0::CSize) $ \destOffPtr ->

234 with (0::CSize) $ \destOffPtr ->

233 with codepoint0 $ \codepointPtr ->

235 with codepoint0 $ \codepointPtr ->

234 with state0 $ \statePtr ->

236 with state0 $ \statePtr ->

237+ with nullPtr $ \curPtrPtr ->

235 let end = ptr `plusPtr` (off + len)

238 let end = ptr `plusPtr` (off + len)

236 loop curPtr = do

239 loop curPtr = do

240+ poke curPtrPtr curPtr

237− curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr curPtr end

241+ curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr

238− codepointPtr statePtr

242+ curPtrPtr end codepointPtr statePtr

239 state <- peek statePtr

243 state <- peek statePtr

240 case state of

244 case state of

241 UTF8_REJECT -> do

245 UTF8_REJECT -> do

@@ -258,7 +262,10 @@ streamDecodeUtf8With onErr = decodeChunk 0 0

258 chunkText <- unsafeSTToIO $ do

262 chunkText <- unsafeSTToIO $ do

259 arr <- A.unsafeFreeze dest

263 arr <- A.unsafeFreeze dest

260 return $! textP arr 0 (fromIntegral n)

264 return $! textP arr 0 (fromIntegral n)

265+ lastPtr <- peek curPtrPtr

266+ let left = lastPtr `minusPtr` curPtr

267+ return $ Some chunkText (B.drop left bs)

261− return $ Some chunkText (decodeChunk codepoint state)

268+ (decodeChunk codepoint state)

262 in loop (ptr `plusPtr` off)

269 in loop (ptr `plusPtr` off)

263 desc = "Data.Text.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"

270 desc = "Data.Text.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"

264

271

@@ -421,7 +428,7 @@ foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

421

428

422foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state

429foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state

423 :: MutableByteArray# s -> Ptr CSize

430 :: MutableByteArray# s -> Ptr CSize

424− -> Ptr Word8 -> Ptr Word8

431+ -> Ptr (Ptr Word8) -> Ptr Word8

425 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)

432 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)

426

433

427foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

434foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

cbits/cbits.c+15−8

@@ -104,10 +104,13 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

104 * A best-effort decoder. Runs until it hits either end of input or

105 * the start of an invalid byte sequence.

106 *

107− * At exit, updates *destoff with the next offset to write to, and

107+ * At exit, we update *destoff with the next offset to write to, *src

108− * returns the next source offset to read from. Moreover, this function

108+ * with the next source location past the last one successfully

109+ * decoded, and return the next source location to read from.

110+ *

109− * exposes the internal decoder state (state0 and codepoint0), allowing one

111+ * Moreover, we expose the internal decoder state (state0 and

112+ * codepoint0), allowing one to restart the decoder after it

110− * to restart the decoder after it terminates (say, due to a partial codepoint).

113+ * terminates (say, due to a partial codepoint).

111 *

114 *

112 * In particular, there are a few possible outcomes,

115 * In particular, there are a few possible outcomes,

113 *

116 *

@@ -126,11 +129,12 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

126 */

129 */

127const uint8_t *

130const uint8_t *

128_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

131_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

132+ const uint8_t **const src,

129− const uint8_t *const src, const uint8_t *const srcend,

133+ const uint8_t *const srcend,

130 uint32_t *codepoint0, uint32_t *state0)

134 uint32_t *codepoint0, uint32_t *state0)

131{

135{

132 uint16_t *d = dest + *destoff;

136 uint16_t *d = dest + *destoff;

133− const uint8_t *s = src;

137+ const uint8_t *s = *src, *last = *src;

134 uint32_t state = *state0;

138 uint32_t state = *state0;

135 uint32_t codepoint = *codepoint0;

139 uint32_t codepoint = *codepoint0;

136

140

@@ -162,6 +166,7 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

162 *d++ = (uint16_t) ((codepoint >> 16) & 0xff);

166 *d++ = (uint16_t) ((codepoint >> 16) & 0xff);

163 *d++ = (uint16_t) ((codepoint >> 24) & 0xff);

167 *d++ = (uint16_t) ((codepoint >> 24) & 0xff);

164 }

168 }

169+ last = s;

165 }

170 }

166#endif

171#endif

167

172

@@ -177,6 +182,7 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

177 *d++ = (uint16_t) (0xD7C0 + (codepoint >> 10));

182 *d++ = (uint16_t) (0xD7C0 + (codepoint >> 10));

178 *d++ = (uint16_t) (0xDC00 + (codepoint & 0x3FF));

183 *d++ = (uint16_t) (0xDC00 + (codepoint & 0x3FF));

179 }

184 }

185+ last = s;

180 }

186 }

181

187

182 /* Invalid encoding, back up to the errant character */

188 /* Invalid encoding, back up to the errant character */

@@ -186,6 +192,7 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

186 *destoff = d - dest;

192 *destoff = d - dest;

187 *codepoint0 = codepoint;

193 *codepoint0 = codepoint;

188 *state0 = state;

194 *state0 = state;

195+ *src = last;

189

196

190 return s;

197 return s;

191}

198}

@@ -195,9 +202,9 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

195 */

202 */

196const uint8_t *

203const uint8_t *

197_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

204_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

198− const uint8_t *const src, const uint8_t *const srcend)

205+ const uint8_t *src, const uint8_t *const srcend)

199{

206{

200 uint32_t codepoint;

207 uint32_t codepoint;

201 uint32_t state = UTF8_ACCEPT;

208 uint32_t state = UTF8_ACCEPT;

202− return _hs_text_decode_utf8_state(dest, destoff, src, srcend, &codepoint, &state);

209+ return _hs_text_decode_utf8_state(dest, destoff, &src, srcend, &codepoint, &state);

203}

210}

tests/Tests/Properties.hs+1−1

@@ -107,7 +107,7 @@ t_utf8_incr = do

107 feedChunksOf n f bs

108 | B.null bs = []

109 | otherwise = let (a,b) = B.splitAt n bs

110− E.Some t f' = f a

110+ E.Some t _ f' = f a

111 in t : feedChunksOf n f' b

112

113-- This is a poor attempt to ensure that the error handling paths on