aca0971 · text · git.iko.soy

git.iko.soy/text

Add support for incremental decoding

Decoding multi-byte encodings such as UTF-8 pose difficulty for streaming I/O as one must take care to carry the decoder state between incoming chunks. Here we introduce `decodeUtf8With'` which exposes an interface similar to that provided by cassava's `Data.Csv.Incremental`. To do this, we adapt the C UTF-8 decoder to expose its automaton state and codepoint accumulator.

commitaca09714e2a413e5bc7893e5f6c9ccda995b5c2e authorBen Gamari <bgamari.foss@gmail.com> dateFri 20 Sep 2013, 20:57 (13 years ago) parent0fc9b4d

2 files changed+100−9

compare with master

unifiedsplit

Data/Text/Encoding.hs+61−1 cbits/cbits.c+39−8

Data/Text/Encoding.hs+61−1

@@ -43,6 +43,10 @@ module Data.Text.Encoding

4343 , decodeUtf32LEWith

4444 , decodeUtf32BEWith

4545

46+ -- ** Streaming decoding with controllable error handling

47+ , decodeUtf8With'

48+ , Decoder(..)

49+

4650 -- * Encoding Text to ByteStrings

4751 , encodeUtf8

4852 , encodeUtf16LE

@@ -57,6 +61,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

5761#else

5862import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

5963#endif

64+import Control.Monad.ST (runST)

6065import Data.Bits ((.&.))

6166import Data.ByteString as B

6267import Data.ByteString.Internal as B

@@ -65,7 +70,7 @@ import Data.Text.Internal (Text(..), safe, textP)

6570import Data.Text.Private (runText)

6671import Data.Text.UnsafeChar (ord, unsafeWrite)

6772import Data.Text.UnsafeShift (shiftL, shiftR)

68−import Data.Word (Word8)

73+import Data.Word (Word8, Word32)

6974import Foreign.C.Types (CSize)

7075import Foreign.ForeignPtr (withForeignPtr)

7176import Foreign.Marshal.Utils (with)

@@ -139,6 +144,56 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

139144 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"

140145{- INLINE[0] decodeUtf8With #-}

141146

147+data Decoder = Some !Text (ByteString -> Decoder)

148+

149+-- | (codepoint, state)

150+type DecoderState = (Word32, Word32)

151+

152+decodeUtf8With' :: OnDecodeError -> ByteString -> Decoder

153+decodeUtf8With' onErr = decodeChunk (0,0)

154+ where

155+ -- We create a slightly larger than necessary buffer to accomodate a

156+ -- potential surrogate pair started in the last buffer

157+ decodeChunk :: DecoderState -> ByteString -> Decoder

158+ decodeChunk (codepoint0,state0) (PS fp off len) =

159+ runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)

160+ where

161+ decodeChunkToBuffer :: A.MArray s -> IO Decoder

162+ decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->

163+ with (0::CSize) $ \destOffPtr ->

164+ with codepoint0 $ \codepointPtr ->

165+ with state0 $ \statePtr ->

166+ let end = ptr `plusPtr` (off + len)

167+ loop curPtr = do

168+ curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr curPtr end

169+ codepointPtr statePtr

170+ state <- peek statePtr

171+ case state of

172+ 12 -> do

173+ -- We encountered an encoding error

174+ x <- peek curPtr'

175+ case onErr desc (Just x) of

176+ Nothing -> loop $ curPtr' `plusPtr` 1

177+ Just c -> do

178+ destOff <- peek destOffPtr

179+ w <- unsafeSTToIO $

180+ unsafeWrite dest (fromIntegral destOff) (safe c)

181+ poke destOffPtr (destOff + fromIntegral w)

182+ poke statePtr 0

183+ loop $ curPtr' `plusPtr` 1

184+

185+ _ -> do

186+ -- We encountered the end of the buffer while decoding

187+ n <- peek destOffPtr

188+ codepoint <- peek codepointPtr

189+ chunkText <- unsafeSTToIO $ do

190+ arr <- A.unsafeFreeze dest

191+ return $! textP arr 0 (fromIntegral n)

192+ return $ Some chunkText $ decodeChunk (codepoint, state)

193+ in loop (ptr `plusPtr` off)

194+ desc = "Data.Text.Encoding.decodeUtf8With': Invalid UTF-8 stream"

195+{- INLINE[0] decodeUtf8With' #-}

196+

142197-- | Decode a 'ByteString' containing UTF-8 encoded text that is known

143198-- to be valid.

144199--

@@ -296,5 +351,10 @@ foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8

296351 :: MutableByteArray# s -> Ptr CSize

297352 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)

298353

354+foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state

355+ :: MutableByteArray# s -> Ptr CSize

356+ -> Ptr Word8 -> Ptr Word8

357+ -> Ptr Word32 -> Ptr Word32 -> IO (Ptr Word8)

358+

299359foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1

300360 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()

cbits/cbits.c+39−8

@@ -104,19 +104,36 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,

104104 * the start of an invalid byte sequence.

105105 *

106106 * At exit, updates *destoff with the next offset to write to, and

107− * returns the next source offset to read from.

107+ * returns the next source offset to read from. Moreover, this function

108+ * exposes the internal decoder state (state0 and codepoint0), allowing one

109+ * to restart the decoder after it terminates (say, due to a partial codepoint).

110+ *

111+ * In particular, there are a few possible outcomes,

112+ *

113+ * 1) We decoded the buffer entirely:

114+ * In this case we return srcend

115+ * state0 == UTF8_ACCEPT

116+ *

117+ * 2) We met an invalid encoding

118+ * In this case we return the address of the first invalid byte

119+ * state0 == UTF8_REJECT

120+ *

121+ * 3) We reached the end of the buffer while decoding a codepoint

122+ * In this case we return a pointer to the first byte of the partial codepoint

123+ * state0 != UTF8_ACCEPT, UTF8_REJECT

124+ *

108125 */

109126uint8_t const *

110−_hs_text_decode_utf8(uint16_t *dest, size_t *destoff,

111− const uint8_t const *src, const uint8_t const *srcend)

127+_hs_text_decode_utf8_state(uint16_t *dest, size_t *destoff,

128+ const uint8_t const *src, const uint8_t const *srcend,

129+ uint32_t *codepoint0, uint32_t *state0)

112130{

113131 uint16_t *d = dest + *destoff;

114132 const uint8_t const *s = src;

115− uint32_t state = UTF8_ACCEPT;

133+ uint32_t state = *state0;

134+ uint32_t codepoint = *codepoint0;

116135

117136 while (s < srcend) {

118− uint32_t codepoint;

119−

120137#if defined(__i386__) || defined(__x86_64__)

121138 /*

122139 * This code will only work on a little-endian system that

@@ -161,11 +178,25 @@ _hs_text_decode_utf8(uint16_t *dest, size_t *destoff,

161178 }

162179 }

163180

164− /* Error recovery - if we're not in a valid finishing state, back up. */

165− if (state != UTF8_ACCEPT)

181+ /* Invalid encoding, back up to the errant character */

182+ if (state == UTF8_REJECT)

166183 s -= 1;

167184

168185 *destoff = d - dest;

186+ *codepoint0 = codepoint;

187+ *state0 = state;

169188

170189 return s;

171190}

191+

192+/*

193+ * Helper to decode buffer and discard final decoder state

194+ */

195+uint8_t const *

196+_hs_text_decode_utf8(uint16_t *dest, size_t *destoff,

197+ const uint8_t const *src, const uint8_t const *srcend)

198+{

199+ uint32_t codepoint;

200+ uint32_t state = UTF8_ACCEPT;

201+ return _hs_text_decode_utf8_state(dest, destoff, src, srcend, &codepoint, &state);

202+}