git.iko.soy/text
on master help sign in
Merge pull request #55 from bgamari/incremental

Introduce incremental UTF8 decoding interface

commiteb0c06f37369c55b9b57c7b1b713bac42582594c authorBryan O'Sullivan <bos@serpentine.com> dateSun 22 Sep 2013, 11:09 (13 years ago) parent509c320df58e31
3 files changed+114−11
compare with master
unifiedsplit
Data/Text/Encoding.hs+61−1 cbits/cbits.c+41−10 tests/Tests/Properties.hs+12
Data/Text/Encoding.hs+61−1
@@ -43,6 +43,10 @@ module Data.Text.Encoding
4343 , decodeUtf32LEWith
4444 , decodeUtf32BEWith
4545
46+ -- ** Streaming decoding with controllable error handling
47+ , decodeUtf8With'
48+ , Decoder(..)
49+
4650 -- * Encoding Text to ByteStrings
4751 , encodeUtf8
4852 , encodeUtf16LE
@@ -57,6 +61,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
5761#else
5862import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
5963#endif
64+import Control.Monad.ST (runST)
6065import Data.Bits ((.&.))
6166import Data.ByteString as B
6267import Data.ByteString.Internal as B
@@ -65,7 +70,7 @@ import Data.Text.Internal (Text(..), safe, textP)
6570import Data.Text.Private (runText)
6671import Data.Text.UnsafeChar (ord, unsafeWrite)
6772import Data.Text.UnsafeShift (shiftL, shiftR)
68−import Data.Word (Word8)
73+import Data.Word (Word8, Word32)
6974import Foreign.C.Types (CSize)
7075import Foreign.ForeignPtr (withForeignPtr)
7176import Foreign.Marshal.Utils (with)
@@ -139,6 +144,56 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
139144 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"
140145{- INLINE[0] decodeUtf8With #-}
141146
147+data Decoder = Some !Text (ByteString -> Decoder)
148+
149+-- | (codepoint, state)
150+type DecoderState = (Word32, Word32)
151+
152+decodeUtf8With' :: OnDecodeError -> ByteString -> Decoder
153+decodeUtf8With' onErr = decodeChunk (0,0)
154+ where
155+ -- We create a slightly larger than necessary buffer to accomodate a
156+ -- potential surrogate pair started in the last buffer
157+ decodeChunk :: DecoderState -> ByteString -> Decoder
158+ decodeChunk (codepoint0,state0) (PS fp off len) =
159+ runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
160+ where
161+ decodeChunkToBuffer :: A.MArray s -> IO Decoder
162+ decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
163+ with (0::CSize) $ \destOffPtr ->
164+ with codepoint0 $ \codepointPtr ->
165+ with state0 $ \statePtr ->
166+ let end = ptr `plusPtr` (off + len)
167+ loop curPtr = do
168+ curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr curPtr end
169+ codepointPtr statePtr
170+ state <- peek statePtr
171+ case state of
172+ 12 -> do
173+ -- We encountered an encoding error
174+ x <- peek curPtr'
175+ case onErr desc (Just x) of
176+ Nothing -> loop $ curPtr' `plusPtr` 1
177+ Just c -> do
178+ destOff <- peek destOffPtr
179+ w <- unsafeSTToIO $
180+ unsafeWrite dest (fromIntegral destOff) (safe c)
181+ poke destOffPtr (destOff + fromIntegral w)
182+ poke statePtr 0
183+ loop $ curPtr' `plusPtr` 1
184+
185+ _ -> do
186+ -- We encountered the end of the buffer while decoding
187+ n <- peek destOffPtr
188+ codepoint <- peek codepointPtr
189+ chunkText <- unsafeSTToIO $ do
190+ arr <- A.unsafeFreeze dest
191+ return $! textP arr 0 (fromIntegral n)
192+ return $ Some chunkText $ decodeChunk (codepoint, state)
193+ in loop (ptr `plusPtr` off)
194+ desc = "Data.Text.Encoding.decodeUtf8With': Invalid UTF-8 stream"
195+{- INLINE[0] decodeUtf8With' #-}
196+
142197-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
143198-- to be valid.
144199--
@@ -296,5 +351,10 @@ foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
296351 :: MutableByteArray# s -> Ptr CSize
297352 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
298353
354+foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state
355+ :: MutableByteArray# s -> Ptr CSize
356+ -> Ptr Word8 -> Ptr Word8
357+ -> Ptr Word32 -> Ptr Word32 -> IO (Ptr Word8)
358+
299359foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
300360 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
cbits/cbits.c+41−10
@@ -104,19 +104,36 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,
104104 * the start of an invalid byte sequence.
105105 *
106106 * At exit, updates *destoff with the next offset to write to, and
107− * returns the next source offset to read from.
107+ * returns the next source offset to read from. Moreover, this function
108+ * exposes the internal decoder state (state0 and codepoint0), allowing one
109+ * to restart the decoder after it terminates (say, due to a partial codepoint).
110+ *
111+ * In particular, there are a few possible outcomes,
112+ *
113+ * 1) We decoded the buffer entirely:
114+ * In this case we return srcend
115+ * state0 == UTF8_ACCEPT
116+ *
117+ * 2) We met an invalid encoding
118+ * In this case we return the address of the first invalid byte
119+ * state0 == UTF8_REJECT
120+ *
121+ * 3) We reached the end of the buffer while decoding a codepoint
122+ * In this case we return a pointer to the first byte of the partial codepoint
123+ * state0 != UTF8_ACCEPT, UTF8_REJECT
124+ *
108125 */
109−uint8_t const *
110−_hs_text_decode_utf8(uint16_t *dest, size_t *destoff,
111− const uint8_t const *src, const uint8_t const *srcend)
126+const uint8_t *
127+_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
128+ const uint8_t *const src, const uint8_t *const srcend,
129+ uint32_t *codepoint0, uint32_t *state0)
112130{
113131 uint16_t *d = dest + *destoff;
114− const uint8_t const *s = src;
115− uint32_t state = UTF8_ACCEPT;
132+ const uint8_t *s = src;
133+ uint32_t state = *state0;
134+ uint32_t codepoint = *codepoint0;
116135
117136 while (s < srcend) {
118− uint32_t codepoint;
119−
120137#if defined(__i386__) || defined(__x86_64__)
121138 /*
122139 * This code will only work on a little-endian system that
@@ -161,11 +178,25 @@ _hs_text_decode_utf8(uint16_t *dest, size_t *destoff,
161178 }
162179 }
163180
164− /* Error recovery - if we're not in a valid finishing state, back up. */
165− if (state != UTF8_ACCEPT)
181+ /* Invalid encoding, back up to the errant character */
182+ if (state == UTF8_REJECT)
166183 s -= 1;
167184
168185 *destoff = d - dest;
186+ *codepoint0 = codepoint;
187+ *state0 = state;
169188
170189 return s;
171190}
191+
192+/*
193+ * Helper to decode buffer and discard final decoder state
194+ */
195+const uint8_t *
196+_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
197+ const uint8_t *const src, const uint8_t *const srcend)
198+{
199+ uint32_t codepoint;
200+ uint32_t state = UTF8_ACCEPT;
201+ return _hs_text_decode_utf8_state(dest, destoff, src, srcend, &codepoint, &state);
202+}
tests/Tests/Properties.hs+12
@@ -93,6 +93,17 @@ tl_utf32LE = forAll genUnicode $ (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id
9393t_utf32BE = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
9494tl_utf32BE = forAll genUnicode $ (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id
9595
96+t_utf8_incr = do
97+ Positive n <- arbitrary
98+ forAll genUnicode $ recode n `eq` id
99+ where recode n = T.concat . feedChunksOf n (E.decodeUtf8With' strictDecode) . E.encodeUtf8
100+ feedChunksOf :: Int -> (B.ByteString -> E.Decoder) -> B.ByteString -> [T.Text]
101+ feedChunksOf n f bs
102+ | B.null bs = []
103+ | otherwise = let (a,b) = B.splitAt n bs
104+ E.Some t f' = f a
105+ in t : feedChunksOf n f' b
106+
96107-- This is a poor attempt to ensure that the error handling paths on
97108-- decode are exercised in some way. Proper testing would be rather
98109-- more involved.
@@ -771,6 +782,7 @@ tests =
771782 testProperty "tl_latin1" tl_latin1,
772783 testProperty "t_utf8" t_utf8,
773784 testProperty "t_utf8'" t_utf8',
785+ testProperty "t_utf8_incr" t_utf8_incr,
774786 testProperty "tl_utf8" tl_utf8,
775787 testProperty "tl_utf8'" tl_utf8',
776788 testProperty "t_utf16LE" t_utf16LE,