git.iko.soy/text
on master help sign in
Merge pull request #55 from bgamari/incremental
eb0c06fBryan O'Sullivan
Data/Text/Encoding.hs+61−1
compare with master
unifiedsplit
@@ -43,6 +43,10 @@ module Data.Text.Encoding
43 , decodeUtf32LEWith
43 , decodeUtf32LEWith
44 , decodeUtf32BEWith
44 , decodeUtf32BEWith
45
45
46+ -- ** Streaming decoding with controllable error handling
47+ , decodeUtf8With'
48+ , Decoder(..)
49+
46 -- * Encoding Text to ByteStrings
50 -- * Encoding Text to ByteStrings
47 , encodeUtf8
51 , encodeUtf8
48 , encodeUtf16LE
52 , encodeUtf16LE
@@ -57,6 +61,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
57#else
61#else
58import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
62import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
59#endif
63#endif
64+import Control.Monad.ST (runST)
60import Data.Bits ((.&.))
65import Data.Bits ((.&.))
61import Data.ByteString as B
66import Data.ByteString as B
62import Data.ByteString.Internal as B
67import Data.ByteString.Internal as B
@@ -65,7 +70,7 @@ import Data.Text.Internal (Text(..), safe, textP)
65import Data.Text.Private (runText)
70import Data.Text.Private (runText)
66import Data.Text.UnsafeChar (ord, unsafeWrite)
71import Data.Text.UnsafeChar (ord, unsafeWrite)
67import Data.Text.UnsafeShift (shiftL, shiftR)
72import Data.Text.UnsafeShift (shiftL, shiftR)
68−import Data.Word (Word8)
73+import Data.Word (Word8, Word32)
69import Foreign.C.Types (CSize)
74import Foreign.C.Types (CSize)
70import Foreign.ForeignPtr (withForeignPtr)
75import Foreign.ForeignPtr (withForeignPtr)
71import Foreign.Marshal.Utils (with)
76import Foreign.Marshal.Utils (with)
@@ -139,6 +144,56 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
139 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"
144 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"
140{- INLINE[0] decodeUtf8With #-}
145{- INLINE[0] decodeUtf8With #-}
141
146
147+data Decoder = Some !Text (ByteString -> Decoder)
148+
149+-- | (codepoint, state)
150+type DecoderState = (Word32, Word32)
151+
152+decodeUtf8With' :: OnDecodeError -> ByteString -> Decoder
153+decodeUtf8With' onErr = decodeChunk (0,0)
154+ where
155+ -- We create a slightly larger than necessary buffer to accomodate a
156+ -- potential surrogate pair started in the last buffer
157+ decodeChunk :: DecoderState -> ByteString -> Decoder
158+ decodeChunk (codepoint0,state0) (PS fp off len) =
159+ runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
160+ where
161+ decodeChunkToBuffer :: A.MArray s -> IO Decoder
162+ decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
163+ with (0::CSize) $ \destOffPtr ->
164+ with codepoint0 $ \codepointPtr ->
165+ with state0 $ \statePtr ->
166+ let end = ptr `plusPtr` (off + len)
167+ loop curPtr = do
168+ curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr curPtr end
169+ codepointPtr statePtr
170+ state <- peek statePtr
171+ case state of
172+ 12 -> do
173+ -- We encountered an encoding error
174+ x <- peek curPtr'
175+ case onErr desc (Just x) of
176+ Nothing -> loop $ curPtr' `plusPtr` 1
177+ Just c -> do
178+ destOff <- peek destOffPtr
179+ w <- unsafeSTToIO $
180+ unsafeWrite dest (fromIntegral destOff) (safe c)
181+ poke destOffPtr (destOff + fromIntegral w)
182+ poke statePtr 0
183+ loop $ curPtr' `plusPtr` 1
184+
185+ _ -> do
186+ -- We encountered the end of the buffer while decoding
187+ n <- peek destOffPtr
188+ codepoint <- peek codepointPtr
189+ chunkText <- unsafeSTToIO $ do
190+ arr <- A.unsafeFreeze dest
191+ return $! textP arr 0 (fromIntegral n)
192+ return $ Some chunkText $ decodeChunk (codepoint, state)
193+ in loop (ptr `plusPtr` off)
194+ desc = "Data.Text.Encoding.decodeUtf8With': Invalid UTF-8 stream"
195+{- INLINE[0] decodeUtf8With' #-}
196+
142-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
197-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
143-- to be valid.
198-- to be valid.
144--
199--
@@ -296,5 +351,10 @@ foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
296 :: MutableByteArray# s -> Ptr CSize
351 :: MutableByteArray# s -> Ptr CSize
297 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
352 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
298
353
354+foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state
355+ :: MutableByteArray# s -> Ptr CSize
356+ -> Ptr Word8 -> Ptr Word8
357+ -> Ptr Word32 -> Ptr Word32 -> IO (Ptr Word8)
358+
299foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
359foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
300 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
360 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
cbits/cbits.c+41−10
@@ -104,19 +104,36 @@ _hs_text_decode_latin1(uint16_t *dest, const uint8_t const *src,
104 * the start of an invalid byte sequence.
104 * the start of an invalid byte sequence.
105 *
105 *
106 * At exit, updates *destoff with the next offset to write to, and
106 * At exit, updates *destoff with the next offset to write to, and
107− * returns the next source offset to read from.
107+ * returns the next source offset to read from. Moreover, this function
108+ * exposes the internal decoder state (state0 and codepoint0), allowing one
109+ * to restart the decoder after it terminates (say, due to a partial codepoint).
110+ *
111+ * In particular, there are a few possible outcomes,
112+ *
113+ * 1) We decoded the buffer entirely:
114+ * In this case we return srcend
115+ * state0 == UTF8_ACCEPT
116+ *
117+ * 2) We met an invalid encoding
118+ * In this case we return the address of the first invalid byte
119+ * state0 == UTF8_REJECT
120+ *
121+ * 3) We reached the end of the buffer while decoding a codepoint
122+ * In this case we return a pointer to the first byte of the partial codepoint
123+ * state0 != UTF8_ACCEPT, UTF8_REJECT
124+ *
108 */
125 */
109−uint8_t const *
126+const uint8_t *
110−_hs_text_decode_utf8(uint16_t *dest, size_t *destoff,
127+_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
111− const uint8_t const *src, const uint8_t const *srcend)
128+ const uint8_t *const src, const uint8_t *const srcend,
129+ uint32_t *codepoint0, uint32_t *state0)
112{
130{
113 uint16_t *d = dest + *destoff;
131 uint16_t *d = dest + *destoff;
114− const uint8_t const *s = src;
132+ const uint8_t *s = src;
115− uint32_t state = UTF8_ACCEPT;
133+ uint32_t state = *state0;
134+ uint32_t codepoint = *codepoint0;
116
135
117 while (s < srcend) {
136 while (s < srcend) {
118− uint32_t codepoint;
119−
120#if defined(__i386__) || defined(__x86_64__)
137#if defined(__i386__) || defined(__x86_64__)
121 /*
138 /*
122 * This code will only work on a little-endian system that
139 * This code will only work on a little-endian system that
@@ -161,11 +178,25 @@ _hs_text_decode_utf8(uint16_t *dest, size_t *destoff,
161 }
178 }
162 }
179 }
163
180
164− /* Error recovery - if we're not in a valid finishing state, back up. */
181+ /* Invalid encoding, back up to the errant character */
165− if (state != UTF8_ACCEPT)
182+ if (state == UTF8_REJECT)
166 s -= 1;
183 s -= 1;
167
184
168 *destoff = d - dest;
185 *destoff = d - dest;
186+ *codepoint0 = codepoint;
187+ *state0 = state;
169
188
170 return s;
189 return s;
171}
190}
191+
192+/*
193+ * Helper to decode buffer and discard final decoder state
194+ */
195+const uint8_t *
196+_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,
197+ const uint8_t *const src, const uint8_t *const srcend)
198+{
199+ uint32_t codepoint;
200+ uint32_t state = UTF8_ACCEPT;
201+ return _hs_text_decode_utf8_state(dest, destoff, src, srcend, &codepoint, &state);
202+}
tests/Tests/Properties.hs+12
@@ -93,6 +93,17 @@ tl_utf32LE = forAll genUnicode $ (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id
93t_utf32BE = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
93t_utf32BE = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
94tl_utf32BE = forAll genUnicode $ (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id
94tl_utf32BE = forAll genUnicode $ (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id
95
95
96+t_utf8_incr = do
97+ Positive n <- arbitrary
98+ forAll genUnicode $ recode n `eq` id
99+ where recode n = T.concat . feedChunksOf n (E.decodeUtf8With' strictDecode) . E.encodeUtf8
100+ feedChunksOf :: Int -> (B.ByteString -> E.Decoder) -> B.ByteString -> [T.Text]
101+ feedChunksOf n f bs
102+ | B.null bs = []
103+ | otherwise = let (a,b) = B.splitAt n bs
104+ E.Some t f' = f a
105+ in t : feedChunksOf n f' b
106+
96-- This is a poor attempt to ensure that the error handling paths on
107-- This is a poor attempt to ensure that the error handling paths on
97-- decode are exercised in some way. Proper testing would be rather
108-- decode are exercised in some way. Proper testing would be rather
98-- more involved.
109-- more involved.
@@ -771,6 +782,7 @@ tests =
771 testProperty "tl_latin1" tl_latin1,
782 testProperty "tl_latin1" tl_latin1,
772 testProperty "t_utf8" t_utf8,
783 testProperty "t_utf8" t_utf8,
773 testProperty "t_utf8'" t_utf8',
784 testProperty "t_utf8'" t_utf8',
785+ testProperty "t_utf8_incr" t_utf8_incr,
774 testProperty "tl_utf8" tl_utf8,
786 testProperty "tl_utf8" tl_utf8,
775 testProperty "tl_utf8'" tl_utf8',
787 testProperty "tl_utf8'" tl_utf8',
776 testProperty "t_utf16LE" t_utf16LE,
788 testProperty "t_utf16LE" t_utf16LE,