@@ -43,9 +43,11 @@ module Data.Text.Encoding
4343 , decodeUtf32LEWith
4444 , decodeUtf32BEWith
4545
46− -- ** Streaming decoding with controllable error handling
47− , decodeUtf8With'
48− , Decoder(..)
46+ -- ** Stream oriented decoding
47+ -- $stream
48+ , streamDecodeUtf8
49+ , streamDecodeUtf8With
50+ , Decoding(..)
4951
5052 -- * Encoding Text to ByteStrings
5153 , encodeUtf8
@@ -65,6 +67,7 @@ import Control.Monad.ST (runST)
6567import Data.Bits ((.&.))
6668import Data.ByteString as B
6769import Data.ByteString.Internal as B
70+import Data.Text ()
6871import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
6972import Data.Text.Internal (Text(..), safe, textP)
7073import Data.Text.Private (runText)
@@ -146,21 +149,85 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
146149 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"
147150{- INLINE[0] decodeUtf8With #-}
148151
149−data Decoder = Some !Text (ByteString -> Decoder)
152+-- $stream
153+--
154+-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept
155+-- a 'ByteString' that represents a possibly incomplete input (e.g. a
156+-- packet from a network stream) that may not end on a UTF-8 boundary.
157+--
158+-- The first element of the result is the maximal chunk of 'Text' that
159+-- can be decoded from the given input. The second is a function which
160+-- accepts another 'ByteString'. That string will be assumed to
161+-- directly follow the string that was passed as input to the original
162+-- function, and it will in turn be decoded.
163+--
164+-- To help understand the use of these functions, consider the Unicode
165+-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi
166+-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.
167+--
168+-- Now suppose that we receive this encoded string as 3 packets that
169+-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",
170+-- \"\\x83\"]@. We cannot decode the entire Unicode string until we
171+-- have received all three packets, but we would like to make progress
172+-- as we receive each one.
173+--
174+-- @
175+-- let 'Some' t0 f0 = 'streamDecodeUtf8' \"hi \\xe2\"
176+-- t0 == \"hi \" :: 'Text'
177+-- @
178+--
179+-- We use the continuation @f0@ to decode our second packet.
180+--
181+-- @
182+-- let 'Some' t1 f1 = f0 \"\\x98\"
183+-- t1 == \"\"
184+-- @
185+--
186+-- We could not give @f0@ enough input to decode anything, so it
187+-- returned an empty string. Once we feed our second continuation @f1@
188+-- the last byte of input, it will make progress.
189+--
190+-- @
191+-- let 'Some' t2 f2 = f1 \"\\x83\"
192+-- t2 == \"☃\"
193+-- @
194+--
195+-- If given invalid input, an exception will be thrown by the function
196+-- or continuation where it is encountered.
197+
198+-- | A stream oriented decoding result.
199+data Decoding = Some Text (ByteString -> Decoding)
200+
201+instance Show Decoding where
202+ showsPrec d (Some t _) = showParen (d > prec) $
203+ showString "Some " . showsPrec (prec+1) t
204+ where prec = 10
150205
151206newtype CodePoint = CodePoint Word32 deriving (Num, Storable)
152207newtype DecoderState = DecoderState Word32 deriving (Eq, Num, Storable)
153208
154−decodeUtf8With' :: OnDecodeError -> ByteString -> Decoder
155−decodeUtf8With' onErr = decodeChunk 0 0
209+-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
210+-- encoded text that is known to be valid.
211+--
212+-- If the input contains any invalid UTF-8 data, an exception will be
213+-- thrown (either by this function or a continuation) that cannot be
214+-- caught in pure code. For more control over the handling of invalid
215+-- data, use 'streamDecodeUtf8With'.
216+streamDecodeUtf8 :: ByteString -> Decoding
217+streamDecodeUtf8 = streamDecodeUtf8With strictDecode
218+
219+-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
220+-- encoded text.
221+streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding
222+streamDecodeUtf8With onErr = decodeChunk 0 0
156223 where
157224 -- We create a slightly larger than necessary buffer to accommodate a
158225 -- potential surrogate pair started in the last buffer
159− decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoder
226+ decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding
160227 decodeChunk codepoint0 state0 (PS fp off len) =
161228 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
162229 where
163− decodeChunkToBuffer :: A.MArray s -> IO Decoder
230+ decodeChunkToBuffer :: A.MArray s -> IO Decoding
164231 decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
165232 with (0::CSize) $ \destOffPtr ->
166233 with codepoint0 $ \codepointPtr ->
@@ -191,10 +258,9 @@ decodeUtf8With' onErr = decodeChunk 0 0
191258 chunkText <- unsafeSTToIO $ do
192259 arr <- A.unsafeFreeze dest
193260 return $! textP arr 0 (fromIntegral n)
194− return $ Some chunkText $ decodeChunk codepoint state
261+ return $ Some chunkText (decodeChunk codepoint state)
195262 in loop (ptr `plusPtr` off)
196− desc = "Data.Text.Encoding.decodeUtf8With': Invalid UTF-8 stream"
197−{- INLINE[0] decodeUtf8With' #-}
263+ desc = "Data.Text.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"
198264
199265-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
200266-- to be valid.