@@ -66,7 +66,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
66import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
66import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
69−import Control.Exception (evaluate, try, throwIO, ErrorCall(ErrorCall))
69+import Control.Exception (evaluate, try)
70import Control.Monad.ST (runST)
70import Control.Monad.ST (runST)
71import Data.Bits ((.&.))
71import Data.Bits ((.&.))
72import Data.ByteString as B
72import Data.ByteString as B
@@ -140,13 +140,6 @@ decodeLatin1 (PS fp off len) = text a 0 len
140 return dest
140 return dest
142-- | Decode a 'ByteString' containing UTF-8 encoded text.
142-- | Decode a 'ByteString' containing UTF-8 encoded text.
144−-- __NOTE__: The replacement character returned by 'OnDecodeError'
145−-- MUST be within the BMP plane; surrogate code points will
146−-- automatically be remapped to the replacement char @U+FFFD@
147−-- (/since 0.11.3.0/), whereas code points beyond the BMP will throw an
148−-- 'error' (/since 1.2.3.1/); For earlier versions of @text@ using
149−-- those unsupported code points would result in undefined behavior.
150decodeUtf8With :: OnDecodeError -> ByteString -> Text
143decodeUtf8With :: OnDecodeError -> ByteString -> Text
151decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
144decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
152 let go dest = withForeignPtr fp $ \ptr ->
145 let go dest = withForeignPtr fp $ \ptr ->
@@ -162,52 +155,16 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
162 x <- peek curPtr'
155 x <- peek curPtr'
163 case onErr desc (Just x) of
156 case onErr desc (Just x) of
164 Nothing -> loop $ curPtr' `plusPtr` 1
157 Nothing -> loop $ curPtr' `plusPtr` 1
165− Just c
158+ Just c -> do
166− | c > '\xFFFF' -> throwUnsupportedReplChar
168− destOff <- peek destOffPtr
159+ destOff <- peek destOffPtr
169− w <- unsafeSTToIO $
160+ w <- unsafeSTToIO $
170− unsafeWrite dest (fromIntegral destOff)
161+ unsafeWrite dest (fromIntegral destOff) (safe c)
172− poke destOffPtr (destOff + fromIntegral w)
162+ poke destOffPtr (destOff + fromIntegral w)
173− loop $ curPtr' `plusPtr` 1
163+ loop $ curPtr' `plusPtr` 1
174 loop (ptr `plusPtr` off)
164 loop (ptr `plusPtr` off)
175 (unsafeIOToST . go) =<< A.new len
165 (unsafeIOToST . go) =<< A.new len
177 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"
167 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"
179− throwUnsupportedReplChar = throwIO $
180− ErrorCall "decodeUtf8With: non-BMP replacement characters not supported"
181− -- TODO: The code currently assumes that the transcoded UTF-16
182− -- stream is at most twice as long (in bytes) as the input UTF-8
183− -- stream. To justify this assumption one has to assume that the
184− -- error handler replacement character also satisfies this
185− -- invariant, by emitting at most one UTF16 code unit.
187− -- One easy way to support the full range of code-points for
188− -- replacement characters in the error handler is to simply change
189− -- the (over-)allocation to `A.new (2*len)` and then shrink back the
190− -- `ByteArray#` to the real size (recent GHCs have a cheap
191− -- `ByteArray#` resize-primop for that which allow the GC to reclaim
192− -- the overallocation). However, this would require 4 times as much
193− -- (temporary) storage as the original UTF-8 required.
195− -- Another strategy would be to optimistically assume that
196− -- replacement characters are within the BMP, and if the case of a
197− -- non-BMP replacement occurs reallocate the target buffer (or throw
198− -- an exception, and fallback to a pessimistic codepath, like e.g.
199− -- `decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)`)
201− -- Alternatively, `OnDecodeError` could become a datastructure which
202− -- statically encodes the replacement-character range,
203− -- e.g. something isomorphic to
205− -- Either (... -> Maybe Word16) (... -> Maybe Char)
207− -- And allow to statically switch between the BMP/non-BMP
208− -- replacement-character codepaths. There's multiple ways to address
209− -- this with different tradeoffs; but ideally we should optimise for
210− -- the optimistic/error-free case.
211{- INLINE[0] decodeUtf8With #-}
168{- INLINE[0] decodeUtf8With #-}
213-- $stream
170-- $stream