@@ -66,7 +66,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
6666import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
6767#endif
6868
69−import Control.Exception (evaluate, try, throwIO, ErrorCall(ErrorCall))
69+import Control.Exception (evaluate, try)
7070import Control.Monad.ST (runST)
7171import Data.Bits ((.&.))
7272import Data.ByteString as B
@@ -140,13 +140,6 @@ decodeLatin1 (PS fp off len) = text a 0 len
140140 return dest
141141
142142-- | Decode a 'ByteString' containing UTF-8 encoded text.
143−--
144−-- __NOTE__: The replacement character returned by 'OnDecodeError'
145−-- MUST be within the BMP plane; surrogate code points will
146−-- automatically be remapped to the replacement char @U+FFFD@
147−-- (/since 0.11.3.0/), whereas code points beyond the BMP will throw an
148−-- 'error' (/since 1.2.3.1/); For earlier versions of @text@ using
149−-- those unsupported code points would result in undefined behavior.
150143decodeUtf8With :: OnDecodeError -> ByteString -> Text
151144decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
152145 let go dest = withForeignPtr fp $ \ptr ->
@@ -162,52 +155,16 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
162155 x <- peek curPtr'
163156 case onErr desc (Just x) of
164157 Nothing -> loop $ curPtr' `plusPtr` 1
165− Just c
166− | c > '\xFFFF' -> throwUnsupportedReplChar
167− | otherwise -> do
168− destOff <- peek destOffPtr
169− w <- unsafeSTToIO $
170− unsafeWrite dest (fromIntegral destOff)
171− (safe c)
172− poke destOffPtr (destOff + fromIntegral w)
173− loop $ curPtr' `plusPtr` 1
158+ Just c -> do
159+ destOff <- peek destOffPtr
160+ w <- unsafeSTToIO $
161+ unsafeWrite dest (fromIntegral destOff) (safe c)
162+ poke destOffPtr (destOff + fromIntegral w)
163+ loop $ curPtr' `plusPtr` 1
174164 loop (ptr `plusPtr` off)
175165 (unsafeIOToST . go) =<< A.new len
176166 where
177167 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"
178−
179− throwUnsupportedReplChar = throwIO $
180− ErrorCall "decodeUtf8With: non-BMP replacement characters not supported"
181− -- TODO: The code currently assumes that the transcoded UTF-16
182− -- stream is at most twice as long (in bytes) as the input UTF-8
183− -- stream. To justify this assumption one has to assume that the
184− -- error handler replacement character also satisfies this
185− -- invariant, by emitting at most one UTF16 code unit.
186− --
187− -- One easy way to support the full range of code-points for
188− -- replacement characters in the error handler is to simply change
189− -- the (over-)allocation to `A.new (2*len)` and then shrink back the
190− -- `ByteArray#` to the real size (recent GHCs have a cheap
191− -- `ByteArray#` resize-primop for that which allow the GC to reclaim
192− -- the overallocation). However, this would require 4 times as much
193− -- (temporary) storage as the original UTF-8 required.
194− --
195− -- Another strategy would be to optimistically assume that
196− -- replacement characters are within the BMP, and if the case of a
197− -- non-BMP replacement occurs reallocate the target buffer (or throw
198− -- an exception, and fallback to a pessimistic codepath, like e.g.
199− -- `decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)`)
200− --
201− -- Alternatively, `OnDecodeError` could become a datastructure which
202− -- statically encodes the replacement-character range,
203− -- e.g. something isomorphic to
204− --
205− -- Either (... -> Maybe Word16) (... -> Maybe Char)
206− --
207− -- And allow to statically switch between the BMP/non-BMP
208− -- replacement-character codepaths. There's multiple ways to address
209− -- this with different tradeoffs; but ideally we should optimise for
210− -- the optimistic/error-free case.
211168{- INLINE[0] decodeUtf8With #-}
212169
213170-- $stream