1{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, GeneralizedNewtypeDeriving, MagicHash,
3#if __GLASGOW_HASKELL__ >= 702
4{-# LANGUAGE Trustworthy #-}
7-- Module : Data.Text.Encoding
8-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan,
9-- (c) 2009 Duncan Coutts,
10-- (c) 2008, 2009 Tom Harper
13-- Maintainer : bos@serpentine.com
14-- Stability : experimental
15-- Portability : portable
17-- Functions for converting 'Text' values to and from 'ByteString',
18-- using several standard encodings.
20-- To gain access to a much larger family of encodings, use the
21-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>
23module Data.Text.Encoding
25 -- * Decoding ByteStrings to Text
35 -- ** Catchable failure
38 -- ** Controllable error handling
45 -- ** Stream oriented decoding
48 , streamDecodeUtf8With
51 -- * Encoding Text to ByteStrings
58#if MIN_VERSION_bytestring(0,10,4)
59 -- * Encoding Text using ByteString Builders
60 -- | /Note/ that these functions are only available if built against
61 -- @bytestring >= 0.10.4.0@.
63 , encodeUtf8BuilderEscaped
67#if __GLASGOW_HASKELL__ >= 702
68import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
70import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
73#if MIN_VERSION_bytestring(0,10,4)
74import Data.Bits ((.&.))
75import Data.Text.Internal.Unsafe.Char (ord)
76import qualified Data.ByteString.Builder as B
77import qualified Data.ByteString.Builder.Extra as B
78import qualified Data.ByteString.Builder.Internal as B hiding (empty)
79import qualified Data.ByteString.Builder.Prim as BP
80import qualified Data.ByteString.Builder.Prim.Internal as BP
81import qualified Data.Text.Internal.Encoding.Utf16 as U16
84import Control.Exception (evaluate, try)
85import Control.Monad.ST (runST)
86import Data.ByteString as B
87import Data.ByteString.Internal as B hiding (c2w)
89import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
90import Data.Text.Internal (Text(..), safe, textP)
91import Data.Text.Internal.Private (runText)
92import Data.Text.Internal.Unsafe.Char (unsafeWrite)
93import Data.Text.Internal.Unsafe.Shift (shiftR)
94import Data.Text.Unsafe (unsafeDupablePerformIO)
95import Data.Word (Word8, Word32)
96import Foreign.C.Types (CSize(..))
97import Foreign.ForeignPtr (withForeignPtr)
98import Foreign.Marshal.Utils (with)
99import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)
100import Foreign.Storable (Storable, peek, poke)
101import GHC.Base (ByteArray#, MutableByteArray#)
102import qualified Data.Text.Array as A
103import qualified Data.Text.Internal.Encoding.Fusion as E
104import qualified Data.Text.Internal.Fusion as F
106#include "text_cbits.h"
110-- All of the single-parameter functions for decoding bytestrings
111-- encoded in one of the Unicode Transformation Formats (UTF) operate
112-- in a /strict/ mode: each will throw an exception if given invalid
115-- Each function has a variant, whose name is suffixed with -'With',
116-- that gives greater control over the handling of decoding errors.
117-- For instance, 'decodeUtf8' will throw an exception, but
118-- 'decodeUtf8With' allows the programmer to determine what to do on a
121-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
124-- This function is deprecated. Use 'decodeLatin1' instead.
125decodeASCII :: ByteString -> Text
126decodeASCII = decodeUtf8
127{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
129-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
131-- 'decodeLatin1' is semantically equivalent to
132-- @Data.Text.pack . Data.ByteString.Char8.unpack@
133decodeLatin1 :: ByteString -> Text
134decodeLatin1 (PS fp off len) = textP a 0 len
136 a = A.run (A.new len >>= unsafeIOToST . go)
137 go dest = withForeignPtr fp $ \ptr -> do
138 c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))
141-- | Decode a 'ByteString' containing UTF-8 encoded text.
142decodeUtf8With :: OnDecodeError -> ByteString -> Text
143decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
144 let go dest = withForeignPtr fp $ \ptr ->
145 with (0::CSize) $ \destOffPtr -> do
146 let end = ptr `plusPtr` (off + len)
148 curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end
152 unsafeSTToIO (done dest (fromIntegral n))
155 case onErr desc (Just x) of
156 Nothing -> loop $ curPtr' `plusPtr` 1
158 destOff <- peek destOffPtr
160 unsafeWrite dest (fromIntegral destOff) (safe c)
161 poke destOffPtr (destOff + fromIntegral w)
162 loop $ curPtr' `plusPtr` 1
163 loop (ptr `plusPtr` off)
164 (unsafeIOToST . go) =<< A.new len
166 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"
167{- INLINE[0] decodeUtf8With #-}
171-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept
172-- a 'ByteString' that represents a possibly incomplete input (e.g. a
173-- packet from a network stream) that may not end on a UTF-8 boundary.
175-- The first element of the result is the maximal chunk of 'Text' that
176-- can be decoded from the given input. The second is a function which
177-- accepts another 'ByteString'. That string will be assumed to
178-- directly follow the string that was passed as input to the original
179-- function, and it will in turn be decoded.
181-- To help understand the use of these functions, consider the Unicode
182-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi
183-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.
185-- Now suppose that we receive this encoded string as 3 packets that
186-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",
187-- \"\\x83\"]@. We cannot decode the entire Unicode string until we
188-- have received all three packets, but we would like to make progress
189-- as we receive each one.
192-- let 'Some' t0 f0 = 'streamDecodeUtf8' \"hi \\xe2\"
193-- t0 == \"hi \" :: 'Text'
196-- We use the continuation @f0@ to decode our second packet.
199-- let 'Some' t1 f1 = f0 \"\\x98\"
203-- We could not give @f0@ enough input to decode anything, so it
204-- returned an empty string. Once we feed our second continuation @f1@
205-- the last byte of input, it will make progress.
208-- let 'Some' t2 f2 = f1 \"\\x83\"
212-- If given invalid input, an exception will be thrown by the function
213-- or continuation where it is encountered.
215-- | A stream oriented decoding result.
216data Decoding = Some Text ByteString (ByteString -> Decoding)
218instance Show Decoding where
219 showsPrec d (Some t bs _) = showParen (d > prec) $
220 showString "Some " . showsPrec prec' t .
221 showChar ' ' . showsPrec prec' bs .
223 where prec = 10; prec' = prec + 1
225newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)
226newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)
228-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
229-- encoded text that is known to be valid.
231-- If the input contains any invalid UTF-8 data, an exception will be
232-- thrown (either by this function or a continuation) that cannot be
233-- caught in pure code. For more control over the handling of invalid
234-- data, use 'streamDecodeUtf8With'.
235streamDecodeUtf8 :: ByteString -> Decoding
236streamDecodeUtf8 = streamDecodeUtf8With strictDecode
238-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
240streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding
241streamDecodeUtf8With onErr = decodeChunk 0 0
243 -- We create a slightly larger than necessary buffer to accommodate a
244 -- potential surrogate pair started in the last buffer
245 decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding
246 decodeChunk codepoint0 state0 bs@(PS fp off len) =
247 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
249 decodeChunkToBuffer :: A.MArray s -> IO Decoding
250 decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
251 with (0::CSize) $ \destOffPtr ->
252 with codepoint0 $ \codepointPtr ->
253 with state0 $ \statePtr ->
254 with nullPtr $ \curPtrPtr ->
255 let end = ptr `plusPtr` (off + len)
257 poke curPtrPtr curPtr
258 curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr
259 curPtrPtr end codepointPtr statePtr
260 state <- peek statePtr
263 -- We encountered an encoding error
265 case onErr desc (Just x) of
266 Nothing -> loop $ curPtr' `plusPtr` 1
268 destOff <- peek destOffPtr
270 unsafeWrite dest (fromIntegral destOff) (safe c)
271 poke destOffPtr (destOff + fromIntegral w)
273 loop $ curPtr' `plusPtr` 1
276 -- We encountered the end of the buffer while decoding
278 codepoint <- peek codepointPtr
279 chunkText <- unsafeSTToIO $ do
280 arr <- A.unsafeFreeze dest
281 return $! textP arr 0 (fromIntegral n)
282 lastPtr <- peek curPtrPtr
283 let left = lastPtr `minusPtr` curPtr
284 return $ Some chunkText (B.drop left bs)
285 (decodeChunk codepoint state)
286 in loop (ptr `plusPtr` off)
287 desc = "Data.Text.Internal.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"
289-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
292-- If the input contains any invalid UTF-8 data, an exception will be
293-- thrown that cannot be caught in pure code. For more control over
294-- the handling of invalid data, use 'decodeUtf8'' or
296decodeUtf8 :: ByteString -> Text
297decodeUtf8 = decodeUtf8With strictDecode
298{-# INLINE[0] decodeUtf8 #-}
299{-# RULES "STREAM stream/decodeUtf8 fusion" [1]
300 forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-}
302-- | Decode a 'ByteString' containing UTF-8 encoded text.
304-- If the input contains any invalid UTF-8 data, the relevant
305-- exception will be returned, otherwise the decoded text.
306decodeUtf8' :: ByteString -> Either UnicodeException Text
307decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode
308{-# INLINE decodeUtf8' #-}
310#if MIN_VERSION_bytestring(0,10,4)
312-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
313encodeUtf8Builder :: Text -> B.Builder
314encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)
316-- | Encode text using UTF-8 encoding and escape the ASCII characters using
317-- a 'BP.BoundedPrim'.
319-- Use this function is to implement efficient encoders for text-based formats
321{-# INLINE encodeUtf8BuilderEscaped #-}
322-- TODO: Extend documentation with references to source code in @blaze-html@
323-- or @aeson@ that uses this function.
324encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
325encodeUtf8BuilderEscaped be =
326 -- manual eta-expansion to ensure inlining works as expected
327 \txt -> B.builder (mkBuildstep txt)
329 bound = max 4 $ BP.sizeBound be
331 mkBuildstep (Text arr off len) !k =
336 outerLoop !i0 !br@(B.BufferRange op0 ope)
338 | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)
339 -- TODO: Use a loop with an integrated bound's check if outRemaining
340 -- is smaller than 8, as this will save on divisions.
341 | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)
343 outRemaining = (ope `minusPtr` op0) `div` bound
344 inpRemaining = iend - i0
346 goPartial !iendTmp = go i0 op0
349 | i < iendTmp = case A.unsafeIndex arr i of
351 BP.runB be (fromIntegral w) op >>= go (i + 1)
353 poke8 0 $ (w `shiftR` 6) + 0xC0
354 poke8 1 $ (w .&. 0x3f) + 0x80
355 go (i + 1) (op `plusPtr` 2)
356 | 0xD800 <= w && w <= 0xDBFF -> do
357 let c = ord $ U16.chr2 w (A.unsafeIndex arr (i+1))
358 poke8 0 $ (c `shiftR` 18) + 0xF0
359 poke8 1 $ ((c `shiftR` 12) .&. 0x3F) + 0x80
360 poke8 2 $ ((c `shiftR` 6) .&. 0x3F) + 0x80
361 poke8 3 $ (c .&. 0x3F) + 0x80
362 go (i + 2) (op `plusPtr` 4)
364 poke8 0 $ (w `shiftR` 12) + 0xE0
365 poke8 1 $ ((w `shiftR` 6) .&. 0x3F) + 0x80
366 poke8 2 $ (w .&. 0x3F) + 0x80
367 go (i + 1) (op `plusPtr` 3)
369 outerLoop i (B.BufferRange op ope)
371 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)
374-- | Encode text using UTF-8 encoding.
375encodeUtf8 :: Text -> ByteString
376encodeUtf8 (Text arr off len)
378 | otherwise = unsafeDupablePerformIO $ do
379 fp <- mallocByteString (len*4)
380 withForeignPtr fp $ \ptr ->
381 with ptr $ \destPtr -> do
382 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)
383 newDest <- peek destPtr
384 let utf8len = newDest `minusPtr` ptr
385 if utf8len >= len `shiftR` 1
386 then return (PS fp 0 utf8len)
388 fp' <- mallocByteString utf8len
389 withForeignPtr fp' $ \ptr' -> do
390 memcpy ptr' ptr (fromIntegral utf8len)
391 return (PS fp' 0 utf8len)
393-- | Decode text from little endian UTF-16 encoding.
394decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
395decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
396{-# INLINE decodeUtf16LEWith #-}
398-- | Decode text from little endian UTF-16 encoding.
400-- If the input contains any invalid little endian UTF-16 data, an
401-- exception will be thrown. For more control over the handling of
402-- invalid data, use 'decodeUtf16LEWith'.
403decodeUtf16LE :: ByteString -> Text
404decodeUtf16LE = decodeUtf16LEWith strictDecode
405{-# INLINE decodeUtf16LE #-}
407-- | Decode text from big endian UTF-16 encoding.
408decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text
409decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
410{-# INLINE decodeUtf16BEWith #-}
412-- | Decode text from big endian UTF-16 encoding.
414-- If the input contains any invalid big endian UTF-16 data, an
415-- exception will be thrown. For more control over the handling of
416-- invalid data, use 'decodeUtf16BEWith'.
417decodeUtf16BE :: ByteString -> Text
418decodeUtf16BE = decodeUtf16BEWith strictDecode
419{-# INLINE decodeUtf16BE #-}
421-- | Encode text using little endian UTF-16 encoding.
422encodeUtf16LE :: Text -> ByteString
423encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))
424{-# INLINE encodeUtf16LE #-}
426-- | Encode text using big endian UTF-16 encoding.
427encodeUtf16BE :: Text -> ByteString
428encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))
429{-# INLINE encodeUtf16BE #-}
431-- | Decode text from little endian UTF-32 encoding.
432decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text
433decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
434{-# INLINE decodeUtf32LEWith #-}
436-- | Decode text from little endian UTF-32 encoding.
438-- If the input contains any invalid little endian UTF-32 data, an
439-- exception will be thrown. For more control over the handling of
440-- invalid data, use 'decodeUtf32LEWith'.
441decodeUtf32LE :: ByteString -> Text
442decodeUtf32LE = decodeUtf32LEWith strictDecode
443{-# INLINE decodeUtf32LE #-}
445-- | Decode text from big endian UTF-32 encoding.
446decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text
447decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
448{-# INLINE decodeUtf32BEWith #-}
450-- | Decode text from big endian UTF-32 encoding.
452-- If the input contains any invalid big endian UTF-32 data, an
453-- exception will be thrown. For more control over the handling of
454-- invalid data, use 'decodeUtf32BEWith'.
455decodeUtf32BE :: ByteString -> Text
456decodeUtf32BE = decodeUtf32BEWith strictDecode
457{-# INLINE decodeUtf32BE #-}
459-- | Encode text using little endian UTF-32 encoding.
460encodeUtf32LE :: Text -> ByteString
461encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))
462{-# INLINE encodeUtf32LE #-}
464-- | Encode text using big endian UTF-32 encoding.
465encodeUtf32BE :: Text -> ByteString
466encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
467{-# INLINE encodeUtf32BE #-}
469foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
470 :: MutableByteArray# s -> Ptr CSize
471 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
473foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state
474 :: MutableByteArray# s -> Ptr CSize
475 -> Ptr (Ptr Word8) -> Ptr Word8
476 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)
478foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
479 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
481foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8
482 :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO ()
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list