1{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, GeneralizedNewtypeDeriving, MagicHash,
3#if __GLASGOW_HASKELL__ >= 702
4{-# LANGUAGE Trustworthy #-}
7-- Module : Data.Text.Encoding
8-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan,
9-- (c) 2009 Duncan Coutts,
10-- (c) 2008, 2009 Tom Harper
13-- Maintainer : bos@serpentine.com
14-- Stability : experimental
15-- Portability : portable
17-- Functions for converting 'Text' values to and from 'ByteString',
18-- using several standard encodings.
20-- To gain access to a much larger family of encodings, use the
21-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>
23module Data.Text.Encoding
25 -- * Decoding ByteStrings to Text
35 -- ** Catchable failure
38 -- ** Controllable error handling
45 -- ** Stream oriented decoding
48 , streamDecodeUtf8With
51 -- * Encoding Text to ByteStrings
58#if MIN_VERSION_bytestring(0,10,4)
59 -- * Encoding Text using ByteString Builders
60 -- | /Note/ that these functions are only available if built against
61 -- @bytestring >= 0.10.4.0@.
63 , encodeUtf8BuilderEscaped
67#if __GLASGOW_HASKELL__ >= 702
68import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
70import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
73#if MIN_VERSION_bytestring(0,10,4)
74import Data.Bits ((.&.))
75import Data.Text.Internal.Unsafe.Char (ord)
76import qualified Data.ByteString.Builder as B
77import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)
78import qualified Data.ByteString.Builder.Prim as BP
79import qualified Data.ByteString.Builder.Prim.Internal as BP
80import qualified Data.Text.Internal.Encoding.Utf16 as U16
83import Control.Exception (evaluate, try)
84import Control.Monad.ST (runST)
85import Data.ByteString as B
86import Data.ByteString.Internal as B hiding (c2w)
88import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
89import Data.Text.Internal (Text(..), safe, text)
90import Data.Text.Internal.Private (runText)
91import Data.Text.Internal.Unsafe.Char (unsafeWrite)
92import Data.Text.Internal.Unsafe.Shift (shiftR)
93import Data.Text.Unsafe (unsafeDupablePerformIO)
94import Data.Word (Word8, Word32)
95import Foreign.C.Types (CSize(..))
96import Foreign.ForeignPtr (withForeignPtr)
97import Foreign.Marshal.Utils (with)
98import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)
99import Foreign.Storable (Storable, peek, poke)
100import GHC.Base (ByteArray#, MutableByteArray#)
101import qualified Data.Text.Array as A
102import qualified Data.Text.Internal.Encoding.Fusion as E
103import qualified Data.Text.Internal.Fusion as F
105#include "text_cbits.h"
109-- All of the single-parameter functions for decoding bytestrings
110-- encoded in one of the Unicode Transformation Formats (UTF) operate
111-- in a /strict/ mode: each will throw an exception if given invalid
114-- Each function has a variant, whose name is suffixed with -'With',
115-- that gives greater control over the handling of decoding errors.
116-- For instance, 'decodeUtf8' will throw an exception, but
117-- 'decodeUtf8With' allows the programmer to determine what to do on a
120-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
122decodeASCII :: ByteString -> Text
123decodeASCII = decodeUtf8
124{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
126-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
128-- 'decodeLatin1' is semantically equivalent to
129-- @Data.Text.pack . Data.ByteString.Char8.unpack@
130decodeLatin1 :: ByteString -> Text
131decodeLatin1 (PS fp off len) = text a 0 len
133 a = A.run (A.new len >>= unsafeIOToST . go)
134 go dest = withForeignPtr fp $ \ptr -> do
135 c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))
138-- | Decode a 'ByteString' containing UTF-8 encoded text.
139decodeUtf8With :: OnDecodeError -> ByteString -> Text
140decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
141 let go dest = withForeignPtr fp $ \ptr ->
142 with (0::CSize) $ \destOffPtr -> do
143 let end = ptr `plusPtr` (off + len)
145 curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end
149 unsafeSTToIO (done dest (fromIntegral n))
152 case onErr desc (Just x) of
153 Nothing -> loop $ curPtr' `plusPtr` 1
155 destOff <- peek destOffPtr
157 unsafeWrite dest (fromIntegral destOff) (safe c)
158 poke destOffPtr (destOff + fromIntegral w)
159 loop $ curPtr' `plusPtr` 1
160 loop (ptr `plusPtr` off)
161 (unsafeIOToST . go) =<< A.new len
163 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"
164{- INLINE[0] decodeUtf8With #-}
168-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept
169-- a 'ByteString' that represents a possibly incomplete input (e.g. a
170-- packet from a network stream) that may not end on a UTF-8 boundary.
172-- 1. The maximal prefix of 'Text' that could be decoded from the
175-- 2. The suffix of the 'ByteString' that could not be decoded due to
176-- insufficient input.
178-- 3. A function that accepts another 'ByteString'. That string will
179-- be assumed to directly follow the string that was passed as
180-- input to the original function, and it will in turn be decoded.
182-- To help understand the use of these functions, consider the Unicode
183-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi
184-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.
186-- Now suppose that we receive this encoded string as 3 packets that
187-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",
188-- \"\\x83\"]@. We cannot decode the entire Unicode string until we
189-- have received all three packets, but we would like to make progress
190-- as we receive each one.
193-- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\"
195-- 'Some' \"hi \" \"\\xe2\" _
198-- We use the continuation @f0@ to decode our second packet.
201-- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\"
203-- 'Some' \"\" \"\\xe2\\x98\"
206-- We could not give @f0@ enough input to decode anything, so it
207-- returned an empty string. Once we feed our second continuation @f1@
208-- the last byte of input, it will make progress.
211-- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\"
213-- 'Some' \"\\x2603\" \"\" _
216-- If given invalid input, an exception will be thrown by the function
217-- or continuation where it is encountered.
219-- | A stream oriented decoding result.
220data Decoding = Some Text ByteString (ByteString -> Decoding)
222instance Show Decoding where
223 showsPrec d (Some t bs _) = showParen (d > prec) $
224 showString "Some " . showsPrec prec' t .
225 showChar ' ' . showsPrec prec' bs .
227 where prec = 10; prec' = prec + 1
229newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)
230newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)
232-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
233-- encoded text that is known to be valid.
235-- If the input contains any invalid UTF-8 data, an exception will be
236-- thrown (either by this function or a continuation) that cannot be
237-- caught in pure code. For more control over the handling of invalid
238-- data, use 'streamDecodeUtf8With'.
239streamDecodeUtf8 :: ByteString -> Decoding
240streamDecodeUtf8 = streamDecodeUtf8With strictDecode
242-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
244streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding
245streamDecodeUtf8With onErr = decodeChunk B.empty 0 0
247 -- We create a slightly larger than necessary buffer to accommodate a
248 -- potential surrogate pair started in the last buffer
249 decodeChunk :: ByteString -> CodePoint -> DecoderState -> ByteString
251 decodeChunk undecoded0 codepoint0 state0 bs@(PS fp off len) =
252 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
254 decodeChunkToBuffer :: A.MArray s -> IO Decoding
255 decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
256 with (0::CSize) $ \destOffPtr ->
257 with codepoint0 $ \codepointPtr ->
258 with state0 $ \statePtr ->
259 with nullPtr $ \curPtrPtr ->
260 let end = ptr `plusPtr` (off + len)
262 poke curPtrPtr curPtr
263 curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr
264 curPtrPtr end codepointPtr statePtr
265 state <- peek statePtr
268 -- We encountered an encoding error
271 case onErr desc (Just x) of
272 Nothing -> loop $ curPtr' `plusPtr` 1
274 destOff <- peek destOffPtr
276 unsafeWrite dest (fromIntegral destOff) (safe c)
277 poke destOffPtr (destOff + fromIntegral w)
278 loop $ curPtr' `plusPtr` 1
281 -- We encountered the end of the buffer while decoding
283 codepoint <- peek codepointPtr
284 chunkText <- unsafeSTToIO $ do
285 arr <- A.unsafeFreeze dest
286 return $! text arr 0 (fromIntegral n)
287 lastPtr <- peek curPtrPtr
288 let left = lastPtr `minusPtr` curPtr
289 undecoded = case state of
290 UTF8_ACCEPT -> B.empty
291 _ -> B.append undecoded0 (B.drop left bs)
292 return $ Some chunkText undecoded
293 (decodeChunk undecoded codepoint state)
294 in loop (ptr `plusPtr` off)
295 desc = "Data.Text.Internal.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"
297-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
300-- If the input contains any invalid UTF-8 data, an exception will be
301-- thrown that cannot be caught in pure code. For more control over
302-- the handling of invalid data, use 'decodeUtf8'' or
304decodeUtf8 :: ByteString -> Text
305decodeUtf8 = decodeUtf8With strictDecode
306{-# INLINE[0] decodeUtf8 #-}
307{-# RULES "STREAM stream/decodeUtf8 fusion" [1]
308 forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-}
310-- | Decode a 'ByteString' containing UTF-8 encoded text.
312-- If the input contains any invalid UTF-8 data, the relevant
313-- exception will be returned, otherwise the decoded text.
314decodeUtf8' :: ByteString -> Either UnicodeException Text
315decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode
316{-# INLINE decodeUtf8' #-}
318#if MIN_VERSION_bytestring(0,10,4)
320-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
321encodeUtf8Builder :: Text -> B.Builder
322encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)
324-- | Encode text using UTF-8 encoding and escape the ASCII characters using
325-- a 'BP.BoundedPrim'.
327-- Use this function is to implement efficient encoders for text-based formats
329{-# INLINE encodeUtf8BuilderEscaped #-}
330-- TODO: Extend documentation with references to source code in @blaze-html@
331-- or @aeson@ that uses this function.
332encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
333encodeUtf8BuilderEscaped be =
334 -- manual eta-expansion to ensure inlining works as expected
335 \txt -> B.builder (mkBuildstep txt)
337 bound = max 4 $ BP.sizeBound be
339 mkBuildstep (Text arr off len) !k =
344 outerLoop !i0 !br@(B.BufferRange op0 ope)
346 | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)
347 -- TODO: Use a loop with an integrated bound's check if outRemaining
348 -- is smaller than 8, as this will save on divisions.
349 | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)
351 outRemaining = (ope `minusPtr` op0) `div` bound
352 inpRemaining = iend - i0
354 goPartial !iendTmp = go i0 op0
357 | i < iendTmp = case A.unsafeIndex arr i of
359 BP.runB be (fromIntegral w) op >>= go (i + 1)
361 poke8 0 $ (w `shiftR` 6) + 0xC0
362 poke8 1 $ (w .&. 0x3f) + 0x80
363 go (i + 1) (op `plusPtr` 2)
364 | 0xD800 <= w && w <= 0xDBFF -> do
365 let c = ord $ U16.chr2 w (A.unsafeIndex arr (i+1))
366 poke8 0 $ (c `shiftR` 18) + 0xF0
367 poke8 1 $ ((c `shiftR` 12) .&. 0x3F) + 0x80
368 poke8 2 $ ((c `shiftR` 6) .&. 0x3F) + 0x80
369 poke8 3 $ (c .&. 0x3F) + 0x80
370 go (i + 2) (op `plusPtr` 4)
372 poke8 0 $ (w `shiftR` 12) + 0xE0
373 poke8 1 $ ((w `shiftR` 6) .&. 0x3F) + 0x80
374 poke8 2 $ (w .&. 0x3F) + 0x80
375 go (i + 1) (op `plusPtr` 3)
377 outerLoop i (B.BufferRange op ope)
379 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)
382-- | Encode text using UTF-8 encoding.
383encodeUtf8 :: Text -> ByteString
384encodeUtf8 (Text arr off len)
386 | otherwise = unsafeDupablePerformIO $ do
387 fp <- mallocByteString (len*4)
388 withForeignPtr fp $ \ptr ->
389 with ptr $ \destPtr -> do
390 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)
391 newDest <- peek destPtr
392 let utf8len = newDest `minusPtr` ptr
393 if utf8len >= len `shiftR` 1
394 then return (PS fp 0 utf8len)
396 fp' <- mallocByteString utf8len
397 withForeignPtr fp' $ \ptr' -> do
398 memcpy ptr' ptr (fromIntegral utf8len)
399 return (PS fp' 0 utf8len)
401-- | Decode text from little endian UTF-16 encoding.
402decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
403decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
404{-# INLINE decodeUtf16LEWith #-}
406-- | Decode text from little endian UTF-16 encoding.
408-- If the input contains any invalid little endian UTF-16 data, an
409-- exception will be thrown. For more control over the handling of
410-- invalid data, use 'decodeUtf16LEWith'.
411decodeUtf16LE :: ByteString -> Text
412decodeUtf16LE = decodeUtf16LEWith strictDecode
413{-# INLINE decodeUtf16LE #-}
415-- | Decode text from big endian UTF-16 encoding.
416decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text
417decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
418{-# INLINE decodeUtf16BEWith #-}
420-- | Decode text from big endian UTF-16 encoding.
422-- If the input contains any invalid big endian UTF-16 data, an
423-- exception will be thrown. For more control over the handling of
424-- invalid data, use 'decodeUtf16BEWith'.
425decodeUtf16BE :: ByteString -> Text
426decodeUtf16BE = decodeUtf16BEWith strictDecode
427{-# INLINE decodeUtf16BE #-}
429-- | Encode text using little endian UTF-16 encoding.
430encodeUtf16LE :: Text -> ByteString
431encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))
432{-# INLINE encodeUtf16LE #-}
434-- | Encode text using big endian UTF-16 encoding.
435encodeUtf16BE :: Text -> ByteString
436encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))
437{-# INLINE encodeUtf16BE #-}
439-- | Decode text from little endian UTF-32 encoding.
440decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text
441decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
442{-# INLINE decodeUtf32LEWith #-}
444-- | Decode text from little endian UTF-32 encoding.
446-- If the input contains any invalid little endian UTF-32 data, an
447-- exception will be thrown. For more control over the handling of
448-- invalid data, use 'decodeUtf32LEWith'.
449decodeUtf32LE :: ByteString -> Text
450decodeUtf32LE = decodeUtf32LEWith strictDecode
451{-# INLINE decodeUtf32LE #-}
453-- | Decode text from big endian UTF-32 encoding.
454decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text
455decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
456{-# INLINE decodeUtf32BEWith #-}
458-- | Decode text from big endian UTF-32 encoding.
460-- If the input contains any invalid big endian UTF-32 data, an
461-- exception will be thrown. For more control over the handling of
462-- invalid data, use 'decodeUtf32BEWith'.
463decodeUtf32BE :: ByteString -> Text
464decodeUtf32BE = decodeUtf32BEWith strictDecode
465{-# INLINE decodeUtf32BE #-}
467-- | Encode text using little endian UTF-32 encoding.
468encodeUtf32LE :: Text -> ByteString
469encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))
470{-# INLINE encodeUtf32LE #-}
472-- | Encode text using big endian UTF-32 encoding.
473encodeUtf32BE :: Text -> ByteString
474encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
475{-# INLINE encodeUtf32BE #-}
477foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
478 :: MutableByteArray# s -> Ptr CSize
479 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
481foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state
482 :: MutableByteArray# s -> Ptr CSize
483 -> Ptr (Ptr Word8) -> Ptr Word8
484 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)
486foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
487 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
489foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8
490 :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO ()
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list