1{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, GeneralizedNewtypeDeriving, MagicHash,
3#if __GLASGOW_HASKELL__ >= 702
4{-# LANGUAGE Trustworthy #-}
7-- Module : Data.Text.Encoding
8-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan,
9-- (c) 2009 Duncan Coutts,
10-- (c) 2008, 2009 Tom Harper
13-- Maintainer : bos@serpentine.com
14-- Stability : experimental
15-- Portability : portable
17-- Functions for converting 'Text' values to and from 'ByteString',
18-- using several standard encodings.
20-- To gain access to a much larger family of encodings, use the
21-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>
23module Data.Text.Encoding
25 -- * Decoding ByteStrings to Text
35 -- ** Catchable failure
38 -- ** Controllable error handling
45 -- ** Stream oriented decoding
48 , streamDecodeUtf8With
51 -- * Encoding Text to ByteStrings
59import Control.Exception (evaluate, try)
60#if __GLASGOW_HASKELL__ >= 702
61import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
63import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
65import Control.Monad.ST (runST)
66import Data.Bits ((.&.))
67import Data.ByteString as B
68import Data.ByteString.Internal as B
70import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
71import Data.Text.Internal (Text(..), safe, textP)
72import Data.Text.Internal.Private (runText)
73import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
74import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)
75import Data.Word (Word8, Word32)
76import Foreign.C.Types (CSize)
77import Foreign.ForeignPtr (withForeignPtr)
78import Foreign.Marshal.Utils (with)
79import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)
80import Foreign.Storable (Storable, peek, poke)
81import GHC.Base (MutableByteArray#)
82import qualified Data.Text.Array as A
83import qualified Data.Text.Internal.Encoding.Fusion as E
84import qualified Data.Text.Internal.Encoding.Utf16 as U16
85import qualified Data.Text.Internal.Fusion as F
86import Data.Text.Unsafe (unsafeDupablePerformIO)
88#include "text_cbits.h"
92-- All of the single-parameter functions for decoding bytestrings
93-- encoded in one of the Unicode Transformation Formats (UTF) operate
94-- in a /strict/ mode: each will throw an exception if given invalid
97-- Each function has a variant, whose name is suffixed with -'With',
98-- that gives greater control over the handling of decoding errors.
99-- For instance, 'decodeUtf8' will throw an exception, but
100-- 'decodeUtf8With' allows the programmer to determine what to do on a
103-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
106-- This function is deprecated. Use 'decodeLatin1' instead.
107decodeASCII :: ByteString -> Text
108decodeASCII = decodeUtf8
109{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
111-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
113-- 'decodeLatin1' is semantically equivalent to
114-- @Data.Text.pack . Data.ByteString.Char8.unpack@
115decodeLatin1 :: ByteString -> Text
116decodeLatin1 (PS fp off len) = textP a 0 len
118 a = A.run (A.new len >>= unsafeIOToST . go)
119 go dest = withForeignPtr fp $ \ptr -> do
120 c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))
123-- | Decode a 'ByteString' containing UTF-8 encoded text.
124decodeUtf8With :: OnDecodeError -> ByteString -> Text
125decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
126 let go dest = withForeignPtr fp $ \ptr ->
127 with (0::CSize) $ \destOffPtr -> do
128 let end = ptr `plusPtr` (off + len)
130 curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end
134 unsafeSTToIO (done dest (fromIntegral n))
137 case onErr desc (Just x) of
138 Nothing -> loop $ curPtr' `plusPtr` 1
140 destOff <- peek destOffPtr
142 unsafeWrite dest (fromIntegral destOff) (safe c)
143 poke destOffPtr (destOff + fromIntegral w)
144 loop $ curPtr' `plusPtr` 1
145 loop (ptr `plusPtr` off)
146 (unsafeIOToST . go) =<< A.new len
148 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"
149{- INLINE[0] decodeUtf8With #-}
153-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept
154-- a 'ByteString' that represents a possibly incomplete input (e.g. a
155-- packet from a network stream) that may not end on a UTF-8 boundary.
157-- The first element of the result is the maximal chunk of 'Text' that
158-- can be decoded from the given input. The second is a function which
159-- accepts another 'ByteString'. That string will be assumed to
160-- directly follow the string that was passed as input to the original
161-- function, and it will in turn be decoded.
163-- To help understand the use of these functions, consider the Unicode
164-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi
165-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.
167-- Now suppose that we receive this encoded string as 3 packets that
168-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",
169-- \"\\x83\"]@. We cannot decode the entire Unicode string until we
170-- have received all three packets, but we would like to make progress
171-- as we receive each one.
174-- let 'Some' t0 f0 = 'streamDecodeUtf8' \"hi \\xe2\"
175-- t0 == \"hi \" :: 'Text'
178-- We use the continuation @f0@ to decode our second packet.
181-- let 'Some' t1 f1 = f0 \"\\x98\"
185-- We could not give @f0@ enough input to decode anything, so it
186-- returned an empty string. Once we feed our second continuation @f1@
187-- the last byte of input, it will make progress.
190-- let 'Some' t2 f2 = f1 \"\\x83\"
194-- If given invalid input, an exception will be thrown by the function
195-- or continuation where it is encountered.
197-- | A stream oriented decoding result.
198data Decoding = Some Text ByteString (ByteString -> Decoding)
200instance Show Decoding where
201 showsPrec d (Some t bs _) = showParen (d > prec) $
202 showString "Some " . showsPrec prec' t .
203 showChar ' ' . showsPrec prec' bs .
205 where prec = 10; prec' = prec + 1
207newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)
208newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)
210-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
211-- encoded text that is known to be valid.
213-- If the input contains any invalid UTF-8 data, an exception will be
214-- thrown (either by this function or a continuation) that cannot be
215-- caught in pure code. For more control over the handling of invalid
216-- data, use 'streamDecodeUtf8With'.
217streamDecodeUtf8 :: ByteString -> Decoding
218streamDecodeUtf8 = streamDecodeUtf8With strictDecode
220-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
222streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding
223streamDecodeUtf8With onErr = decodeChunk 0 0
225 -- We create a slightly larger than necessary buffer to accommodate a
226 -- potential surrogate pair started in the last buffer
227 decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding
228 decodeChunk codepoint0 state0 bs@(PS fp off len) =
229 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
231 decodeChunkToBuffer :: A.MArray s -> IO Decoding
232 decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
233 with (0::CSize) $ \destOffPtr ->
234 with codepoint0 $ \codepointPtr ->
235 with state0 $ \statePtr ->
236 with nullPtr $ \curPtrPtr ->
237 let end = ptr `plusPtr` (off + len)
239 poke curPtrPtr curPtr
240 curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr
241 curPtrPtr end codepointPtr statePtr
242 state <- peek statePtr
245 -- We encountered an encoding error
247 case onErr desc (Just x) of
248 Nothing -> loop $ curPtr' `plusPtr` 1
250 destOff <- peek destOffPtr
252 unsafeWrite dest (fromIntegral destOff) (safe c)
253 poke destOffPtr (destOff + fromIntegral w)
255 loop $ curPtr' `plusPtr` 1
258 -- We encountered the end of the buffer while decoding
260 codepoint <- peek codepointPtr
261 chunkText <- unsafeSTToIO $ do
262 arr <- A.unsafeFreeze dest
263 return $! textP arr 0 (fromIntegral n)
264 lastPtr <- peek curPtrPtr
265 let left = lastPtr `minusPtr` curPtr
266 return $ Some chunkText (B.drop left bs)
267 (decodeChunk codepoint state)
268 in loop (ptr `plusPtr` off)
269 desc = "Data.Text.Internal.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"
271-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
274-- If the input contains any invalid UTF-8 data, an exception will be
275-- thrown that cannot be caught in pure code. For more control over
276-- the handling of invalid data, use 'decodeUtf8'' or
278decodeUtf8 :: ByteString -> Text
279decodeUtf8 = decodeUtf8With strictDecode
280{-# INLINE[0] decodeUtf8 #-}
281{-# RULES "STREAM stream/decodeUtf8 fusion" [1]
282 forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-}
284-- | Decode a 'ByteString' containing UTF-8 encoded text.
286-- If the input contains any invalid UTF-8 data, the relevant
287-- exception will be returned, otherwise the decoded text.
288decodeUtf8' :: ByteString -> Either UnicodeException Text
289decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode
290{-# INLINE decodeUtf8' #-}
292-- | Encode text using UTF-8 encoding.
293encodeUtf8 :: Text -> ByteString
294encodeUtf8 (Text arr off len) = unsafeDupablePerformIO $ do
295 let size0 = max len 4
296 mallocByteString size0 >>= start size0 off 0
298 start size n0 m0 fp = withForeignPtr fp $ loop n0 m0
300 loop n1 m1 ptr = go n1 m1
304 | n == offLen = return (PS fp 0 m)
306 let poke8 k v = poke (ptr `plusPtr` k) (fromIntegral v :: Word8)
309 | otherwise = {-# SCC "resizeUtf8/ensure" #-} do
310 let newSize = size `shiftL` 1
311 fp' <- mallocByteString newSize
312 withForeignPtr fp' $ \ptr' ->
313 memcpy ptr' ptr (fromIntegral m)
314 start newSize n m fp'
315 {-# INLINE ensure #-}
316 case A.unsafeIndex arr n of
317 w| w <= 0x7F -> ensure 1 $ do
318 poke (ptr `plusPtr` m) (fromIntegral w :: Word8)
319 -- A single ASCII octet is likely to start a run of
320 -- them. We see better performance when we
321 -- special-case this assumption.
322 let end = ptr `plusPtr` size
324 | t == offLen || u == end || v >= 0x80 =
325 go t (u `minusPtr` ptr)
327 poke u (fromIntegral v :: Word8)
328 ascii (t+1) (u `plusPtr` 1)
329 where v = A.unsafeIndex arr t
330 ascii (n+1) (ptr `plusPtr` (m+1))
331 | w <= 0x7FF -> ensure 2 $ do
332 poke8 m $ (w `shiftR` 6) + 0xC0
333 poke8 (m+1) $ (w .&. 0x3f) + 0x80
335 | 0xD800 <= w && w <= 0xDBFF -> ensure 4 $ do
336 let c = ord $ U16.chr2 w (A.unsafeIndex arr (n+1))
337 poke8 m $ (c `shiftR` 18) + 0xF0
338 poke8 (m+1) $ ((c `shiftR` 12) .&. 0x3F) + 0x80
339 poke8 (m+2) $ ((c `shiftR` 6) .&. 0x3F) + 0x80
340 poke8 (m+3) $ (c .&. 0x3F) + 0x80
342 | otherwise -> ensure 3 $ do
343 poke8 m $ (w `shiftR` 12) + 0xE0
344 poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80
345 poke8 (m+2) $ (w .&. 0x3F) + 0x80
348-- | Decode text from little endian UTF-16 encoding.
349decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
350decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
351{-# INLINE decodeUtf16LEWith #-}
353-- | Decode text from little endian UTF-16 encoding.
355-- If the input contains any invalid little endian UTF-16 data, an
356-- exception will be thrown. For more control over the handling of
357-- invalid data, use 'decodeUtf16LEWith'.
358decodeUtf16LE :: ByteString -> Text
359decodeUtf16LE = decodeUtf16LEWith strictDecode
360{-# INLINE decodeUtf16LE #-}
362-- | Decode text from big endian UTF-16 encoding.
363decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text
364decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
365{-# INLINE decodeUtf16BEWith #-}
367-- | Decode text from big endian UTF-16 encoding.
369-- If the input contains any invalid big endian UTF-16 data, an
370-- exception will be thrown. For more control over the handling of
371-- invalid data, use 'decodeUtf16BEWith'.
372decodeUtf16BE :: ByteString -> Text
373decodeUtf16BE = decodeUtf16BEWith strictDecode
374{-# INLINE decodeUtf16BE #-}
376-- | Encode text using little endian UTF-16 encoding.
377encodeUtf16LE :: Text -> ByteString
378encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))
379{-# INLINE encodeUtf16LE #-}
381-- | Encode text using big endian UTF-16 encoding.
382encodeUtf16BE :: Text -> ByteString
383encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))
384{-# INLINE encodeUtf16BE #-}
386-- | Decode text from little endian UTF-32 encoding.
387decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text
388decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
389{-# INLINE decodeUtf32LEWith #-}
391-- | Decode text from little endian UTF-32 encoding.
393-- If the input contains any invalid little endian UTF-32 data, an
394-- exception will be thrown. For more control over the handling of
395-- invalid data, use 'decodeUtf32LEWith'.
396decodeUtf32LE :: ByteString -> Text
397decodeUtf32LE = decodeUtf32LEWith strictDecode
398{-# INLINE decodeUtf32LE #-}
400-- | Decode text from big endian UTF-32 encoding.
401decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text
402decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
403{-# INLINE decodeUtf32BEWith #-}
405-- | Decode text from big endian UTF-32 encoding.
407-- If the input contains any invalid big endian UTF-32 data, an
408-- exception will be thrown. For more control over the handling of
409-- invalid data, use 'decodeUtf32BEWith'.
410decodeUtf32BE :: ByteString -> Text
411decodeUtf32BE = decodeUtf32BEWith strictDecode
412{-# INLINE decodeUtf32BE #-}
414-- | Encode text using little endian UTF-32 encoding.
415encodeUtf32LE :: Text -> ByteString
416encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))
417{-# INLINE encodeUtf32LE #-}
419-- | Encode text using big endian UTF-32 encoding.
420encodeUtf32BE :: Text -> ByteString
421encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
422{-# INLINE encodeUtf32BE #-}
424foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
425 :: MutableByteArray# s -> Ptr CSize
426 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
428foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state
429 :: MutableByteArray# s -> Ptr CSize
430 -> Ptr (Ptr Word8) -> Ptr Word8
431 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)
433foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
434 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list