1{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, GeneralizedNewtypeDeriving, MagicHash,
3#if __GLASGOW_HASKELL__ >= 702
4{-# LANGUAGE Trustworthy #-}
7-- Module : Data.Text.Encoding
8-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan,
9-- (c) 2009 Duncan Coutts,
10-- (c) 2008, 2009 Tom Harper
13-- Maintainer : bos@serpentine.com, rtomharper@googlemail.com,
15-- Stability : experimental
16-- Portability : portable
18-- Functions for converting 'Text' values to and from 'ByteString',
19-- using several standard encodings.
21-- To gain access to a much larger family of encodings, use the
22-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>
24module Data.Text.Encoding
26 -- * Decoding ByteStrings to Text
36 -- ** Catchable failure
39 -- ** Controllable error handling
46 -- ** Stream oriented decoding
49 , streamDecodeUtf8With
52 -- * Encoding Text to ByteStrings
60import Control.Exception (evaluate, try)
61#if __GLASGOW_HASKELL__ >= 702
62import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
64import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
66import Control.Monad.ST (runST)
67import Data.Bits ((.&.))
68import Data.ByteString as B
69import Data.ByteString.Internal as B
71import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
72import Data.Text.Internal (Text(..), safe, textP)
73import Data.Text.Private (runText)
74import Data.Text.UnsafeChar (ord, unsafeWrite)
75import Data.Text.UnsafeShift (shiftL, shiftR)
76import Data.Word (Word8, Word32)
77import Foreign.C.Types (CSize)
78import Foreign.ForeignPtr (withForeignPtr)
79import Foreign.Marshal.Utils (with)
80import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)
81import Foreign.Storable (Storable, peek, poke)
82import GHC.Base (MutableByteArray#)
83import qualified Data.Text.Array as A
84import qualified Data.Text.Encoding.Fusion as E
85import qualified Data.Text.Encoding.Utf16 as U16
86import qualified Data.Text.Fusion as F
87import Data.Text.Unsafe (unsafeDupablePerformIO)
89#include "text_cbits.h"
93-- All of the single-parameter functions for decoding bytestrings
94-- encoded in one of the Unicode Transformation Formats (UTF) operate
95-- in a /strict/ mode: each will throw an exception if given invalid
98-- Each function has a variant, whose name is suffixed with -'With',
99-- that gives greater control over the handling of decoding errors.
100-- For instance, 'decodeUtf8' will throw an exception, but
101-- 'decodeUtf8With' allows the programmer to determine what to do on a
104-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
107-- This function is deprecated. Use 'decodeLatin1' instead.
108decodeASCII :: ByteString -> Text
109decodeASCII = decodeUtf8
110{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
112-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
114-- 'decodeLatin1' is semantically equivalent to
115-- @Data.Text.pack . Data.ByteString.Char8.unpack@
116decodeLatin1 :: ByteString -> Text
117decodeLatin1 (PS fp off len) = textP a 0 len
119 a = A.run (A.new len >>= unsafeIOToST . go)
120 go dest = withForeignPtr fp $ \ptr -> do
121 c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))
124-- | Decode a 'ByteString' containing UTF-8 encoded text.
125decodeUtf8With :: OnDecodeError -> ByteString -> Text
126decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
127 let go dest = withForeignPtr fp $ \ptr ->
128 with (0::CSize) $ \destOffPtr -> do
129 let end = ptr `plusPtr` (off + len)
131 curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end
135 unsafeSTToIO (done dest (fromIntegral n))
138 case onErr desc (Just x) of
139 Nothing -> loop $ curPtr' `plusPtr` 1
141 destOff <- peek destOffPtr
143 unsafeWrite dest (fromIntegral destOff) (safe c)
144 poke destOffPtr (destOff + fromIntegral w)
145 loop $ curPtr' `plusPtr` 1
146 loop (ptr `plusPtr` off)
147 (unsafeIOToST . go) =<< A.new len
149 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"
150{- INLINE[0] decodeUtf8With #-}
154-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept
155-- a 'ByteString' that represents a possibly incomplete input (e.g. a
156-- packet from a network stream) that may not end on a UTF-8 boundary.
158-- The first element of the result is the maximal chunk of 'Text' that
159-- can be decoded from the given input. The second is a function which
160-- accepts another 'ByteString'. That string will be assumed to
161-- directly follow the string that was passed as input to the original
162-- function, and it will in turn be decoded.
164-- To help understand the use of these functions, consider the Unicode
165-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi
166-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.
168-- Now suppose that we receive this encoded string as 3 packets that
169-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",
170-- \"\\x83\"]@. We cannot decode the entire Unicode string until we
171-- have received all three packets, but we would like to make progress
172-- as we receive each one.
175-- let 'Some' t0 f0 = 'streamDecodeUtf8' \"hi \\xe2\"
176-- t0 == \"hi \" :: 'Text'
179-- We use the continuation @f0@ to decode our second packet.
182-- let 'Some' t1 f1 = f0 \"\\x98\"
186-- We could not give @f0@ enough input to decode anything, so it
187-- returned an empty string. Once we feed our second continuation @f1@
188-- the last byte of input, it will make progress.
191-- let 'Some' t2 f2 = f1 \"\\x83\"
195-- If given invalid input, an exception will be thrown by the function
196-- or continuation where it is encountered.
198-- | A stream oriented decoding result.
199data Decoding = Some Text ByteString (ByteString -> Decoding)
201instance Show Decoding where
202 showsPrec d (Some t bs _) = showParen (d > prec) $
203 showString "Some " . showsPrec prec' t .
204 showChar ' ' . showsPrec prec' bs .
206 where prec = 10; prec' = prec + 1
208newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)
209newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)
211-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
212-- encoded text that is known to be valid.
214-- If the input contains any invalid UTF-8 data, an exception will be
215-- thrown (either by this function or a continuation) that cannot be
216-- caught in pure code. For more control over the handling of invalid
217-- data, use 'streamDecodeUtf8With'.
218streamDecodeUtf8 :: ByteString -> Decoding
219streamDecodeUtf8 = streamDecodeUtf8With strictDecode
221-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8
223streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding
224streamDecodeUtf8With onErr = decodeChunk 0 0
226 -- We create a slightly larger than necessary buffer to accommodate a
227 -- potential surrogate pair started in the last buffer
228 decodeChunk :: CodePoint -> DecoderState -> ByteString -> Decoding
229 decodeChunk codepoint0 state0 bs@(PS fp off len) =
230 runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)
232 decodeChunkToBuffer :: A.MArray s -> IO Decoding
233 decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->
234 with (0::CSize) $ \destOffPtr ->
235 with codepoint0 $ \codepointPtr ->
236 with state0 $ \statePtr ->
237 with nullPtr $ \curPtrPtr ->
238 let end = ptr `plusPtr` (off + len)
240 poke curPtrPtr curPtr
241 curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr
242 curPtrPtr end codepointPtr statePtr
243 state <- peek statePtr
246 -- We encountered an encoding error
248 case onErr desc (Just x) of
249 Nothing -> loop $ curPtr' `plusPtr` 1
251 destOff <- peek destOffPtr
253 unsafeWrite dest (fromIntegral destOff) (safe c)
254 poke destOffPtr (destOff + fromIntegral w)
256 loop $ curPtr' `plusPtr` 1
259 -- We encountered the end of the buffer while decoding
261 codepoint <- peek codepointPtr
262 chunkText <- unsafeSTToIO $ do
263 arr <- A.unsafeFreeze dest
264 return $! textP arr 0 (fromIntegral n)
265 lastPtr <- peek curPtrPtr
266 let left = lastPtr `minusPtr` curPtr
267 return $ Some chunkText (B.drop left bs)
268 (decodeChunk codepoint state)
269 in loop (ptr `plusPtr` off)
270 desc = "Data.Text.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"
272-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
275-- If the input contains any invalid UTF-8 data, an exception will be
276-- thrown that cannot be caught in pure code. For more control over
277-- the handling of invalid data, use 'decodeUtf8'' or
279decodeUtf8 :: ByteString -> Text
280decodeUtf8 = decodeUtf8With strictDecode
281{-# INLINE[0] decodeUtf8 #-}
282{-# RULES "STREAM stream/decodeUtf8 fusion" [1]
283 forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-}
285-- | Decode a 'ByteString' containing UTF-8 encoded text.
287-- If the input contains any invalid UTF-8 data, the relevant
288-- exception will be returned, otherwise the decoded text.
289decodeUtf8' :: ByteString -> Either UnicodeException Text
290decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode
291{-# INLINE decodeUtf8' #-}
293-- | Encode text using UTF-8 encoding.
294encodeUtf8 :: Text -> ByteString
295encodeUtf8 (Text arr off len) = unsafeDupablePerformIO $ do
296 let size0 = max len 4
297 mallocByteString size0 >>= start size0 off 0
299 start size n0 m0 fp = withForeignPtr fp $ loop n0 m0
301 loop n1 m1 ptr = go n1 m1
305 | n == offLen = return (PS fp 0 m)
307 let poke8 k v = poke (ptr `plusPtr` k) (fromIntegral v :: Word8)
310 | otherwise = {-# SCC "resizeUtf8/ensure" #-} do
311 let newSize = size `shiftL` 1
312 fp' <- mallocByteString newSize
313 withForeignPtr fp' $ \ptr' ->
314 memcpy ptr' ptr (fromIntegral m)
315 start newSize n m fp'
316 {-# INLINE ensure #-}
317 case A.unsafeIndex arr n of
318 w| w <= 0x7F -> ensure 1 $ do
319 poke (ptr `plusPtr` m) (fromIntegral w :: Word8)
320 -- A single ASCII octet is likely to start a run of
321 -- them. We see better performance when we
322 -- special-case this assumption.
323 let end = ptr `plusPtr` size
325 | t == offLen || u == end || v >= 0x80 =
326 go t (u `minusPtr` ptr)
328 poke u (fromIntegral v :: Word8)
329 ascii (t+1) (u `plusPtr` 1)
330 where v = A.unsafeIndex arr t
331 ascii (n+1) (ptr `plusPtr` (m+1))
332 | w <= 0x7FF -> ensure 2 $ do
333 poke8 m $ (w `shiftR` 6) + 0xC0
334 poke8 (m+1) $ (w .&. 0x3f) + 0x80
336 | 0xD800 <= w && w <= 0xDBFF -> ensure 4 $ do
337 let c = ord $ U16.chr2 w (A.unsafeIndex arr (n+1))
338 poke8 m $ (c `shiftR` 18) + 0xF0
339 poke8 (m+1) $ ((c `shiftR` 12) .&. 0x3F) + 0x80
340 poke8 (m+2) $ ((c `shiftR` 6) .&. 0x3F) + 0x80
341 poke8 (m+3) $ (c .&. 0x3F) + 0x80
343 | otherwise -> ensure 3 $ do
344 poke8 m $ (w `shiftR` 12) + 0xE0
345 poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80
346 poke8 (m+2) $ (w .&. 0x3F) + 0x80
349-- | Decode text from little endian UTF-16 encoding.
350decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
351decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
352{-# INLINE decodeUtf16LEWith #-}
354-- | Decode text from little endian UTF-16 encoding.
356-- If the input contains any invalid little endian UTF-16 data, an
357-- exception will be thrown. For more control over the handling of
358-- invalid data, use 'decodeUtf16LEWith'.
359decodeUtf16LE :: ByteString -> Text
360decodeUtf16LE = decodeUtf16LEWith strictDecode
361{-# INLINE decodeUtf16LE #-}
363-- | Decode text from big endian UTF-16 encoding.
364decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text
365decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
366{-# INLINE decodeUtf16BEWith #-}
368-- | Decode text from big endian UTF-16 encoding.
370-- If the input contains any invalid big endian UTF-16 data, an
371-- exception will be thrown. For more control over the handling of
372-- invalid data, use 'decodeUtf16BEWith'.
373decodeUtf16BE :: ByteString -> Text
374decodeUtf16BE = decodeUtf16BEWith strictDecode
375{-# INLINE decodeUtf16BE #-}
377-- | Encode text using little endian UTF-16 encoding.
378encodeUtf16LE :: Text -> ByteString
379encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))
380{-# INLINE encodeUtf16LE #-}
382-- | Encode text using big endian UTF-16 encoding.
383encodeUtf16BE :: Text -> ByteString
384encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))
385{-# INLINE encodeUtf16BE #-}
387-- | Decode text from little endian UTF-32 encoding.
388decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text
389decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
390{-# INLINE decodeUtf32LEWith #-}
392-- | Decode text from little endian UTF-32 encoding.
394-- If the input contains any invalid little endian UTF-32 data, an
395-- exception will be thrown. For more control over the handling of
396-- invalid data, use 'decodeUtf32LEWith'.
397decodeUtf32LE :: ByteString -> Text
398decodeUtf32LE = decodeUtf32LEWith strictDecode
399{-# INLINE decodeUtf32LE #-}
401-- | Decode text from big endian UTF-32 encoding.
402decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text
403decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
404{-# INLINE decodeUtf32BEWith #-}
406-- | Decode text from big endian UTF-32 encoding.
408-- If the input contains any invalid big endian UTF-32 data, an
409-- exception will be thrown. For more control over the handling of
410-- invalid data, use 'decodeUtf32BEWith'.
411decodeUtf32BE :: ByteString -> Text
412decodeUtf32BE = decodeUtf32BEWith strictDecode
413{-# INLINE decodeUtf32BE #-}
415-- | Encode text using little endian UTF-32 encoding.
416encodeUtf32LE :: Text -> ByteString
417encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))
418{-# INLINE encodeUtf32LE #-}
420-- | Encode text using big endian UTF-32 encoding.
421encodeUtf32BE :: Text -> ByteString
422encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
423{-# INLINE encodeUtf32BE #-}
425foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8
426 :: MutableByteArray# s -> Ptr CSize
427 -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)
429foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state
430 :: MutableByteArray# s -> Ptr CSize
431 -> Ptr (Ptr Word8) -> Ptr Word8
432 -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)
434foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
435 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list