git.iko.soy/text
on cedee36 help sign in
text/Data/Text/Encoding.hs
182 lines 6.3 kB history raw
1{-# LANGUAGE BangPatterns #-}
2-- |
3-- Module : Data.Text.Encoding
4-- Copyright : (c) Tom Harper 2008-2009,
5-- (c) Bryan O'Sullivan 2009,
6-- (c) Duncan Coutts 2009
7--
8-- License : BSD-style
9-- Maintainer : bos@serpentine.com, rtomharper@googlemail.com,
10-- duncan@haskell.org
11-- Stability : experimental
12-- Portability : portable
13--
14-- Functions for converting 'Text' values to and from 'ByteString',
15-- using several standard encodings.
16--
17-- To make use of a much larger variety of encodings, use the @text-icu@
18-- package.
20module Data.Text.Encoding
21 (
22 -- * Decoding ByteStrings to Text
23 decodeASCII
24 , decodeUtf8
25 , decodeUtf16LE
26 , decodeUtf16BE
27 , decodeUtf32LE
28 , decodeUtf32BE
29 -- ** Controllable error handling
30 , decodeUtf8With
31 , decodeUtf16LEWith
32 , decodeUtf16BEWith
33 , decodeUtf32LEWith
34 , decodeUtf32BEWith
36 -- * Encoding Text to ByteStrings
37 , encodeUtf8
38 , encodeUtf8'
39 , encodeUtf16LE
40 , encodeUtf16BE
41 , encodeUtf32LE
42 , encodeUtf32BE
43 ) where
45import Data.Bits ((.&.))
46import Data.ByteString as B
47import Data.ByteString.Internal as B
48import Data.Text.Encoding.Error (OnDecodeError, strictDecode)
49import Data.Text.Encoding.Utf16 (chr2)
50import Data.Text.Internal (Text(..))
51import Data.Text.UnsafeChar (ord)
52import Data.Text.UnsafeShift (shiftL, shiftR)
53import Data.Word (Word8)
54import Foreign.ForeignPtr (withForeignPtr)
55import Foreign.Ptr (plusPtr)
56import Foreign.Storable (poke)
57import System.IO.Unsafe (unsafePerformIO)
58import qualified Data.Text.Array as A
59import qualified Data.Text.Encoding.Fusion as E
60import qualified Data.Text.Fusion as F
62-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
63decodeASCII :: ByteString -> Text
64decodeASCII bs = F.unstream (E.streamASCII bs)
65{-# INLINE decodeASCII #-}
67-- | Decode a 'ByteString' containing UTF-8 encoded text.
68decodeUtf8With :: OnDecodeError -> ByteString -> Text
69decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)
70{-# INLINE decodeUtf8With #-}
72-- | Decode a 'ByteString' containing UTF-8 encoded text.
73decodeUtf8 :: ByteString -> Text
74decodeUtf8 = decodeUtf8With strictDecode
75{-# INLINE decodeUtf8 #-}
77-- | Encode text using UTF-8 encoding.
78encodeUtf8' :: Text -> ByteString
79encodeUtf8' txt = E.unstream (E.restreamUtf8 (F.stream txt))
80{-# INLINE encodeUtf8' #-}
82-- | Encode text using UTF-8 encoding.
83encodeUtf8 :: Text -> ByteString
84encodeUtf8 (Text arr off len) = unsafePerformIO $ do
85 let size0 = min len 4
86 mallocByteString size0 >>= start size0 off 0
87 where
88 start size n0 m0 fp = withForeignPtr fp $ loop n0 m0
89 where
90 loop n1 m1 ptr = go n1 m1
91 where
92 go !n !m
93 | n-off == len = return $! PS fp 0 m
94 | size-m < 4 = {-# SCC "encodeUtf8/resize" #-} do
95 let newSize = size `shiftL` 1
96 fp' <- mallocByteString newSize
97 withForeignPtr fp' $ \ptr' -> memcpy ptr' ptr (fromIntegral m)
98 start newSize n m fp'
99 | otherwise = do
100 let poke8 k v = poke (ptr `plusPtr` k) (fromIntegral v :: Word8)
101 w = A.unsafeIndex arr n
102 case undefined of
103 _| w <= 0x7F -> do
104 poke8 m w
105 go (n+1) (m+1)
106 | w <= 0x7FF -> do
107 poke8 m $ (w `shiftR` 6) + 0xC0
108 poke8 (m+1) $ (w .&. 0x3f) + 0x80
109 go (n+1) (m+2)
110 | 0xD800 <= w && w <= 0xDBFF -> do
111 let c = ord $ chr2 w (A.unsafeIndex arr (n+1))
112 poke8 m $ (c `shiftR` 18) + 0xF0
113 poke8 (m+1) $ ((c `shiftR` 12) .&. 0x3F) + 0x80
114 poke8 (m+2) $ ((c `shiftR` 6) .&. 0x3F) + 0x80
115 poke8 (m+3) $ (c .&. 0x3F) + 0x80
116 go (n+2) (m+4)
117 | otherwise -> do
118 poke8 m $ (w `shiftR` 12) + 0xE0
119 poke8 (m+1) $ ((w `shiftR` 6) .&. 0x3F) + 0x80
120 poke8 (m+2) $ (w .&. 0x3F) + 0x80
121 go (n+1) (m+3)
122{- INLINE encodeUtf8 #-}
124-- | Decode text from little endian UTF-16 encoding.
125decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
126decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
127{-# INLINE decodeUtf16LEWith #-}
129-- | Decode text from little endian UTF-16 encoding.
130decodeUtf16LE :: ByteString -> Text
131decodeUtf16LE = decodeUtf16LEWith strictDecode
132{-# INLINE decodeUtf16LE #-}
134-- | Decode text from big endian UTF-16 encoding.
135decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text
136decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
137{-# INLINE decodeUtf16BEWith #-}
139-- | Decode text from big endian UTF-16 encoding.
140decodeUtf16BE :: ByteString -> Text
141decodeUtf16BE = decodeUtf16BEWith strictDecode
142{-# INLINE decodeUtf16BE #-}
144-- | Encode text using little endian UTF-16 encoding.
145encodeUtf16LE :: Text -> ByteString
146encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))
147{-# INLINE encodeUtf16LE #-}
149-- | Encode text using big endian UTF-16 encoding.
150encodeUtf16BE :: Text -> ByteString
151encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))
152{-# INLINE encodeUtf16BE #-}
154-- | Decode text from little endian UTF-32 encoding.
155decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text
156decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
157{-# INLINE decodeUtf32LEWith #-}
159-- | Decode text from little endian UTF-32 encoding.
160decodeUtf32LE :: ByteString -> Text
161decodeUtf32LE = decodeUtf32LEWith strictDecode
162{-# INLINE decodeUtf32LE #-}
164-- | Decode text from big endian UTF-32 encoding.
165decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text
166decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
167{-# INLINE decodeUtf32BEWith #-}
169-- | Decode text from big endian UTF-32 encoding.
170decodeUtf32BE :: ByteString -> Text
171decodeUtf32BE = decodeUtf32BEWith strictDecode
172{-# INLINE decodeUtf32BE #-}
174-- | Encode text using little endian UTF-32 encoding.
175encodeUtf32LE :: Text -> ByteString
176encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))
177{-# INLINE encodeUtf32LE #-}
179-- | Encode text using big endian UTF-32 encoding.
180encodeUtf32BE :: Text -> ByteString
181encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
182{-# INLINE encodeUtf32BE #-}