git.iko.soy/text
on 3cf8564 help sign in
text/Data/Text/Lazy/Encoding.hs
216 lines 7.8 kB history raw
1{-# LANGUAGE BangPatterns,CPP #-}
2#if __GLASGOW_HASKELL__ >= 702
3{-# LANGUAGE Trustworthy #-}
4#endif
5-- |
6-- Module : Data.Text.Lazy.Encoding
7-- Copyright : (c) 2009, 2010 Bryan O'Sullivan
8--
9-- License : BSD-style
10-- Maintainer : bos@serpentine.com, rtomharper@googlemail.com,
11-- duncan@haskell.org
12-- Stability : experimental
13-- Portability : portable
14--
15-- Functions for converting lazy 'Text' values to and from lazy
16-- 'ByteString', using several standard encodings.
17--
18-- To gain access to a much larger variety of encodings, use the
19-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>
21module Data.Text.Lazy.Encoding
22 (
23 -- * Decoding ByteStrings to Text
24 -- $strict
25 decodeASCII
26 , decodeLatin1
27 , decodeUtf8
28 , decodeUtf16LE
29 , decodeUtf16BE
30 , decodeUtf32LE
31 , decodeUtf32BE
33 -- ** Catchable failure
34 , decodeUtf8'
36 -- ** Controllable error handling
37 , decodeUtf8With
38 , decodeUtf16LEWith
39 , decodeUtf16BEWith
40 , decodeUtf32LEWith
41 , decodeUtf32BEWith
43 -- * Encoding Text to ByteStrings
44 , encodeUtf8
45 , encodeUtf16LE
46 , encodeUtf16BE
47 , encodeUtf32LE
48 , encodeUtf32BE
49 ) where
51import Control.Exception (evaluate, try)
52import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
53import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldrChunks)
54import qualified Data.ByteString as S
55import qualified Data.ByteString.Lazy as B
56import qualified Data.ByteString.Lazy.Internal as B
57import qualified Data.ByteString.Unsafe as B
58import qualified Data.Text.Encoding as TE
59import qualified Data.Text.Lazy as L
60import qualified Data.Text.Lazy.Encoding.Fusion as E
61import qualified Data.Text.Lazy.Fusion as F
62import Data.Text.Unsafe (unsafeDupablePerformIO)
64-- $strict
65--
66-- All of the single-parameter functions for decoding bytestrings
67-- encoded in one of the Unicode Transformation Formats (UTF) operate
68-- in a /strict/ mode: each will throw an exception if given invalid
69-- input.
70--
71-- Each function has a variant, whose name is suffixed with -'With',
72-- that gives greater control over the handling of decoding errors.
73-- For instance, 'decodeUtf8' will throw an exception, but
74-- 'decodeUtf8With' allows the programmer to determine what to do on a
75-- decoding error.
77-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
78-- encoded text.
79--
80-- This function is deprecated. Use 'decodeLatin1' instead.
81decodeASCII :: B.ByteString -> Text
82decodeASCII = decodeUtf8
83{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
85-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
86decodeLatin1 :: B.ByteString -> Text
87decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks
89-- | Decode a 'ByteString' containing UTF-8 encoded text.
90decodeUtf8With :: OnDecodeError -> B.ByteString -> Text
91decodeUtf8With onErr (B.Chunk b0 bs0) =
92 case TE.streamDecodeUtf8With onErr b0 of
93 TE.Some t l f -> chunk t (go f l bs0)
94 where
95 go f0 _ (B.Chunk b bs) =
96 case f0 b of
97 TE.Some t l f -> chunk t (go f l bs)
98 go _ l _
99 | S.null l = empty
100 | otherwise = case onErr desc (Just (B.unsafeHead l)) of
101 Nothing -> empty
102 Just c -> L.singleton c
103 desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream"
104decodeUtf8With _ _ = empty
106-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
107-- to be valid.
108--
109-- If the input contains any invalid UTF-8 data, an exception will be
110-- thrown that cannot be caught in pure code. For more control over
111-- the handling of invalid data, use 'decodeUtf8'' or
112-- 'decodeUtf8With'.
113decodeUtf8 :: B.ByteString -> Text
114decodeUtf8 = decodeUtf8With strictDecode
115{-# INLINE[0] decodeUtf8 #-}
117-- This rule seems to cause performance loss.
118{- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1]
119 forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-}
121-- | Decode a 'ByteString' containing UTF-8 encoded text..
122--
123-- If the input contains any invalid UTF-8 data, the relevant
124-- exception will be returned, otherwise the decoded text.
125--
126-- /Note/: this function is /not/ lazy, as it must decode its entire
127-- input before it can return a result. If you need lazy (streaming)
128-- decoding, use 'decodeUtf8With' in lenient mode.
129decodeUtf8' :: B.ByteString -> Either UnicodeException Text
130decodeUtf8' bs = unsafeDupablePerformIO $ do
131 let t = decodeUtf8 bs
132 try (evaluate (rnf t `seq` t))
133 where
134 rnf Empty = ()
135 rnf (Chunk _ ts) = rnf ts
136{-# INLINE decodeUtf8' #-}
138encodeUtf8 :: Text -> B.ByteString
139encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
140encodeUtf8 Empty = B.Empty
142-- | Decode text from little endian UTF-16 encoding.
143decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text
144decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
145{-# INLINE decodeUtf16LEWith #-}
147-- | Decode text from little endian UTF-16 encoding.
148--
149-- If the input contains any invalid little endian UTF-16 data, an
150-- exception will be thrown. For more control over the handling of
151-- invalid data, use 'decodeUtf16LEWith'.
152decodeUtf16LE :: B.ByteString -> Text
153decodeUtf16LE = decodeUtf16LEWith strictDecode
154{-# INLINE decodeUtf16LE #-}
156-- | Decode text from big endian UTF-16 encoding.
157decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text
158decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
159{-# INLINE decodeUtf16BEWith #-}
161-- | Decode text from big endian UTF-16 encoding.
162--
163-- If the input contains any invalid big endian UTF-16 data, an
164-- exception will be thrown. For more control over the handling of
165-- invalid data, use 'decodeUtf16BEWith'.
166decodeUtf16BE :: B.ByteString -> Text
167decodeUtf16BE = decodeUtf16BEWith strictDecode
168{-# INLINE decodeUtf16BE #-}
170-- | Encode text using little endian UTF-16 encoding.
171encodeUtf16LE :: Text -> B.ByteString
172encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt)
173{-# INLINE encodeUtf16LE #-}
175-- | Encode text using big endian UTF-16 encoding.
176encodeUtf16BE :: Text -> B.ByteString
177encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt)
178{-# INLINE encodeUtf16BE #-}
180-- | Decode text from little endian UTF-32 encoding.
181decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text
182decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
183{-# INLINE decodeUtf32LEWith #-}
185-- | Decode text from little endian UTF-32 encoding.
186--
187-- If the input contains any invalid little endian UTF-32 data, an
188-- exception will be thrown. For more control over the handling of
189-- invalid data, use 'decodeUtf32LEWith'.
190decodeUtf32LE :: B.ByteString -> Text
191decodeUtf32LE = decodeUtf32LEWith strictDecode
192{-# INLINE decodeUtf32LE #-}
194-- | Decode text from big endian UTF-32 encoding.
195decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text
196decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
197{-# INLINE decodeUtf32BEWith #-}
199-- | Decode text from big endian UTF-32 encoding.
200--
201-- If the input contains any invalid big endian UTF-32 data, an
202-- exception will be thrown. For more control over the handling of
203-- invalid data, use 'decodeUtf32BEWith'.
204decodeUtf32BE :: B.ByteString -> Text
205decodeUtf32BE = decodeUtf32BEWith strictDecode
206{-# INLINE decodeUtf32BE #-}
208-- | Encode text using little endian UTF-32 encoding.
209encodeUtf32LE :: Text -> B.ByteString
210encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt)
211{-# INLINE encodeUtf32LE #-}
213-- | Encode text using big endian UTF-32 encoding.
214encodeUtf32BE :: Text -> B.ByteString
215encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt)
216{-# INLINE encodeUtf32BE #-}