@@ -1,3 +1,4 @@
1+{-# LANGUAGE BangPatterns #-}
12-- |
23-- Module : Data.Text.Lazy.Encoding
34-- Copyright : (c) 2009, 2010 Bryan O'Sullivan
@@ -19,12 +20,14 @@ module Data.Text.Lazy.Encoding
1920 -- * Decoding ByteStrings to Text
2021 decodeASCII
2122 , decodeUtf8
23+ , decodeUtf8'
2224 , decodeUtf16LE
2325 , decodeUtf16BE
2426 , decodeUtf32LE
2527 , decodeUtf32BE
2628 -- ** Controllable error handling
2729 , decodeUtf8With
30+ , decodeUtf8With'
2831 , decodeUtf16LEWith
2932 , decodeUtf16BEWith
3033 , decodeUtf32LEWith
@@ -38,17 +41,21 @@ module Data.Text.Lazy.Encoding
3841 , encodeUtf32BE
3942 ) where
4043
44+import Data.Bits ((.&.))
4145import Data.Text.Encoding.Error (OnDecodeError, strictDecode)
42−import Data.Text.Lazy.Internal (Text(..), chunk, foldrChunks)
46+import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldrChunks)
47+import qualified Data.ByteString as S
4348import qualified Data.ByteString.Lazy as B
4449import qualified Data.ByteString.Lazy.Internal as B
50+import qualified Data.ByteString.Unsafe as S
51+import qualified Data.Text as T
4552import qualified Data.Text.Encoding as TE
4653import qualified Data.Text.Lazy.Encoding.Fusion as E
4754import qualified Data.Text.Lazy.Fusion as F
4855
4956-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
5057decodeASCII :: B.ByteString -> Text
51−decodeASCII bs = foldr (chunk . TE.decodeASCII) Empty (B.toChunks bs)
58+decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)
5259{-# INLINE decodeASCII #-}
5360
5461-- | Decode a 'ByteString' containing UTF-8 encoded text.
@@ -61,6 +68,53 @@ decodeUtf8 :: B.ByteString -> Text
6168decodeUtf8 = decodeUtf8With strictDecode
6269{-# INLINE decodeUtf8 #-}
6370
71+-- | Decode a 'ByteString' containing UTF-8 encoded text.
72+decodeUtf8With' :: OnDecodeError -> B.ByteString -> Text
73+decodeUtf8With' onErr bs0 = fast bs0
74+ where
75+ decode = TE.decodeUtf8With onErr
76+ fast (B.Chunk p ps) | isComplete p = chunk (decode p) (fast ps)
77+ | otherwise = chunk (decode h) (slow t ps)
78+ where (h,t) = S.splitAt pivot p
79+ pivot | at 1 = len-1
80+ | at 2 = len-2
81+ | otherwise = len-3
82+ len = S.length p
83+ at n = len >= n && S.unsafeIndex p (len-n) .&. 0xc0 == 0xc0
84+ fast B.Empty = empty
85+ slow i bs = {-# SCC "decodeUtf8With'/slow" #-}
86+ case B.uncons bs of
87+ Just (w,bs') | isComplete i' -> chunk (decode i') (fast bs')
88+ | otherwise -> slow i' bs'
89+ where i' = S.snoc i w
90+ Nothing -> case S.uncons i of
91+ Just (j,i') ->
92+ case onErr desc (Just j) of
93+ Nothing -> slow i' bs
94+ Just c -> Chunk (T.singleton c) (slow i' bs)
95+ Nothing ->
96+ case onErr desc Nothing of
97+ Nothing -> empty
98+ Just c -> Chunk (T.singleton c) empty
99+ isComplete bs = {-# SCC "decodeUtf8With'/isComplete" #-}
100+ ix 1 .&. 0x80 == 0 ||
101+ (len >= 2 && ix 2 .&. 0xe0 == 0xc0) ||
102+ (len >= 3 && ix 3 .&. 0xf0 == 0xe0) ||
103+ (len >= 4 && ix 4 .&. 0xf8 == 0xf0)
104+ where len = S.length bs
105+ ix n = S.unsafeIndex bs (len-n)
106+ desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream"
107+{-# INLINE[0] decodeUtf8With' #-}
108+
109+-- | Decode a 'ByteString' containing UTF-8 encoded text.
110+decodeUtf8' :: B.ByteString -> Text
111+decodeUtf8' = decodeUtf8With' strictDecode
112+{-# INLINE[0] decodeUtf8' #-}
113+
114+-- This rule seems to cause performance loss.
115+{- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1]
116+ forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-}
117+
64118encodeUtf8 :: Text -> B.ByteString
65119encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
66120encodeUtf8 Empty = B.Empty