git.iko.soy/text
on master help sign in
Add a faster lazy UTF-8 decode (roughly 2x)
be727f0Bryan O'Sullivan
Data/Text/Lazy/Encoding.hs+56−2
compare with master
unifiedsplit
@@ -1,3 +1,4 @@
1+{-# LANGUAGE BangPatterns #-}
1-- |
2-- |
2-- Module : Data.Text.Lazy.Encoding
3-- Module : Data.Text.Lazy.Encoding
3-- Copyright : (c) 2009, 2010 Bryan O'Sullivan
4-- Copyright : (c) 2009, 2010 Bryan O'Sullivan
@@ -19,12 +20,14 @@ module Data.Text.Lazy.Encoding
19 -- * Decoding ByteStrings to Text
20 -- * Decoding ByteStrings to Text
20 decodeASCII
21 decodeASCII
21 , decodeUtf8
22 , decodeUtf8
23+ , decodeUtf8'
22 , decodeUtf16LE
24 , decodeUtf16LE
23 , decodeUtf16BE
25 , decodeUtf16BE
24 , decodeUtf32LE
26 , decodeUtf32LE
25 , decodeUtf32BE
27 , decodeUtf32BE
26 -- ** Controllable error handling
28 -- ** Controllable error handling
27 , decodeUtf8With
29 , decodeUtf8With
30+ , decodeUtf8With'
28 , decodeUtf16LEWith
31 , decodeUtf16LEWith
29 , decodeUtf16BEWith
32 , decodeUtf16BEWith
30 , decodeUtf32LEWith
33 , decodeUtf32LEWith
@@ -38,17 +41,21 @@ module Data.Text.Lazy.Encoding
38 , encodeUtf32BE
41 , encodeUtf32BE
39 ) where
42 ) where
40
43
44+import Data.Bits ((.&.))
41import Data.Text.Encoding.Error (OnDecodeError, strictDecode)
45import Data.Text.Encoding.Error (OnDecodeError, strictDecode)
42−import Data.Text.Lazy.Internal (Text(..), chunk, foldrChunks)
46+import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldrChunks)
47+import qualified Data.ByteString as S
43import qualified Data.ByteString.Lazy as B
48import qualified Data.ByteString.Lazy as B
44import qualified Data.ByteString.Lazy.Internal as B
49import qualified Data.ByteString.Lazy.Internal as B
50+import qualified Data.ByteString.Unsafe as S
51+import qualified Data.Text as T
45import qualified Data.Text.Encoding as TE
52import qualified Data.Text.Encoding as TE
46import qualified Data.Text.Lazy.Encoding.Fusion as E
53import qualified Data.Text.Lazy.Encoding.Fusion as E
47import qualified Data.Text.Lazy.Fusion as F
54import qualified Data.Text.Lazy.Fusion as F
48
55
49-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
56-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
50decodeASCII :: B.ByteString -> Text
57decodeASCII :: B.ByteString -> Text
51−decodeASCII bs = foldr (chunk . TE.decodeASCII) Empty (B.toChunks bs)
58+decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)
52{-# INLINE decodeASCII #-}
59{-# INLINE decodeASCII #-}
53
60
54-- | Decode a 'ByteString' containing UTF-8 encoded text.
61-- | Decode a 'ByteString' containing UTF-8 encoded text.
@@ -61,6 +68,53 @@ decodeUtf8 :: B.ByteString -> Text
61decodeUtf8 = decodeUtf8With strictDecode
68decodeUtf8 = decodeUtf8With strictDecode
62{-# INLINE decodeUtf8 #-}
69{-# INLINE decodeUtf8 #-}
63
70
71+-- | Decode a 'ByteString' containing UTF-8 encoded text.
72+decodeUtf8With' :: OnDecodeError -> B.ByteString -> Text
73+decodeUtf8With' onErr bs0 = fast bs0
74+ where
75+ decode = TE.decodeUtf8With onErr
76+ fast (B.Chunk p ps) | isComplete p = chunk (decode p) (fast ps)
77+ | otherwise = chunk (decode h) (slow t ps)
78+ where (h,t) = S.splitAt pivot p
79+ pivot | at 1 = len-1
80+ | at 2 = len-2
81+ | otherwise = len-3
82+ len = S.length p
83+ at n = len >= n && S.unsafeIndex p (len-n) .&. 0xc0 == 0xc0
84+ fast B.Empty = empty
85+ slow i bs = {-# SCC "decodeUtf8With'/slow" #-}
86+ case B.uncons bs of
87+ Just (w,bs') | isComplete i' -> chunk (decode i') (fast bs')
88+ | otherwise -> slow i' bs'
89+ where i' = S.snoc i w
90+ Nothing -> case S.uncons i of
91+ Just (j,i') ->
92+ case onErr desc (Just j) of
93+ Nothing -> slow i' bs
94+ Just c -> Chunk (T.singleton c) (slow i' bs)
95+ Nothing ->
96+ case onErr desc Nothing of
97+ Nothing -> empty
98+ Just c -> Chunk (T.singleton c) empty
99+ isComplete bs = {-# SCC "decodeUtf8With'/isComplete" #-}
100+ ix 1 .&. 0x80 == 0 ||
101+ (len >= 2 && ix 2 .&. 0xe0 == 0xc0) ||
102+ (len >= 3 && ix 3 .&. 0xf0 == 0xe0) ||
103+ (len >= 4 && ix 4 .&. 0xf8 == 0xf0)
104+ where len = S.length bs
105+ ix n = S.unsafeIndex bs (len-n)
106+ desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream"
107+{-# INLINE[0] decodeUtf8With' #-}
108+
109+-- | Decode a 'ByteString' containing UTF-8 encoded text.
110+decodeUtf8' :: B.ByteString -> Text
111+decodeUtf8' = decodeUtf8With' strictDecode
112+{-# INLINE[0] decodeUtf8' #-}
113+
114+-- This rule seems to cause performance loss.
115+{- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1]
116+ forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-}
117+
64encodeUtf8 :: Text -> B.ByteString
118encodeUtf8 :: Text -> B.ByteString
65encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
119encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
66encodeUtf8 Empty = B.Empty
120encodeUtf8 Empty = B.Empty
tests/benchmarks/DecodeUtf8.hs+20
@@ -37,10 +37,26 @@ lazy h = do
37 bs <- BL.hGetContents h
37 bs <- BL.hGetContents h
38 rnf (TL.decodeUtf8 bs) `seq` return ()
38 rnf (TL.decodeUtf8 bs) `seq` return ()
39
39
40+lazy_ h = do
41+ bs <- BL.hGetContents h
42+ rnf (TL.decodeUtf8' bs) `seq` return ()
43+
40lazy_len h = do
44lazy_len h = do
41 bs <- BL.hGetContents h
45 bs <- BL.hGetContents h
42 print . TL.length . TL.decodeUtf8 $ bs
46 print . TL.length . TL.decodeUtf8 $ bs
43
47
48+lazy__len h = do
49+ bs <- BL.hGetContents h
50+ print . TL.length . TL.decodeUtf8' $ bs
51+
52+lazy_init_len h = do
53+ bs <- BL.hGetContents h
54+ print . TL.length . TL.init . TL.decodeUtf8 $ bs
55+
56+lazy__init_len h = do
57+ bs <- BL.hGetContents h
58+ print . TL.length . TL.init . TL.decodeUtf8' $ bs
59+
44lazy_io h = do
60lazy_io h = do
45 hSetEncoding h utf8
61 hSetEncoding h utf8
46 t <- TL.hGetContents h
62 t <- TL.hGetContents h
@@ -92,6 +108,10 @@ main = do
92 "strict_len_io" -> strict_len_io h
108 "strict_len_io" -> strict_len_io h
93 "lazy" -> lazy h
109 "lazy" -> lazy h
94 "lazy_len" -> lazy_len h
110 "lazy_len" -> lazy_len h
111+ "lazy_init_len" -> lazy_init_len h
112+ "lazy__init_len" -> lazy__init_len h
113+ "lazy_" -> lazy_ h
114+ "lazy__len" -> lazy__len h
95 "lazy_io" -> lazy_io h
115 "lazy_io" -> lazy_io h
96 "lazy_len_io" -> lazy_len_io h
116 "lazy_len_io" -> lazy_len_io h
97 "string" -> string h
117 "string" -> string h