git.iko.soy/text
on master help sign in
Improve small string performance for UTF-8 encoding to bytestrings
1c65333Simon Meier
Data/Text/Encoding.hs+17−6
compare with master
unifiedsplit
@@ -81,13 +81,11 @@ import Data.ByteString.Internal as B hiding (c2w)
81
81
82#if MIN_VERSION_bytestring(0,10,4)
82#if MIN_VERSION_bytestring(0,10,4)
83import qualified Data.ByteString.Builder as B
83import qualified Data.ByteString.Builder as B
84+import qualified Data.ByteString.Builder.Extra as B
84import qualified Data.ByteString.Builder.Internal as B hiding (empty)
85import qualified Data.ByteString.Builder.Internal as B hiding (empty)
85import qualified Data.ByteString.Builder.Prim as BP
86import qualified Data.ByteString.Builder.Prim as BP
86import qualified Data.ByteString.Builder.Prim.Internal as BP
87import qualified Data.ByteString.Builder.Prim.Internal as BP
87import qualified Data.ByteString.Lazy as BL
88import qualified Data.ByteString.Lazy as BL
88−import Data.Text.Internal.Unsafe.Shift (shiftR)
89−#else
90−import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)
91#endif
89#endif
92
90
93#if __GLASGOW_HASKELL__ >= 706
91#if __GLASGOW_HASKELL__ >= 706
@@ -99,6 +97,7 @@ import Foreign.ForeignPtr (unsafeForeignPtrToPtr)
99import Data.Text ()
97import Data.Text ()
100import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
98import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
101import Data.Text.Internal (Text(..), safe, textP)
99import Data.Text.Internal (Text(..), safe, textP)
100+import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)
102import Data.Text.Internal.Private (runText)
101import Data.Text.Internal.Private (runText)
103import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
102import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
104import Data.Word (Word8, Word32)
103import Data.Word (Word8, Word32)
@@ -322,7 +321,17 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDec
322encodeUtf8 :: Text -> ByteString
321encodeUtf8 :: Text -> ByteString
323#if MIN_VERSION_bytestring(0,10,4)
322#if MIN_VERSION_bytestring(0,10,4)
324
323
324+encodeUtf8 t@(Text _arr _off len) =
325+ B.copy -- copy to trim and avoid wasting memory
326+ $ BL.toStrict
327+ $ B.toLazyByteStringWith strategy BL.empty
325−encodeUtf8 = BL.toStrict . B.toLazyByteString . encodeUtf8Builder
328+ $ encodeUtf8Builder t
329+ where
330+ -- We use a strategy that allocates a buffer that is guaranteed to be
331+ -- large enough for the whole result. This ensures that we always stay in
332+ -- the fast 'goPartial' loop in the 'encodeUtf8BuilderEscaped' function.
333+ strategy = B.untrimmedStrategy (4 * (len + 1)) B.defaultChunkSize
334+
326
335
327-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
336-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
328encodeUtf8Builder :: Text -> B.Builder
337encodeUtf8Builder :: Text -> B.Builder
@@ -351,6 +360,8 @@ encodeUtf8BuilderEscaped be =
351 outerLoop !i0 !br@(B.BufferRange op0 ope)
360 outerLoop !i0 !br@(B.BufferRange op0 ope)
352 | i0 >= iend = k br
361 | i0 >= iend = k br
353 | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)
362 | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)
363+ -- TODO: Use a loop with an integrated bound's check if outRemaining
364+ -- is smaller than 8, as this will save on divisions.
354 | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)
365 | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)
355 where
366 where
356 outRemaining = (ope `minusPtr` op0) `div` bound
367 outRemaining = (ope `minusPtr` op0) `div` bound
@@ -387,6 +398,8 @@ encodeUtf8BuilderEscaped be =
387
398
388encodeUtf8 = encodeUtf8_0
399encodeUtf8 = encodeUtf8_0
389
400
401+#endif
402+
390encodeUtf8_0 :: Text -> ByteString
403encodeUtf8_0 :: Text -> ByteString
391encodeUtf8_0 (Text arr off len) = unsafeDupablePerformIO $ do
404encodeUtf8_0 (Text arr off len) = unsafeDupablePerformIO $ do
392 let size0 = max len 4
405 let size0 = max len 4
@@ -431,8 +444,6 @@ encodeUtf8_0 (Text arr off len) = unsafeDupablePerformIO $ do
431 poke8 (m+2) $ (w .&. 0x3F) + 0x80
444 poke8 (m+2) $ (w .&. 0x3F) + 0x80
432 go (n+1) (m+3)
445 go (n+1) (m+3)
433
446
434−#endif
435−
436encodeUtf8_1 :: Text -> ByteString
447encodeUtf8_1 :: Text -> ByteString
437encodeUtf8_1 (Text arr off len)
448encodeUtf8_1 (Text arr off len)
438 | len == 0 = B.empty
449 | len == 0 = B.empty
Data/Text/Lazy/Encoding.hs+13−1
@@ -64,7 +64,9 @@ import qualified Data.ByteString.Unsafe as B
64import Data.Word (Word8)
64import Data.Word (Word8)
65import Data.Monoid (mempty, (<>))
65import Data.Monoid (mempty, (<>))
66import qualified Data.ByteString.Builder as B
66import qualified Data.ByteString.Builder as B
67+import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)
67import qualified Data.ByteString.Builder.Prim as BP
68import qualified Data.ByteString.Builder.Prim as BP
69+import qualified Data.Text as T
68#endif
70#endif
69import qualified Data.Text.Encoding as TE
71import qualified Data.Text.Encoding as TE
70import qualified Data.Text.Lazy as L
72import qualified Data.Text.Lazy as L
@@ -148,7 +150,17 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do
148
150
149encodeUtf8 :: Text -> B.ByteString
151encodeUtf8 :: Text -> B.ByteString
150#if MIN_VERSION_bytestring(0,10,4)
152#if MIN_VERSION_bytestring(0,10,4)
151−encodeUtf8 = B.toLazyByteString . encodeUtf8Builder
153+encodeUtf8 Empty = B.empty
154+encodeUtf8 lt@(Chunk t _) =
155+ B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt
156+ where
157+ -- To improve our small string performance, we use a strategy that
158+ -- allocates a buffer that is guaranteed to be large enough for the
159+ -- encoding of the first chunk, but not larger than the default
160+ -- B.smallChunkSize. We clamp the firstChunkSize to ensure that we don't
161+ -- generate too large buffers which hamper streaming.
162+ firstChunkSize = min B.smallChunkSize (4 * (T.length t + 1))
163+ strategy = B.safeStrategy firstChunkSize B.defaultChunkSize
152
164
153encodeUtf8Builder :: Text -> B.Builder
165encodeUtf8Builder :: Text -> B.Builder
154encodeUtf8Builder = foldrChunks (\c b -> TE.encodeUtf8Builder c <> b) mempty
166encodeUtf8Builder = foldrChunks (\c b -> TE.encodeUtf8Builder c <> b) mempty