git.iko.soy/text
on master help sign in
Polish UTF-8 bytestring builder support
3e915b2Simon Meier
Data/Text/Encoding.hs+38−54
compare with master
unifiedsplit
@@ -56,37 +56,44 @@ module Data.Text.Encoding
56 , encodeUtf32BE
56 , encodeUtf32BE
57
57
58#if MIN_VERSION_bytestring(0,10,4)
58#if MIN_VERSION_bytestring(0,10,4)
59− -- * Generic encoding of Text
59+ -- * Encoding Text using ByteString Builders
60− -- , encodeStreamWithB
60+ -- | /Note/ that these functions are only available if built against
61− -- , encodeTextWithB
61+ -- @bytestring >= 0.10.4.0@.
62− -- , encodeUtf8Builder
62+ , encodeUtf8Builder
63− , encodeUtf8Escaped
63+ , encodeUtf8BuilderEscaped
64#endif
64#endif
65 ) where
65 ) where
66
66
67import Control.Exception (evaluate, try)
67import Control.Exception (evaluate, try)
68+
68#if __GLASGOW_HASKELL__ >= 702
69#if __GLASGOW_HASKELL__ >= 702
69import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
70import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
70#else
71#else
71import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
72import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
72#endif
73#endif
74+
73import Control.Monad.ST (runST)
75import Control.Monad.ST (runST)
74import Data.Bits ((.&.))
76import Data.Bits ((.&.))
75import Data.ByteString as B
77import Data.ByteString as B
76−import Data.ByteString.Internal as B
78+import Data.ByteString.Internal as B hiding (c2w)
79+
77#if MIN_VERSION_bytestring(0,10,4)
80#if MIN_VERSION_bytestring(0,10,4)
81+import Data.Monoid ((<>))
78import qualified Data.ByteString.Builder as B
82import qualified Data.ByteString.Builder as B
79import qualified Data.ByteString.Builder.Internal as B
83import qualified Data.ByteString.Builder.Internal as B
80import qualified Data.ByteString.Builder.Prim as BP
84import qualified Data.ByteString.Builder.Prim as BP
81import qualified Data.ByteString.Builder.Prim.Internal as BP
85import qualified Data.ByteString.Builder.Prim.Internal as BP
82import qualified Data.ByteString.Lazy as BL
86import qualified Data.ByteString.Lazy as BL
87+import Data.Text.Internal.Unsafe.Shift (shiftR)
88+#else
89+import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)
83#endif
90#endif
91+
84import Data.Text ()
92import Data.Text ()
85import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
93import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
86import Data.Text.Internal (Text(..), safe, textP)
94import Data.Text.Internal (Text(..), safe, textP)
87import Data.Text.Internal.Private (runText)
95import Data.Text.Internal.Private (runText)
88import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
96import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
89−import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)
90import Data.Word (Word8, Word32)
97import Data.Word (Word8, Word32)
91import Foreign.C.Types (CSize)
98import Foreign.C.Types (CSize)
92import Foreign.ForeignPtr (withForeignPtr)
99import Foreign.ForeignPtr (withForeignPtr)
@@ -308,26 +315,36 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDec
308encodeUtf8 :: Text -> ByteString
315encodeUtf8 :: Text -> ByteString
309#if MIN_VERSION_bytestring(0,10,4)
316#if MIN_VERSION_bytestring(0,10,4)
310
317
311−encodeUtf8 =
312− BL.toStrict . B.toLazyByteString
318+encodeUtf8 = BL.toStrict . B.toLazyByteString . encodeUtf8Builder
319+
320+-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
321+encodeUtf8Builder :: Text -> B.Builder
313− . encodeUtf8Escaped (BP.liftFixedToBounded BP.word8)
322+encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)
314
323
315-- | Encode text using UTF-8 encoding and escape the ASCII characters using
324-- | Encode text using UTF-8 encoding and escape the ASCII characters using
316−-- a 'BP.PrimBounded'.
325+-- a 'BP.BoundedPrim'.
326+--
327+-- Use this function is to implement efficient encoders for text-based formats
328+-- like JSON or HTML.
329+{-# INLINE encodeUtf8BuilderEscaped #-}
330+-- TODO: Extend documentation with references to source code in @blaze-html@
331+-- or @aeson@ that uses this function.
317−encodeUtf8Escaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
332+encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
318−encodeUtf8Escaped be (Text arr off len) =
333+encodeUtf8BuilderEscaped be =
319− B.builder step
334+ -- manual eta-expansion to ensure inlining works as expected
335+ \txt -> B.builder (mkBuildstep txt)
320 where
336 where
321− bound = max 4 $ BP.sizeBound be
337+ bound = max 4 $ BP.sizeBound be
322− iend = off + len
338+
323− step !k =
339+ mkBuildstep (Text arr off len) !k =
324 outerLoop off
340 outerLoop off
325 where
341 where
342+ iend = off + len
343+
326 outerLoop !i0 !br@(B.BufferRange op0 ope)
344 outerLoop !i0 !br@(B.BufferRange op0 ope)
327− | i0 >= iend = k br
345+ | i0 >= iend = k br
328− | op0 `plusPtr` bound < ope =
329− goPartial (i0 + min outRemaining inpRemaining)
346+ | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)
330− | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)
347+ | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)
331 where
348 where
332 outRemaining = (ope `minusPtr` op0) `div` bound
349 outRemaining = (ope `minusPtr` op0) `div` bound
333 inpRemaining = iend - i0
350 inpRemaining = iend - i0
@@ -492,36 +509,3 @@ foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_stat
492
509
493foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
510foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
494 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
511 :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
495−
496−
497−{-
498−-- | Encode all elements of a 'F.Stream' using a 'B.BoundedEncoding'.
499−{-# INLINE encodeStreamWithB #-}
500−encodeStreamWithB :: B.BoundedEncoding a -> F.Stream a -> B.Builder
501−encodeStreamWithB be =
502− \(F.Stream next s0 _) -> B.builder $ step next s0
503− where
504− bound = B.sizeBound be
505− step next s0 k (B.BufferRange op0 ope0) =
506− go s0 op0
507− where
508− go s !op = case next s of
509− F.Done -> k (B.BufferRange op ope0)
510− F.Skip s' -> go s' op
511− F.Yield x s'
512− | op `plusPtr` bound <= ope0 -> B.runB be x op >>= go s'
513− | otherwise ->
514− return $ B.bufferFull bound op (step next s k)
515−
516−
517−-- |
518−-- | /Subject to fusion./
519−-- Encode all 'Char's of a 'T.Text' using a 'B.BoundedEncoding'.
520−{-# INLINE encodeTextWithB #-}
521−encodeTextWithB :: B.BoundedEncoding Char -> Text -> B.Builder
522−encodeTextWithB be = encodeStreamWithB be . F.stream
523−
524−-- | Encode text using UTF-8 encoding.
525−encodeUtf8Builder :: Text -> B.Builder
526−encodeUtf8Builder = encodeUtf8Escaped (B.fromF B.word8)
527−-}
Data/Text/Lazy/Encoding.hs+17−6
@@ -45,6 +45,12 @@ module Data.Text.Lazy.Encoding
45 , encodeUtf16BE
45 , encodeUtf16BE
46 , encodeUtf32LE
46 , encodeUtf32LE
47 , encodeUtf32BE
47 , encodeUtf32BE
48+
49+#if MIN_VERSION_bytestring(0,10,4)
50+ -- * Encoding Text using ByteString Builders
51+ , encodeUtf8Builder
52+ , encodeUtf8BuilderEscaped
53+#endif
48 ) where
54 ) where
49
55
50import Control.Exception (evaluate, try)
56import Control.Exception (evaluate, try)
@@ -55,6 +61,7 @@ import qualified Data.ByteString.Lazy as B
55import qualified Data.ByteString.Lazy.Internal as B
61import qualified Data.ByteString.Lazy.Internal as B
56import qualified Data.ByteString.Unsafe as B
62import qualified Data.ByteString.Unsafe as B
57#if MIN_VERSION_bytestring(0,10,4)
63#if MIN_VERSION_bytestring(0,10,4)
64+import Data.Word (Word8)
58import Data.Monoid (mempty, (<>))
65import Data.Monoid (mempty, (<>))
59import qualified Data.ByteString.Builder as B
66import qualified Data.ByteString.Builder as B
60import qualified Data.ByteString.Builder.Prim as BP
67import qualified Data.ByteString.Builder.Prim as BP
@@ -141,12 +148,16 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do
141
148
142encodeUtf8 :: Text -> B.ByteString
149encodeUtf8 :: Text -> B.ByteString
143#if MIN_VERSION_bytestring(0,10,4)
150#if MIN_VERSION_bytestring(0,10,4)
144−encodeUtf8 =
145− B.toLazyByteString . go
151+encodeUtf8 = B.toLazyByteString . encodeUtf8Builder
146− where
152+
147− go Empty = mempty
153+encodeUtf8Builder :: Text -> B.Builder
148− go (Chunk c cs) =
154+encodeUtf8Builder = foldrChunks (\c b -> TE.encodeUtf8Builder c <> b) mempty
149− TE.encodeUtf8Escaped (BP.liftFixedToBounded BP.word8) c <> go cs
155+
156+{-# INLINE encodeUtf8BuilderEscaped #-}
157+encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
158+encodeUtf8BuilderEscaped prim =
159+ foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c <> b) mempty
160+
150#else
161#else
151encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
162encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
152encodeUtf8 Empty = B.Empty
163encodeUtf8 Empty = B.Empty