git.iko.soy/text
on master help sign in
An experimental attempt to speed up lazy decoding
c8ddf0fBryan O'Sullivan
Data/Text/Encoding/Fusion/Common.hs+49−4
compare with master
unifiedsplit
@@ -25,16 +25,46 @@ module Data.Text.Encoding.Fusion.Common
25 , restreamUtf16BE
25 , restreamUtf16BE
26 , restreamUtf32LE
26 , restreamUtf32LE
27 , restreamUtf32BE
27 , restreamUtf32BE
28+ , istreamUtf8
28 ) where
29 ) where
29
30
31+import Control.Monad.ST
30−import Data.Bits ((.&.))
32+import Data.Bits ((.&.), (.|.))
33+import Data.Text.Array
31import Data.Text.Fusion (Step(..), Stream(..))
34import Data.Text.Fusion (Step(..), Stream(..))
32import Data.Text.Fusion.Internal (M(..), S(..))
35import Data.Text.Fusion.Internal (M(..), S(..))
33−import Data.Text.UnsafeChar (ord)
36+import Data.Text.UnsafeChar (ord, unsafeChr32)
34−import Data.Text.UnsafeShift (shiftR)
37+import Data.Text.UnsafeShift (shiftL, shiftR)
35−import Data.Word (Word8)
38+import Data.Word (Word8, Word32)
36import qualified Data.Text.Encoding.Utf8 as U8
39import qualified Data.Text.Encoding.Utf8 as U8
37
40
41+accept, reject :: Word32
42+accept = 0
43+reject = 12
44+
45+data S8 s = S8 !s {-# UNPACK #-} !Word32 {-# UNPACK #-} !Word32
46+
47+istreamUtf8 :: Stream Word8 -> Stream Char
48+istreamUtf8 (Stream next0 s0 len) =
49+ Stream next (S8 s0 accept 0) len
50+ where
51+ next (S8 s state code) =
52+ case next0 s of
53+ Done -> Done
54+ Skip s' -> Skip (S8 s' state code)
55+ Yield w s'
56+ | state' == accept -> Yield (unsafeChr32 code') (S8 s' accept code')
57+ | state' /= reject -> Skip (S8 s' state' code')
58+ where
59+ word = fromIntegral w
60+ peeku :: Int -> Word32
61+ peeku n = fromIntegral (unsafeIndexWord8 utf8d n)
62+ !kind = peeku (fromIntegral word)
63+ !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)
64+ | otherwise = (0xff `shiftR` fromIntegral kind) .&. word
65+ !state' = peeku (256 + fromIntegral (state + kind))
66+
67+
38-- | /O(n)/ Convert a Stream Char into a UTF-8 encoded Stream Word8.
68-- | /O(n)/ Convert a Stream Char into a UTF-8 encoded Stream Word8.
39restreamUtf8 :: Stream Char -> Stream Word8
69restreamUtf8 :: Stream Char -> Stream Word8
40restreamUtf8 (Stream next0 s0 len) =
70restreamUtf8 (Stream next0 s0 len) =
@@ -160,3 +190,18 @@ restreamUtf32LE (Stream next0 s0 len) =
160internalError :: String -> a
190internalError :: String -> a
161internalError func =
191internalError func =
162 error $ "Data.Text.Encoding.Fusion.Common." ++ func ++ ": internal error"
192 error $ "Data.Text.Encoding.Fusion.Common." ++ func ++ ": internal error"
193+
194+utf8d :: Array
195+{-# NOINLINE utf8d #-}
196+utf8d = runST fill where
197+ fill = do
198+ ary <- unsafeNew . (`div` 2) . sum . map fst $ xs
199+ mapM_ (uncurry (unsafeWriteWord8 ary))
200+ (zip [0..] (concatMap (uncurry replicate) xs))
201+ unsafeFreeze ary
202+ xs = [(128,0),(16,1),(16,9),(32,7),(2,8),(30,2),(1,10),(12,3),(1,4),(2,3),
203+ (1,11),(3,6),(1,5),(11,8),(1,0),(1,12),(1,24),(1,36),(1,60),(1,96),
204+ (1,84),(3,12),(1,48),(1,72),(13,12),(1,0),(5,12),(1,0),(1,12),(1,0),
205+ (3,12),(1,24),(5,12),(1,24),(1,12),(1,24),(9,12),(1,24),(5,12),(1,24),
206+ (7,12),(1,24),(9,12),(1,36),(1,12),(1,36),(3,12),(1,36),(5,12),(1,36),
207+ (1,12),(1,36),(3,12),(1,36),(10,12)]
Data/Text/Lazy/Encoding/Fusion.hs+15−1
@@ -57,12 +57,26 @@ data S = S0
57 | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
57 | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
58 | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
58 | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
59
59
60+data SS = SS ByteString {-# UNPACK #-} !Int
61+
62+streamBytes :: ByteString -> Stream Word8
63+streamBytes bs0 = Stream next (SS bs0 0) unknownSize
64+ where
65+ next (SS bs@(Chunk s ss) i)
66+ | i < len = Yield (B.unsafeIndex s i) (SS bs (i+1))
67+ | otherwise = next (SS ss 0)
68+ where len = B.length s
69+ next _ = Done
70+
60data T = T {-# UNPACK #-} !ByteString {-# UNPACK #-} !S {-# UNPACK #-} !Int
71data T = T {-# UNPACK #-} !ByteString {-# UNPACK #-} !S {-# UNPACK #-} !Int
61
72
62-- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using
73-- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using
63-- UTF-8 encoding.
74-- UTF-8 encoding.
64streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
75streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
76+streamUtf8 onErr = istreamUtf8 . streamBytes
77+
78+streamUtf8' :: OnDecodeError -> ByteString -> Stream Char
65−streamUtf8 onErr bs0 = Stream next (T bs0 S0 0) unknownSize
79+streamUtf8' onErr bs0 = Stream next (T bs0 S0 0) unknownSize
66 where
80 where
67 next (T bs@(Chunk ps _) S0 i)
81 next (T bs@(Chunk ps _) S0 i)
68 | i < len && U8.validate1 a =
82 | i < len && U8.validate1 a =