git.iko.soy/text
on master help sign in
Roll back unsuccessful speedup attempts

--HG-- extra : convert_revision : ec4654bd81889688106a4482e946cb1742f48ee3

commit4e56f87783a8edacb37de0e62e145e28d629c64e authorBryan O'Sullivan <bos@serpentine.com> dateFri 20 Aug 2010, 06:32 (16 years ago) parentc8ddf0f
4 files changed+10−133
compare with master
unifiedsplit
Data/Text/Array.hs+3−22 Data/Text/Encoding/Fusion.hs+2−47 Data/Text/Encoding/Fusion/Common.hs+4−49 Data/Text/Lazy/Encoding/Fusion.hs+1−15
Data/Text/Array.hs+3−22
@@ -39,10 +39,8 @@ module Data.Text.Array
3939 , toList
4040 , unsafeFreeze
4141 , unsafeIndex
42− , unsafeIndexWord8
4342 , unsafeNew
4443 , unsafeWrite
45− , unsafeWriteWord8
4644 ) where
4745
4846#if defined(ASSERTS)
@@ -62,11 +60,11 @@ import Control.Exception (assert)
6260import Data.Bits ((.&.))
6361import Data.Text.UnsafeShift (shiftL, shiftR)
6462import GHC.Base (ByteArray#, MutableByteArray#, Int(..),
65− indexWord16Array#, indexWord8Array#, indexWordArray#, newByteArray#,
63+ indexWord16Array#, indexWordArray#, newByteArray#,
6664 readWord16Array#, readWordArray#, unsafeCoerce#,
67− writeWord16Array#, writeWord8Array#, writeWordArray#)
65+ writeWord16Array#, writeWordArray#)
6866import GHC.ST (ST(..), runST)
69−import GHC.Word (Word16(..), Word8(..), Word(..))
67+import GHC.Word (Word16(..), Word(..))
7068import Prelude hiding (length, read)
7169
7270-- | Immutable array type.
@@ -136,14 +134,6 @@ unsafeIndexWord (Array len ba#) i@(I# i#) =
136134 case indexWordArray# ba# i# of r# -> (W# r#)
137135{-# INLINE unsafeIndexWord #-}
138136
139−-- | Unchecked read of an immutable array. May return garbage or
140−-- crash on an out-of-bounds access.
141−unsafeIndexWord8 :: Array -> Int -> Word8
142−unsafeIndexWord8 (Array len ba#) i@(I# i#) =
143− CHECK_BOUNDS("unsafeIndexWord8",len,i)
144− case indexWord8Array# ba# i# of r# -> (W8# r#)
145−{-# INLINE unsafeIndexWord8 #-}
146−
147137-- | Unchecked read of a mutable array. May return garbage or
148138-- crash on an out-of-bounds access.
149139unsafeRead :: MArray s -> Int -> ST s Word16
@@ -180,15 +170,6 @@ unsafeWriteWord (MArray len marr#) i@(I# i#) (W# e#) = ST $ \s1# ->
180170 s2# -> (# s2#, () #)
181171{-# INLINE unsafeWriteWord #-}
182172
183−-- | Unchecked write of a mutable array. May return garbage or crash
184−-- on an out-of-bounds access.
185−unsafeWriteWord8 :: MArray s -> Int -> Word8 -> ST s ()
186−unsafeWriteWord8 (MArray len marr#) i@(I# i#) (W8# e#) = ST $ \s1# ->
187− CHECK_BOUNDS("unsafeWriteWord8",len,i)
188− case writeWord8Array# marr# i# e# s1# of
189− s2# -> (# s2#, () #)
190−{-# INLINE unsafeWriteWord8 #-}
191−
192173-- | Convert an immutable array to a list.
193174toList :: Array -> [Word16]
194175toList a = loop 0
Data/Text/Encoding/Fusion.hs+2−47
@@ -34,10 +34,7 @@ module Data.Text.Encoding.Fusion
3434#if defined(ASSERTS)
3535import Control.Exception (assert)
3636#endif
37−import Data.Bits ((.&.), (.|.))
3837import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)
39−import Control.Monad.ST
40−import Data.Text.Array
4138import Data.Text.Fusion (Step(..), Stream(..))
4239import Data.Text.Fusion.Size
4340import Data.Text.Encoding.Error
@@ -68,8 +65,8 @@ streamASCII bs = Stream next 0 (maxSize l)
6865
6966-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8
7067-- encoding.
71−streamUtf8' :: OnDecodeError -> ByteString -> Stream Char
72−streamUtf8' onErr bs = Stream next 0 (maxSize l)
68+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
69+streamUtf8 onErr bs = Stream next 0 (maxSize l)
7370 where
7471 l = B.length bs
7572 next i
@@ -87,33 +84,6 @@ streamUtf8' onErr bs = Stream next 0 (maxSize l)
8784 idx = B.unsafeIndex bs
8885{-# INLINE [0] streamUtf8 #-}
8986
90−accept, reject :: Word32
91−accept = 0
92−reject = 12
93−
94−data S8 = S8 {-# UNPACK #-} !Word32 {-# UNPACK #-} !Int {-# UNPACK #-} !Word32
95−
96−streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
97−streamUtf8 onErr bs = Stream next (S8 accept 0 0) (maxSize l)
98− where
99− l = B.length bs
100− next (S8 state i code)
101− | i >= l = Done
102− | state' == accept = Yield (unsafeChr32 code') (s' accept)
103− | state' /= reject = Skip (s' state')
104− | otherwise = decodeError "streamUtf8" "UTF-8" onErr
105− (Just byte) (s' accept)
106− where s' s = S8 s (i+1) code'
107− byte = B.unsafeIndex bs i
108− word = fromIntegral byte
109− peeku :: Int -> Word32
110− peeku n = fromIntegral (unsafeIndexWord8 utf8d n)
111− !kind = peeku (fromIntegral word)
112− !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)
113− | otherwise = (0xff `shiftR` fromIntegral kind) .&. word
114− !state' = peeku (256 + fromIntegral (state + kind))
115−
116−
11787-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little
11888-- endian UTF-16 encoding.
11989streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char
@@ -232,18 +202,3 @@ decodeError func kind onErr mb i =
232202 Just c -> Yield c i
233203 where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " ++
234204 kind ++ " stream"
235−
236−utf8d :: Array
237−{-# NOINLINE utf8d #-}
238−utf8d = runST fill where
239− fill = do
240− ary <- unsafeNew . (`div` 2) . sum . map fst $ xs
241− mapM_ (uncurry (unsafeWriteWord8 ary))
242− (zip [0..] (concatMap (uncurry replicate) xs))
243− unsafeFreeze ary
244− xs = [(128,0),(16,1),(16,9),(32,7),(2,8),(30,2),(1,10),(12,3),(1,4),(2,3),
245− (1,11),(3,6),(1,5),(11,8),(1,0),(1,12),(1,24),(1,36),(1,60),(1,96),
246− (1,84),(3,12),(1,48),(1,72),(13,12),(1,0),(5,12),(1,0),(1,12),(1,0),
247− (3,12),(1,24),(5,12),(1,24),(1,12),(1,24),(9,12),(1,24),(5,12),(1,24),
248− (7,12),(1,24),(9,12),(1,36),(1,12),(1,36),(3,12),(1,36),(5,12),(1,36),
249− (1,12),(1,36),(3,12),(1,36),(10,12)]
Data/Text/Encoding/Fusion/Common.hs+4−49
@@ -25,46 +25,16 @@ module Data.Text.Encoding.Fusion.Common
2525 , restreamUtf16BE
2626 , restreamUtf32LE
2727 , restreamUtf32BE
28− , istreamUtf8
2928 ) where
3029
31−import Control.Monad.ST
32−import Data.Bits ((.&.), (.|.))
33−import Data.Text.Array
30+import Data.Bits ((.&.))
3431import Data.Text.Fusion (Step(..), Stream(..))
3532import Data.Text.Fusion.Internal (M(..), S(..))
36−import Data.Text.UnsafeChar (ord, unsafeChr32)
37−import Data.Text.UnsafeShift (shiftL, shiftR)
38−import Data.Word (Word8, Word32)
33+import Data.Text.UnsafeChar (ord)
34+import Data.Text.UnsafeShift (shiftR)
35+import Data.Word (Word8)
3936import qualified Data.Text.Encoding.Utf8 as U8
4037
41−accept, reject :: Word32
42−accept = 0
43−reject = 12
44−
45−data S8 s = S8 !s {-# UNPACK #-} !Word32 {-# UNPACK #-} !Word32
46−
47−istreamUtf8 :: Stream Word8 -> Stream Char
48−istreamUtf8 (Stream next0 s0 len) =
49− Stream next (S8 s0 accept 0) len
50− where
51− next (S8 s state code) =
52− case next0 s of
53− Done -> Done
54− Skip s' -> Skip (S8 s' state code)
55− Yield w s'
56− | state' == accept -> Yield (unsafeChr32 code') (S8 s' accept code')
57− | state' /= reject -> Skip (S8 s' state' code')
58− where
59− word = fromIntegral w
60− peeku :: Int -> Word32
61− peeku n = fromIntegral (unsafeIndexWord8 utf8d n)
62− !kind = peeku (fromIntegral word)
63− !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)
64− | otherwise = (0xff `shiftR` fromIntegral kind) .&. word
65− !state' = peeku (256 + fromIntegral (state + kind))
66−
67−
6838-- | /O(n)/ Convert a Stream Char into a UTF-8 encoded Stream Word8.
6939restreamUtf8 :: Stream Char -> Stream Word8
7040restreamUtf8 (Stream next0 s0 len) =
@@ -190,18 +160,3 @@ restreamUtf32LE (Stream next0 s0 len) =
190160internalError :: String -> a
191161internalError func =
192162 error $ "Data.Text.Encoding.Fusion.Common." ++ func ++ ": internal error"
193−
194−utf8d :: Array
195−{-# NOINLINE utf8d #-}
196−utf8d = runST fill where
197− fill = do
198− ary <- unsafeNew . (`div` 2) . sum . map fst $ xs
199− mapM_ (uncurry (unsafeWriteWord8 ary))
200− (zip [0..] (concatMap (uncurry replicate) xs))
201− unsafeFreeze ary
202− xs = [(128,0),(16,1),(16,9),(32,7),(2,8),(30,2),(1,10),(12,3),(1,4),(2,3),
203− (1,11),(3,6),(1,5),(11,8),(1,0),(1,12),(1,24),(1,36),(1,60),(1,96),
204− (1,84),(3,12),(1,48),(1,72),(13,12),(1,0),(5,12),(1,0),(1,12),(1,0),
205− (3,12),(1,24),(5,12),(1,24),(1,12),(1,24),(9,12),(1,24),(5,12),(1,24),
206− (7,12),(1,24),(9,12),(1,36),(1,12),(1,36),(3,12),(1,36),(5,12),(1,36),
207− (1,12),(1,36),(3,12),(1,36),(10,12)]
Data/Text/Lazy/Encoding/Fusion.hs+1−15
@@ -57,26 +57,12 @@ data S = S0
5757 | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
5858 | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
5959
60−data SS = SS ByteString {-# UNPACK #-} !Int
61−
62−streamBytes :: ByteString -> Stream Word8
63−streamBytes bs0 = Stream next (SS bs0 0) unknownSize
64− where
65− next (SS bs@(Chunk s ss) i)
66− | i < len = Yield (B.unsafeIndex s i) (SS bs (i+1))
67− | otherwise = next (SS ss 0)
68− where len = B.length s
69− next _ = Done
70−
7160data T = T {-# UNPACK #-} !ByteString {-# UNPACK #-} !S {-# UNPACK #-} !Int
7261
7362-- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using
7463-- UTF-8 encoding.
7564streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
76−streamUtf8 onErr = istreamUtf8 . streamBytes
77−
78−streamUtf8' :: OnDecodeError -> ByteString -> Stream Char
79−streamUtf8' onErr bs0 = Stream next (T bs0 S0 0) unknownSize
65+streamUtf8 onErr bs0 = Stream next (T bs0 S0 0) unknownSize
8066 where
8167 next (T bs@(Chunk ps _) S0 i)
8268 | i < len && U8.validate1 a =