a8b2bd6 · text · git.iko.soy

git.iko.soy/text

on master help sign in

Merge

a8b2bd6Bryan O'Sullivan14 years ago

Data/Text/Encoding.hs+24−23

compare with master

unifiedsplit

@@ -57,7 +57,8 @@ import Data.Bits ((.&.))

57import Data.ByteString as B

58import Data.ByteString.Internal as B

59import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

60−import Data.Text.Internal (Text(..), textP)

60+import Data.Text.Internal (Text(..))

61+import Data.Text.Private (runText)

61import Data.Text.UnsafeChar (ord, unsafeWrite)

62import Data.Text.UnsafeChar (ord, unsafeWrite)

62import Data.Text.UnsafeShift (shiftL, shiftR)

63import Data.Text.UnsafeShift (shiftL, shiftR)

63import Data.Word (Word8)

64import Data.Word (Word8)

@@ -96,30 +97,30 @@ decodeASCII = decodeUtf8

97-- | Decode a 'ByteString' containing UTF-8 encoded text.

98-- | Decode a 'ByteString' containing UTF-8 encoded text.

98decodeUtf8With :: OnDecodeError -> ByteString -> Text

99decodeUtf8With :: OnDecodeError -> ByteString -> Text

99−decodeUtf8With onErr (PS fp off len) = textP (fst a) 0 (snd a)

100+decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

101+ let go dest = withForeignPtr fp $ \ptr ->

102+ with (0::CSize) $ \destOffPtr -> do

103+ let end = ptr `plusPtr` (off + len)

104+ loop curPtr = do

105+ curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end

106+ if curPtr' == end

107+ then do

108+ n <- peek destOffPtr

109+ unsafeSTToIO (done dest (fromIntegral n))

110+ else do

111+ x <- peek curPtr'

112+ case onErr desc (Just x) of

113+ Nothing -> loop $ curPtr' `plusPtr` 1

114+ Just c -> do

115+ destOff <- peek destOffPtr

116+ w <- unsafeSTToIO $

117+ unsafeWrite dest (fromIntegral destOff) c

118+ poke destOffPtr (destOff + fromIntegral w)

119+ loop $ curPtr' `plusPtr` 1

120+ loop (ptr `plusPtr` off)

121+ (unsafeIOToST . go) =<< A.new len

100 where

122 where

101− a = A.run2 (A.new len >>= unsafeIOToST . go)

102 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"

123 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"

103− go dest = withForeignPtr fp $ \ptr ->

104− with (0::CSize) $ \destOffPtr -> do

105− let end = ptr `plusPtr` (off + len)

106− loop curPtr = do

107− curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end

108− if curPtr' == end

109− then do

110− n <- peek destOffPtr

111− return (dest,fromIntegral n)

112− else do

113− x <- peek curPtr'

114− case onErr desc (Just x) of

115− Nothing -> loop $ curPtr' `plusPtr` 1

116− Just c -> do

117− destOff <- peek destOffPtr

118− w <- unsafeSTToIO $

119− unsafeWrite dest (fromIntegral destOff) c

120− poke destOffPtr (destOff + fromIntegral w)

121− loop $ curPtr' `plusPtr` 1

122− loop (ptr `plusPtr` off)

123{- INLINE[0] decodeUtf8With #-}

124{- INLINE[0] decodeUtf8With #-}

124

125

125-- | Decode a 'ByteString' containing UTF-8 encoded text that is known

126-- | Decode a 'ByteString' containing UTF-8 encoded text that is known

Data/Text/Fusion.hs+8−7

@@ -51,6 +51,7 @@ import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int,

51 fromIntegral, otherwise)

52import Data.Bits ((.&.))

53import Data.Text.Internal (Text(..))

54+import Data.Text.Private (runText)

54import Data.Text.UnsafeChar (ord, unsafeChr, unsafeWrite)

55import Data.Text.UnsafeChar (ord, unsafeChr, unsafeWrite)

55import Data.Text.UnsafeShift (shiftL, shiftR)

56import Data.Text.UnsafeShift (shiftL, shiftR)

56import qualified Data.Text.Array as A

57import qualified Data.Text.Array as A

@@ -94,15 +95,14 @@ reverseStream (Text arr off len) = Stream next (off+len-1) (maxSize len)

95-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.

96-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.

96unstream :: Stream Char -> Text

97unstream :: Stream Char -> Text

97−unstream (Stream next0 s0 len) = I.textP (P.fst a) 0 (P.snd a)

98+unstream (Stream next0 s0 len) = runText $ \done -> do

98− where

99− a = A.run2 (A.new mlen >>= \arr -> outer arr mlen s0 0)

100− where mlen = upperBound 4 len

99+ let mlen = upperBound 4 len

100+ arr0 <- A.new mlen

101− outer arr top = loop

101+ let outer arr top = loop

102− where

102+ where

103 loop !s !i =

104 case next0 s of

105− Done -> return (arr, i)

105+ Done -> done arr i

106 Skip s' -> loop s' i

107 Yield x s'

108 | j >= top -> {-# SCC "unstream/resize" #-} do

@@ -114,6 +114,7 @@ unstream (Stream next0 s0 len) = I.textP (P.fst a) 0 (P.snd a)

114 loop s' (i+d)

115 where j | ord x < 0x10000 = i

116 | otherwise = i + 1

117+ outer arr0 mlen s0 0

117{-# INLINE [0] unstream #-}

118{-# INLINE [0] unstream #-}

118{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}

119{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}

119

120

Data/Text/Private.hs+11−2

@@ -1,4 +1,4 @@

1−{-# LANGUAGE BangPatterns, UnboxedTuples #-}

1+{-# LANGUAGE BangPatterns, Rank2Types, UnboxedTuples #-}

3-- |

4-- Module : Data.Text.Private

@@ -11,11 +11,14 @@

12module Data.Text.Private

13 (

14+ runText

14− span_

15+ , span_

15 ) where

16 ) where

18+import Control.Monad.ST (ST, runST)

17import Data.Text.Internal (Text(..), textP)

19import Data.Text.Internal (Text(..), textP)

18import Data.Text.Unsafe (Iter(..), iter)

20import Data.Text.Unsafe (Iter(..), iter)

21+import qualified Data.Text.Array as A

20span_ :: (Char -> Bool) -> Text -> (# Text, Text #)

23span_ :: (Char -> Bool) -> Text -> (# Text, Text #)

21span_ p t@(Text arr off len) = (# hd,tl #)

24span_ p t@(Text arr off len) = (# hd,tl #)

@@ -26,3 +29,9 @@ span_ p t@(Text arr off len) = (# hd,tl #)

26 | otherwise = i

29 | otherwise = i

27 where Iter c d = iter t i

30 where Iter c d = iter t i

28{-# INLINE span_ #-}

31{-# INLINE span_ #-}

32+

33+runText :: (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text

34+runText act = runST (act $ \ !marr !len -> do

35+ arr <- A.unsafeFreeze marr

36+ return $! textP arr 0 len)

37+{-# INLINE runText #-}