git.iko.soy/text
on master help sign in
Merge
commita8b2bd62235790448d717596eda76ed0c8f730a3 authorBryan O'Sullivan <bos@serpentine.com> dateSat 23 Jun 2012, 22:24 (14 years ago) parent7f2801bb131ab8
3 files changed+43−32
compare with master
unifiedsplit
Data/Text/Encoding.hs+24−23 Data/Text/Fusion.hs+8−7 Data/Text/Private.hs+11−2
Data/Text/Encoding.hs+24−23
@@ -57,7 +57,8 @@ import Data.Bits ((.&.))
5757import Data.ByteString as B
5858import Data.ByteString.Internal as B
5959import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
60−import Data.Text.Internal (Text(..), textP)
60+import Data.Text.Internal (Text(..))
61+import Data.Text.Private (runText)
6162import Data.Text.UnsafeChar (ord, unsafeWrite)
6263import Data.Text.UnsafeShift (shiftL, shiftR)
6364import Data.Word (Word8)
@@ -96,30 +97,30 @@ decodeASCII = decodeUtf8
9697
9798-- | Decode a 'ByteString' containing UTF-8 encoded text.
9899decodeUtf8With :: OnDecodeError -> ByteString -> Text
99−decodeUtf8With onErr (PS fp off len) = textP (fst a) 0 (snd a)
100+decodeUtf8With onErr (PS fp off len) = runText $ \done -> do
101+ let go dest = withForeignPtr fp $ \ptr ->
102+ with (0::CSize) $ \destOffPtr -> do
103+ let end = ptr `plusPtr` (off + len)
104+ loop curPtr = do
105+ curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end
106+ if curPtr' == end
107+ then do
108+ n <- peek destOffPtr
109+ unsafeSTToIO (done dest (fromIntegral n))
110+ else do
111+ x <- peek curPtr'
112+ case onErr desc (Just x) of
113+ Nothing -> loop $ curPtr' `plusPtr` 1
114+ Just c -> do
115+ destOff <- peek destOffPtr
116+ w <- unsafeSTToIO $
117+ unsafeWrite dest (fromIntegral destOff) c
118+ poke destOffPtr (destOff + fromIntegral w)
119+ loop $ curPtr' `plusPtr` 1
120+ loop (ptr `plusPtr` off)
121+ (unsafeIOToST . go) =<< A.new len
100122 where
101− a = A.run2 (A.new len >>= unsafeIOToST . go)
102123 desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream"
103− go dest = withForeignPtr fp $ \ptr ->
104− with (0::CSize) $ \destOffPtr -> do
105− let end = ptr `plusPtr` (off + len)
106− loop curPtr = do
107− curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end
108− if curPtr' == end
109− then do
110− n <- peek destOffPtr
111− return (dest,fromIntegral n)
112− else do
113− x <- peek curPtr'
114− case onErr desc (Just x) of
115− Nothing -> loop $ curPtr' `plusPtr` 1
116− Just c -> do
117− destOff <- peek destOffPtr
118− w <- unsafeSTToIO $
119− unsafeWrite dest (fromIntegral destOff) c
120− poke destOffPtr (destOff + fromIntegral w)
121− loop $ curPtr' `plusPtr` 1
122− loop (ptr `plusPtr` off)
123124{- INLINE[0] decodeUtf8With #-}
124125
125126-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
Data/Text/Fusion.hs+8−7
@@ -51,6 +51,7 @@ import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int,
5151 fromIntegral, otherwise)
5252import Data.Bits ((.&.))
5353import Data.Text.Internal (Text(..))
54+import Data.Text.Private (runText)
5455import Data.Text.UnsafeChar (ord, unsafeChr, unsafeWrite)
5556import Data.Text.UnsafeShift (shiftL, shiftR)
5657import qualified Data.Text.Array as A
@@ -94,15 +95,14 @@ reverseStream (Text arr off len) = Stream next (off+len-1) (maxSize len)
9495
9596-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.
9697unstream :: Stream Char -> Text
97−unstream (Stream next0 s0 len) = I.textP (P.fst a) 0 (P.snd a)
98− where
99− a = A.run2 (A.new mlen >>= \arr -> outer arr mlen s0 0)
100− where mlen = upperBound 4 len
101− outer arr top = loop
102− where
98+unstream (Stream next0 s0 len) = runText $ \done -> do
99+ let mlen = upperBound 4 len
100+ arr0 <- A.new mlen
101+ let outer arr top = loop
102+ where
103103 loop !s !i =
104104 case next0 s of
105− Done -> return (arr, i)
105+ Done -> done arr i
106106 Skip s' -> loop s' i
107107 Yield x s'
108108 | j >= top -> {-# SCC "unstream/resize" #-} do
@@ -114,6 +114,7 @@ unstream (Stream next0 s0 len) = I.textP (P.fst a) 0 (P.snd a)
114114 loop s' (i+d)
115115 where j | ord x < 0x10000 = i
116116 | otherwise = i + 1
117+ outer arr0 mlen s0 0
117118{-# INLINE [0] unstream #-}
118119{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}
119120
Data/Text/Private.hs+11−2
@@ -1,4 +1,4 @@
1−{-# LANGUAGE BangPatterns, UnboxedTuples #-}
1+{-# LANGUAGE BangPatterns, Rank2Types, UnboxedTuples #-}
22
33-- |
44-- Module : Data.Text.Private
@@ -11,11 +11,14 @@
1111
1212module Data.Text.Private
1313 (
14− span_
14+ runText
15+ , span_
1516 ) where
1617
18+import Control.Monad.ST (ST, runST)
1719import Data.Text.Internal (Text(..), textP)
1820import Data.Text.Unsafe (Iter(..), iter)
21+import qualified Data.Text.Array as A
1922
2023span_ :: (Char -> Bool) -> Text -> (# Text, Text #)
2124span_ p t@(Text arr off len) = (# hd,tl #)
@@ -26,3 +29,9 @@ span_ p t@(Text arr off len) = (# hd,tl #)
2629 | otherwise = i
2730 where Iter c d = iter t i
2831{-# INLINE span_ #-}
32+
33+runText :: (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text
34+runText act = runST (act $ \ !marr !len -> do
35+ arr <- A.unsafeFreeze marr
36+ return $! textP arr 0 len)
37+{-# INLINE runText #-}