git.iko.soy/text
on master help sign in
More progress on laziness
cc5a5d2Bryan O'Sullivan
Data/Text.hs+2−6
compare with master
unifiedsplit
@@ -140,7 +140,7 @@ module Data.Text
140 , zipWith
140 , zipWith
141
141
142 -- -* Ordered text
142 -- -* Ordered text
143− , -- sort
143+ -- , sort
144 ) where
144 ) where
145
145
146import Prelude (Char, Bool(..), Functor(..), Int, Maybe(..), String,
146import Prelude (Char, Bool(..), Functor(..), Int, Maybe(..), String,
@@ -209,11 +209,7 @@ unpack = S.unstreamList . stream
209-- | /O(1)/ Convert a character into a Text.
209-- | /O(1)/ Convert a character into a Text.
210-- Subject to array fusion.
210-- Subject to array fusion.
211singleton :: Char -> Text
211singleton :: Char -> Text
212−singleton c = unstream (Stream next (c:[]) 1)
212+singleton = unstream . S.singleton
213− where
214− {-# INLINE next #-}
215− next (k:ks) = Yield k ks
216− next [] = Done
217{-# INLINE [1] singleton #-}
213{-# INLINE [1] singleton #-}
218
214
219-- -----------------------------------------------------------------------------
215-- -----------------------------------------------------------------------------
Data/Text/Fusion.hs+1−1
@@ -152,7 +152,7 @@ reverseStream (Text arr off len) = Stream next (off+len-1) len
152 n2 = A.unsafeIndex arr (i - 1)
152 n2 = A.unsafeIndex arr (i - 1)
153{-# INLINE [0] reverseStream #-}
153{-# INLINE [0] reverseStream #-}
154
154
155−-- | /O(n)/ Convert a Stream Char into a Text.
155+-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.
156unstream :: Stream Char -> Text
156unstream :: Stream Char -> Text
157unstream (Stream next0 s0 len)
157unstream (Stream next0 s0 len)
158 | len == 0 = I.empty
158 | len == 0 = I.empty
Data/Text/Fusion/Internal.hs+7
@@ -19,6 +19,7 @@ module Data.Text.Fusion.Internal
19 , Switch(..)
19 , Switch(..)
20 , Step(..)
20 , Step(..)
21 , Stream(..)
21 , Stream(..)
22+ , singleton
22 , empty
23 , empty
23 , streamList
24 , streamList
24 , unstreamList
25 , unstreamList
@@ -53,6 +54,12 @@ data Stream a =
53 !s -- current state
54 !s -- current state
54 {-# UNPACK #-}!Int -- length hint
55 {-# UNPACK #-}!Int -- length hint
55
56
57+singleton :: Char -> Stream Char
58+singleton c = Stream next False 1
59+ where next False = Yield c True
60+ next True = Done
61+{-# INLINE singleton #-}
62+
56-- | The empty stream.
63-- | The empty stream.
57empty :: Stream a
64empty :: Stream a
58empty = Stream next () 0
65empty = Stream next () 0
Data/Text/Lazy.hs+151−2
@@ -1,17 +1,155 @@
1{-# OPTIONS_GHC -fno-warn-orphans #-}
1{-# OPTIONS_GHC -fno-warn-orphans #-}
2+-- |
3+-- Module : Data.Text.Lazy
4+-- Copyright : (c) Bryan O'Sullivan 2009,
5+-- (c) Tom Harper 2008-2009,
6+-- (c) Duncan Coutts 2008
7+--
8+-- License : BSD-style
9+-- Maintainer : rtharper@aftereternity.co.uk, bos@serpentine.com,
10+-- duncan@haskell.org
11+-- Stability : experimental
12+-- Portability : GHC
13+--
14+-- A time and space-efficient implementation of Unicode text using
15+-- lists of packed arrays. This representation is suitable for high
16+-- performance use and for streaming large quantities of data. It
17+-- provides a means to manipulate a large body of text without
18+-- requiring that the entire content be resident in memory.
19+--
20+-- Some operations, such as 'concat', 'append', 'reverse' and 'cons',
21+-- have better complexity than their "Data.Text" equivalents, due to
22+-- optimisations resulting from the list spine structure. And for
23+-- other operations lazy 'Text's are usually within a few percent of
24+-- strict ones, but with better heap usage. For data larger than
25+-- available memory, or if you have tight memory constraints, this
26+-- module will be the only option.
27+--
28+-- This module is intended to be imported @qualified@, to avoid name
29+-- clashes with "Prelude" functions. eg.
30+--
31+-- > import qualified Data.Text.Lazy as B
2
32
3module Data.Text.Lazy
33module Data.Text.Lazy
4 (
34 (
5 Text
35 Text
36+ -- * Creation and elimination
6 , pack
37 , pack
7 , unpack
38 , unpack
39+ , singleton
40+ , empty
41+
8 -- * Basic interface
42 -- * Basic interface
43+ -- , cons
44+ -- , snoc
9 , append
45 , append
46+ -- , uncons
47+ -- , head
48+ -- , last
49+ -- , tail
50+ -- , init
51+ -- , null
52+ -- , length
53+
54+ -- * Transformations
55+ -- , map
56+ -- , intercalate
57+ -- , intersperse
58+ -- , transpose
59+ -- , reverse
60+
61+ -- * Folds
62+ -- , foldl
63+ -- , foldl'
64+ -- , foldl1
65+ -- , foldl1'
66+ -- , foldr
67+ -- , foldr1
68+
69+ -- ** Special folds
70+ -- , concat
71+ -- , concatMap
72+ -- , any
73+ -- , all
74+ -- , maximum
75+ -- , minimum
76+
77+ -- * Construction
78+
79+ -- ** Scans
80+ -- , scanl
81+ -- , scanl1
82+ -- , scanr
83+ -- , scanr1
84+
85+ -- ** Accumulating maps
86+ -- , mapAccumL
87+ -- , mapAccumR
88+
89+ -- ** Generation and unfolding
90+ -- , replicate
91+ -- , unfoldr
92+ -- , unfoldrN
93+
94+ -- * Substrings
95+
96+ -- ** Breaking strings
97+ -- , take
98+ -- , drop
99+ -- , takeWhile
100+ -- , dropWhile
101+ -- , splitAt
102+ -- , span
103+ -- , break
104+ -- , group
105+ -- , groupBy
106+ -- , inits
107+ -- , tails
108+
109+ -- ** Breaking into many substrings
110+ -- , split
111+ -- , splitWith
112+ -- , breakSubstring
113+
114+ -- ** Breaking into lines and words
115+ -- , lines
116+ --, lines'
117+ -- , words
118+ -- , unlines
119+ -- , unwords
120+
121+ -- * Predicates
122+ -- , isPrefixOf
123+ -- , isSuffixOf
124+ -- , isInfixOf
125+
126+ -- * Searching
127+ -- , elem
128+ -- , filter
129+ -- , find
130+ -- , partition
131+
132+ -- , findSubstring
133+
134+ -- * Indexing
135+ -- , index
136+ -- , findIndex
137+ -- , findIndices
138+ -- , elemIndex
139+ -- , elemIndices
140+ -- , count
141+
142+ -- * Zipping and unzipping
143+ -- , zipWith
144+
145+ -- -* Ordered text
146+ -- , sort
10 ) where
147 ) where
11
148
12import Data.String (IsString(..))
149import Data.String (IsString(..))
150+import qualified Data.Text as T
13import qualified Data.Text.Fusion as S
151import qualified Data.Text.Fusion as S
14−import Data.Text.Fusion.Internal as S
152+import qualified Data.Text.Fusion.Internal as S
15import Data.Text.Lazy.Fusion
153import Data.Text.Lazy.Fusion
16import Data.Text.Lazy.Internal
154import Data.Text.Lazy.Internal
17
155
@@ -37,9 +175,20 @@ pack = unstream . S.streamList
37-- | /O(n)/ Convert a 'Text' into a 'String'.
175-- | /O(n)/ Convert a 'Text' into a 'String'.
38-- Subject to array fusion.
176-- Subject to array fusion.
39unpack :: Text -> String
177unpack :: Text -> String
40−unpack = unstreamList . stream
178+unpack = S.unstreamList . stream
41{-# INLINE [1] unpack #-}
179{-# INLINE [1] unpack #-}
42
180
181+singleton :: Char -> Text
182+singleton c = Chunk (T.singleton c) Empty
183+{-# INLINE [1] singleton #-}
184+
185+{-# RULES
186+"TEXT singleton -> fused" [~1] forall c.
187+ singleton c = unstream (S.singleton c)
188+"TEXT singleton -> unfused" [1] forall c.
189+ unstream (S.singleton c) = singleton c
190+ #-}
191+
43-- | /O(n\/c)/ Append two 'Text's
192-- | /O(n\/c)/ Append two 'Text's
44append :: Text -> Text -> Text
193append :: Text -> Text -> Text
45append xs ys = foldrChunks Chunk ys xs
194append xs ys = foldrChunks Chunk ys xs
Data/Text/Lazy/Fusion.hs+24−15
@@ -1,8 +1,6 @@
1-- |
1-- |
2-- Module : Data.Text.Lazy.Fusion
2-- Module : Data.Text.Lazy.Fusion
3−-- Copyright : (c) Tom Harper 2008-2009,
4−-- (c) Bryan O'Sullivan 2009,
3+-- Copyright : (c) Bryan O'Sullivan 2009
5−-- (c) Duncan Coutts 2009
6--
4--
7-- License : BSD-style
5-- License : BSD-style
8-- Maintainer : rtharper@aftereternity.co.uk, bos@serpentine.com,
6-- Maintainer : rtharper@aftereternity.co.uk, bos@serpentine.com,
@@ -16,6 +14,7 @@ module Data.Text.Lazy.Fusion
16 (
14 (
17 stream
15 stream
18 , unstream
16 , unstream
17+ , unstreamChunks
19 ) where
18 ) where
20
19
21import Data.Text.Fusion.Internal
20import Data.Text.Fusion.Internal
@@ -28,6 +27,7 @@ import Data.Text.Unsafe (iter)
28
27
29default(Int)
28default(Int)
30
29
30+-- | /O(n)/ Convert a 'Text' into a 'Stream Char'.
31stream :: Text -> Stream Char
31stream :: Text -> Stream Char
32stream text = Stream next (text :!: 0) 4
32stream text = Stream next (text :!: 0) 4
33 where
33 where
@@ -38,23 +38,25 @@ stream text = Stream next (text :!: 0) 4
38 where (c,d) = iter t i
38 where (c,d) = iter t i
39{-# INLINE [0] stream #-}
39{-# INLINE [0] stream #-}
40
40
41+-- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given
42+-- chunk size.
41−unstream :: Stream Char -> Text
43+unstreamChunks :: Int -> Stream Char -> Text
42−unstream (Stream next s0 len0)
44+unstreamChunks chunkSize (Stream next s0 len0)
43 | len0 == 0 = Empty
45 | len0 == 0 = Empty
44 | otherwise = outer s0
46 | otherwise = outer s0
45 where
47 where
46 outer s = case next s of
48 outer s = case next s of
47− Done -> Empty
49+ Done -> Empty
48− Skip s' -> outer s'
50+ Skip s' -> outer s'
49− Yield x s' -> chunk t (outer s'')
51+ Yield x s' -> I.Text arr 0 len `chunk` outer s''
50− where (t,s'') = fill x s'
52+ where (arr,(s'',len)) = A.run2 fill
51− fill x s = (I.Text a 0 l,s')
53+ fill = do arr <- A.unsafeNew unknownLength
52− where (a,(s',l)) = A.run2 (A.unsafeNew initLen >>= (\arr -> inner arr initLen x s 0))
54+ inner arr unknownLength x s' 0
53− initLen = 8
55+ unknownLength = 4
54 inner marr len x s i
56 inner marr len x s i
55− | i + 1 >= defaultChunkSize = return (marr, (s,i))
57+ | i + 1 >= chunkSize = return (marr, (s,i))
56− | i + 1 >= len = do
58+ | i + 1 >= len = do
57− let newLen = min (len * 2) defaultChunkSize
59+ let newLen = min (len * 2) chunkSize
58 marr' <- A.unsafeNew newLen
60 marr' <- A.unsafeNew newLen
59 A.copy marr marr'
61 A.copy marr marr'
60 inner marr' newLen x s i
62 inner marr' newLen x s i
@@ -64,5 +66,12 @@ unstream (Stream next s0 len0)
64 return (marr,(s,i'))
66 return (marr,(s,i'))
65 Skip s' -> inner marr len x s i
67 Skip s' -> inner marr len x s i
66 Yield x' s' -> unsafeWrite marr i x >>= inner marr len x' s'
68 Yield x' s' -> unsafeWrite marr i x >>= inner marr len x' s'
69+{-# INLINE [0] unstreamChunks #-}
70+
71+-- | /O(n)/ Convert a 'Stream Char' into a 'Text', using
72+-- 'defaultChunkSize'.
73+unstream :: Stream Char -> Text
74+unstream = unstreamChunks defaultChunkSize
67{-# INLINE [0] unstream #-}
75{-# INLINE [0] unstream #-}
76+
68{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}
77{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}
Data/Text/Lazy/Internal.hs+6
@@ -16,6 +16,7 @@ module Data.Text.Lazy.Internal
16 (
16 (
17 Text(..)
17 Text(..)
18 , chunk
18 , chunk
19+ , empty
19 , foldrChunks
20 , foldrChunks
20 , foldlChunks
21 , foldlChunks
21 -- * Data type invariant and abstraction functions
22 -- * Data type invariant and abstraction functions
@@ -65,6 +66,11 @@ chunk :: T.Text -> Text -> Text
65chunk t@(T.Text _ _ len) ts | len == 0 = ts
66chunk t@(T.Text _ _ len) ts | len == 0 = ts
66 | otherwise = Chunk t ts
67 | otherwise = Chunk t ts
67
68
69+-- | Smart constructor for 'Empty'.
70+empty :: Text
71+{-# INLINE [0] empty #-}
72+empty = Empty
73+
68-- | Consume the chunks of a lazy 'Text' with a natural right fold.
74-- | Consume the chunks of a lazy 'Text' with a natural right fold.
69foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a
75foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a
70foldrChunks f z = go
76foldrChunks f z = go
tests/Properties.hs+2
@@ -39,6 +39,7 @@ prop_T_stream_unstream = (S.unstream . S.stream) `eq` id
39prop_TL_stream_unstream = (SL.unstream . SL.stream) `eq` id
39prop_TL_stream_unstream = (SL.unstream . SL.stream) `eq` id
40prop_T_reverse_stream t = (S.reverse . S.reverseStream) t == t
40prop_T_reverse_stream t = (S.reverse . S.reverseStream) t == t
41prop_T_singleton c = [c] == (T.unpack . T.singleton) c
41prop_T_singleton c = [c] == (T.unpack . T.singleton) c
42+prop_TL_singleton c = [c] == (TL.unpack . TL.singleton) c
42
43
43prop_T_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
44prop_T_ascii t = E.decodeASCII (E.encodeUtf8 a) == a
44 where a = T.map (\c -> chr (ord c `mod` 128)) t
45 where a = T.map (\c -> chr (ord c `mod` 128)) t
@@ -236,6 +237,7 @@ tests = [
236 ("prop_TL_stream_unstream", mytest prop_TL_stream_unstream),
237 ("prop_TL_stream_unstream", mytest prop_TL_stream_unstream),
237 ("prop_T_reverse_stream", mytest prop_T_reverse_stream),
238 ("prop_T_reverse_stream", mytest prop_T_reverse_stream),
238 ("prop_T_singleton", mytest prop_T_singleton),
239 ("prop_T_singleton", mytest prop_T_singleton),
240+ ("prop_TL_singleton", mytest prop_TL_singleton),
239
241
240 ("prop_T_ascii", mytest prop_T_ascii),
242 ("prop_T_ascii", mytest prop_T_ascii),
241 ("prop_T_utf8", mytest prop_T_utf8),
243 ("prop_T_utf8", mytest prop_T_utf8),