@@ -1,15 +1,45 @@
11{-# LANGUAGE BangPatterns #-}
22{-# OPTIONS_GHC -fno-warn-orphans #-}
33
4+-- |
5+-- Module : Data.Text
6+-- Copyright : (c) Tom Harper 2008-2009,
7+-- (c) Bryan O'Sullivan 2009,
8+-- (c) Duncan Coutts 2009
9+--
10+-- License : BSD-style
11+-- Maintainer : rtharper@aftereternity.co.uk, bos@serpentine.com,
12+-- duncan@haskell.org
13+-- Stability : experimental
14+-- Portability : GHC
15+--
16+-- A time and space-efficient implementation of Unicode text using
17+-- packed Word16 arrays. Suitable for performance critical use, both
18+-- in terms of large data quantities and high speed.
19+--
20+-- This module is intended to be imported @qualified@, to avoid name
21+-- clashes with "Prelude" functions, e.g.
22+--
23+-- > import qualified Data.Text as T
24+
425module Data.Text
526 (
27+ -- * Fusion
28+ -- $fusion
29+
30+ -- * Types
631 Text
732 , Encoding(..)
33+
34+ -- * Creation and elimination
835 , pack
936 , unpack
1037 , singleton
38+ , empty
1139 , encode
1240 , decode
41+
42+ -- * Basic interface
1343 , cons
1444 , snoc
1545 , append
@@ -19,35 +49,57 @@ module Data.Text
1949 , init
2050 , null
2151 , length
52+
53+ -- * Transformations
2254 , map
2355 , intersperse
2456 , transpose
57+
58+ -- * Folds
2559 , foldl
2660 , foldl'
2761 , foldl1
2862 , foldl1'
2963 , foldr
3064 , foldr1
65+
66+ -- ** Special folds
3167 , concat
3268 , concatMap
3369 , any
3470 , all
3571 , maximum
3672 , minimum
73+
74+ -- * Construction
3775 , unfoldr
3876 , unfoldrN
77+
78+ -- * Substrings
79+
80+ -- ** Breaking strings
3981 , take
4082 , drop
4183 , takeWhile
4284 , dropWhile
85+
86+ -- ** Breaking into lines and words
87+ , words
88+
89+ -- * Searching
4390 , elem
44− , find
4591 , filter
92+
93+ -- * Indexing
94+ , find
4695 , index
4796 , findIndex
4897 , elemIndex
98+
99+ -- * Zipping and unzipping
49100 , zipWith
50− , words
101+
102+ -- * I/O
51103 , readFile
52104 ) where
53105
@@ -58,7 +110,6 @@ import Prelude (Char,Bool,Int,Maybe,String,
58110 (&&),(||),(+),(-),(<),(>),(<=),(>=),(.),(>>=),
59111 return,otherwise,
60112 IO, FilePath)
61−
62113import Data.Char (isSpace)
63114import Control.Monad.ST(ST)
64115import Data.Array.Base(unsafeNewArray_,unsafeWrite,unsafeAt)
@@ -78,6 +129,12 @@ import qualified Prelude as P
78129import Data.Text.UnsafeChar(unsafeChr)
79130import qualified Data.Text.Utf16 as U16
80131
132+-- $fusion
133+--
134+-- Most of the functions in this module are subject to /array fusion/,
135+-- meaning that a pipeline of functions will usually allocate at most
136+-- one 'Text' value.
137+
81138instance Eq Text where
82139 t1 == t2 = (stream t1) `S.eq` (stream t2)
83140
@@ -98,11 +155,9 @@ instance IsString Text where
98155-- -----------------------------------------------------------------------------
99156-- * Conversion to/from 'Text'
100157
101−-- | /O(n)/ Convert a String into a Text.
158+-- | /O(n)/ Convert a 'String' into a 'Text'.
102159--
103−-- This function is subject to array fusion, so calling other fusible
104−-- function(s) on a packed string will only cause one 'Text' to be written
105−-- out at the end of the pipeline, instead of one before and one after.
160+-- This function is subject to array fusion.
106161pack :: String -> Text
107162pack str = (unstream (stream_list str))
108163 where
@@ -126,7 +181,7 @@ unpack txt = (unstream_list (stream txt))
126181 S.Yield x s' -> x : unfold s'
127182{-# INLINE [1] unpack #-}
128183
129−-- | Convert a character into a Text.
184+-- | /O(1)/ Convert a character into a Text.
130185-- Subject to array fusion.
131186singleton :: Char -> Text
132187singleton c = unstream (Stream next (c:[]) 1)
@@ -147,23 +202,21 @@ encode enc txt = unstream_bs (restream enc (stream txt))
147202-- -----------------------------------------------------------------------------
148203-- * Basic functions
149204
150−-- | /O(n)/ Adds a character to the front of a 'Text'. This function is more
151−-- costly than its 'List' counterpart because it requires copying a new array.
152−-- Subject to array fusion.
205+-- | /O(n)/ Adds a character to the front of a 'Text'. This function
206+-- is more costly than its 'List' counterpart because it requires
207+-- copying a new array. Subject to array fusion.
153208cons :: Char -> Text -> Text
154209cons c t = unstream (S.cons c (stream t))
155210{-# INLINE cons #-}
156211
157−-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the entire
158−-- array in the process.
159−-- Subject to array fusion.
212+-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the
213+-- entire array in the process. Subject to array fusion.
160214snoc :: Text -> Char -> Text
161215snoc t c = unstream (S.snoc (stream t) c)
162216{-# INLINE snoc #-}
163217
164−-- | /O(n)/ Appends one Text to the other by copying both of them into a new
165−-- Text.
166−-- Subject to array fusion
218+-- | /O(n)/ Appends one 'Text' to the other by copying both of them
219+-- into a new 'Text'. Subject to array fusion.
167220append :: Text -> Text -> Text
168221append (Text arr1 off1 len1) (Text arr2 off2 len2) = Text (runSTUArray x) 0 len
169222 where
@@ -187,14 +240,14 @@ append (Text arr1 off1 len1) (Text arr2 off2 len2) = Text (runSTUArray x) 0 len
187240 unstream (S.append (stream t1) (stream t2)) = append t1 t2
188241 #-}
189242
190−-- | /O(1)/ Returns the first character of a Text, which must be non-empty.
191−-- Subject to array fusion.
243+-- | /O(1)/ Returns the first character of a 'Text', which must be
244+-- non-empty. Subject to array fusion.
192245head :: Text -> Char
193246head t = S.head (stream t)
194247{-# INLINE head #-}
195248
196−-- | /O(n)/ Returns the last character of a Text, which must be non-empty.
197−-- Subject to array fusion.
249+-- | /O(n)/ Returns the last character of a 'Text', which must be
250+-- non-empty. Subject to array fusion.
198251last :: Text -> Char
199252last (Text arr off len)
200253 | len <= 0 = errorEmptyList "last"
@@ -213,9 +266,8 @@ last (Text arr off len)
213266 #-}
214267
215268
216−-- | /O(1)/ Returns all characters after the head of a Text, which must
217−-- be non-empty.
218−-- Subject to array fusion.
269+-- | /O(1)/ Returns all characters after the head of a 'Text', which
270+-- must be non-empty. Subject to array fusion.
219271tail :: Text -> Text
220272tail (Text arr off len)
221273 | len <= 0 = errorEmptyList "tail"
@@ -227,9 +279,8 @@ tail (Text arr off len)
227279
228280
229281
230−-- | /O(1)/ Returns all but the last character of a Text, which
231−-- must be non-empty.
232−-- Subject to array fusion.
282+-- | /O(1)/ Returns all but the last character of a 'Text', which must
283+-- be non-empty. Subject to array fusion.
233284init :: Text -> Text
234285init (Text arr off len) | len <= 0 = errorEmptyList "init"
235286 | n >= 0xDC00 && n <= 0xDFFF = Text arr off (len-2)
@@ -245,13 +296,13 @@ init (Text arr off len) | len <= 0 = errorEmptyList "init"
245296 unstream (S.init (stream t)) = init t
246297 #-}
247298
248−-- | /O(1)/ Tests whether a Text is empty or not.
249−-- Subject to array fusion.
299+-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to array
300+-- fusion.
250301null :: Text -> Bool
251302null t = S.null (stream t)
252303{-# INLINE null #-}
253304
254−-- | /O(n)/ Returns the number of characters in a text.
305+-- | /O(n)/ Returns the number of characters in a 'Text'.
255306-- Subject to array fusion.
256307length :: Text -> Int
257308length (Text _arr _off len) = len
@@ -266,32 +317,31 @@ length (Text _arr _off len) = len
266317
267318-- -----------------------------------------------------------------------------
268319-- * Transformations
269−-- | /O(n)/ 'map' @f @xs is the Text obtained by applying @f@ to each
270−-- element of @xs@.
271−-- Subject to array fusion.
320+-- | /O(n)/ 'map' @f @xs is the 'Text' obtained by applying @f@ to
321+-- each element of @xs@. Subject to array fusion.
272322map :: (Char -> Char) -> Text -> Text
273323map f t = unstream (S.map f (stream t))
274324{-# INLINE [1] map #-}
275325
276−-- | /O(n)/ The 'intersperse' function takes a character and places it between
277−-- the characters of a Text.
278−-- Subject to array fusion.
326+-- | /O(n)/ The 'intersperse' function takes a character and places it
327+-- between the characters of a 'Text'. Subject to array fusion.
279328intersperse :: Char -> Text -> Text
280329intersperse c t = unstream (S.intersperse c (stream t))
281330{-# INLINE intersperse #-}
282331
283−-- | /O(n)/ The 'transpose' function transposes the rows and columns of its
284−-- Text argument. Note that this function uses pack, unpack, and the 'List'
285−-- version of transpose and is thus not very efficient.
332+-- | /O(n)/ The 'transpose' function transposes the rows and columns
333+-- of its 'Text' argument. Note that this function uses 'pack',
334+-- 'unpack', and the 'List' version of transpose and is thus not very
335+-- efficient.
286336transpose :: [Text] -> [Text]
287337transpose ts = P.map pack (L.transpose (P.map unpack ts))
288338
289339-- -----------------------------------------------------------------------------
290340-- * Reducing 'Text's (folds)
291341
292−-- | 'foldl', applied to a binary operator, a starting value (typically the
293−-- left-identity of the operator), and a Text, reduces the Text using the
294−-- binary operator, from left to right.
342+-- | 'foldl', applied to a binary operator, a starting value
343+-- (typically the left-identity of the operator), and a 'Text',
344+-- reduces the 'Text' using the binary operator, from left to right.
295345-- Subject to array fusion.
296346foldl :: (b -> Char -> b) -> b -> Text -> b
297347foldl f z t = S.foldl f z (stream t)
@@ -303,9 +353,9 @@ foldl' :: (b -> Char -> b) -> b -> Text -> b
303353foldl' f z t = S.foldl' f z (stream t)
304354{-# INLINE foldl' #-}
305355
306−-- | 'foldl1' is a variant of 'foldl' that has no starting value argument,
307−-- and thus must be applied to non-empty 'Text's.
308−-- Subject to array fusion.
356+-- | A variant of 'foldl' that has no starting value argument, and
357+-- thus must be applied to a non-empty 'Text'. Subject to array
358+-- fusion.
309359foldl1 :: (Char -> Char -> Char) -> Text -> Char
310360foldl1 f t = S.foldl1 f (stream t)
311361{-# INLINE foldl1 #-}
@@ -316,17 +366,17 @@ foldl1' :: (Char -> Char -> Char) -> Text -> Char
316366foldl1' f t = S.foldl1' f (stream t)
317367{-# INLINE foldl1' #-}
318368
319−-- | 'foldr', applied to a binary operator, a starting value (typically the
320−-- right-identity of the operator), and a Text, reduces the Text using the
321−-- binary operator, from right to left.
369+-- | 'foldr', applied to a binary operator, a starting value
370+-- (typically the right-identity of the operator), and a 'Text',
371+-- reduces the 'Text' using the binary operator, from right to left.
322372-- Subject to array fusion.
323373foldr :: (Char -> b -> b) -> b -> Text -> b
324374foldr f z t = S.foldr f z (stream t)
325375{-# INLINE foldr #-}
326376
327−-- | 'foldr1' is a variant of 'foldr' that has no starting value argument,
328−-- and thust must be applied to non-empty 'Text's.
329−-- Subject to array fusion.
377+-- | A variant of 'foldr' that has no starting value argument, and
378+-- thust must be applied to a non-empty 'Text'. Subject to array
379+-- fusion.
330380foldr1 :: (Char -> Char -> Char) -> Text -> Char
331381foldr1 f t = S.foldr1 f (stream t)
332382{-# INLINE foldr1 #-}
@@ -339,34 +389,35 @@ concat :: [Text] -> Text
339389concat ts = unstream (S.concat (L.map stream ts))
340390{-# INLINE concat #-}
341391
342−-- | Map a function over a Text that results in a Text and concatenate the
343−-- results. This function is subject to array fusion, and note that if in
344−-- 'concatMap' @f @xs, @f@ is defined in terms of fusible functions it will
345−-- also be fusible.
392+-- | Map a function over a 'Text' that results in a 'Text', and concatenate the
393+-- results. This function is subject to array fusion.
394+--
395+-- Note: if in 'concatMap' @f @xs, @f@ is defined in terms of fusible
396+-- functions, it will also be fusible.
346397concatMap :: (Char -> Text) -> Text -> Text
347398concatMap f t = unstream (S.concatMap (stream . f) (stream t))
348399{-# INLINE concatMap #-}
349400
350−-- | 'any' @p @xs determines if any character in the 'Text' @xs@ satisifes the
351−-- predicate @p@. Subject to array fusion.
401+-- | 'any' @p @xs determines whether any character in the 'Text' @xs@
402+-- satisifes the predicate @p@. Subject to array fusion.
352403any :: (Char -> Bool) -> Text -> Bool
353404any p t = S.any p (stream t)
354405{-# INLINE any #-}
355406
356−-- | 'all' @p @xs determines if all characters in the 'Text' @xs@ satisify the
357−-- predicate @p@. Subject to array fusion.
407+-- | 'all' @p @xs determines whether all characters in the 'Text' @xs@
408+-- satisify the predicate @p@. Subject to array fusion.
358409all :: (Char -> Bool) -> Text -> Bool
359410all p t = S.all p (stream t)
360411{-# INLINE all #-}
361412
362−-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which must be
363−-- non-empty. Subject to array fusion.
413+-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
414+-- must be non-empty. Subject to array fusion.
364415maximum :: Text -> Char
365416maximum t = S.maximum (stream t)
366417{-# INLINE maximum #-}
367418
368−-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which must be
369−-- non-empty. Subject to array fusion.
419+-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
420+-- must be non-empty. Subject to array fusion.
370421minimum :: Text -> Char
371422minimum t = S.minimum (stream t)
372423{-# INLINE minimum #-}
@@ -377,21 +428,21 @@ minimum t = S.minimum (stream t)
377428-- -----------------------------------------------------------------------------
378429-- ** Generating and unfolding 'Text's
379430
380−-- /O(n)/, where @n@ is the length of the result. The unfoldr function
381−-- is analogous to the List 'unfoldr'. unfoldr builds a Text
382−-- from a seed value. The function takes the element and returns
383−-- Nothing if it is done producing the Text or returns Just
384−-- (a,b), in which case, a is the next Char in the string, and b is
385−-- the seed value for further production.
431+-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'
432+-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a
433+-- 'Text' from a seed value. The function takes the element and
434+-- returns 'Nothing' if it is done producing the 'Text', otherwise
435+-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the
436+-- string, and @b@ is the seed value for further production.
386437unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
387438unfoldr f s = unstream (S.unfoldr f s)
388439{-# INLINE unfoldr #-}
389440
390−-- O(n) Like unfoldr, unfoldrN builds a Text from a seed
441+-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
391442-- value. However, the length of the result should be limited by the
392−-- first argument to unfoldrN. This function is more efficient than
393−-- unfoldr when the maximum length of the result and correct,
394−-- otherwise its complexity performance is similar to 'unfoldr'
443+-- first argument to 'unfoldrN'. This function is more efficient than
444+-- 'unfoldr' when the maximum length of the result is known and
445+-- correct, otherwise its performance is similar to 'unfoldr'.
395446unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text
396447unfoldrN n f s = unstream (S.unfoldrN n f s)
397448{-# INLINE unfoldrN #-}
@@ -399,12 +450,15 @@ unfoldrN n f s = unstream (S.unfoldrN n f s)
399450-- -----------------------------------------------------------------------------
400451-- * Substrings
401452
402−-- O(n) 'take' @n, applied to a Text, returns the prefix of the
403−-- Text of length n, or the Text itself if n is greater than the
404−-- length of the Text.
453+-- /O(n) 'take' @n@, applied to a 'Text', returns the prefix of the
454+-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than
455+-- the length of the Text.
405456take :: Int -> Text -> Text
406−take n (Text arr off len) = Text arr off (loop off 0)
407− where
457+take n t@(Text arr off len)
458+ | n <= 0 = empty
459+ | n >= len = t
460+ | otherwise = Text arr off (loop off 0)
461+ where
408462 end = off+len
409463 loop !i !count
410464 | i >= end || count >= n = i - off
@@ -421,12 +475,15 @@ take n (Text arr off len) = Text arr off (loop off 0)
421475 unstream (S.take n (stream t)) = take n t
422476 #-}
423477
424−-- /O(n)/ 'drop' @n, applied to a Text, returns the suffix of the
425−-- Text of length @n, or the empty Text if @n is greater than the
426−-- length of the Text.
478+-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the
479+-- 'Text' of length @n@, or the empty 'Text' if @n@ is greater than the
480+-- length of the 'Text'.
427481drop :: Int -> Text -> Text
428−drop n (Text arr off len) = (Text arr newOff newLen)
429− where
482+drop n t@(Text arr off len)
483+ | n <= 0 = t
484+ | n >= len = empty
485+ | otherwise = Text arr newOff newLen
486+ where
430487 (newOff, newLen) = loop off 0 len
431488 end = off + len
432489 loop !i !count !l
@@ -444,12 +501,13 @@ drop n (Text arr off len) = (Text arr newOff newLen)
444501 unstream (S.drop n (stream t)) = drop n t
445502 #-}
446503
447−-- | 'takeWhile', applied to a predicate @p@ and a stream, returns the
448−-- longest prefix (possibly empty) of elements that satisfy p.
504+-- | 'takeWhile', applied to a predicate @p@ and a 'Text', returns the
505+-- longest prefix (possibly empty) of elements that satisfy @p@.
449506takeWhile :: (Char -> Bool) -> Text -> Text
450507takeWhile p t = unstream (S.takeWhile p (stream t))
451508
452−-- | 'dropWhile' @p @xs returns the suffix remaining after 'takeWhile' @p @xs.
509+-- | 'dropWhile' @p@ @xs@ returns the suffix remaining after
510+-- 'takeWhile' @p@ @xs@.
453511dropWhile :: (Char -> Bool) -> Text -> Text
454512dropWhile p t = unstream (S.dropWhile p (stream t))
455513