@@ -1,15 +1,45 @@
1{-# LANGUAGE BangPatterns #-}
1{-# LANGUAGE BangPatterns #-}
2{-# OPTIONS_GHC -fno-warn-orphans #-}
2{-# OPTIONS_GHC -fno-warn-orphans #-}
6+-- Copyright : (c) Tom Harper 2008-2009,
7+-- (c) Bryan O'Sullivan 2009,
8+-- (c) Duncan Coutts 2009
10+-- License : BSD-style
11+-- Maintainer : rtharper@aftereternity.co.uk, bos@serpentine.com,
13+-- Stability : experimental
16+-- A time and space-efficient implementation of Unicode text using
17+-- packed Word16 arrays. Suitable for performance critical use, both
18+-- in terms of large data quantities and high speed.
20+-- This module is intended to be imported @qualified@, to avoid name
21+-- clashes with "Prelude" functions, e.g.
23+-- > import qualified Data.Text as T
4module Data.Text
25module Data.Text
7 , Encoding(..)
32 , Encoding(..)
34+ -- * Creation and elimination
10 , singleton
37 , singleton
@@ -19,35 +49,57 @@ module Data.Text
23 , intersperse
55 , intersperse
24 , transpose
56 , transpose
32 , concatMap
68 , concatMap
38 , unfoldrN
76 , unfoldrN
80+ -- ** Breaking strings
41 , takeWhile
83 , takeWhile
42 , dropWhile
84 , dropWhile
86+ -- ** Breaking into lines and words
47 , findIndex
96 , findIndex
48 , elemIndex
97 , elemIndex
99+ -- * Zipping and unzipping
49 , zipWith
100 , zipWith
51 , readFile
103 , readFile
@@ -58,7 +110,6 @@ import Prelude (Char,Bool,Int,Maybe,String,
58 (&&),(||),(+),(-),(<),(>),(<=),(>=),(.),(>>=),
110 (&&),(||),(+),(-),(<),(>),(<=),(>=),(.),(>>=),
59 return,otherwise,
111 return,otherwise,
60 IO, FilePath)
112 IO, FilePath)
62import Data.Char (isSpace)
113import Data.Char (isSpace)
63import Control.Monad.ST(ST)
114import Control.Monad.ST(ST)
64import Data.Array.Base(unsafeNewArray_,unsafeWrite,unsafeAt)
115import Data.Array.Base(unsafeNewArray_,unsafeWrite,unsafeAt)
@@ -78,6 +129,12 @@ import qualified Prelude as P
78import Data.Text.UnsafeChar(unsafeChr)
129import Data.Text.UnsafeChar(unsafeChr)
79import qualified Data.Text.Utf16 as U16
130import qualified Data.Text.Utf16 as U16
134+-- Most of the functions in this module are subject to /array fusion/,
135+-- meaning that a pipeline of functions will usually allocate at most
81instance Eq Text where
138instance Eq Text where
82 t1 == t2 = (stream t1) `S.eq` (stream t2)
139 t1 == t2 = (stream t1) `S.eq` (stream t2)
@@ -98,11 +155,9 @@ instance IsString Text where
98-- -----------------------------------------------------------------------------
155-- -----------------------------------------------------------------------------
99-- * Conversion to/from 'Text'
156-- * Conversion to/from 'Text'
101−-- | /O(n)/ Convert a String into a Text.
158+-- | /O(n)/ Convert a 'String' into a 'Text'.
103−-- This function is subject to array fusion, so calling other fusible
160+-- This function is subject to array fusion.
104−-- function(s) on a packed string will only cause one 'Text' to be written
105−-- out at the end of the pipeline, instead of one before and one after.
106pack :: String -> Text
161pack :: String -> Text
107pack str = (unstream (stream_list str))
162pack str = (unstream (stream_list str))
@@ -126,7 +181,7 @@ unpack txt = (unstream_list (stream txt))
126 S.Yield x s' -> x : unfold s'
181 S.Yield x s' -> x : unfold s'
127{-# INLINE [1] unpack #-}
182{-# INLINE [1] unpack #-}
129−-- | Convert a character into a Text.
184+-- | /O(1)/ Convert a character into a Text.
130-- Subject to array fusion.
185-- Subject to array fusion.
131singleton :: Char -> Text
186singleton :: Char -> Text
132singleton c = unstream (Stream next (c:[]) 1)
187singleton c = unstream (Stream next (c:[]) 1)
@@ -147,23 +202,21 @@ encode enc txt = unstream_bs (restream enc (stream txt))
147-- -----------------------------------------------------------------------------
202-- -----------------------------------------------------------------------------
148-- * Basic functions
203-- * Basic functions
150−-- | /O(n)/ Adds a character to the front of a 'Text'. This function is more
205+-- | /O(n)/ Adds a character to the front of a 'Text'. This function
151−-- costly than its 'List' counterpart because it requires copying a new array.
206+-- is more costly than its 'List' counterpart because it requires
152−-- Subject to array fusion.
207+-- copying a new array. Subject to array fusion.
153cons :: Char -> Text -> Text
208cons :: Char -> Text -> Text
154cons c t = unstream (S.cons c (stream t))
209cons c t = unstream (S.cons c (stream t))
155{-# INLINE cons #-}
210{-# INLINE cons #-}
157−-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the entire
212+-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the
158−-- array in the process.
159−-- Subject to array fusion.
213+-- entire array in the process. Subject to array fusion.
160snoc :: Text -> Char -> Text
214snoc :: Text -> Char -> Text
161snoc t c = unstream (S.snoc (stream t) c)
215snoc t c = unstream (S.snoc (stream t) c)
162{-# INLINE snoc #-}
216{-# INLINE snoc #-}
164−-- | /O(n)/ Appends one Text to the other by copying both of them into a new
218+-- | /O(n)/ Appends one 'Text' to the other by copying both of them
166−-- Subject to array fusion
219+-- into a new 'Text'. Subject to array fusion.
167append :: Text -> Text -> Text
220append :: Text -> Text -> Text
168append (Text arr1 off1 len1) (Text arr2 off2 len2) = Text (runSTUArray x) 0 len
221append (Text arr1 off1 len1) (Text arr2 off2 len2) = Text (runSTUArray x) 0 len
@@ -187,14 +240,14 @@ append (Text arr1 off1 len1) (Text arr2 off2 len2) = Text (runSTUArray x) 0 len
187 unstream (S.append (stream t1) (stream t2)) = append t1 t2
240 unstream (S.append (stream t1) (stream t2)) = append t1 t2
190−-- | /O(1)/ Returns the first character of a Text, which must be non-empty.
243+-- | /O(1)/ Returns the first character of a 'Text', which must be
191−-- Subject to array fusion.
244+-- non-empty. Subject to array fusion.
192head :: Text -> Char
245head :: Text -> Char
193head t = S.head (stream t)
246head t = S.head (stream t)
194{-# INLINE head #-}
247{-# INLINE head #-}
196−-- | /O(n)/ Returns the last character of a Text, which must be non-empty.
249+-- | /O(n)/ Returns the last character of a 'Text', which must be
197−-- Subject to array fusion.
250+-- non-empty. Subject to array fusion.
198last :: Text -> Char
251last :: Text -> Char
199last (Text arr off len)
252last (Text arr off len)
200 | len <= 0 = errorEmptyList "last"
253 | len <= 0 = errorEmptyList "last"
@@ -213,9 +266,8 @@ last (Text arr off len)
216−-- | /O(1)/ Returns all characters after the head of a Text, which must
269+-- | /O(1)/ Returns all characters after the head of a 'Text', which
218−-- Subject to array fusion.
270+-- must be non-empty. Subject to array fusion.
219tail :: Text -> Text
271tail :: Text -> Text
220tail (Text arr off len)
272tail (Text arr off len)
221 | len <= 0 = errorEmptyList "tail"
273 | len <= 0 = errorEmptyList "tail"
@@ -227,9 +279,8 @@ tail (Text arr off len)
230−-- | /O(1)/ Returns all but the last character of a Text, which
282+-- | /O(1)/ Returns all but the last character of a 'Text', which must
231−-- must be non-empty.
232−-- Subject to array fusion.
283+-- be non-empty. Subject to array fusion.
233init :: Text -> Text
284init :: Text -> Text
234init (Text arr off len) | len <= 0 = errorEmptyList "init"
285init (Text arr off len) | len <= 0 = errorEmptyList "init"
235 | n >= 0xDC00 && n <= 0xDFFF = Text arr off (len-2)
286 | n >= 0xDC00 && n <= 0xDFFF = Text arr off (len-2)
@@ -245,13 +296,13 @@ init (Text arr off len) | len <= 0 = errorEmptyList "init"
245 unstream (S.init (stream t)) = init t
296 unstream (S.init (stream t)) = init t
248−-- | /O(1)/ Tests whether a Text is empty or not.
299+-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to array
249−-- Subject to array fusion.
300+-- fusion.
250null :: Text -> Bool
301null :: Text -> Bool
251null t = S.null (stream t)
302null t = S.null (stream t)
252{-# INLINE null #-}
303{-# INLINE null #-}
254−-- | /O(n)/ Returns the number of characters in a text.
305+-- | /O(n)/ Returns the number of characters in a 'Text'.
255-- Subject to array fusion.
306-- Subject to array fusion.
256length :: Text -> Int
307length :: Text -> Int
257length (Text _arr _off len) = len
308length (Text _arr _off len) = len
@@ -266,32 +317,31 @@ length (Text _arr _off len) = len
267-- -----------------------------------------------------------------------------
318-- -----------------------------------------------------------------------------
268-- * Transformations
319-- * Transformations
269−-- | /O(n)/ 'map' @f @xs is the Text obtained by applying @f@ to each
320+-- | /O(n)/ 'map' @f @xs is the 'Text' obtained by applying @f@ to
271−-- Subject to array fusion.
321+-- each element of @xs@. Subject to array fusion.
272map :: (Char -> Char) -> Text -> Text
322map :: (Char -> Char) -> Text -> Text
273map f t = unstream (S.map f (stream t))
323map f t = unstream (S.map f (stream t))
274{-# INLINE [1] map #-}
324{-# INLINE [1] map #-}
276−-- | /O(n)/ The 'intersperse' function takes a character and places it between
326+-- | /O(n)/ The 'intersperse' function takes a character and places it
277−-- the characters of a Text.
327+-- between the characters of a 'Text'. Subject to array fusion.
278−-- Subject to array fusion.
279intersperse :: Char -> Text -> Text
328intersperse :: Char -> Text -> Text
280intersperse c t = unstream (S.intersperse c (stream t))
329intersperse c t = unstream (S.intersperse c (stream t))
281{-# INLINE intersperse #-}
330{-# INLINE intersperse #-}
283−-- | /O(n)/ The 'transpose' function transposes the rows and columns of its
332+-- | /O(n)/ The 'transpose' function transposes the rows and columns
284−-- Text argument. Note that this function uses pack, unpack, and the 'List'
333+-- of its 'Text' argument. Note that this function uses 'pack',
285−-- version of transpose and is thus not very efficient.
334+-- 'unpack', and the 'List' version of transpose and is thus not very
286transpose :: [Text] -> [Text]
336transpose :: [Text] -> [Text]
287transpose ts = P.map pack (L.transpose (P.map unpack ts))
337transpose ts = P.map pack (L.transpose (P.map unpack ts))
289-- -----------------------------------------------------------------------------
339-- -----------------------------------------------------------------------------
290-- * Reducing 'Text's (folds)
340-- * Reducing 'Text's (folds)
292−-- | 'foldl', applied to a binary operator, a starting value (typically the
342+-- | 'foldl', applied to a binary operator, a starting value
293−-- left-identity of the operator), and a Text, reduces the Text using the
343+-- (typically the left-identity of the operator), and a 'Text',
294−-- binary operator, from left to right.
344+-- reduces the 'Text' using the binary operator, from left to right.
295-- Subject to array fusion.
345-- Subject to array fusion.
296foldl :: (b -> Char -> b) -> b -> Text -> b
346foldl :: (b -> Char -> b) -> b -> Text -> b
297foldl f z t = S.foldl f z (stream t)
347foldl f z t = S.foldl f z (stream t)
@@ -303,9 +353,9 @@ foldl' :: (b -> Char -> b) -> b -> Text -> b
303foldl' f z t = S.foldl' f z (stream t)
353foldl' f z t = S.foldl' f z (stream t)
304{-# INLINE foldl' #-}
354{-# INLINE foldl' #-}
306−-- | 'foldl1' is a variant of 'foldl' that has no starting value argument,
356+-- | A variant of 'foldl' that has no starting value argument, and
307−-- and thus must be applied to non-empty 'Text's.
357+-- thus must be applied to a non-empty 'Text'. Subject to array
308−-- Subject to array fusion.
358+-- fusion.
309foldl1 :: (Char -> Char -> Char) -> Text -> Char
359foldl1 :: (Char -> Char -> Char) -> Text -> Char
310foldl1 f t = S.foldl1 f (stream t)
360foldl1 f t = S.foldl1 f (stream t)
311{-# INLINE foldl1 #-}
361{-# INLINE foldl1 #-}
@@ -316,17 +366,17 @@ foldl1' :: (Char -> Char -> Char) -> Text -> Char
316foldl1' f t = S.foldl1' f (stream t)
366foldl1' f t = S.foldl1' f (stream t)
317{-# INLINE foldl1' #-}
367{-# INLINE foldl1' #-}
319−-- | 'foldr', applied to a binary operator, a starting value (typically the
369+-- | 'foldr', applied to a binary operator, a starting value
320−-- right-identity of the operator), and a Text, reduces the Text using the
370+-- (typically the right-identity of the operator), and a 'Text',
321−-- binary operator, from right to left.
371+-- reduces the 'Text' using the binary operator, from right to left.
322-- Subject to array fusion.
372-- Subject to array fusion.
323foldr :: (Char -> b -> b) -> b -> Text -> b
373foldr :: (Char -> b -> b) -> b -> Text -> b
324foldr f z t = S.foldr f z (stream t)
374foldr f z t = S.foldr f z (stream t)
325{-# INLINE foldr #-}
375{-# INLINE foldr #-}
327−-- | 'foldr1' is a variant of 'foldr' that has no starting value argument,
377+-- | A variant of 'foldr' that has no starting value argument, and
328−-- and thust must be applied to non-empty 'Text's.
378+-- thust must be applied to a non-empty 'Text'. Subject to array
329−-- Subject to array fusion.
379+-- fusion.
330foldr1 :: (Char -> Char -> Char) -> Text -> Char
380foldr1 :: (Char -> Char -> Char) -> Text -> Char
331foldr1 f t = S.foldr1 f (stream t)
381foldr1 f t = S.foldr1 f (stream t)
332{-# INLINE foldr1 #-}
382{-# INLINE foldr1 #-}
@@ -339,34 +389,35 @@ concat :: [Text] -> Text
339concat ts = unstream (S.concat (L.map stream ts))
389concat ts = unstream (S.concat (L.map stream ts))
340{-# INLINE concat #-}
390{-# INLINE concat #-}
342−-- | Map a function over a Text that results in a Text and concatenate the
392+-- | Map a function over a 'Text' that results in a 'Text', and concatenate the
343−-- results. This function is subject to array fusion, and note that if in
393+-- results. This function is subject to array fusion.
344−-- 'concatMap' @f @xs, @f@ is defined in terms of fusible functions it will
395+-- Note: if in 'concatMap' @f @xs, @f@ is defined in terms of fusible
345−-- also be fusible.
396+-- functions, it will also be fusible.
346concatMap :: (Char -> Text) -> Text -> Text
397concatMap :: (Char -> Text) -> Text -> Text
347concatMap f t = unstream (S.concatMap (stream . f) (stream t))
398concatMap f t = unstream (S.concatMap (stream . f) (stream t))
348{-# INLINE concatMap #-}
399{-# INLINE concatMap #-}
350−-- | 'any' @p @xs determines if any character in the 'Text' @xs@ satisifes the
401+-- | 'any' @p @xs determines whether any character in the 'Text' @xs@
351−-- predicate @p@. Subject to array fusion.
402+-- satisifes the predicate @p@. Subject to array fusion.
352any :: (Char -> Bool) -> Text -> Bool
403any :: (Char -> Bool) -> Text -> Bool
353any p t = S.any p (stream t)
404any p t = S.any p (stream t)
354{-# INLINE any #-}
405{-# INLINE any #-}
356−-- | 'all' @p @xs determines if all characters in the 'Text' @xs@ satisify the
407+-- | 'all' @p @xs determines whether all characters in the 'Text' @xs@
357−-- predicate @p@. Subject to array fusion.
408+-- satisify the predicate @p@. Subject to array fusion.
358all :: (Char -> Bool) -> Text -> Bool
409all :: (Char -> Bool) -> Text -> Bool
359all p t = S.all p (stream t)
410all p t = S.all p (stream t)
360{-# INLINE all #-}
411{-# INLINE all #-}
362−-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which must be
413+-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
363−-- non-empty. Subject to array fusion.
414+-- must be non-empty. Subject to array fusion.
364maximum :: Text -> Char
415maximum :: Text -> Char
365maximum t = S.maximum (stream t)
416maximum t = S.maximum (stream t)
366{-# INLINE maximum #-}
417{-# INLINE maximum #-}
368−-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which must be
419+-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
369−-- non-empty. Subject to array fusion.
420+-- must be non-empty. Subject to array fusion.
370minimum :: Text -> Char
421minimum :: Text -> Char
371minimum t = S.minimum (stream t)
422minimum t = S.minimum (stream t)
372{-# INLINE minimum #-}
423{-# INLINE minimum #-}
@@ -377,21 +428,21 @@ minimum t = S.minimum (stream t)
377-- -----------------------------------------------------------------------------
428-- -----------------------------------------------------------------------------
378-- ** Generating and unfolding 'Text's
429-- ** Generating and unfolding 'Text's
380−-- /O(n)/, where @n@ is the length of the result. The unfoldr function
431+-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'
381−-- is analogous to the List 'unfoldr'. unfoldr builds a Text
432+-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a
382−-- from a seed value. The function takes the element and returns
433+-- 'Text' from a seed value. The function takes the element and
383−-- Nothing if it is done producing the Text or returns Just
434+-- returns 'Nothing' if it is done producing the 'Text', otherwise
384−-- (a,b), in which case, a is the next Char in the string, and b is
435+-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the
385−-- the seed value for further production.
436+-- string, and @b@ is the seed value for further production.
386unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
437unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
387unfoldr f s = unstream (S.unfoldr f s)
438unfoldr f s = unstream (S.unfoldr f s)
388{-# INLINE unfoldr #-}
439{-# INLINE unfoldr #-}
390−-- O(n) Like unfoldr, unfoldrN builds a Text from a seed
441+-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
391-- value. However, the length of the result should be limited by the
442-- value. However, the length of the result should be limited by the
392−-- first argument to unfoldrN. This function is more efficient than
443+-- first argument to 'unfoldrN'. This function is more efficient than
393−-- unfoldr when the maximum length of the result and correct,
444+-- 'unfoldr' when the maximum length of the result is known and
394−-- otherwise its complexity performance is similar to 'unfoldr'
445+-- correct, otherwise its performance is similar to 'unfoldr'.
395unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text
446unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text
396unfoldrN n f s = unstream (S.unfoldrN n f s)
447unfoldrN n f s = unstream (S.unfoldrN n f s)
397{-# INLINE unfoldrN #-}
448{-# INLINE unfoldrN #-}
@@ -399,12 +450,15 @@ unfoldrN n f s = unstream (S.unfoldrN n f s)
399-- -----------------------------------------------------------------------------
450-- -----------------------------------------------------------------------------
400-- * Substrings
451-- * Substrings
402−-- O(n) 'take' @n, applied to a Text, returns the prefix of the
453+-- /O(n) 'take' @n@, applied to a 'Text', returns the prefix of the
403−-- Text of length n, or the Text itself if n is greater than the
454+-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than
404−-- length of the Text.
455+-- the length of the Text.
405take :: Int -> Text -> Text
456take :: Int -> Text -> Text
406−take n (Text arr off len) = Text arr off (loop off 0)
457+take n t@(Text arr off len)
460+ | otherwise = Text arr off (loop off 0)
408 end = off+len
462 end = off+len
409 loop !i !count
463 loop !i !count
410 | i >= end || count >= n = i - off
464 | i >= end || count >= n = i - off
@@ -421,12 +475,15 @@ take n (Text arr off len) = Text arr off (loop off 0)
421 unstream (S.take n (stream t)) = take n t
475 unstream (S.take n (stream t)) = take n t
424−-- /O(n)/ 'drop' @n, applied to a Text, returns the suffix of the
478+-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the
425−-- Text of length @n, or the empty Text if @n is greater than the
479+-- 'Text' of length @n@, or the empty 'Text' if @n@ is greater than the
426−-- length of the Text.
480+-- length of the 'Text'.
427drop :: Int -> Text -> Text
481drop :: Int -> Text -> Text
428−drop n (Text arr off len) = (Text arr newOff newLen)
482+drop n t@(Text arr off len)
485+ | otherwise = Text arr newOff newLen
430 (newOff, newLen) = loop off 0 len
487 (newOff, newLen) = loop off 0 len
431 end = off + len
488 end = off + len
432 loop !i !count !l
489 loop !i !count !l
@@ -444,12 +501,13 @@ drop n (Text arr off len) = (Text arr newOff newLen)
444 unstream (S.drop n (stream t)) = drop n t
501 unstream (S.drop n (stream t)) = drop n t
447−-- | 'takeWhile', applied to a predicate @p@ and a stream, returns the
504+-- | 'takeWhile', applied to a predicate @p@ and a 'Text', returns the
448−-- longest prefix (possibly empty) of elements that satisfy p.
505+-- longest prefix (possibly empty) of elements that satisfy @p@.
449takeWhile :: (Char -> Bool) -> Text -> Text
506takeWhile :: (Char -> Bool) -> Text -> Text
450takeWhile p t = unstream (S.takeWhile p (stream t))
507takeWhile p t = unstream (S.takeWhile p (stream t))
452−-- | 'dropWhile' @p @xs returns the suffix remaining after 'takeWhile' @p @xs.
509+-- | 'dropWhile' @p@ @xs@ returns the suffix remaining after
510+-- 'takeWhile' @p@ @xs@.
453dropWhile :: (Char -> Bool) -> Text -> Text
511dropWhile :: (Char -> Bool) -> Text -> Text
454dropWhile p t = unstream (S.dropWhile p (stream t))
512dropWhile p t = unstream (S.dropWhile p (stream t))