@@ -11,18 +11,23 @@
1111-- Portability : GHC
1212--
1313-- A time and space-efficient implementation of Unicode text using
14−-- lists of packed arrays. This representation is suitable for high
14+-- lists of packed arrays.
15+--
16+-- /Note/: Read below the synopsis for important notes on the use of
17+-- this module.
18+--
19+-- The representation used by this module is suitable for high
1520-- performance use and for streaming large quantities of data. It
1621-- provides a means to manipulate a large body of text without
1722-- requiring that the entire content be resident in memory.
1823--
1924-- Some operations, such as 'concat', 'append', 'reverse' and 'cons',
20−-- have better complexity than their "Data.Text" equivalents, due to
21−-- optimisations resulting from the list spine structure. And for
22−-- other operations lazy 'Text's are usually within a few percent of
23−-- strict ones, but with better heap usage. For data larger than
24−-- available memory, or if you have tight memory constraints, this
25−-- module will be the only option.
25+-- have better time complexity than their "Data.Text" equivalents, due
26+-- to the underlying representation being a list of chunks. For other
27+-- operations, lazy 'Text's are usually within a few percent of strict
28+-- ones, but often with better heap usage if used in a streaming
29+-- fashion. For data larger than available memory, or if you have
30+-- tight memory constraints, this module will be the only option.
2631--
2732-- This module is intended to be imported @qualified@, to avoid name
2833-- clashes with "Prelude" functions. eg.
@@ -31,7 +36,15 @@
3136
3237module Data.Text.Lazy
3338 (
39+ -- * Fusion
40+ -- $fusion
41+
42+ -- * Acceptable data
43+ -- $replacement
44+
45+ -- * Types
3446 Text
47+
3548 -- * Creation and elimination
3649 , pack
3750 , unpack
@@ -200,10 +213,62 @@ import qualified Data.Text.Lazy.Fusion as S
200213import Data.Text.Fusion.Internal (PairS(..))
201214import Data.Text.Lazy.Fusion (stream, unstream)
202215import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)
203−import Data.Text.Internal (textP)
216+import Data.Text.Internal (firstf, safe, textP)
204217import qualified Data.Text.Util as U
205218import Data.Text.Lazy.Search (indices)
206219
220+-- $fusion
221+--
222+-- Most of the functions in this module are subject to /fusion/,
223+-- meaning that a pipeline of such functions will usually allocate at
224+-- most one 'Text' value.
225+--
226+-- As an example, consider the following pipeline:
227+--
228+-- > import Data.Text.Lazy as T
229+-- > import Data.Text.Lazy.Encoding as E
230+-- > import Data.ByteString.Lazy (ByteString)
231+-- >
232+-- > countChars :: ByteString -> Int
233+-- > countChars = T.length . T.toUpper . E.decodeUtf8
234+--
235+-- From the type signatures involved, this looks like it should
236+-- allocate one 'ByteString' value, and two 'Text' values. However,
237+-- when a module is compiled with optimisation enabled under GHC, the
238+-- two intermediate 'Text' values will be optimised away, and the
239+-- function will be compiled down to a single loop over the source
240+-- 'ByteString'.
241+--
242+-- Functions that can be fused by the compiler are documented with the
243+-- phrase \"Subject to fusion\".
244+
245+-- $replacement
246+--
247+-- A 'Text' value is a sequence of Unicode scalar values, as defined
248+-- in §3.9, definition D76 of the Unicode 5.2 standard:
249+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As
250+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF
251+-- inclusive. Haskell implementations admit all Unicode code points
252+-- (§3.4, definition D10) as 'Char' values, including code points
253+-- from this invalid range. This means that there are some 'Char'
254+-- values that are not valid Unicode scalar values, and the functions
255+-- in this module must handle those cases.
256+--
257+-- Within this module, many functions construct a 'Text' from one or
258+-- more 'Char' values. Those functions will substitute 'Char' values
259+-- that are not valid Unicode scalar values with the replacement
260+-- character \"�\" (U+FFFD). Functions that perform this
261+-- inspection and replacement are documented with the phrase
262+-- \"Performs replacement on invalid scalar values\".
263+--
264+-- (One reason for this policy of replacement is that internally, a
265+-- 'Text' value is represented as packed UTF-16 data. Values in the
266+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate
267+-- code points, and so cannot be represented. The functions replace
268+-- invalid scalar values, instead of dropping them, as a security
269+-- measure. For details, see Unicode Technical Report 36, §3.5:
270+-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)
271+
207272equal :: Text -> Text -> Bool
208273equal Empty Empty = True
209274equal Empty _ = False
@@ -274,19 +339,19 @@ instance Data Text where
274339
275340-- | /O(n)/ Convert a 'String' into a 'Text'.
276341--
277−-- This function is subject to array fusion.
342+-- Subject to fusion. Performs replacement on invalid scalar values.
278343pack :: String -> Text
279−pack s = unstream (S.streamList s)
344+pack = unstream . S.streamList . L.map safe
280345{-# INLINE [1] pack #-}
281346
282347-- | /O(n)/ Convert a 'Text' into a 'String'.
283−-- Subject to array fusion.
348+-- Subject to fusion.
284349unpack :: Text -> String
285350unpack t = S.unstreamList (stream t)
286351{-# INLINE [1] unpack #-}
287352
288−-- | /O(1)/ Convert a character into a Text.
289−-- Subject to fusion.
353+-- | /O(1)/ Convert a character into a Text. Subject to fusion.
354+-- Performs replacement on invalid scalar values.
290355singleton :: Char -> Text
291356singleton c = Chunk (T.singleton c) Empty
292357{-# INLINE [1] singleton #-}
@@ -346,8 +411,7 @@ snoc t c = foldrChunks Chunk (singleton c) t
346411 unstream (S.snoc (stream t) c) = snoc t c
347412 #-}
348413
349−-- | /O(n\/c)/ Appends one 'Text' to another. Subject to array
350−-- fusion.
414+-- | /O(n\/c)/ Appends one 'Text' to another. Subject to fusion.
351415append :: Text -> Text -> Text
352416append xs ys = foldrChunks Chunk ys xs
353417{-# INLINE [1] append #-}
@@ -360,7 +424,7 @@ append xs ys = foldrChunks Chunk ys xs
360424 #-}
361425
362426-- | /O(1)/ Returns the first character and rest of a 'Text', or
363−-- 'Nothing' if empty. Subject to array fusion.
427+-- 'Nothing' if empty. Subject to fusion.
364428uncons :: Text -> Maybe (Char, Text)
365429uncons Empty = Nothing
366430uncons (Chunk t ts) = Just (T.unsafeHead t, ts')
@@ -369,13 +433,13 @@ uncons (Chunk t ts) = Just (T.unsafeHead t, ts')
369433{-# INLINE uncons #-}
370434
371435-- | /O(1)/ Returns the first character of a 'Text', which must be
372−-- non-empty. Subject to array fusion.
436+-- non-empty. Subject to fusion.
373437head :: Text -> Char
374438head t = S.head (stream t)
375439{-# INLINE head #-}
376440
377441-- | /O(1)/ Returns all characters after the head of a 'Text', which
378−-- must be non-empty. Subject to array fusion.
442+-- must be non-empty. Subject to fusion.
379443tail :: Text -> Text
380444tail (Chunk t ts) = chunk (T.tail t) ts
381445tail Empty = emptyError "tail"
@@ -389,7 +453,7 @@ tail Empty = emptyError "tail"
389453 #-}
390454
391455-- | /O(1)/ Returns all but the last character of a 'Text', which must
392−-- be non-empty. Subject to array fusion.
456+-- be non-empty. Subject to fusion.
393457init :: Text -> Text
394458init (Chunk t0 ts0) = go t0 ts0
395459 where go t (Chunk t' ts) = Chunk t (go t' ts)
@@ -404,7 +468,7 @@ init Empty = emptyError "init"
404468 unstream (S.init (stream t)) = init t
405469 #-}
406470
407−-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to array
471+-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to
408472-- fusion.
409473null :: Text -> Bool
410474null Empty = True
@@ -425,7 +489,7 @@ isSingleton = S.isSingleton . stream
425489{-# INLINE isSingleton #-}
426490
427491-- | /O(1)/ Returns the last character of a 'Text', which must be
428−-- non-empty. Subject to array fusion.
492+-- non-empty. Subject to fusion.
429493last :: Text -> Char
430494last Empty = emptyError "last"
431495last (Chunk t ts) = go t ts
@@ -469,9 +533,10 @@ compareLength t n = S.compareLengthI (stream t) n
469533-- properties of code.
470534
471535-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to
472−-- each element of @t@. Subject to array fusion.
536+-- each element of @t@. Subject to fusion. Performs replacement on
537+-- invalid scalar values.
473538map :: (Char -> Char) -> Text -> Text
474−map f t = unstream (S.map f (stream t))
539+map f t = unstream (S.map (safe . f) (stream t))
475540{-# INLINE [1] map #-}
476541
477542-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of
@@ -482,13 +547,17 @@ intercalate t = concat . (U.intersperse t)
482547{-# INLINE intercalate #-}
483548
484549-- | /O(n)/ The 'intersperse' function takes a character and places it
485−-- between the characters of a 'Text'. Subject to array fusion.
486−intersperse :: Char -> Text -> Text
487−intersperse c t = unstream (S.intersperse c (stream t))
550+-- between the characters of a 'Text'. Subject to fusion. Performs
551+-- replacement on invalid scalar values.
552+intersperse :: Char -> Text -> Text
553+intersperse c t = unstream (S.intersperse (safe c) (stream t))
488554{-# INLINE intersperse #-}
489555
490556-- | /O(n)/ Left-justify a string to the given length, using the
491−-- specified fill character on the right. Subject to fusion. Examples:
557+-- specified fill character on the right. Subject to fusion. Performs
558+-- replacement on invalid scalar values.
559+--
560+-- Examples:
492561--
493562-- > justifyLeft 7 'x' "foo" == "fooxxxx"
494563-- > justifyLeft 3 'x' "foobar" == "foobar"
@@ -507,7 +576,10 @@ justifyLeft k c t
507576 #-}
508577
509578-- | /O(n)/ Right-justify a string to the given length, using the
510−-- specified fill character on the left. Examples:
579+-- specified fill character on the left. Performs replacement on
580+-- invalid scalar values.
581+--
582+-- Examples:
511583--
512584-- > justifyRight 7 'x' "bar" == "xxxxbar"
513585-- > justifyRight 3 'x' "foobar" == "foobar"
@@ -518,8 +590,11 @@ justifyRight k c t
518590 where len = length t
519591{-# INLINE justifyRight #-}
520592
521−-- | /O(n)/ Center a string to the given length, using the
522−-- specified fill character on either side. Examples:
593+-- | /O(n)/ Center a string to the given length, using the specified
594+-- fill character on either side. Performs replacement on invalid
595+-- scalar values.
596+--
597+-- Examples:
523598--
524599-- > center 8 'x' "HS" = "xxxHSxxx"
525600center :: Int64 -> Char -> Text -> Text
@@ -608,26 +683,24 @@ toUpper t = unstream (S.toUpper (stream t))
608683-- | /O(n)/ 'foldl', applied to a binary operator, a starting value
609684-- (typically the left-identity of the operator), and a 'Text',
610685-- reduces the 'Text' using the binary operator, from left to right.
611−-- Subject to array fusion.
686+-- Subject to fusion.
612687foldl :: (a -> Char -> a) -> a -> Text -> a
613688foldl f z t = S.foldl f z (stream t)
614689{-# INLINE foldl #-}
615690
616691-- | /O(n)/ A strict version of 'foldl'.
617−-- Subject to array fusion.
692+-- Subject to fusion.
618693foldl' :: (a -> Char -> a) -> a -> Text -> a
619694foldl' f z t = S.foldl' f z (stream t)
620695{-# INLINE foldl' #-}
621696
622697-- | /O(n)/ A variant of 'foldl' that has no starting value argument,
623−-- and thus must be applied to a non-empty 'Text'. Subject to array
624−-- fusion.
698+-- and thus must be applied to a non-empty 'Text'. Subject to fusion.
625699foldl1 :: (Char -> Char -> Char) -> Text -> Char
626700foldl1 f t = S.foldl1 f (stream t)
627701{-# INLINE foldl1 #-}
628702
629−-- | /O(n)/ A strict version of 'foldl1'.
630−-- Subject to array fusion.
703+-- | /O(n)/ A strict version of 'foldl1'. Subject to fusion.
631704foldl1' :: (Char -> Char -> Char) -> Text -> Char
632705foldl1' f t = S.foldl1' f (stream t)
633706{-# INLINE foldl1' #-}
@@ -635,13 +708,13 @@ foldl1' f t = S.foldl1' f (stream t)
635708-- | /O(n)/ 'foldr', applied to a binary operator, a starting value
636709-- (typically the right-identity of the operator), and a 'Text',
637710-- reduces the 'Text' using the binary operator, from right to left.
638−-- Subject to array fusion.
711+-- Subject to fusion.
639712foldr :: (Char -> a -> a) -> a -> Text -> a
640713foldr f z t = S.foldr f z (stream t)
641714{-# INLINE foldr #-}
642715
643−-- | /O(n)/ A variant of 'foldr' that has no starting value argument, and
644−-- thust must be applied to a non-empty 'Text'. Subject to array
716+-- | /O(n)/ A variant of 'foldr' that has no starting value argument,
717+-- and thust must be applied to a non-empty 'Text'. Subject to
645718-- fusion.
646719foldr1 :: (Char -> Char -> Char) -> Text -> Char
647720foldr1 f t = S.foldr1 f (stream t)
@@ -664,32 +737,32 @@ concatMap f = concat . foldr ((:) . f) []
664737{-# INLINE concatMap #-}
665738
666739-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
667−-- 'Text' @t@ satisifes the predicate @p@. Subject to array fusion.
740+-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion.
668741any :: (Char -> Bool) -> Text -> Bool
669742any p t = S.any p (stream t)
670743{-# INLINE any #-}
671744
672745-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
673−-- 'Text' @t@ satisify the predicate @p@. Subject to array fusion.
746+-- 'Text' @t@ satisify the predicate @p@. Subject to fusion.
674747all :: (Char -> Bool) -> Text -> Bool
675748all p t = S.all p (stream t)
676749{-# INLINE all #-}
677750
678751-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
679−-- must be non-empty. Subject to array fusion.
752+-- must be non-empty. Subject to fusion.
680753maximum :: Text -> Char
681754maximum t = S.maximum (stream t)
682755{-# INLINE maximum #-}
683756
684757-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
685−-- must be non-empty. Subject to array fusion.
758+-- must be non-empty. Subject to fusion.
686759minimum :: Text -> Char
687760minimum t = S.minimum (stream t)
688761{-# INLINE minimum #-}
689762
690763-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of
691−-- successive reduced values from the left. This function is subject
692−-- to array fusion.
764+-- successive reduced values from the left. Subject to fusion.
765+-- Performs replacement on invalid scalar values.
693766--
694767-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]
695768--
@@ -697,11 +770,13 @@ minimum t = S.minimum (stream t)
697770--
698771-- > last (scanl f z xs) == foldl f z xs.
699772scanl :: (Char -> Char -> Char) -> Char -> Text -> Text
700−scanl f z t = unstream (S.scanl f z (stream t))
773+scanl f z t = unstream (S.scanl g z (stream t))
774+ where g a b = safe (f a b)
701775{-# INLINE scanl #-}
702776
703777-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting
704−-- value argument. This function is subject to array fusion.
778+-- value argument. Subject to fusion. Performs replacement on
779+-- invalid scalar values.
705780--
706781-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]
707782scanl1 :: (Char -> Char -> Char) -> Text -> Text
@@ -710,21 +785,24 @@ scanl1 f t0 = case uncons t0 of
710785 Just (t,ts) -> scanl f t ts
711786{-# INLINE scanl1 #-}
712787
713−-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.
788+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs
789+-- replacement on invalid scalar values.
714790--
715791-- > scanr f v == reverse . scanl (flip f) v . reverse
716792scanr :: (Char -> Char -> Char) -> Char -> Text -> Text
717−scanr f v = reverse . scanl (flip f) v . reverse
793+scanr f v = reverse . scanl g v . reverse
794+ where g a b = safe (f b a)
718795
719796-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting
720−-- value argument.
797+-- value argument. Performs replacement on invalid scalar values.
721798scanr1 :: (Char -> Char -> Char) -> Text -> Text
722799scanr1 f t | null t = empty
723800 | otherwise = scanr f (last t) (init t)
724801
725802-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a
726803-- function to each element of a 'Text', passing an accumulating
727−-- parameter from left to right, and returns a final 'Text'.
804+-- parameter from left to right, and returns a final 'Text'. Performs
805+-- replacement on invalid scalar values.
728806mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
729807mapAccumL f = go
730808 where
@@ -738,7 +816,7 @@ mapAccumL f = go
738816-- a strict 'foldr'; it applies a function to each element of a
739817-- 'Text', passing an accumulating parameter from right to left, and
740818-- returning a final value of this accumulator together with the new
741−-- 'Text'.
819+-- 'Text'. Performs replacement on invalid scalar values.
742820mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
743821mapAccumR f = go
744822 where
@@ -762,7 +840,7 @@ replicate n t
762840-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the
763841-- value of every element. Subject to fusion.
764842replicateChar :: Int64 -> Char -> Text
765−replicateChar n c = unstream (S.replicateCharI n c)
843+replicateChar n c = unstream (S.replicateCharI n (safe c))
766844{-# INLINE replicateChar #-}
767845
768846{-# RULES
@@ -775,9 +853,10 @@ replicateChar n c = unstream (S.replicateCharI n c)
775853-- 'Text' from a seed value. The function takes the element and
776854-- returns 'Nothing' if it is done producing the 'Text', otherwise
777855-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the
778−-- string, and @b@ is the seed value for further production.
779−unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
780−unfoldr f s = unstream (S.unfoldr f s)
856+-- string, and @b@ is the seed value for further production. Performs
857+-- replacement on invalid scalar values.
858+unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
859+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)
781860{-# INLINE unfoldr #-}
782861
783862-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
@@ -785,8 +864,9 @@ unfoldr f s = unstream (S.unfoldr f s)
785864-- first argument to 'unfoldrN'. This function is more efficient than
786865-- 'unfoldr' when the maximum length of the result is known and
787866-- correct, otherwise its performance is similar to 'unfoldr'.
788−unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text
789−unfoldrN n f s = unstream (S.unfoldrN n f s)
867+-- Performs replacement on invalid scalar values.
868+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text
869+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)
790870{-# INLINE unfoldrN #-}
791871
792872-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the
@@ -847,9 +927,9 @@ dropWords i t0
847927 where len' = fromIntegral len
848928 n' = fromIntegral n
849929
850−-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', returns
851−-- the longest prefix (possibly empty) of elements that satisfy @p@.
852−-- This function is subject to array fusion.
930+-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',
931+-- returns the longest prefix (possibly empty) of elements that
932+-- satisfy @p@. Subject to fusion.
853933takeWhile :: (Char -> Bool) -> Text -> Text
854934takeWhile p t0 = takeWhile' t0
855935 where takeWhile' Empty = Empty
@@ -868,7 +948,7 @@ takeWhile p t0 = takeWhile' t0
868948 #-}
869949
870950-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
871−-- 'takeWhile' @p@ @t@. This function is subject to array fusion.
951+-- 'takeWhile' @p@ @t@. Subject to fusion.
872952dropWhile :: (Char -> Bool) -> Text -> Text
873953dropWhile p t0 = dropWhile' t0
874954 where dropWhile' Empty = Empty
@@ -1193,8 +1273,7 @@ unwords = intercalate (singleton ' ')
11931273{-# INLINE unwords #-}
11941274
11951275-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns
1196−-- 'True' iff the first is a prefix of the second. This function is
1197−-- subject to fusion.
1276+-- 'True' iff the first is a prefix of the second. Subject to fusion.
11981277isPrefixOf :: Text -> Text -> Bool
11991278isPrefixOf Empty _ = True
12001279isPrefixOf _ Empty = False
@@ -1340,8 +1419,7 @@ count pat src
13401419 #-}
13411420
13421421-- | /O(n)/ The 'countChar' function returns the number of times the
1343−-- query element appears in the given 'Text'. This function is subject
1344−-- to fusion.
1422+-- query element appears in the given 'Text'. Subject to fusion.
13451423countChar :: Char -> Text -> Int64
13461424countChar c t = S.countChar c (stream t)
13471425
@@ -1355,8 +1433,10 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)
13551433
13561434-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function
13571435-- given as the first argument, instead of a tupling function.
1436+-- Performs replacement on invalid scalar values.
13581437zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text
1359−zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))
1438+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))
1439+ where g a b = safe (f a b)
13601440{-# INLINE [0] zipWith #-}
13611441
13621442revChunks :: [T.Text] -> Text