@@ -11,18 +11,23 @@
11-- Portability : GHC
11-- Portability : GHC
13-- A time and space-efficient implementation of Unicode text using
13-- A time and space-efficient implementation of Unicode text using
14+-- lists of packed arrays.
16+-- /Note/: Read below the synopsis for important notes on the use of
14−-- lists of packed arrays. This representation is suitable for high
19+-- The representation used by this module is suitable for high
15-- performance use and for streaming large quantities of data. It
20-- performance use and for streaming large quantities of data. It
16-- provides a means to manipulate a large body of text without
21-- provides a means to manipulate a large body of text without
17-- requiring that the entire content be resident in memory.
22-- requiring that the entire content be resident in memory.
19-- Some operations, such as 'concat', 'append', 'reverse' and 'cons',
24-- Some operations, such as 'concat', 'append', 'reverse' and 'cons',
20−-- have better complexity than their "Data.Text" equivalents, due to
25+-- have better time complexity than their "Data.Text" equivalents, due
21−-- optimisations resulting from the list spine structure. And for
26+-- to the underlying representation being a list of chunks. For other
22−-- other operations lazy 'Text's are usually within a few percent of
27+-- operations, lazy 'Text's are usually within a few percent of strict
23−-- strict ones, but with better heap usage. For data larger than
28+-- ones, but often with better heap usage if used in a streaming
24−-- available memory, or if you have tight memory constraints, this
29+-- fashion. For data larger than available memory, or if you have
25−-- module will be the only option.
30+-- tight memory constraints, this module will be the only option.
27-- This module is intended to be imported @qualified@, to avoid name
32-- This module is intended to be imported @qualified@, to avoid name
28-- clashes with "Prelude" functions. eg.
33-- clashes with "Prelude" functions. eg.
@@ -31,7 +36,15 @@
32module Data.Text.Lazy
37module Data.Text.Lazy
35 -- * Creation and elimination
48 -- * Creation and elimination
@@ -200,10 +213,62 @@ import qualified Data.Text.Lazy.Fusion as S
200import Data.Text.Fusion.Internal (PairS(..))
213import Data.Text.Fusion.Internal (PairS(..))
201import Data.Text.Lazy.Fusion (stream, unstream)
214import Data.Text.Lazy.Fusion (stream, unstream)
202import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)
215import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)
203−import Data.Text.Internal (textP)
216+import Data.Text.Internal (firstf, safe, textP)
204import qualified Data.Text.Util as U
217import qualified Data.Text.Util as U
205import Data.Text.Lazy.Search (indices)
218import Data.Text.Lazy.Search (indices)
222+-- Most of the functions in this module are subject to /fusion/,
223+-- meaning that a pipeline of such functions will usually allocate at
224+-- most one 'Text' value.
226+-- As an example, consider the following pipeline:
228+-- > import Data.Text.Lazy as T
229+-- > import Data.Text.Lazy.Encoding as E
230+-- > import Data.ByteString.Lazy (ByteString)
232+-- > countChars :: ByteString -> Int
233+-- > countChars = T.length . T.toUpper . E.decodeUtf8
235+-- From the type signatures involved, this looks like it should
236+-- allocate one 'ByteString' value, and two 'Text' values. However,
237+-- when a module is compiled with optimisation enabled under GHC, the
238+-- two intermediate 'Text' values will be optimised away, and the
239+-- function will be compiled down to a single loop over the source
242+-- Functions that can be fused by the compiler are documented with the
243+-- phrase \"Subject to fusion\".
247+-- A 'Text' value is a sequence of Unicode scalar values, as defined
248+-- in §3.9, definition D76 of the Unicode 5.2 standard:
249+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As
250+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF
251+-- inclusive. Haskell implementations admit all Unicode code points
252+-- (§3.4, definition D10) as 'Char' values, including code points
253+-- from this invalid range. This means that there are some 'Char'
254+-- values that are not valid Unicode scalar values, and the functions
255+-- in this module must handle those cases.
257+-- Within this module, many functions construct a 'Text' from one or
258+-- more 'Char' values. Those functions will substitute 'Char' values
259+-- that are not valid Unicode scalar values with the replacement
260+-- character \"�\" (U+FFFD). Functions that perform this
261+-- inspection and replacement are documented with the phrase
262+-- \"Performs replacement on invalid scalar values\".
264+-- (One reason for this policy of replacement is that internally, a
265+-- 'Text' value is represented as packed UTF-16 data. Values in the
266+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate
267+-- code points, and so cannot be represented. The functions replace
268+-- invalid scalar values, instead of dropping them, as a security
269+-- measure. For details, see Unicode Technical Report 36, §3.5:
270+-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)
207equal :: Text -> Text -> Bool
272equal :: Text -> Text -> Bool
208equal Empty Empty = True
273equal Empty Empty = True
209equal Empty _ = False
274equal Empty _ = False
@@ -274,19 +339,19 @@ instance Data Text where
275-- | /O(n)/ Convert a 'String' into a 'Text'.
340-- | /O(n)/ Convert a 'String' into a 'Text'.
277−-- This function is subject to array fusion.
342+-- Subject to fusion. Performs replacement on invalid scalar values.
278pack :: String -> Text
343pack :: String -> Text
279−pack s = unstream (S.streamList s)
344+pack = unstream . S.streamList . L.map safe
280{-# INLINE [1] pack #-}
345{-# INLINE [1] pack #-}
282-- | /O(n)/ Convert a 'Text' into a 'String'.
347-- | /O(n)/ Convert a 'Text' into a 'String'.
283−-- Subject to array fusion.
348+-- Subject to fusion.
284unpack :: Text -> String
349unpack :: Text -> String
285unpack t = S.unstreamList (stream t)
350unpack t = S.unstreamList (stream t)
286{-# INLINE [1] unpack #-}
351{-# INLINE [1] unpack #-}
288−-- | /O(1)/ Convert a character into a Text.
353+-- | /O(1)/ Convert a character into a Text. Subject to fusion.
289−-- Subject to fusion.
354+-- Performs replacement on invalid scalar values.
290singleton :: Char -> Text
355singleton :: Char -> Text
291singleton c = Chunk (T.singleton c) Empty
356singleton c = Chunk (T.singleton c) Empty
292{-# INLINE [1] singleton #-}
357{-# INLINE [1] singleton #-}
@@ -346,8 +411,7 @@ snoc t c = foldrChunks Chunk (singleton c) t
346 unstream (S.snoc (stream t) c) = snoc t c
411 unstream (S.snoc (stream t) c) = snoc t c
349−-- | /O(n\/c)/ Appends one 'Text' to another. Subject to array
414+-- | /O(n\/c)/ Appends one 'Text' to another. Subject to fusion.
351append :: Text -> Text -> Text
415append :: Text -> Text -> Text
352append xs ys = foldrChunks Chunk ys xs
416append xs ys = foldrChunks Chunk ys xs
353{-# INLINE [1] append #-}
417{-# INLINE [1] append #-}
@@ -360,7 +424,7 @@ append xs ys = foldrChunks Chunk ys xs
362-- | /O(1)/ Returns the first character and rest of a 'Text', or
426-- | /O(1)/ Returns the first character and rest of a 'Text', or
363−-- 'Nothing' if empty. Subject to array fusion.
427+-- 'Nothing' if empty. Subject to fusion.
364uncons :: Text -> Maybe (Char, Text)
428uncons :: Text -> Maybe (Char, Text)
365uncons Empty = Nothing
429uncons Empty = Nothing
366uncons (Chunk t ts) = Just (T.unsafeHead t, ts')
430uncons (Chunk t ts) = Just (T.unsafeHead t, ts')
@@ -369,13 +433,13 @@ uncons (Chunk t ts) = Just (T.unsafeHead t, ts')
369{-# INLINE uncons #-}
433{-# INLINE uncons #-}
371-- | /O(1)/ Returns the first character of a 'Text', which must be
435-- | /O(1)/ Returns the first character of a 'Text', which must be
372−-- non-empty. Subject to array fusion.
436+-- non-empty. Subject to fusion.
373head :: Text -> Char
437head :: Text -> Char
374head t = S.head (stream t)
438head t = S.head (stream t)
375{-# INLINE head #-}
439{-# INLINE head #-}
377-- | /O(1)/ Returns all characters after the head of a 'Text', which
441-- | /O(1)/ Returns all characters after the head of a 'Text', which
378−-- must be non-empty. Subject to array fusion.
442+-- must be non-empty. Subject to fusion.
379tail :: Text -> Text
443tail :: Text -> Text
380tail (Chunk t ts) = chunk (T.tail t) ts
444tail (Chunk t ts) = chunk (T.tail t) ts
381tail Empty = emptyError "tail"
445tail Empty = emptyError "tail"
@@ -389,7 +453,7 @@ tail Empty = emptyError "tail"
391-- | /O(1)/ Returns all but the last character of a 'Text', which must
455-- | /O(1)/ Returns all but the last character of a 'Text', which must
392−-- be non-empty. Subject to array fusion.
456+-- be non-empty. Subject to fusion.
393init :: Text -> Text
457init :: Text -> Text
394init (Chunk t0 ts0) = go t0 ts0
458init (Chunk t0 ts0) = go t0 ts0
395 where go t (Chunk t' ts) = Chunk t (go t' ts)
459 where go t (Chunk t' ts) = Chunk t (go t' ts)
@@ -404,7 +468,7 @@ init Empty = emptyError "init"
404 unstream (S.init (stream t)) = init t
468 unstream (S.init (stream t)) = init t
407−-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to array
471+-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to
408-- fusion.
472-- fusion.
409null :: Text -> Bool
473null :: Text -> Bool
410null Empty = True
474null Empty = True
@@ -425,7 +489,7 @@ isSingleton = S.isSingleton . stream
425{-# INLINE isSingleton #-}
489{-# INLINE isSingleton #-}
427-- | /O(1)/ Returns the last character of a 'Text', which must be
491-- | /O(1)/ Returns the last character of a 'Text', which must be
428−-- non-empty. Subject to array fusion.
492+-- non-empty. Subject to fusion.
429last :: Text -> Char
493last :: Text -> Char
430last Empty = emptyError "last"
494last Empty = emptyError "last"
431last (Chunk t ts) = go t ts
495last (Chunk t ts) = go t ts
@@ -469,9 +533,10 @@ compareLength t n = S.compareLengthI (stream t) n
469-- properties of code.
533-- properties of code.
471-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to
535-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to
472−-- each element of @t@. Subject to array fusion.
536+-- each element of @t@. Subject to fusion. Performs replacement on
537+-- invalid scalar values.
473map :: (Char -> Char) -> Text -> Text
538map :: (Char -> Char) -> Text -> Text
474−map f t = unstream (S.map f (stream t))
539+map f t = unstream (S.map (safe . f) (stream t))
475{-# INLINE [1] map #-}
540{-# INLINE [1] map #-}
477-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of
542-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of
@@ -482,13 +547,17 @@ intercalate t = concat . (U.intersperse t)
482{-# INLINE intercalate #-}
547{-# INLINE intercalate #-}
484-- | /O(n)/ The 'intersperse' function takes a character and places it
549-- | /O(n)/ The 'intersperse' function takes a character and places it
485−-- between the characters of a 'Text'. Subject to array fusion.
550+-- between the characters of a 'Text'. Subject to fusion. Performs
551+-- replacement on invalid scalar values.
486−intersperse :: Char -> Text -> Text
552+intersperse :: Char -> Text -> Text
487−intersperse c t = unstream (S.intersperse c (stream t))
553+intersperse c t = unstream (S.intersperse (safe c) (stream t))
488{-# INLINE intersperse #-}
554{-# INLINE intersperse #-}
490-- | /O(n)/ Left-justify a string to the given length, using the
556-- | /O(n)/ Left-justify a string to the given length, using the
491−-- specified fill character on the right. Subject to fusion. Examples:
557+-- specified fill character on the right. Subject to fusion. Performs
558+-- replacement on invalid scalar values.
493-- > justifyLeft 7 'x' "foo" == "fooxxxx"
562-- > justifyLeft 7 'x' "foo" == "fooxxxx"
494-- > justifyLeft 3 'x' "foobar" == "foobar"
563-- > justifyLeft 3 'x' "foobar" == "foobar"
@@ -507,7 +576,10 @@ justifyLeft k c t
509-- | /O(n)/ Right-justify a string to the given length, using the
578-- | /O(n)/ Right-justify a string to the given length, using the
510−-- specified fill character on the left. Examples:
579+-- specified fill character on the left. Performs replacement on
580+-- invalid scalar values.
512-- > justifyRight 7 'x' "bar" == "xxxxbar"
584-- > justifyRight 7 'x' "bar" == "xxxxbar"
513-- > justifyRight 3 'x' "foobar" == "foobar"
585-- > justifyRight 3 'x' "foobar" == "foobar"
@@ -518,8 +590,11 @@ justifyRight k c t
518 where len = length t
590 where len = length t
519{-# INLINE justifyRight #-}
591{-# INLINE justifyRight #-}
521−-- | /O(n)/ Center a string to the given length, using the
593+-- | /O(n)/ Center a string to the given length, using the specified
522−-- specified fill character on either side. Examples:
594+-- fill character on either side. Performs replacement on invalid
524-- > center 8 'x' "HS" = "xxxHSxxx"
599-- > center 8 'x' "HS" = "xxxHSxxx"
525center :: Int64 -> Char -> Text -> Text
600center :: Int64 -> Char -> Text -> Text
@@ -608,26 +683,24 @@ toUpper t = unstream (S.toUpper (stream t))
608-- | /O(n)/ 'foldl', applied to a binary operator, a starting value
683-- | /O(n)/ 'foldl', applied to a binary operator, a starting value
609-- (typically the left-identity of the operator), and a 'Text',
684-- (typically the left-identity of the operator), and a 'Text',
610-- reduces the 'Text' using the binary operator, from left to right.
685-- reduces the 'Text' using the binary operator, from left to right.
611−-- Subject to array fusion.
686+-- Subject to fusion.
612foldl :: (a -> Char -> a) -> a -> Text -> a
687foldl :: (a -> Char -> a) -> a -> Text -> a
613foldl f z t = S.foldl f z (stream t)
688foldl f z t = S.foldl f z (stream t)
614{-# INLINE foldl #-}
689{-# INLINE foldl #-}
616-- | /O(n)/ A strict version of 'foldl'.
691-- | /O(n)/ A strict version of 'foldl'.
617−-- Subject to array fusion.
692+-- Subject to fusion.
618foldl' :: (a -> Char -> a) -> a -> Text -> a
693foldl' :: (a -> Char -> a) -> a -> Text -> a
619foldl' f z t = S.foldl' f z (stream t)
694foldl' f z t = S.foldl' f z (stream t)
620{-# INLINE foldl' #-}
695{-# INLINE foldl' #-}
622-- | /O(n)/ A variant of 'foldl' that has no starting value argument,
697-- | /O(n)/ A variant of 'foldl' that has no starting value argument,
623−-- and thus must be applied to a non-empty 'Text'. Subject to array
698+-- and thus must be applied to a non-empty 'Text'. Subject to fusion.
625foldl1 :: (Char -> Char -> Char) -> Text -> Char
699foldl1 :: (Char -> Char -> Char) -> Text -> Char
626foldl1 f t = S.foldl1 f (stream t)
700foldl1 f t = S.foldl1 f (stream t)
627{-# INLINE foldl1 #-}
701{-# INLINE foldl1 #-}
629−-- | /O(n)/ A strict version of 'foldl1'.
703+-- | /O(n)/ A strict version of 'foldl1'. Subject to fusion.
630−-- Subject to array fusion.
631foldl1' :: (Char -> Char -> Char) -> Text -> Char
704foldl1' :: (Char -> Char -> Char) -> Text -> Char
632foldl1' f t = S.foldl1' f (stream t)
705foldl1' f t = S.foldl1' f (stream t)
633{-# INLINE foldl1' #-}
706{-# INLINE foldl1' #-}
@@ -635,13 +708,13 @@ foldl1' f t = S.foldl1' f (stream t)
635-- | /O(n)/ 'foldr', applied to a binary operator, a starting value
708-- | /O(n)/ 'foldr', applied to a binary operator, a starting value
636-- (typically the right-identity of the operator), and a 'Text',
709-- (typically the right-identity of the operator), and a 'Text',
637-- reduces the 'Text' using the binary operator, from right to left.
710-- reduces the 'Text' using the binary operator, from right to left.
638−-- Subject to array fusion.
711+-- Subject to fusion.
639foldr :: (Char -> a -> a) -> a -> Text -> a
712foldr :: (Char -> a -> a) -> a -> Text -> a
640foldr f z t = S.foldr f z (stream t)
713foldr f z t = S.foldr f z (stream t)
641{-# INLINE foldr #-}
714{-# INLINE foldr #-}
643−-- | /O(n)/ A variant of 'foldr' that has no starting value argument, and
716+-- | /O(n)/ A variant of 'foldr' that has no starting value argument,
644−-- thust must be applied to a non-empty 'Text'. Subject to array
717+-- and thust must be applied to a non-empty 'Text'. Subject to
645-- fusion.
718-- fusion.
646foldr1 :: (Char -> Char -> Char) -> Text -> Char
719foldr1 :: (Char -> Char -> Char) -> Text -> Char
647foldr1 f t = S.foldr1 f (stream t)
720foldr1 f t = S.foldr1 f (stream t)
@@ -664,32 +737,32 @@ concatMap f = concat . foldr ((:) . f) []
664{-# INLINE concatMap #-}
737{-# INLINE concatMap #-}
666-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
739-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
667−-- 'Text' @t@ satisifes the predicate @p@. Subject to array fusion.
740+-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion.
668any :: (Char -> Bool) -> Text -> Bool
741any :: (Char -> Bool) -> Text -> Bool
669any p t = S.any p (stream t)
742any p t = S.any p (stream t)
670{-# INLINE any #-}
743{-# INLINE any #-}
672-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
745-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
673−-- 'Text' @t@ satisify the predicate @p@. Subject to array fusion.
746+-- 'Text' @t@ satisify the predicate @p@. Subject to fusion.
674all :: (Char -> Bool) -> Text -> Bool
747all :: (Char -> Bool) -> Text -> Bool
675all p t = S.all p (stream t)
748all p t = S.all p (stream t)
676{-# INLINE all #-}
749{-# INLINE all #-}
678-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
751-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
679−-- must be non-empty. Subject to array fusion.
752+-- must be non-empty. Subject to fusion.
680maximum :: Text -> Char
753maximum :: Text -> Char
681maximum t = S.maximum (stream t)
754maximum t = S.maximum (stream t)
682{-# INLINE maximum #-}
755{-# INLINE maximum #-}
684-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
757-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
685−-- must be non-empty. Subject to array fusion.
758+-- must be non-empty. Subject to fusion.
686minimum :: Text -> Char
759minimum :: Text -> Char
687minimum t = S.minimum (stream t)
760minimum t = S.minimum (stream t)
688{-# INLINE minimum #-}
761{-# INLINE minimum #-}
690-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of
763-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of
691−-- successive reduced values from the left. This function is subject
764+-- successive reduced values from the left. Subject to fusion.
692−-- to array fusion.
765+-- Performs replacement on invalid scalar values.
694-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]
767-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]
@@ -697,11 +770,13 @@ minimum t = S.minimum (stream t)
698-- > last (scanl f z xs) == foldl f z xs.
771-- > last (scanl f z xs) == foldl f z xs.
699scanl :: (Char -> Char -> Char) -> Char -> Text -> Text
772scanl :: (Char -> Char -> Char) -> Char -> Text -> Text
700−scanl f z t = unstream (S.scanl f z (stream t))
773+scanl f z t = unstream (S.scanl g z (stream t))
774+ where g a b = safe (f a b)
701{-# INLINE scanl #-}
775{-# INLINE scanl #-}
703-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting
777-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting
704−-- value argument. This function is subject to array fusion.
778+-- value argument. Subject to fusion. Performs replacement on
779+-- invalid scalar values.
706-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]
781-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]
707scanl1 :: (Char -> Char -> Char) -> Text -> Text
782scanl1 :: (Char -> Char -> Char) -> Text -> Text
@@ -710,21 +785,24 @@ scanl1 f t0 = case uncons t0 of
710 Just (t,ts) -> scanl f t ts
785 Just (t,ts) -> scanl f t ts
711{-# INLINE scanl1 #-}
786{-# INLINE scanl1 #-}
713−-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.
788+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs
789+-- replacement on invalid scalar values.
715-- > scanr f v == reverse . scanl (flip f) v . reverse
791-- > scanr f v == reverse . scanl (flip f) v . reverse
716scanr :: (Char -> Char -> Char) -> Char -> Text -> Text
792scanr :: (Char -> Char -> Char) -> Char -> Text -> Text
717−scanr f v = reverse . scanl (flip f) v . reverse
793+scanr f v = reverse . scanl g v . reverse
794+ where g a b = safe (f b a)
719-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting
796-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting
720−-- value argument.
797+-- value argument. Performs replacement on invalid scalar values.
721scanr1 :: (Char -> Char -> Char) -> Text -> Text
798scanr1 :: (Char -> Char -> Char) -> Text -> Text
722scanr1 f t | null t = empty
799scanr1 f t | null t = empty
723 | otherwise = scanr f (last t) (init t)
800 | otherwise = scanr f (last t) (init t)
725-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a
802-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a
726-- function to each element of a 'Text', passing an accumulating
803-- function to each element of a 'Text', passing an accumulating
727−-- parameter from left to right, and returns a final 'Text'.
804+-- parameter from left to right, and returns a final 'Text'. Performs
805+-- replacement on invalid scalar values.
728mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
806mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
729mapAccumL f = go
807mapAccumL f = go
@@ -738,7 +816,7 @@ mapAccumL f = go
738-- a strict 'foldr'; it applies a function to each element of a
816-- a strict 'foldr'; it applies a function to each element of a
739-- 'Text', passing an accumulating parameter from right to left, and
817-- 'Text', passing an accumulating parameter from right to left, and
740-- returning a final value of this accumulator together with the new
818-- returning a final value of this accumulator together with the new
741−-- 'Text'.
819+-- 'Text'. Performs replacement on invalid scalar values.
742mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
820mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
743mapAccumR f = go
821mapAccumR f = go
@@ -762,7 +840,7 @@ replicate n t
762-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the
840-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the
763-- value of every element. Subject to fusion.
841-- value of every element. Subject to fusion.
764replicateChar :: Int64 -> Char -> Text
842replicateChar :: Int64 -> Char -> Text
765−replicateChar n c = unstream (S.replicateCharI n c)
843+replicateChar n c = unstream (S.replicateCharI n (safe c))
766{-# INLINE replicateChar #-}
844{-# INLINE replicateChar #-}
@@ -775,9 +853,10 @@ replicateChar n c = unstream (S.replicateCharI n c)
775-- 'Text' from a seed value. The function takes the element and
853-- 'Text' from a seed value. The function takes the element and
776-- returns 'Nothing' if it is done producing the 'Text', otherwise
854-- returns 'Nothing' if it is done producing the 'Text', otherwise
777-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the
855-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the
778−-- string, and @b@ is the seed value for further production.
856+-- string, and @b@ is the seed value for further production. Performs
857+-- replacement on invalid scalar values.
779−unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
858+unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
780−unfoldr f s = unstream (S.unfoldr f s)
859+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)
781{-# INLINE unfoldr #-}
860{-# INLINE unfoldr #-}
783-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
862-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
@@ -785,8 +864,9 @@ unfoldr f s = unstream (S.unfoldr f s)
785-- first argument to 'unfoldrN'. This function is more efficient than
864-- first argument to 'unfoldrN'. This function is more efficient than
786-- 'unfoldr' when the maximum length of the result is known and
865-- 'unfoldr' when the maximum length of the result is known and
787-- correct, otherwise its performance is similar to 'unfoldr'.
866-- correct, otherwise its performance is similar to 'unfoldr'.
867+-- Performs replacement on invalid scalar values.
788−unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text
868+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text
789−unfoldrN n f s = unstream (S.unfoldrN n f s)
869+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)
790{-# INLINE unfoldrN #-}
870{-# INLINE unfoldrN #-}
792-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the
872-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the
@@ -847,9 +927,9 @@ dropWords i t0
847 where len' = fromIntegral len
927 where len' = fromIntegral len
848 n' = fromIntegral n
928 n' = fromIntegral n
850−-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', returns
930+-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',
851−-- the longest prefix (possibly empty) of elements that satisfy @p@.
931+-- returns the longest prefix (possibly empty) of elements that
852−-- This function is subject to array fusion.
932+-- satisfy @p@. Subject to fusion.
853takeWhile :: (Char -> Bool) -> Text -> Text
933takeWhile :: (Char -> Bool) -> Text -> Text
854takeWhile p t0 = takeWhile' t0
934takeWhile p t0 = takeWhile' t0
855 where takeWhile' Empty = Empty
935 where takeWhile' Empty = Empty
@@ -868,7 +948,7 @@ takeWhile p t0 = takeWhile' t0
870-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
950-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
871−-- 'takeWhile' @p@ @t@. This function is subject to array fusion.
951+-- 'takeWhile' @p@ @t@. Subject to fusion.
872dropWhile :: (Char -> Bool) -> Text -> Text
952dropWhile :: (Char -> Bool) -> Text -> Text
873dropWhile p t0 = dropWhile' t0
953dropWhile p t0 = dropWhile' t0
874 where dropWhile' Empty = Empty
954 where dropWhile' Empty = Empty
@@ -1193,8 +1273,7 @@ unwords = intercalate (singleton ' ')
1193{-# INLINE unwords #-}
1273{-# INLINE unwords #-}
1195-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns
1275-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns
1196−-- 'True' iff the first is a prefix of the second. This function is
1276+-- 'True' iff the first is a prefix of the second. Subject to fusion.
1197−-- subject to fusion.
1198isPrefixOf :: Text -> Text -> Bool
1277isPrefixOf :: Text -> Text -> Bool
1199isPrefixOf Empty _ = True
1278isPrefixOf Empty _ = True
1200isPrefixOf _ Empty = False
1279isPrefixOf _ Empty = False
@@ -1340,8 +1419,7 @@ count pat src
1342-- | /O(n)/ The 'countChar' function returns the number of times the
1421-- | /O(n)/ The 'countChar' function returns the number of times the
1343−-- query element appears in the given 'Text'. This function is subject
1422+-- query element appears in the given 'Text'. Subject to fusion.
1345countChar :: Char -> Text -> Int64
1423countChar :: Char -> Text -> Int64
1346countChar c t = S.countChar c (stream t)
1424countChar c t = S.countChar c (stream t)
@@ -1355,8 +1433,10 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)
1356-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function
1434-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function
1357-- given as the first argument, instead of a tupling function.
1435-- given as the first argument, instead of a tupling function.
1436+-- Performs replacement on invalid scalar values.
1358zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text
1437zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text
1359−zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))
1438+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))
1439+ where g a b = safe (f a b)
1360{-# INLINE [0] zipWith #-}
1440{-# INLINE [0] zipWith #-}
1362revChunks :: [T.Text] -> Text
1442revChunks :: [T.Text] -> Text