d7d9bc6 · text · git.iko.soy

git.iko.soy/text

Implement and document the new replacement behaviour.

commitd7d9bc6327f9a74b891160520d290efbe4f4e59d authorBryan O'Sullivan <bos@serpentine.com> dateTue 30 Nov 2010, 00:39 (16 years ago) parentf97e59a

3 files changed+234−62

compare with master

unifiedsplit

Data/Text.hs+98−34 Data/Text/Internal.hs+24 Data/Text/Lazy.hs+112−28

Data/Text.hs+98−34

@@ -13,20 +13,31 @@

1313-- Stability : experimental

1414-- Portability : GHC

1515--

16−-- A time and space-efficient implementation of Unicode text using

17−-- packed Word16 arrays. Suitable for performance critical use, both

18−-- in terms of large data quantities and high speed.

16+-- A time and space-efficient implementation of Unicode text.

17+-- Suitable for performance critical use, both in terms of large data

18+-- quantities and high speed.

19+--

20+-- /Note/: Read below the synopsis for important notes on the use of

21+-- this module.

1922--

2023-- This module is intended to be imported @qualified@, to avoid name

2124-- clashes with "Prelude" functions, e.g.

2225--

2326-- > import qualified Data.Text as T

27+--

28+-- To use an extended and very rich family of functions for working

29+-- with Unicode text (including normalization, regular expressions,

30+-- non-standard encodings, text breaking, and locales), see the

31+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

2432

2533module Data.Text

2634 (

2735 -- * Strict vs lazy types

2836 -- $strict

2937

38+ -- * Acceptable data

39+ -- $replacement

40+

3041 -- * Fusion

3142 -- $fusion

3243

@@ -196,7 +207,7 @@ import Data.String (IsString(..))

196207import qualified Data.Text.Fusion as S

197208import qualified Data.Text.Fusion.Common as S

198209import Data.Text.Fusion (stream, reverseStream, unstream)

199−import Data.Text.Internal (Text(..), empty, text, textP)

210+import Data.Text.Internal (Text(..), empty, firstf, safe, text, textP)

200211import qualified Prelude as P

201212import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,

202213 unsafeHead, unsafeTail)

@@ -230,6 +241,33 @@ import Data.Int (Int64)

230241-- difference being that the strict module uses 'Int' values for

231242-- lengths and counts, while the lazy module uses 'Int64' lengths.

232243

244+-- $replacement

245+--

246+-- A 'Text' value is a sequence of Unicode scalar values, as defined

247+-- in §3.9, definition D76 of the Unicode 5.2 standard:

248+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

249+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

250+-- inclusive. Haskell implementations admit all Unicode code points

251+-- (§3.4, definition D10) as 'Char' values, including code points

252+-- from this invalid range. This means that there are some 'Char'

253+-- values that are not valid Unicode scalar values, and the functions

254+-- in this module must handle those cases.

255+--

256+-- Within this module, many functions construct a 'Text' from one or

257+-- more 'Char' values. Those functions will substitute 'Char' values

258+-- that are not valid Unicode scalar values with the replacement

259+-- character \"�\" (U+FFFD). Functions that perform this

260+-- inspection and replacement are documented with the phrase

261+-- \"Performs replacement on invalid scalar values\".

262+--

263+-- (One reason for this policy of replacement is that internally, a

264+-- 'Text' value is represented as packed UTF-16 data. Values in the

265+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

266+-- code points, and so cannot be represented. The functions replace

267+-- invalid scalar values, instead of dropping them, as a security

268+-- measure. For details, see Unicode Technical Report 36, §3.5:

269+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters>)

270+

233271-- $fusion

234272--

235273-- Most of the functions in this module are subject to /fusion/,

@@ -320,9 +358,10 @@ compareText ta@(Text _arrA _offA lenA) tb@(Text _arrB _offB lenB)

320358-- -----------------------------------------------------------------------------

321359-- * Conversion to/from 'Text'

322360

323−-- | /O(n)/ Convert a 'String' into a 'Text'. Subject to fusion.

361+-- | /O(n)/ Convert a 'String' into a 'Text'. Subject to

362+-- fusion. Performs replacement on invalid scalar values.

324363pack :: String -> Text

325−pack = unstream . S.streamList

364+pack = unstream . S.streamList . L.map safe

326365{-# INLINE [1] pack #-}

327366

328367-- | /O(n)/ Convert a Text into a String. Subject to fusion.

@@ -330,10 +369,10 @@ unpack :: Text -> String

330369unpack = S.unstreamList . stream

331370{-# INLINE [1] unpack #-}

332371

333−-- | /O(1)/ Convert a character into a Text.

334−-- Subject to fusion.

372+-- | /O(1)/ Convert a character into a Text. Subject to fusion.

373+-- Performs replacement on invalid scalar values.

335374singleton :: Char -> Text

336−singleton = unstream . S.singleton

375+singleton = unstream . S.singleton . safe

337376{-# INLINE [1] singleton #-}

338377

339378-- -----------------------------------------------------------------------------

@@ -341,15 +380,17 @@ singleton = unstream . S.singleton

341380

342381-- | /O(n)/ Adds a character to the front of a 'Text'. This function

343382-- is more costly than its 'List' counterpart because it requires

344−-- copying a new array. Subject to fusion.

383+-- copying a new array. Subject to fusion. Performs replacement on

384+-- invalid scalar values.

345385cons :: Char -> Text -> Text

346−cons c t = unstream (S.cons c (stream t))

386+cons c t = unstream (S.cons (safe c) (stream t))

347387{-# INLINE cons #-}

348388

349389-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the

350390-- entire array in the process, unless fused. Subject to fusion.

391+-- Performs replacement on invalid scalar values.

351392snoc :: Text -> Char -> Text

352−snoc t c = unstream (S.snoc (stream t) c)

393+snoc t c = unstream (S.snoc (stream t) (safe c))

353394{-# INLINE snoc #-}

354395

355396-- | /O(n)/ Appends one 'Text' to the other by copying both of them

@@ -523,9 +564,10 @@ compareLength t n = S.compareLengthI (stream t) n

523564-- -----------------------------------------------------------------------------

524565-- * Transformations

525566-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

526−-- each element of @t@. Subject to fusion.

567+-- each element of @t@. Subject to fusion. Performs replacement on

568+-- invalid scalar values.

527569map :: (Char -> Char) -> Text -> Text

528−map f t = unstream (S.map f (stream t))

570+map f t = unstream (S.map (safe . f) (stream t))

529571{-# INLINE [1] map #-}

530572

531573-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

@@ -536,9 +578,10 @@ intercalate t = concat . (U.intersperse t)

536578{-# INLINE intercalate #-}

537579

538580-- | /O(n)/ The 'intersperse' function takes a character and places it

539−-- between the characters of a 'Text'. Subject to fusion.

581+-- between the characters of a 'Text'. Subject to fusion. Performs

582+-- replacement on invalid scalar values.

540583intersperse :: Char -> Text -> Text

541−intersperse c t = unstream (S.intersperse c (stream t))

584+intersperse c t = unstream (S.intersperse (safe c) (stream t))

542585{-# INLINE intersperse #-}

543586

544587-- | /O(n)/ Reverse the characters of a string. Subject to fusion.

@@ -615,7 +658,10 @@ toUpper t = unstream (S.toUpper (stream t))

615658{-# INLINE toUpper #-}

616659

617660-- | /O(n)/ Left-justify a string to the given length, using the

618−-- specified fill character on the right. Subject to fusion. Examples:

661+-- specified fill character on the right. Subject to fusion.

662+-- Performs replacement on invalid scalar values.

663+--

664+-- Examples:

619665--

620666-- > justifyLeft 7 'x' "foo" == "fooxxxx"

621667-- > justifyLeft 3 'x' "foobar" == "foobar"

@@ -634,7 +680,10 @@ justifyLeft k c t

634680 #-}

635681

636682-- | /O(n)/ Right-justify a string to the given length, using the

637−-- specified fill character on the left. Examples:

683+-- specified fill character on the left. Performs replacement on

684+-- invalid scalar values.

685+--

686+-- Examples:

638687--

639688-- > justifyRight 7 'x' "bar" == "xxxxbar"

640689-- > justifyRight 3 'x' "foobar" == "foobar"

@@ -645,8 +694,11 @@ justifyRight k c t

645694 where len = length t

646695{-# INLINE justifyRight #-}

647696

648−-- | /O(n)/ Center a string to the given length, using the

649−-- specified fill character on either side. Examples:

697+-- | /O(n)/ Center a string to the given length, using the specified

698+-- fill character on either side. Performs replacement on invalid

699+-- scalar values.

700+--

701+-- Examples:

650702--

651703-- > center 8 'x' "HS" = "xxxHSxxx"

652704center :: Int -> Char -> Text -> Text

@@ -761,6 +813,7 @@ minimum t = S.minimum (stream t)

761813

762814-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of

763815-- successive reduced values from the left. Subject to fusion.

816+-- Performs replacement on invalid scalar values.

764817--

765818-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]

766819--

@@ -768,11 +821,13 @@ minimum t = S.minimum (stream t)

768821--

769822-- > last (scanl f z xs) == foldl f z xs.

770823scanl :: (Char -> Char -> Char) -> Char -> Text -> Text

771−scanl f z t = unstream (S.scanl f z (stream t))

824+scanl f z t = unstream (S.scanl g z (stream t))

825+ where g a b = safe (f a b)

772826{-# INLINE scanl #-}

773827

774828-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting

775−-- value argument. Subject to fusion.

829+-- value argument. Subject to fusion. Performs replacement on

830+-- invalid scalar values.

776831--

777832-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]

778833scanl1 :: (Char -> Char -> Char) -> Text -> Text

@@ -780,15 +835,18 @@ scanl1 f t | null t = empty

780835 | otherwise = scanl f (unsafeHead t) (unsafeTail t)

781836{-# INLINE scanl1 #-}

782837

783−-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.

838+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs

839+-- replacement on invalid scalar values.

784840--

785841-- > scanr f v == reverse . scanl (flip f) v . reverse

786842scanr :: (Char -> Char -> Char) -> Char -> Text -> Text

787−scanr f z = S.reverse . S.reverseScanr f z . reverseStream

843+scanr f z = S.reverse . S.reverseScanr g z . reverseStream

844+ where g a b = safe (f a b)

788845{-# INLINE scanr #-}

789846

790847-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting

791−-- value argument. Subject to fusion.

848+-- value argument. Subject to fusion. Performs replacement on

849+-- invalid scalar values.

792850scanr1 :: (Char -> Char -> Char) -> Text -> Text

793851scanr1 f t | null t = empty

794852 | otherwise = scanr f (last t) (init t)

@@ -796,9 +854,11 @@ scanr1 f t | null t = empty

796854

797855-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a

798856-- function to each element of a 'Text', passing an accumulating

799−-- parameter from left to right, and returns a final 'Text'.

857+-- parameter from left to right, and returns a final 'Text'. Performs

858+-- replacement on invalid scalar values.

800859mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

801−mapAccumL f z0 = S.mapAccumL f z0 . stream

860+mapAccumL f z0 = S.mapAccumL g z0 . stream

861+ where g a b = second safe (f a b)

802862{-# INLINE mapAccumL #-}

803863

804864-- | The 'mapAccumR' function behaves like a combination of 'map' and

@@ -806,8 +866,10 @@ mapAccumL f z0 = S.mapAccumL f z0 . stream

806866-- 'Text', passing an accumulating parameter from right to left, and

807867-- returning a final value of this accumulator together with the new

808868-- 'Text'.

869+-- Performs replacement on invalid scalar values.

809870mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

810−mapAccumR f z0 = second reverse . S.mapAccumL f z0 . reverseStream

871+mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream

872+ where g a b = second safe (f a b)

811873{-# INLINE mapAccumR #-}

812874

813875-- -----------------------------------------------------------------------------

@@ -840,7 +902,7 @@ replicate n t@(Text a o l)

840902-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the

841903-- value of every element. Subject to fusion.

842904replicateChar :: Int -> Char -> Text

843−replicateChar n c = unstream (S.replicateCharI n c)

905+replicateChar n c = unstream (S.replicateCharI n (safe c))

844906{-# INLINE replicateChar #-}

845907

846908-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'

@@ -849,9 +911,9 @@ replicateChar n c = unstream (S.replicateCharI n c)

849911-- returns 'Nothing' if it is done producing the 'Text', otherwise

850912-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the

851913-- string, and @b@ is the seed value for further production. Subject

852−-- to fusion.

914+-- to fusion. Performs replacement on invalid scalar values.

853915unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

854−unfoldr f s = unstream (S.unfoldr f s)

916+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)

855917{-# INLINE unfoldr #-}

856918

857919-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed

@@ -859,9 +921,9 @@ unfoldr f s = unstream (S.unfoldr f s)

859921-- first argument to 'unfoldrN'. This function is more efficient than

860922-- 'unfoldr' when the maximum length of the result is known and

861923-- correct, otherwise its performance is similar to 'unfoldr'. Subject

862−-- to fusion.

924+-- to fusion. Performs replacement on invalid scalar values.

863925unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text

864−unfoldrN n f s = unstream (S.unfoldrN n f s)

926+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)

865927{-# INLINE unfoldrN #-}

866928

867929-- -----------------------------------------------------------------------------

@@ -1299,8 +1361,10 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)

12991361

13001362-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function

13011363-- given as the first argument, instead of a tupling function.

1364+-- Performs replacement on invalid scalar values.

13021365zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text

1303−zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))

1366+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))

1367+ where g a b = safe (f a b)

13041368{-# INLINE [0] zipWith #-}

13051369

13061370-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's

Data/Text/Internal.hs+24

@@ -22,8 +22,12 @@ module Data.Text.Internal

2222 -- * Construction

2323 , text

2424 , textP

25+ -- * Safety

26+ , safe

2527 -- * Code that must be here for accessibility

2628 , empty

29+ -- * Utilities

30+ , firstf

2731 -- * Debugging

2832 , showText

2933 ) where

@@ -31,7 +35,9 @@ module Data.Text.Internal

3135#if defined(ASSERTS)

3236import Control.Exception (assert)

3337#endif

38+import Data.Bits ((.&.))

3439import qualified Data.Text.Array as A

40+import Data.Text.UnsafeChar (ord)

3541import Data.Typeable (Typeable)

3642

3743-- | A space efficient, packed, unboxed Unicode text type.

@@ -72,3 +78,21 @@ showText :: Text -> String

7278showText (Text arr off len) =

7379 "Text " ++ show (A.toList arr off len) ++ ' ' :

7480 show off ++ ' ' : show len

81+

82+-- | Map a 'Char' to a 'Text'-safe value.

83+--

84+-- UTF-16 surrogate code points are not included in the set of Unicode

85+-- scalar values, but are unfortunately admitted as valid 'Char'

86+-- values by Haskell. They cannot be represented in a 'Text'. This

87+-- function remaps those code points to the Unicode replacement

88+-- character \"�\", and leaves other code points unchanged.

89+safe :: Char -> Char

90+safe c

91+ | ord c .&. 0x1ff800 /= 0xd800 = c

92+ | otherwise = '\xfffd'

93+{-# INLINE safe #-}

94+

95+-- | Apply a function to the first element of an optional pair.

96+firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)

97+firstf f (Just (a, b)) = Just (f a, b)

98+firstf _ Nothing = Nothing

Data/Text/Lazy.hs+112−28

@@ -11,7 +11,12 @@

1111-- Portability : GHC

1212--

1313-- A time and space-efficient implementation of Unicode text using

14−-- lists of packed arrays. This representation is suitable for high

14+-- lists of packed arrays.

15+--

16+-- /Note/: Read below the synopsis for important notes on the use of

17+-- this module.

18+--

19+-- The representation used by this module is suitable for high

1520-- performance use and for streaming large quantities of data. It

1621-- provides a means to manipulate a large body of text without

1722-- requiring that the entire content be resident in memory.

@@ -31,7 +36,15 @@

3136

3237module Data.Text.Lazy

3338 (

39+ -- * Fusion

40+ -- $fusion

41+

42+ -- * Acceptable data

43+ -- $replacement

44+

45+ -- * Types

3446 Text

47+

3548 -- * Creation and elimination

3649 , pack

3750 , unpack

@@ -200,10 +213,62 @@ import qualified Data.Text.Lazy.Fusion as S

200213import Data.Text.Fusion.Internal (PairS(..))

201214import Data.Text.Lazy.Fusion (stream, unstream)

202215import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)

203−import Data.Text.Internal (textP)

216+import Data.Text.Internal (firstf, safe, textP)

204217import qualified Data.Text.Util as U

205218import Data.Text.Lazy.Search (indices)

206219

220+-- $fusion

221+--

222+-- Most of the functions in this module are subject to /fusion/,

223+-- meaning that a pipeline of such functions will usually allocate at

224+-- most one 'Text' value.

225+--

226+-- As an example, consider the following pipeline:

227+--

228+-- > import Data.Text.Lazy as T

229+-- > import Data.Text.Lazy.Encoding as E

230+-- > import Data.ByteString.Lazy (ByteString)

231+-- >

232+-- > countChars :: ByteString -> Int

233+-- > countChars = T.length . T.toUpper . E.decodeUtf8

234+--

235+-- From the type signatures involved, this looks like it should

236+-- allocate one 'ByteString' value, and two 'Text' values. However,

237+-- when a module is compiled with optimisation enabled under GHC, the

238+-- two intermediate 'Text' values will be optimised away, and the

239+-- function will be compiled down to a single loop over the source

240+-- 'ByteString'.

241+--

242+-- Functions that can be fused by the compiler are documented with the

243+-- phrase \"Subject to fusion\".

244+

245+-- $replacement

246+--

247+-- A 'Text' value is a sequence of Unicode scalar values, as defined

248+-- in §3.9, definition D76 of the Unicode 5.2 standard:

249+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

250+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

251+-- inclusive. Haskell implementations admit all Unicode code points

252+-- (§3.4, definition D10) as 'Char' values, including code points

253+-- from this invalid range. This means that there are some 'Char'

254+-- values that are not valid Unicode scalar values, and the functions

255+-- in this module must handle those cases.

256+--

257+-- Within this module, many functions construct a 'Text' from one or

258+-- more 'Char' values. Those functions will substitute 'Char' values

259+-- that are not valid Unicode scalar values with the replacement

260+-- character \"�\" (U+FFFD). Functions that perform this

261+-- inspection and replacement are documented with the phrase

262+-- \"Performs replacement on invalid scalar values\".

263+--

264+-- (One reason for this policy of replacement is that internally, a

265+-- 'Text' value is represented as packed UTF-16 data. Values in the

266+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

267+-- code points, and so cannot be represented. The functions replace

268+-- invalid scalar values, instead of dropping them, as a security

269+-- measure. For details, see Unicode Technical Report 36, §3.5:

270+-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)

271+

207272equal :: Text -> Text -> Bool

208273equal Empty Empty = True

209274equal Empty _ = False

@@ -274,9 +339,9 @@ instance Data Text where

274339

275340-- | /O(n)/ Convert a 'String' into a 'Text'.

276341--

277−-- This function is subject to array fusion.

342+-- Subject to fusion. Performs replacement on invalid scalar values.

278343pack :: String -> Text

279−pack s = unstream (S.streamList s)

344+pack = unstream . S.streamList . L.map safe

280345{-# INLINE [1] pack #-}

281346

282347-- | /O(n)/ Convert a 'Text' into a 'String'.

@@ -285,8 +350,8 @@ unpack :: Text -> String

285350unpack t = S.unstreamList (stream t)

286351{-# INLINE [1] unpack #-}

287352

288−-- | /O(1)/ Convert a character into a Text.

289−-- Subject to fusion.

353+-- | /O(1)/ Convert a character into a Text. Subject to fusion.

354+-- Performs replacement on invalid scalar values.

290355singleton :: Char -> Text

291356singleton c = Chunk (T.singleton c) Empty

292357{-# INLINE [1] singleton #-}

@@ -469,9 +534,10 @@ compareLength t n = S.compareLengthI (stream t) n

469534-- properties of code.

470535

471536-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

472−-- each element of @t@. Subject to array fusion.

537+-- each element of @t@. Subject to fusion. Performs replacement on

538+-- invalid scalar values.

473539map :: (Char -> Char) -> Text -> Text

474−map f t = unstream (S.map f (stream t))

540+map f t = unstream (S.map (safe . f) (stream t))

475541{-# INLINE [1] map #-}

476542

477543-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

@@ -488,7 +554,10 @@ intersperse c t = unstream (S.intersperse c (stream t))

488554{-# INLINE intersperse #-}

489555

490556-- | /O(n)/ Left-justify a string to the given length, using the

491−-- specified fill character on the right. Subject to fusion. Examples:

557+-- specified fill character on the right. Subject to fusion. Performs

558+-- replacement on invalid scalar values.

559+--

560+-- Examples:

492561--

493562-- > justifyLeft 7 'x' "foo" == "fooxxxx"

494563-- > justifyLeft 3 'x' "foobar" == "foobar"

@@ -507,7 +576,10 @@ justifyLeft k c t

507576 #-}

508577

509578-- | /O(n)/ Right-justify a string to the given length, using the

510−-- specified fill character on the left. Examples:

579+-- specified fill character on the left. Performs replacement on

580+-- invalid scalar values.

581+--

582+-- Examples:

511583--

512584-- > justifyRight 7 'x' "bar" == "xxxxbar"

513585-- > justifyRight 3 'x' "foobar" == "foobar"

@@ -518,8 +590,11 @@ justifyRight k c t

518590 where len = length t

519591{-# INLINE justifyRight #-}

520592

521−-- | /O(n)/ Center a string to the given length, using the

522−-- specified fill character on either side. Examples:

593+-- | /O(n)/ Center a string to the given length, using the specified

594+-- fill character on either side. Performs replacement on invalid

595+-- scalar values.

596+--

597+-- Examples:

523598--

524599-- > center 8 'x' "HS" = "xxxHSxxx"

525600center :: Int64 -> Char -> Text -> Text

@@ -688,8 +763,8 @@ minimum t = S.minimum (stream t)

688763{-# INLINE minimum #-}

689764

690765-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of

691−-- successive reduced values from the left. This function is subject

692−-- to array fusion.

766+-- successive reduced values from the left. Subject to fusion.

767+-- Performs replacement on invalid scalar values.

693768--

694769-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]

695770--

@@ -697,11 +772,13 @@ minimum t = S.minimum (stream t)

697772--

698773-- > last (scanl f z xs) == foldl f z xs.

699774scanl :: (Char -> Char -> Char) -> Char -> Text -> Text

700−scanl f z t = unstream (S.scanl f z (stream t))

775+scanl f z t = unstream (S.scanl g z (stream t))

776+ where g a b = safe (f a b)

701777{-# INLINE scanl #-}

702778

703779-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting

704−-- value argument. This function is subject to array fusion.

780+-- value argument. Subject to fusion. Performs replacement on

781+-- invalid scalar values.

705782--

706783-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]

707784scanl1 :: (Char -> Char -> Char) -> Text -> Text

@@ -710,21 +787,24 @@ scanl1 f t0 = case uncons t0 of

710787 Just (t,ts) -> scanl f t ts

711788{-# INLINE scanl1 #-}

712789

713−-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.

790+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs

791+-- replacement on invalid scalar values.

714792--

715793-- > scanr f v == reverse . scanl (flip f) v . reverse

716794scanr :: (Char -> Char -> Char) -> Char -> Text -> Text

717−scanr f v = reverse . scanl (flip f) v . reverse

795+scanr f v = reverse . scanl g v . reverse

796+ where g a b = safe (f b a)

718797

719798-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting

720−-- value argument.

799+-- value argument. Performs replacement on invalid scalar values.

721800scanr1 :: (Char -> Char -> Char) -> Text -> Text

722801scanr1 f t | null t = empty

723802 | otherwise = scanr f (last t) (init t)

724803

725804-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a

726805-- function to each element of a 'Text', passing an accumulating

727−-- parameter from left to right, and returns a final 'Text'.

806+-- parameter from left to right, and returns a final 'Text'. Performs

807+-- replacement on invalid scalar values.

728808mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

729809mapAccumL f = go

730810 where

@@ -738,7 +818,7 @@ mapAccumL f = go

738818-- a strict 'foldr'; it applies a function to each element of a

739819-- 'Text', passing an accumulating parameter from right to left, and

740820-- returning a final value of this accumulator together with the new

741−-- 'Text'.

821+-- 'Text'. Performs replacement on invalid scalar values.

742822mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

743823mapAccumR f = go

744824 where

@@ -762,7 +842,7 @@ replicate n t

762842-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the

763843-- value of every element. Subject to fusion.

764844replicateChar :: Int64 -> Char -> Text

765−replicateChar n c = unstream (S.replicateCharI n c)

845+replicateChar n c = unstream (S.replicateCharI n (safe c))

766846{-# INLINE replicateChar #-}

767847

768848{-# RULES

@@ -775,9 +855,10 @@ replicateChar n c = unstream (S.replicateCharI n c)

775855-- 'Text' from a seed value. The function takes the element and

776856-- returns 'Nothing' if it is done producing the 'Text', otherwise

777857-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the

778−-- string, and @b@ is the seed value for further production.

779−unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

780−unfoldr f s = unstream (S.unfoldr f s)

858+-- string, and @b@ is the seed value for further production. Performs

859+-- replacement on invalid scalar values.

860+unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

861+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)

781862{-# INLINE unfoldr #-}

782863

783864-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed

@@ -785,8 +866,9 @@ unfoldr f s = unstream (S.unfoldr f s)

785866-- first argument to 'unfoldrN'. This function is more efficient than

786867-- 'unfoldr' when the maximum length of the result is known and

787868-- correct, otherwise its performance is similar to 'unfoldr'.

788−unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text

789−unfoldrN n f s = unstream (S.unfoldrN n f s)

869+-- Performs replacement on invalid scalar values.

870+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text

871+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)

790872{-# INLINE unfoldrN #-}

791873

792874-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the

@@ -1355,8 +1437,10 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)

13551437

13561438-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function

13571439-- given as the first argument, instead of a tupling function.

1440+-- Performs replacement on invalid scalar values.

13581441zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text

1359−zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))

1442+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))

1443+ where g a b = safe (f a b)

13601444{-# INLINE [0] zipWith #-}

13611445

13621446revChunks :: [T.Text] -> Text