d7d9bc6 · text · git.iko.soy

git.iko.soy/text

on master help sign in

Implement and document the new replacement behaviour.

d7d9bc6Bryan O'Sullivan16 years ago

Data/Text.hs+98−34

compare with master

unifiedsplit

@@ -13,20 +13,31 @@

13-- Stability : experimental

14-- Portability : GHC

15--

16−-- A time and space-efficient implementation of Unicode text using

16+-- A time and space-efficient implementation of Unicode text.

17−-- packed Word16 arrays. Suitable for performance critical use, both

17+-- Suitable for performance critical use, both in terms of large data

18−-- in terms of large data quantities and high speed.

18+-- quantities and high speed.

19+--

20+-- /Note/: Read below the synopsis for important notes on the use of

21+-- this module.

19--

22--

20-- This module is intended to be imported @qualified@, to avoid name

23-- This module is intended to be imported @qualified@, to avoid name

21-- clashes with "Prelude" functions, e.g.

24-- clashes with "Prelude" functions, e.g.

22--

25--

23-- > import qualified Data.Text as T

26-- > import qualified Data.Text as T

27+--

28+-- To use an extended and very rich family of functions for working

29+-- with Unicode text (including normalization, regular expressions,

30+-- non-standard encodings, text breaking, and locales), see the

31+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

25module Data.Text

33module Data.Text

26 (

34 (

27 -- * Strict vs lazy types

35 -- * Strict vs lazy types

28 -- $strict

36 -- $strict

38+ -- * Acceptable data

39+ -- $replacement

40+

30 -- * Fusion

41 -- * Fusion

31 -- $fusion

42 -- $fusion

@@ -196,7 +207,7 @@ import Data.String (IsString(..))

196import qualified Data.Text.Fusion as S

207import qualified Data.Text.Fusion as S

197import qualified Data.Text.Fusion.Common as S

208import qualified Data.Text.Fusion.Common as S

198import Data.Text.Fusion (stream, reverseStream, unstream)

209import Data.Text.Fusion (stream, reverseStream, unstream)

199−import Data.Text.Internal (Text(..), empty, text, textP)

210+import Data.Text.Internal (Text(..), empty, firstf, safe, text, textP)

200import qualified Prelude as P

211import qualified Prelude as P

201import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,

212import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,

202 unsafeHead, unsafeTail)

213 unsafeHead, unsafeTail)

@@ -230,6 +241,33 @@ import Data.Int (Int64)

230-- difference being that the strict module uses 'Int' values for

241-- difference being that the strict module uses 'Int' values for

231-- lengths and counts, while the lazy module uses 'Int64' lengths.

242-- lengths and counts, while the lazy module uses 'Int64' lengths.

232

243

244+-- $replacement

245+--

246+-- A 'Text' value is a sequence of Unicode scalar values, as defined

247+-- in §3.9, definition D76 of the Unicode 5.2 standard:

248+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

249+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

250+-- inclusive. Haskell implementations admit all Unicode code points

251+-- (§3.4, definition D10) as 'Char' values, including code points

252+-- from this invalid range. This means that there are some 'Char'

253+-- values that are not valid Unicode scalar values, and the functions

254+-- in this module must handle those cases.

255+--

256+-- Within this module, many functions construct a 'Text' from one or

257+-- more 'Char' values. Those functions will substitute 'Char' values

258+-- that are not valid Unicode scalar values with the replacement

259+-- character \"�\" (U+FFFD). Functions that perform this

260+-- inspection and replacement are documented with the phrase

261+-- \"Performs replacement on invalid scalar values\".

262+--

263+-- (One reason for this policy of replacement is that internally, a

264+-- 'Text' value is represented as packed UTF-16 data. Values in the

265+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

266+-- code points, and so cannot be represented. The functions replace

267+-- invalid scalar values, instead of dropping them, as a security

268+-- measure. For details, see Unicode Technical Report 36, §3.5:

269+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters>)

270+

233-- $fusion

271-- $fusion

234--

272--

235-- Most of the functions in this module are subject to /fusion/,

273-- Most of the functions in this module are subject to /fusion/,

@@ -320,9 +358,10 @@ compareText ta@(Text _arrA _offA lenA) tb@(Text _arrB _offB lenB)

320-- -----------------------------------------------------------------------------

358-- -----------------------------------------------------------------------------

321-- * Conversion to/from 'Text'

359-- * Conversion to/from 'Text'

322

360

323−-- | /O(n)/ Convert a 'String' into a 'Text'. Subject to fusion.

361+-- | /O(n)/ Convert a 'String' into a 'Text'. Subject to

362+-- fusion. Performs replacement on invalid scalar values.

324pack :: String -> Text

363pack :: String -> Text

325−pack = unstream . S.streamList

364+pack = unstream . S.streamList . L.map safe

326{-# INLINE [1] pack #-}

365{-# INLINE [1] pack #-}

327

366

328-- | /O(n)/ Convert a Text into a String. Subject to fusion.

367-- | /O(n)/ Convert a Text into a String. Subject to fusion.

@@ -330,10 +369,10 @@ unpack :: Text -> String

330unpack = S.unstreamList . stream

369unpack = S.unstreamList . stream

331{-# INLINE [1] unpack #-}

370{-# INLINE [1] unpack #-}

332

371

333−-- | /O(1)/ Convert a character into a Text.

372+-- | /O(1)/ Convert a character into a Text. Subject to fusion.

334−-- Subject to fusion.

373+-- Performs replacement on invalid scalar values.

335singleton :: Char -> Text

374singleton :: Char -> Text

336−singleton = unstream . S.singleton

375+singleton = unstream . S.singleton . safe

337{-# INLINE [1] singleton #-}

376{-# INLINE [1] singleton #-}

338

377

339-- -----------------------------------------------------------------------------

378-- -----------------------------------------------------------------------------

@@ -341,15 +380,17 @@ singleton = unstream . S.singleton

341

380

342-- | /O(n)/ Adds a character to the front of a 'Text'. This function

381-- | /O(n)/ Adds a character to the front of a 'Text'. This function

343-- is more costly than its 'List' counterpart because it requires

382-- is more costly than its 'List' counterpart because it requires

344−-- copying a new array. Subject to fusion.

383+-- copying a new array. Subject to fusion. Performs replacement on

384+-- invalid scalar values.

345cons :: Char -> Text -> Text

385cons :: Char -> Text -> Text

346−cons c t = unstream (S.cons c (stream t))

386+cons c t = unstream (S.cons (safe c) (stream t))

347{-# INLINE cons #-}

387{-# INLINE cons #-}

348

388

349-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the

389-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the

350-- entire array in the process, unless fused. Subject to fusion.

390-- entire array in the process, unless fused. Subject to fusion.

391+-- Performs replacement on invalid scalar values.

351snoc :: Text -> Char -> Text

392snoc :: Text -> Char -> Text

352−snoc t c = unstream (S.snoc (stream t) c)

393+snoc t c = unstream (S.snoc (stream t) (safe c))

353{-# INLINE snoc #-}

394{-# INLINE snoc #-}

354

395

355-- | /O(n)/ Appends one 'Text' to the other by copying both of them

396-- | /O(n)/ Appends one 'Text' to the other by copying both of them

@@ -523,9 +564,10 @@ compareLength t n = S.compareLengthI (stream t) n

523-- -----------------------------------------------------------------------------

564-- -----------------------------------------------------------------------------

524-- * Transformations

565-- * Transformations

525-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

566-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

526−-- each element of @t@. Subject to fusion.

567+-- each element of @t@. Subject to fusion. Performs replacement on

568+-- invalid scalar values.

527map :: (Char -> Char) -> Text -> Text

569map :: (Char -> Char) -> Text -> Text

528−map f t = unstream (S.map f (stream t))

570+map f t = unstream (S.map (safe . f) (stream t))

529{-# INLINE [1] map #-}

571{-# INLINE [1] map #-}

530

572

531-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

573-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

@@ -536,9 +578,10 @@ intercalate t = concat . (U.intersperse t)

536{-# INLINE intercalate #-}

578{-# INLINE intercalate #-}

537

579

538-- | /O(n)/ The 'intersperse' function takes a character and places it

580-- | /O(n)/ The 'intersperse' function takes a character and places it

539−-- between the characters of a 'Text'. Subject to fusion.

581+-- between the characters of a 'Text'. Subject to fusion. Performs

582+-- replacement on invalid scalar values.

540intersperse :: Char -> Text -> Text

583intersperse :: Char -> Text -> Text

541−intersperse c t = unstream (S.intersperse c (stream t))

584+intersperse c t = unstream (S.intersperse (safe c) (stream t))

542{-# INLINE intersperse #-}

585{-# INLINE intersperse #-}

543

586

544-- | /O(n)/ Reverse the characters of a string. Subject to fusion.

587-- | /O(n)/ Reverse the characters of a string. Subject to fusion.

@@ -615,7 +658,10 @@ toUpper t = unstream (S.toUpper (stream t))

615{-# INLINE toUpper #-}

658{-# INLINE toUpper #-}

616

659

617-- | /O(n)/ Left-justify a string to the given length, using the

660-- | /O(n)/ Left-justify a string to the given length, using the

618−-- specified fill character on the right. Subject to fusion. Examples:

661+-- specified fill character on the right. Subject to fusion.

662+-- Performs replacement on invalid scalar values.

663+--

664+-- Examples:

619--

665--

620-- > justifyLeft 7 'x' "foo" == "fooxxxx"

666-- > justifyLeft 7 'x' "foo" == "fooxxxx"

621-- > justifyLeft 3 'x' "foobar" == "foobar"

667-- > justifyLeft 3 'x' "foobar" == "foobar"

@@ -634,7 +680,10 @@ justifyLeft k c t

634 #-}

680 #-}

635

681

636-- | /O(n)/ Right-justify a string to the given length, using the

682-- | /O(n)/ Right-justify a string to the given length, using the

637−-- specified fill character on the left. Examples:

683+-- specified fill character on the left. Performs replacement on

684+-- invalid scalar values.

685+--

686+-- Examples:

638--

687--

639-- > justifyRight 7 'x' "bar" == "xxxxbar"

688-- > justifyRight 7 'x' "bar" == "xxxxbar"

640-- > justifyRight 3 'x' "foobar" == "foobar"

689-- > justifyRight 3 'x' "foobar" == "foobar"

@@ -645,8 +694,11 @@ justifyRight k c t

645 where len = length t

694 where len = length t

646{-# INLINE justifyRight #-}

695{-# INLINE justifyRight #-}

647

696

648−-- | /O(n)/ Center a string to the given length, using the

697+-- | /O(n)/ Center a string to the given length, using the specified

649−-- specified fill character on either side. Examples:

698+-- fill character on either side. Performs replacement on invalid

699+-- scalar values.

700+--

701+-- Examples:

650--

702--

651-- > center 8 'x' "HS" = "xxxHSxxx"

703-- > center 8 'x' "HS" = "xxxHSxxx"

652center :: Int -> Char -> Text -> Text

704center :: Int -> Char -> Text -> Text

@@ -761,6 +813,7 @@ minimum t = S.minimum (stream t)

761

813

762-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of

814-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of

763-- successive reduced values from the left. Subject to fusion.

815-- successive reduced values from the left. Subject to fusion.

816+-- Performs replacement on invalid scalar values.

764--

817--

765-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]

818-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]

766--

819--

@@ -768,11 +821,13 @@ minimum t = S.minimum (stream t)

768--

821--

769-- > last (scanl f z xs) == foldl f z xs.

822-- > last (scanl f z xs) == foldl f z xs.

770scanl :: (Char -> Char -> Char) -> Char -> Text -> Text

823scanl :: (Char -> Char -> Char) -> Char -> Text -> Text

771−scanl f z t = unstream (S.scanl f z (stream t))

824+scanl f z t = unstream (S.scanl g z (stream t))

825+ where g a b = safe (f a b)

772{-# INLINE scanl #-}

826{-# INLINE scanl #-}

773

827

774-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting

828-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting

775−-- value argument. Subject to fusion.

829+-- value argument. Subject to fusion. Performs replacement on

830+-- invalid scalar values.

776--

831--

777-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]

832-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]

778scanl1 :: (Char -> Char -> Char) -> Text -> Text

833scanl1 :: (Char -> Char -> Char) -> Text -> Text

@@ -780,15 +835,18 @@ scanl1 f t | null t = empty

780 | otherwise = scanl f (unsafeHead t) (unsafeTail t)

835 | otherwise = scanl f (unsafeHead t) (unsafeTail t)

781{-# INLINE scanl1 #-}

836{-# INLINE scanl1 #-}

782

837

783−-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.

838+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs

839+-- replacement on invalid scalar values.

784--

840--

785-- > scanr f v == reverse . scanl (flip f) v . reverse

841-- > scanr f v == reverse . scanl (flip f) v . reverse

786scanr :: (Char -> Char -> Char) -> Char -> Text -> Text

842scanr :: (Char -> Char -> Char) -> Char -> Text -> Text

787−scanr f z = S.reverse . S.reverseScanr f z . reverseStream

843+scanr f z = S.reverse . S.reverseScanr g z . reverseStream

844+ where g a b = safe (f a b)

788{-# INLINE scanr #-}

845{-# INLINE scanr #-}

789

846

790-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting

847-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting

791−-- value argument. Subject to fusion.

848+-- value argument. Subject to fusion. Performs replacement on

849+-- invalid scalar values.

792scanr1 :: (Char -> Char -> Char) -> Text -> Text

850scanr1 :: (Char -> Char -> Char) -> Text -> Text

793scanr1 f t | null t = empty

851scanr1 f t | null t = empty

794 | otherwise = scanr f (last t) (init t)

852 | otherwise = scanr f (last t) (init t)

@@ -796,9 +854,11 @@ scanr1 f t | null t = empty

796

854

797-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a

855-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a

798-- function to each element of a 'Text', passing an accumulating

856-- function to each element of a 'Text', passing an accumulating

799−-- parameter from left to right, and returns a final 'Text'.

857+-- parameter from left to right, and returns a final 'Text'. Performs

858+-- replacement on invalid scalar values.

800mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

859mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

801−mapAccumL f z0 = S.mapAccumL f z0 . stream

860+mapAccumL f z0 = S.mapAccumL g z0 . stream

861+ where g a b = second safe (f a b)

802{-# INLINE mapAccumL #-}

862{-# INLINE mapAccumL #-}

803

863

804-- | The 'mapAccumR' function behaves like a combination of 'map' and

864-- | The 'mapAccumR' function behaves like a combination of 'map' and

@@ -806,8 +866,10 @@ mapAccumL f z0 = S.mapAccumL f z0 . stream

806-- 'Text', passing an accumulating parameter from right to left, and

866-- 'Text', passing an accumulating parameter from right to left, and

807-- returning a final value of this accumulator together with the new

867-- returning a final value of this accumulator together with the new

808-- 'Text'.

868-- 'Text'.

869+-- Performs replacement on invalid scalar values.

809mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

870mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

810−mapAccumR f z0 = second reverse . S.mapAccumL f z0 . reverseStream

871+mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream

872+ where g a b = second safe (f a b)

811{-# INLINE mapAccumR #-}

873{-# INLINE mapAccumR #-}

812

874

813-- -----------------------------------------------------------------------------

875-- -----------------------------------------------------------------------------

@@ -840,7 +902,7 @@ replicate n t@(Text a o l)

840-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the

902-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the

841-- value of every element. Subject to fusion.

903-- value of every element. Subject to fusion.

842replicateChar :: Int -> Char -> Text

904replicateChar :: Int -> Char -> Text

843−replicateChar n c = unstream (S.replicateCharI n c)

905+replicateChar n c = unstream (S.replicateCharI n (safe c))

844{-# INLINE replicateChar #-}

906{-# INLINE replicateChar #-}

845

907

846-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'

908-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'

@@ -849,9 +911,9 @@ replicateChar n c = unstream (S.replicateCharI n c)

849-- returns 'Nothing' if it is done producing the 'Text', otherwise

911-- returns 'Nothing' if it is done producing the 'Text', otherwise

850-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the

912-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the

851-- string, and @b@ is the seed value for further production. Subject

913-- string, and @b@ is the seed value for further production. Subject

852−-- to fusion.

914+-- to fusion. Performs replacement on invalid scalar values.

853unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

915unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

854−unfoldr f s = unstream (S.unfoldr f s)

916+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)

855{-# INLINE unfoldr #-}

917{-# INLINE unfoldr #-}

856

918

857-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed

919-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed

@@ -859,9 +921,9 @@ unfoldr f s = unstream (S.unfoldr f s)

859-- first argument to 'unfoldrN'. This function is more efficient than

921-- first argument to 'unfoldrN'. This function is more efficient than

860-- 'unfoldr' when the maximum length of the result is known and

922-- 'unfoldr' when the maximum length of the result is known and

861-- correct, otherwise its performance is similar to 'unfoldr'. Subject

923-- correct, otherwise its performance is similar to 'unfoldr'. Subject

862−-- to fusion.

924+-- to fusion. Performs replacement on invalid scalar values.

863unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text

925unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text

864−unfoldrN n f s = unstream (S.unfoldrN n f s)

926+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)

865{-# INLINE unfoldrN #-}

927{-# INLINE unfoldrN #-}

866

928

867-- -----------------------------------------------------------------------------

929-- -----------------------------------------------------------------------------

@@ -1299,8 +1361,10 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)

1299

1361

1300-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function

1362-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function

1301-- given as the first argument, instead of a tupling function.

1363-- given as the first argument, instead of a tupling function.

1364+-- Performs replacement on invalid scalar values.

1302zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text

1365zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text

1303−zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))

1366+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))

1367+ where g a b = safe (f a b)

1304{-# INLINE [0] zipWith #-}

1368{-# INLINE [0] zipWith #-}

1305

1369

1306-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's

1370-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's

Data/Text/Internal.hs+24

@@ -22,8 +22,12 @@ module Data.Text.Internal

22 -- * Construction

23 , text

24 , textP

25+ -- * Safety

26+ , safe

25 -- * Code that must be here for accessibility

27 -- * Code that must be here for accessibility

26 , empty

28 , empty

29+ -- * Utilities

30+ , firstf

27 -- * Debugging

31 -- * Debugging

28 , showText

32 , showText

29 ) where

33 ) where

@@ -31,7 +35,9 @@ module Data.Text.Internal

31#if defined(ASSERTS)

35#if defined(ASSERTS)

32import Control.Exception (assert)

36import Control.Exception (assert)

33#endif

37#endif

38+import Data.Bits ((.&.))

34import qualified Data.Text.Array as A

39import qualified Data.Text.Array as A

40+import Data.Text.UnsafeChar (ord)

35import Data.Typeable (Typeable)

41import Data.Typeable (Typeable)

37-- | A space efficient, packed, unboxed Unicode text type.

43-- | A space efficient, packed, unboxed Unicode text type.

@@ -72,3 +78,21 @@ showText :: Text -> String

72showText (Text arr off len) =

78showText (Text arr off len) =

73 "Text " ++ show (A.toList arr off len) ++ ' ' :

79 "Text " ++ show (A.toList arr off len) ++ ' ' :

74 show off ++ ' ' : show len

80 show off ++ ' ' : show len

81+

82+-- | Map a 'Char' to a 'Text'-safe value.

83+--

84+-- UTF-16 surrogate code points are not included in the set of Unicode

85+-- scalar values, but are unfortunately admitted as valid 'Char'

86+-- values by Haskell. They cannot be represented in a 'Text'. This

87+-- function remaps those code points to the Unicode replacement

88+-- character \"�\", and leaves other code points unchanged.

89+safe :: Char -> Char

90+safe c

91+ | ord c .&. 0x1ff800 /= 0xd800 = c

92+ | otherwise = '\xfffd'

93+{-# INLINE safe #-}

94+

95+-- | Apply a function to the first element of an optional pair.

96+firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)

97+firstf f (Just (a, b)) = Just (f a, b)

98+firstf _ Nothing = Nothing

Data/Text/Lazy.hs+112−28

@@ -11,7 +11,12 @@

11-- Portability : GHC

12--

13-- A time and space-efficient implementation of Unicode text using

14+-- lists of packed arrays.

15+--

16+-- /Note/: Read below the synopsis for important notes on the use of

17+-- this module.

18+--

14−-- lists of packed arrays. This representation is suitable for high

19+-- The representation used by this module is suitable for high

15-- performance use and for streaming large quantities of data. It

20-- performance use and for streaming large quantities of data. It

16-- provides a means to manipulate a large body of text without

21-- provides a means to manipulate a large body of text without

17-- requiring that the entire content be resident in memory.

22-- requiring that the entire content be resident in memory.

@@ -31,7 +36,15 @@

32module Data.Text.Lazy

37module Data.Text.Lazy

33 (

38 (

39+ -- * Fusion

40+ -- $fusion

41+

42+ -- * Acceptable data

43+ -- $replacement

44+

45+ -- * Types

34 Text

46 Text

47+

35 -- * Creation and elimination

48 -- * Creation and elimination

36 , pack

49 , pack

37 , unpack

50 , unpack

@@ -200,10 +213,62 @@ import qualified Data.Text.Lazy.Fusion as S

200import Data.Text.Fusion.Internal (PairS(..))

213import Data.Text.Fusion.Internal (PairS(..))

201import Data.Text.Lazy.Fusion (stream, unstream)

214import Data.Text.Lazy.Fusion (stream, unstream)

202import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)

215import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)

203−import Data.Text.Internal (textP)

216+import Data.Text.Internal (firstf, safe, textP)

204import qualified Data.Text.Util as U

217import qualified Data.Text.Util as U

205import Data.Text.Lazy.Search (indices)

218import Data.Text.Lazy.Search (indices)

206

219

220+-- $fusion

221+--

222+-- Most of the functions in this module are subject to /fusion/,

223+-- meaning that a pipeline of such functions will usually allocate at

224+-- most one 'Text' value.

225+--

226+-- As an example, consider the following pipeline:

227+--

228+-- > import Data.Text.Lazy as T

229+-- > import Data.Text.Lazy.Encoding as E

230+-- > import Data.ByteString.Lazy (ByteString)

231+-- >

232+-- > countChars :: ByteString -> Int

233+-- > countChars = T.length . T.toUpper . E.decodeUtf8

234+--

235+-- From the type signatures involved, this looks like it should

236+-- allocate one 'ByteString' value, and two 'Text' values. However,

237+-- when a module is compiled with optimisation enabled under GHC, the

238+-- two intermediate 'Text' values will be optimised away, and the

239+-- function will be compiled down to a single loop over the source

240+-- 'ByteString'.

241+--

242+-- Functions that can be fused by the compiler are documented with the

243+-- phrase \"Subject to fusion\".

244+

245+-- $replacement

246+--

247+-- A 'Text' value is a sequence of Unicode scalar values, as defined

248+-- in §3.9, definition D76 of the Unicode 5.2 standard:

249+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

250+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

251+-- inclusive. Haskell implementations admit all Unicode code points

252+-- (§3.4, definition D10) as 'Char' values, including code points

253+-- from this invalid range. This means that there are some 'Char'

254+-- values that are not valid Unicode scalar values, and the functions

255+-- in this module must handle those cases.

256+--

257+-- Within this module, many functions construct a 'Text' from one or

258+-- more 'Char' values. Those functions will substitute 'Char' values

259+-- that are not valid Unicode scalar values with the replacement

260+-- character \"�\" (U+FFFD). Functions that perform this

261+-- inspection and replacement are documented with the phrase

262+-- \"Performs replacement on invalid scalar values\".

263+--

264+-- (One reason for this policy of replacement is that internally, a

265+-- 'Text' value is represented as packed UTF-16 data. Values in the

266+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

267+-- code points, and so cannot be represented. The functions replace

268+-- invalid scalar values, instead of dropping them, as a security

269+-- measure. For details, see Unicode Technical Report 36, §3.5:

270+-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)

271+

207equal :: Text -> Text -> Bool

272equal :: Text -> Text -> Bool

208equal Empty Empty = True

273equal Empty Empty = True

209equal Empty _ = False

274equal Empty _ = False

@@ -274,9 +339,9 @@ instance Data Text where

274

339

275-- | /O(n)/ Convert a 'String' into a 'Text'.

340-- | /O(n)/ Convert a 'String' into a 'Text'.

276--

341--

277−-- This function is subject to array fusion.

342+-- Subject to fusion. Performs replacement on invalid scalar values.

278pack :: String -> Text

343pack :: String -> Text

279−pack s = unstream (S.streamList s)

344+pack = unstream . S.streamList . L.map safe

280{-# INLINE [1] pack #-}

345{-# INLINE [1] pack #-}

281

346

282-- | /O(n)/ Convert a 'Text' into a 'String'.

347-- | /O(n)/ Convert a 'Text' into a 'String'.

@@ -285,8 +350,8 @@ unpack :: Text -> String

285unpack t = S.unstreamList (stream t)

350unpack t = S.unstreamList (stream t)

286{-# INLINE [1] unpack #-}

351{-# INLINE [1] unpack #-}

287

352

288−-- | /O(1)/ Convert a character into a Text.

353+-- | /O(1)/ Convert a character into a Text. Subject to fusion.

289−-- Subject to fusion.

354+-- Performs replacement on invalid scalar values.

290singleton :: Char -> Text

355singleton :: Char -> Text

291singleton c = Chunk (T.singleton c) Empty

356singleton c = Chunk (T.singleton c) Empty

292{-# INLINE [1] singleton #-}

357{-# INLINE [1] singleton #-}

@@ -469,9 +534,10 @@ compareLength t n = S.compareLengthI (stream t) n

469-- properties of code.

534-- properties of code.

470

535

471-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

536-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

472−-- each element of @t@. Subject to array fusion.

537+-- each element of @t@. Subject to fusion. Performs replacement on

538+-- invalid scalar values.

473map :: (Char -> Char) -> Text -> Text

539map :: (Char -> Char) -> Text -> Text

474−map f t = unstream (S.map f (stream t))

540+map f t = unstream (S.map (safe . f) (stream t))

475{-# INLINE [1] map #-}

541{-# INLINE [1] map #-}

476

542

477-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

543-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

@@ -488,7 +554,10 @@ intersperse c t = unstream (S.intersperse c (stream t))

488{-# INLINE intersperse #-}

554{-# INLINE intersperse #-}

489

555

490-- | /O(n)/ Left-justify a string to the given length, using the

556-- | /O(n)/ Left-justify a string to the given length, using the

491−-- specified fill character on the right. Subject to fusion. Examples:

557+-- specified fill character on the right. Subject to fusion. Performs

558+-- replacement on invalid scalar values.

559+--

560+-- Examples:

492--

561--

493-- > justifyLeft 7 'x' "foo" == "fooxxxx"

562-- > justifyLeft 7 'x' "foo" == "fooxxxx"

494-- > justifyLeft 3 'x' "foobar" == "foobar"

563-- > justifyLeft 3 'x' "foobar" == "foobar"

@@ -507,7 +576,10 @@ justifyLeft k c t

507 #-}

576 #-}

508

577

509-- | /O(n)/ Right-justify a string to the given length, using the

578-- | /O(n)/ Right-justify a string to the given length, using the

510−-- specified fill character on the left. Examples:

579+-- specified fill character on the left. Performs replacement on

580+-- invalid scalar values.

581+--

582+-- Examples:

511--

583--

512-- > justifyRight 7 'x' "bar" == "xxxxbar"

584-- > justifyRight 7 'x' "bar" == "xxxxbar"

513-- > justifyRight 3 'x' "foobar" == "foobar"

585-- > justifyRight 3 'x' "foobar" == "foobar"

@@ -518,8 +590,11 @@ justifyRight k c t

518 where len = length t

590 where len = length t

519{-# INLINE justifyRight #-}

591{-# INLINE justifyRight #-}

520

592

521−-- | /O(n)/ Center a string to the given length, using the

593+-- | /O(n)/ Center a string to the given length, using the specified

522−-- specified fill character on either side. Examples:

594+-- fill character on either side. Performs replacement on invalid

595+-- scalar values.

596+--

597+-- Examples:

523--

598--

524-- > center 8 'x' "HS" = "xxxHSxxx"

599-- > center 8 'x' "HS" = "xxxHSxxx"

525center :: Int64 -> Char -> Text -> Text

600center :: Int64 -> Char -> Text -> Text

@@ -688,8 +763,8 @@ minimum t = S.minimum (stream t)

688{-# INLINE minimum #-}

763{-# INLINE minimum #-}

689

764

690-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of

765-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of

691−-- successive reduced values from the left. This function is subject

766+-- successive reduced values from the left. Subject to fusion.

692−-- to array fusion.

767+-- Performs replacement on invalid scalar values.

693--

768--

694-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]

769-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]

695--

770--

@@ -697,11 +772,13 @@ minimum t = S.minimum (stream t)

697--

772--

698-- > last (scanl f z xs) == foldl f z xs.

773-- > last (scanl f z xs) == foldl f z xs.

699scanl :: (Char -> Char -> Char) -> Char -> Text -> Text

774scanl :: (Char -> Char -> Char) -> Char -> Text -> Text

700−scanl f z t = unstream (S.scanl f z (stream t))

775+scanl f z t = unstream (S.scanl g z (stream t))

776+ where g a b = safe (f a b)

701{-# INLINE scanl #-}

777{-# INLINE scanl #-}

702

778

703-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting

779-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting

704−-- value argument. This function is subject to array fusion.

780+-- value argument. Subject to fusion. Performs replacement on

781+-- invalid scalar values.

705--

782--

706-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]

783-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]

707scanl1 :: (Char -> Char -> Char) -> Text -> Text

784scanl1 :: (Char -> Char -> Char) -> Text -> Text

@@ -710,21 +787,24 @@ scanl1 f t0 = case uncons t0 of

710 Just (t,ts) -> scanl f t ts

787 Just (t,ts) -> scanl f t ts

711{-# INLINE scanl1 #-}

788{-# INLINE scanl1 #-}

712

789

713−-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.

790+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs

791+-- replacement on invalid scalar values.

714--

792--

715-- > scanr f v == reverse . scanl (flip f) v . reverse

793-- > scanr f v == reverse . scanl (flip f) v . reverse

716scanr :: (Char -> Char -> Char) -> Char -> Text -> Text

794scanr :: (Char -> Char -> Char) -> Char -> Text -> Text

717−scanr f v = reverse . scanl (flip f) v . reverse

795+scanr f v = reverse . scanl g v . reverse

796+ where g a b = safe (f b a)

718

797

719-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting

798-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting

720−-- value argument.

799+-- value argument. Performs replacement on invalid scalar values.

721scanr1 :: (Char -> Char -> Char) -> Text -> Text

800scanr1 :: (Char -> Char -> Char) -> Text -> Text

722scanr1 f t | null t = empty

801scanr1 f t | null t = empty

723 | otherwise = scanr f (last t) (init t)

802 | otherwise = scanr f (last t) (init t)

724

803

725-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a

804-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a

726-- function to each element of a 'Text', passing an accumulating

805-- function to each element of a 'Text', passing an accumulating

727−-- parameter from left to right, and returns a final 'Text'.

806+-- parameter from left to right, and returns a final 'Text'. Performs

807+-- replacement on invalid scalar values.

728mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

808mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

729mapAccumL f = go

809mapAccumL f = go

730 where

810 where

@@ -738,7 +818,7 @@ mapAccumL f = go

738-- a strict 'foldr'; it applies a function to each element of a

818-- a strict 'foldr'; it applies a function to each element of a

739-- 'Text', passing an accumulating parameter from right to left, and

819-- 'Text', passing an accumulating parameter from right to left, and

740-- returning a final value of this accumulator together with the new

820-- returning a final value of this accumulator together with the new

741−-- 'Text'.

821+-- 'Text'. Performs replacement on invalid scalar values.

742mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

822mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)

743mapAccumR f = go

823mapAccumR f = go

744 where

824 where

@@ -762,7 +842,7 @@ replicate n t

762-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the

842-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the

763-- value of every element. Subject to fusion.

843-- value of every element. Subject to fusion.

764replicateChar :: Int64 -> Char -> Text

844replicateChar :: Int64 -> Char -> Text

765−replicateChar n c = unstream (S.replicateCharI n c)

845+replicateChar n c = unstream (S.replicateCharI n (safe c))

766{-# INLINE replicateChar #-}

846{-# INLINE replicateChar #-}

767

847

768{-# RULES

848{-# RULES

@@ -775,9 +855,10 @@ replicateChar n c = unstream (S.replicateCharI n c)

775-- 'Text' from a seed value. The function takes the element and

855-- 'Text' from a seed value. The function takes the element and

776-- returns 'Nothing' if it is done producing the 'Text', otherwise

856-- returns 'Nothing' if it is done producing the 'Text', otherwise

777-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the

857-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the

778−-- string, and @b@ is the seed value for further production.

858+-- string, and @b@ is the seed value for further production. Performs

859+-- replacement on invalid scalar values.

779−unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

860+unfoldr :: (a -> Maybe (Char,a)) -> a -> Text

780−unfoldr f s = unstream (S.unfoldr f s)

861+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)

781{-# INLINE unfoldr #-}

862{-# INLINE unfoldr #-}

782

863

783-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed

864-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed

@@ -785,8 +866,9 @@ unfoldr f s = unstream (S.unfoldr f s)

785-- first argument to 'unfoldrN'. This function is more efficient than

866-- first argument to 'unfoldrN'. This function is more efficient than

786-- 'unfoldr' when the maximum length of the result is known and

867-- 'unfoldr' when the maximum length of the result is known and

787-- correct, otherwise its performance is similar to 'unfoldr'.

868-- correct, otherwise its performance is similar to 'unfoldr'.

869+-- Performs replacement on invalid scalar values.

788−unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text

870+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text

789−unfoldrN n f s = unstream (S.unfoldrN n f s)

871+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)

790{-# INLINE unfoldrN #-}

872{-# INLINE unfoldrN #-}

791

873

792-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the

874-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the

@@ -1355,8 +1437,10 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)

1355

1437

1356-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function

1438-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function

1357-- given as the first argument, instead of a tupling function.

1439-- given as the first argument, instead of a tupling function.

1440+-- Performs replacement on invalid scalar values.

1358zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text

1441zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text

1359−zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))

1442+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))

1443+ where g a b = safe (f a b)

1360{-# INLINE [0] zipWith #-}

1444{-# INLINE [0] zipWith #-}

1361

1445

1362revChunks :: [T.Text] -> Text

1446revChunks :: [T.Text] -> Text