master...9fac5db · text

.gitignore+10−3

since splitsnapshots

unifiedsplit

@@ -1,3 +1,10 @@

1−dist

1+/dist/

2+/dist-boot/

3+/dist-install/

4+/dist-newstyle/

2−cabal-dev

5+/cabal-dev/

3−cabal.sandbox.config

6+/cabal.sandbox.config

7+/ghc.mk

8+/GNUmakefile

9+/.ghc.environment.*

10+/cabal.project.local

.hgtags+1

@@ -64,3 +64,4 @@ f8a60279237c1c28e094868740d9188dd94fcc9e 1.2.0.0

64631fd055e27fa10983e17e9ad36d1b285b85c661 1.2.1.3

658221a68ed5ec37028357df9c6c25f52320e54ad9 1.2.2.0

66ce1c5ac65eb9f5580114b4e380092b05a485e95d 1.2.2.1

67+d72c634a91ffe27ec5cd03921dd53efc8129d8f6 1.2.2.2

.travis.ymlnew file+111

@@ -0,0 +1,111 @@

1+# This Travis job script has been generated by a script via

2+#

3+# make_travis_yml_2.hs '-o' '.travis.yml' 'text.cabal'

4+#

5+# For more information, see https://github.com/hvr/multi-ghc-travis

6+#

7+language: c

8+sudo: false

9+

10+# Bryan threatens to turn off Travis-CI if he gets any email :-)

11+notifications:

12+ email: false

13+

14+cache:

15+ directories:

16+ - $HOME/.cabal/packages

17+ - $HOME/.cabal/store

18+

19+before_cache:

20+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/build-reports.log

21+ # remove files that are regenerated by 'cabal update'

22+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/00-index.*

23+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/*.json

24+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.cache

25+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.tar

26+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.tar.idx

27+

28+matrix:

29+ include:

30+ - compiler: "ghc-7.0.4"

31+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

32+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.0.4], sources: [hvr-ghc]}}

33+ - compiler: "ghc-7.2.2"

34+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

35+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.2.2], sources: [hvr-ghc]}}

36+ - compiler: "ghc-7.4.2"

37+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

38+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.4.2], sources: [hvr-ghc]}}

39+ - compiler: "ghc-7.6.3"

40+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

41+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.6.3], sources: [hvr-ghc]}}

42+ - compiler: "ghc-7.8.4"

43+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

44+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.8.4], sources: [hvr-ghc]}}

45+ - compiler: "ghc-7.10.3"

46+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

47+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.10.3], sources: [hvr-ghc]}}

48+ - compiler: "ghc-8.0.2"

49+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

50+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-8.0.2], sources: [hvr-ghc]}}

51+ - compiler: "ghc-8.2.2"

52+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

53+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-8.2.2], sources: [hvr-ghc]}}

54+

55+before_install:

56+ - HC=${CC}

57+ - HCPKG=${HC/ghc/ghc-pkg}

58+ - unset CC

59+ - PATH=/opt/ghc/bin:/opt/ghc-ppa-tools/bin:$PATH

60+ - PKGNAME='text'

61+

62+install:

63+ - cabal --version

64+ - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"

65+ - BENCH=${BENCH---enable-benchmarks}

66+ - TEST=${TEST---enable-tests}

67+ - HADDOCK=${HADDOCK-true}

68+ - INSTALLED=${INSTALLED-true}

69+ - travis_retry cabal update -v

70+ - sed -i.bak 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config

71+ - rm -fv cabal.project.local

72+ - "echo 'packages: .' > cabal.project"

73+ - rm -f cabal.project.freeze

74+ - cabal new-build -w ${HC} ${TEST} ${BENCH} --dep -j2 all

75+ - cabal new-build -w ${HC} --disable-tests --disable-benchmarks --dep -j2 all

76+

77+# Here starts the actual work to be performed for the package under test;

78+# any command which exits with a non-zero exit code causes the build to fail.

79+script:

80+ - if [ -f configure.ac ]; then autoreconf -i; fi

81+ - rm -rf .ghc.environment.* dist/

82+ - cabal sdist # test that a source-distribution can be generated

83+ - cd dist/

84+ - SRCTAR=(${PKGNAME}-*.tar.gz)

85+ - SRC_BASENAME="${SRCTAR/%.tar.gz}"

86+ - tar -xvf "./$SRC_BASENAME.tar.gz"

87+ - cd "$SRC_BASENAME/"

88+## from here on, CWD is inside the extracted source-tarball

89+ - rm -fv cabal.project.local

90+ - "echo 'packages: .' > cabal.project"

91+ # this builds all libraries and executables (without tests/benchmarks)

92+ - rm -f cabal.project.freeze

93+ - cabal new-build -w ${HC} --disable-tests --disable-benchmarks all

94+ # this builds all libraries and executables (including tests/benchmarks)

95+ # - rm -rf ./dist-newstyle

96+

97+ # Build with installed constraints for packages in global-db

98+ - if $INSTALLED; then

99+ echo cabal new-build -w ${HC} --disable-tests --disable-benchmarks $(${HCPKG} list --global --simple-output --names-only | sed 's/$[a-zA-Z0-9-]\{1,\}$ */--constraint="\1 installed" /g') all | sh;

100+ else echo "Not building with installed constraints"; fi

101+

102+ # build & run tests, build benchmarks

103+ - cabal new-build -w ${HC} ${TEST} ${BENCH} all

104+ - if [ "x$TEST" = "x--enable-tests" ]; then cabal new-test -w ${HC} ${TEST} all; fi

105+

106+ # haddock

107+ - rm -rf ./dist-newstyle

108+ - if $HADDOCK; then cabal new-haddock -w ${HC} --disable-tests --disable-benchmarks all; else echo "Skipping haddock generation";fi

109+

110+# REGENDATA ["-o",".travis.yml","text.cabal"]

111+# EOF

Data/Text.hs+177−46

@@ -15,7 +15,6 @@

15--

16-- License : BSD-style

17-- Maintainer : bos@serpentine.com

18−-- Stability : experimental

19-- Portability : GHC

18-- Portability : GHC

20--

19--

21-- A time and space-efficient implementation of Unicode text.

20-- A time and space-efficient implementation of Unicode text.

@@ -32,8 +31,9 @@

32--

31--

33-- To use an extended and very rich family of functions for working

32-- To use an extended and very rich family of functions for working

34-- with Unicode text (including normalization, regular expressions,

33-- with Unicode text (including normalization, regular expressions,

35−-- non-standard encodings, text breaking, and locales), see

34+-- non-standard encodings, text breaking, and locales), see the

36−-- <http://hackage.haskell.org/package/text-icu the text-icu package >.

35+-- <http://hackage.haskell.org/package/text-icu text-icu package >.

36+--

37

38module Data.Text

39 (

@@ -43,6 +43,9 @@ module Data.Text

43 -- * Acceptable data

44 -- $replacement

45

46+ -- * Definition of character

47+ -- $character_definition

48+

46 -- * Fusion

49 -- * Fusion

47 -- $fusion

50 -- $fusion

48

51

@@ -60,6 +63,7 @@ module Data.Text

60 , snoc

63 , snoc

61 , append

64 , append

62 , uncons

65 , uncons

66+ , unsnoc

63 , head

67 , head

64 , last

68 , last

65 , tail

69 , tail

@@ -246,6 +250,18 @@ import qualified GHC.Exts as Exts

246import Text.Printf (PrintfArg, formatArg, formatString)

250import Text.Printf (PrintfArg, formatArg, formatString)

247#endif

251#endif

248

252

253+-- $character_definition

254+--

255+-- This package uses the term /character/ to denote Unicode /code points/.

256+--

257+-- Note that this is not the same thing as a grapheme (e.g. a

258+-- composition of code points that form one visual symbol). For

259+-- instance, consider the grapheme \"ä\". This symbol has two

260+-- Unicode representations: a single code-point representation

261+-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),

262+-- and a two code point representation @U+0061@ (the \"@A@\" code

263+-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).

264+

249-- $strict

265-- $strict

250--

266--

251-- This package provides both strict and lazy 'Text' types. The

267-- This package provides both strict and lazy 'Text' types. The

@@ -334,9 +350,13 @@ instance Read Text where

334 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

350 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

335

351

336#if MIN_VERSION_base(4,9,0)

352#if MIN_VERSION_base(4,9,0)

353+-- | Non-orphan 'Semigroup' instance only defined for

337−-- Semigroup orphan instances for older GHCs are provided by

354+-- @base-4.9.0.0@ and later; orphan instances for older GHCs are

355+-- provided by

356+-- the [semigroups](http://hackage.haskell.org/package/semigroups)

338−-- 'semigroups` package

357+-- package

339−

358+--

359+-- @since 1.2.2.0

340instance Semigroup Text where

360instance Semigroup Text where

341 (<>) = append

361 (<>) = append

342#endif

362#endif

@@ -354,6 +374,7 @@ instance IsString Text where

354 fromString = pack

374 fromString = pack

355

375

356#if __GLASGOW_HASKELL__ >= 708

376#if __GLASGOW_HASKELL__ >= 708

377+-- | @since 1.2.0.0

357instance Exts.IsList Text where

378instance Exts.IsList Text where

358 type Item Text = Char

379 type Item Text = Char

359 fromList = pack

380 fromList = pack

@@ -364,6 +385,7 @@ instance Exts.IsList Text where

364instance NFData Text where rnf !_ = ()

385instance NFData Text where rnf !_ = ()

365#endif

386#endif

366

387

388+-- | @since 1.2.1.0

367instance Binary Text where

389instance Binary Text where

368 put t = put (encodeUtf8 t)

390 put t = put (encodeUtf8 t)

369 get = do

391 get = do

@@ -397,6 +419,8 @@ instance Data Text where

397

419

398#if MIN_VERSION_base(4,7,0)

420#if MIN_VERSION_base(4,7,0)

399-- | Only defined for @base-4.7.0.0@ and later

421-- | Only defined for @base-4.7.0.0@ and later

422+--

423+-- @since 1.2.2.0

400instance PrintfArg Text where

424instance PrintfArg Text where

401 formatArg txt = formatString $ unpack txt

425 formatArg txt = formatString $ unpack txt

402#endif

426#endif

@@ -545,6 +569,19 @@ init (Text arr off len) | len <= 0 = emptyError "init"

545 unstream (S.init (stream t)) = init t

569 unstream (S.init (stream t)) = init t

546 #-}

570 #-}

547

571

572+-- | /O(1)/ Returns all but the last character and the last character of a

573+-- 'Text', or 'Nothing' if empty.

574+--

575+-- @since 1.2.3.0

576+unsnoc :: Text -> Maybe (Text, Char)

577+unsnoc (Text arr off len)

578+ | len <= 0 = Nothing

579+ | n < 0xDC00 || n > 0xDFFF = Just (text arr off (len-1), unsafeChr n)

580+ | otherwise = Just (text arr off (len-2), U16.chr2 n0 n)

581+ where n = A.unsafeIndex arr (off+len-1)

582+ n0 = A.unsafeIndex arr (off+len-2)

583+{-# INLINE [1] unsnoc #-}

584+

548-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

585-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

549-- fusion.

586-- fusion.

550null :: Text -> Bool

587null :: Text -> Bool

@@ -624,8 +661,15 @@ compareLength t n = S.compareLengthI (stream t) n

624-- -----------------------------------------------------------------------------

661-- -----------------------------------------------------------------------------

625-- * Transformations

662-- * Transformations

626-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

663-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

664+-- each element of @t@.

665+--

666+-- Example:

667+--

668+-- >>> let message = pack "I am not angry. Not at all."

669+-- >>> T.map (\c -> if c == '.' then '!' else c) message

670+-- "I am not angry! Not at all!"

671+--

627−-- each element of @t@. Subject to fusion. Performs replacement on

672+-- Subject to fusion. Performs replacement on invalid scalar values.

628−-- invalid scalar values.

629map :: (Char -> Char) -> Text -> Text

673map :: (Char -> Char) -> Text -> Text

630map f t = unstream (S.map (safe . f) (stream t))

674map f t = unstream (S.map (safe . f) (stream t))

631{-# INLINE [1] map #-}

675{-# INLINE [1] map #-}

@@ -633,18 +677,36 @@ map f t = unstream (S.map (safe . f) (stream t))

633-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

677-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

634-- 'Text's and concatenates the list after interspersing the first

678-- 'Text's and concatenates the list after interspersing the first

635-- argument between each element of the list.

679-- argument between each element of the list.

680+--

681+-- Example:

682+--

683+-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]

684+-- "WeNI!seekNI!theNI!HolyNI!Grail"

636intercalate :: Text -> [Text] -> Text

685intercalate :: Text -> [Text] -> Text

637intercalate t = concat . (F.intersperse t)

686intercalate t = concat . (F.intersperse t)

638{-# INLINE intercalate #-}

687{-# INLINE intercalate #-}

639

688

640-- | /O(n)/ The 'intersperse' function takes a character and places it

689-- | /O(n)/ The 'intersperse' function takes a character and places it

641−-- between the characters of a 'Text'. Subject to fusion. Performs

690+-- between the characters of a 'Text'.

691+--

692+-- Example:

693+--

694+-- >>> T.intersperse '.' "SHIELD"

695+-- "S.H.I.E.L.D"

696+--

642−-- replacement on invalid scalar values.

697+-- Subject to fusion. Performs replacement on invalid scalar values.

643intersperse :: Char -> Text -> Text

698intersperse :: Char -> Text -> Text

644intersperse c t = unstream (S.intersperse (safe c) (stream t))

699intersperse c t = unstream (S.intersperse (safe c) (stream t))

645{-# INLINE intersperse #-}

700{-# INLINE intersperse #-}

646

701

647−-- | /O(n)/ Reverse the characters of a string. Subject to fusion.

702+-- | /O(n)/ Reverse the characters of a string.

703+--

704+-- Example:

705+--

706+-- >>> T.reverse "desrever"

707+-- "reversed"

708+--

709+-- Subject to fusion.

648reverse :: Text -> Text

710reverse :: Text -> Text

649reverse t = S.reverse (stream t)

711reverse t = S.reverse (stream t)

650{-# INLINE reverse #-}

712{-# INLINE reverse #-}

@@ -663,12 +725,14 @@ reverse t = S.reverse (stream t)

663-- @needle@ occurs in @replacement@, that occurrence will /not/ itself

725-- @needle@ occurs in @replacement@, that occurrence will /not/ itself

664-- be replaced recursively:

726-- be replaced recursively:

665--

727--

666−-- > replace "oo" "foo" "oo" == "foo"

728+-- >>> replace "oo" "foo" "oo"

729+-- "foo"

667--

730--

668-- In cases where several instances of @needle@ overlap, only the

731-- In cases where several instances of @needle@ overlap, only the

669-- first one will be replaced:

732-- first one will be replaced:

670--

733--

671−-- > replace "ofo" "bar" "ofofo" == "barfo"

734+-- >>> replace "ofo" "bar" "ofofo"

735+-- "barfo"

672--

736--

673-- In (unlikely) bad cases, this function's time complexity degrades

737-- In (unlikely) bad cases, this function's time complexity degrades

674-- towards /O(n*m)/.

738-- towards /O(n*m)/.

@@ -782,6 +846,8 @@ toUpper t = unstream (S.toUpper (stream t))

782-- guides disagree on whether the book name \"The Hill of the Red

846-- guides disagree on whether the book name \"The Hill of the Red

783-- Fox\" is correctly title cased—but this function will

847-- Fox\" is correctly title cased—but this function will

784-- capitalize /every/ word.

848-- capitalize /every/ word.

849+--

850+-- @since 1.0.0.0

785toTitle :: Text -> Text

851toTitle :: Text -> Text

786toTitle t = unstream (S.toTitle (stream t))

852toTitle t = unstream (S.toTitle (stream t))

787{-# INLINE toTitle #-}

853{-# INLINE toTitle #-}

@@ -792,8 +858,11 @@ toTitle t = unstream (S.toTitle (stream t))

792--

858--

793-- Examples:

859-- Examples:

794--

860--

795−-- > justifyLeft 7 'x' "foo" == "fooxxxx"

861+-- >>> justifyLeft 7 'x' "foo"

862+-- "fooxxxx"

863+--

796−-- > justifyLeft 3 'x' "foobar" == "foobar"

864+-- >>> justifyLeft 3 'x' "foobar"

865+-- "foobar"

797justifyLeft :: Int -> Char -> Text -> Text

866justifyLeft :: Int -> Char -> Text -> Text

798justifyLeft k c t

867justifyLeft k c t

799 | len >= k = t

868 | len >= k = t

@@ -814,8 +883,11 @@ justifyLeft k c t

814--

883--

815-- Examples:

884-- Examples:

816--

885--

817−-- > justifyRight 7 'x' "bar" == "xxxxbar"

886+-- >>> justifyRight 7 'x' "bar"

887+-- "xxxxbar"

888+--

818−-- > justifyRight 3 'x' "foobar" == "foobar"

889+-- >>> justifyRight 3 'x' "foobar"

890+-- "foobar"

819justifyRight :: Int -> Char -> Text -> Text

891justifyRight :: Int -> Char -> Text -> Text

820justifyRight k c t

892justifyRight k c t

821 | len >= k = t

893 | len >= k = t

@@ -829,7 +901,8 @@ justifyRight k c t

829--

901--

830-- Examples:

902-- Examples:

831--

903--

832−-- > center 8 'x' "HS" = "xxxHSxxx"

904+-- >>> center 8 'x' "HS"

905+-- "xxxHSxxx"

833center :: Int -> Char -> Text -> Text

906center :: Int -> Char -> Text -> Text

834center k c t

907center k c t

835 | len >= k = t

908 | len >= k = t

@@ -844,6 +917,14 @@ center k c t

844-- of its 'Text' argument. Note that this function uses 'pack',

917-- of its 'Text' argument. Note that this function uses 'pack',

845-- 'unpack', and the list version of transpose, and is thus not very

918-- 'unpack', and the list version of transpose, and is thus not very

846-- efficient.

919-- efficient.

920+--

921+-- Examples:

922+--

923+-- >>> transpose ["green","orange"]

924+-- ["go","rr","ea","en","ng","e"]

925+--

926+-- >>> transpose ["blue","red"]

927+-- ["br","le","ud","e"]

847transpose :: [Text] -> [Text]

928transpose :: [Text] -> [Text]

848transpose ts = P.map pack (L.transpose (P.map unpack ts))

929transpose ts = P.map pack (L.transpose (P.map unpack ts))

849

930

@@ -1088,7 +1169,10 @@ iterN n t@(Text _arr _off len) = loop 0 0

1088--

1169--

1089-- Examples:

1170-- Examples:

1090--

1171--

1091−-- > takeEnd 3 "foobar" == "bar"

1172+-- >>> takeEnd 3 "foobar"

1173+-- "bar"

1174+--

1175+-- @since 1.1.1.0

1092takeEnd :: Int -> Text -> Text

1176takeEnd :: Int -> Text -> Text

1093takeEnd n t@(Text arr off len)

1177takeEnd n t@(Text arr off len)

1094 | n <= 0 = empty

1178 | n <= 0 = empty

@@ -1099,8 +1183,8 @@ takeEnd n t@(Text arr off len)

1099iterNEnd :: Int -> Text -> Int

1183iterNEnd :: Int -> Text -> Int

1100iterNEnd n t@(Text _arr _off len) = loop (len-1) n

1184iterNEnd n t@(Text _arr _off len) = loop (len-1) n

1101 where loop i !m

1185 where loop i !m

1186+ | m <= 0 = i+1

1102 | i <= 0 = 0

1187 | i <= 0 = 0

1103− | m <= 1 = i

1104 | otherwise = loop (i+d) (m-1)

1188 | otherwise = loop (i+d) (m-1)

1105 where d = reverseIter_ t i

1189 where d = reverseIter_ t i

1106

1190

@@ -1127,7 +1211,10 @@ drop n t@(Text arr off len)

1127--

1211--

1128-- Examples:

1212-- Examples:

1129--

1213--

1130−-- > dropEnd 3 "foobar" == "foo"

1214+-- >>> dropEnd 3 "foobar"

1215+-- "foo"

1216+--

1217+-- @since 1.1.1.0

1131dropEnd :: Int -> Text -> Text

1218dropEnd :: Int -> Text -> Text

1132dropEnd n t@(Text arr off len)

1219dropEnd n t@(Text arr off len)

1133 | n <= 0 = t

1220 | n <= 0 = t

@@ -1157,7 +1244,10 @@ takeWhile p t@(Text arr off len) = loop 0

1157-- satisfy @p@. Subject to fusion.

1244-- satisfy @p@. Subject to fusion.

1158-- Examples:

1245-- Examples:

1159--

1246--

1160−-- > takeWhileEnd (=='o') "foo" == "oo"

1247+-- >>> takeWhileEnd (=='o') "foo"

1248+-- "oo"

1249+--

1250+-- @since 1.2.2.0

1161takeWhileEnd :: (Char -> Bool) -> Text -> Text

1251takeWhileEnd :: (Char -> Bool) -> Text -> Text

1162takeWhileEnd p t@(Text arr off len) = loop (len-1) len

1252takeWhileEnd p t@(Text arr off len) = loop (len-1) len

1163 where loop !i !l | l <= 0 = t

1253 where loop !i !l | l <= 0 = t

@@ -1196,7 +1286,8 @@ dropWhile p t@(Text arr off len) = loop 0 0

1196--

1286--

1197-- Examples:

1287-- Examples:

1198--

1288--

1199−-- > dropWhileEnd (=='.') "foo..." == "foo"

1289+-- >>> dropWhileEnd (=='.') "foo..."

1290+-- "foo"

1200dropWhileEnd :: (Char -> Bool) -> Text -> Text

1291dropWhileEnd :: (Char -> Bool) -> Text -> Text

1201dropWhileEnd p t@(Text arr off len) = loop (len-1) len

1292dropWhileEnd p t@(Text arr off len) = loop (len-1) len

1202 where loop !i !l | l <= 0 = empty

1293 where loop !i !l | l <= 0 = empty

@@ -1314,9 +1405,14 @@ tails t | null t = [empty]

1314--

1405--

1315-- Examples:

1406-- Examples:

1316--

1407--

1317−-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1408+-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"

1409+-- ["a","b","d","e"]

1410+--

1318−-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1411+-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa"

1412+-- ["","X","X","X",""]

1413+--

1319−-- > splitOn "x" "x" == ["",""]

1414+-- >>> splitOn "x" "x"

1415+-- ["",""]

1320--

1416--

1321-- and

1417-- and

1322--

1418--

@@ -1352,8 +1448,11 @@ splitOn pat@(Text _ _ l) src@(Text arr off len)

1352-- resulting components do not contain the separators. Two adjacent

1448-- resulting components do not contain the separators. Two adjacent

1353-- separators result in an empty component in the output. eg.

1449-- separators result in an empty component in the output. eg.

1354--

1450--

1355−-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1451+-- >>> split (=='a') "aabbaca"

1452+-- ["","","bb","c",""]

1453+--

1356−-- > split (=='a') "" == [""]

1454+-- >>> split (=='a') ""

1455+-- [""]

1357split :: (Char -> Bool) -> Text -> [Text]

1456split :: (Char -> Bool) -> Text -> [Text]

1358split _ t@(Text _off _arr 0) = [t]

1457split _ t@(Text _off _arr 0) = [t]

1359split p t = loop t

1458split p t = loop t

@@ -1366,8 +1465,11 @@ split p t = loop t

1366-- element may be shorter than the other chunks, depending on the

1465-- element may be shorter than the other chunks, depending on the

1367-- length of the input. Examples:

1466-- length of the input. Examples:

1368--

1467--

1369−-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"]

1468+-- >>> chunksOf 3 "foobarbaz"

1469+-- ["foo","bar","baz"]

1470+--

1370−-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]

1471+-- >>> chunksOf 4 "haskell.org"

1472+-- ["hask","ell.","org"]

1371chunksOf :: Int -> Text -> [Text]

1473chunksOf :: Int -> Text -> [Text]

1372chunksOf k = go

1474chunksOf k = go

1373 where

1475 where

@@ -1412,8 +1514,11 @@ filter p t = unstream (S.filter p (stream t))

1412--

1514--

1413-- Examples:

1515-- Examples:

1414--

1516--

1415−-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1517+-- >>> breakOn "::" "a::b::c"

1518+-- ("a","::b::c")

1519+--

1416−-- > breakOn "/" "foobar" ==> ("foobar", "")

1520+-- >>> breakOn "/" "foobar"

1521+-- ("foobar","")

1417--

1522--

1418-- Laws:

1523-- Laws:

1419--

1524--

@@ -1441,7 +1546,8 @@ breakOn pat src@(Text arr off len)

1441-- up to and including the last match of @needle@. The second is the

1546-- up to and including the last match of @needle@. The second is the

1442-- remainder of @haystack@, following the match.

1547-- remainder of @haystack@, following the match.

1443--

1548--

1444−-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1549+-- >>> breakOnEnd "::" "a::b::c"

1550+-- ("a::b::","c")

1445breakOnEnd :: Text -> Text -> (Text, Text)

1551breakOnEnd :: Text -> Text -> (Text, Text)

1446breakOnEnd pat src = (reverse b, reverse a)

1552breakOnEnd pat src = (reverse b, reverse a)

1447 where (a,b) = breakOn (reverse pat) (reverse src)

1553 where (a,b) = breakOn (reverse pat) (reverse src)

@@ -1456,10 +1562,11 @@ breakOnEnd pat src = (reverse b, reverse a)

1456--

1562--

1457-- Examples:

1563-- Examples:

1458--

1564--

1459−-- > breakOnAll "::" ""

1565+-- >>> breakOnAll "::" ""

1460−-- > ==> []

1566+-- []

1567+--

1461−-- > breakOnAll "/" "a/b/c/"

1568+-- >>> breakOnAll "/" "a/b/c/"

1462−-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]

1569+-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]

1463--

1570--

1464-- In (unlikely) bad cases, this function's time complexity degrades

1571-- In (unlikely) bad cases, this function's time complexity degrades

1465-- towards /O(n*m)/.

1572-- towards /O(n*m)/.

@@ -1658,9 +1765,14 @@ isInfixOf needle haystack

1658--

1765--

1659-- Examples:

1766-- Examples:

1660--

1767--

1661−-- > stripPrefix "foo" "foobar" == Just "bar"

1768+-- >>> stripPrefix "foo" "foobar"

1769+-- Just "bar"

1770+--

1662−-- > stripPrefix "" "baz" == Just "baz"

1771+-- >>> stripPrefix "" "baz"

1772+-- Just "baz"

1773+--

1663−-- > stripPrefix "foo" "quux" == Nothing

1774+-- >>> stripPrefix "foo" "quux"

1775+-- Nothing

1664--

1776--

1665-- This is particularly useful with the @ViewPatterns@ extension to

1777-- This is particularly useful with the @ViewPatterns@ extension to

1666-- GHC, as follows:

1778-- GHC, as follows:

@@ -1685,9 +1797,14 @@ stripPrefix p@(Text _arr _off plen) t@(Text arr off len)

1685--

1797--

1686-- Examples:

1798-- Examples:

1687--

1799--

1688−-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")

1800+-- >>> commonPrefixes "foobar" "fooquux"

1801+-- Just ("foo","bar","quux")

1802+--

1689−-- > commonPrefixes "veeble" "fetzer" == Nothing

1803+-- >>> commonPrefixes "veeble" "fetzer"

1804+-- Nothing

1805+--

1690−-- > commonPrefixes "" "baz" == Nothing

1806+-- >>> commonPrefixes "" "baz"

1807+-- Nothing

1691commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)

1808commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)

1692commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1809commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1693 where

1810 where

@@ -1704,9 +1821,14 @@ commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1704--

1821--

1705-- Examples:

1822-- Examples:

1706--

1823--

1707−-- > stripSuffix "bar" "foobar" == Just "foo"

1824+-- >>> stripSuffix "bar" "foobar"

1825+-- Just "foo"

1826+--

1708−-- > stripSuffix "" "baz" == Just "baz"

1827+-- >>> stripSuffix "" "baz"

1828+-- Just "baz"

1829+--

1709−-- > stripSuffix "foo" "quux" == Nothing

1830+-- >>> stripSuffix "foo" "quux"

1831+-- Nothing

1710--

1832--

1711-- This is particularly useful with the @ViewPatterns@ extension to

1833-- This is particularly useful with the @ViewPatterns@ extension to

1712-- GHC, as follows:

1834-- GHC, as follows:

@@ -1753,3 +1875,12 @@ copy (Text arr off len) = Text (A.run go) 0 len

1753 marr <- A.new len

1875 marr <- A.new len

1754 A.copyI marr 0 arr off len

1876 A.copyI marr 0 arr off len

1755 return marr

1877 return marr

1878+

1879+

1880+-------------------------------------------------

1881+-- NOTE: the named chunk below used by doctest;

1882+-- verify the doctests via `doctest -fobject-code Data/Text.hs`

1883+

1884+-- $setup

1885+-- >>> :set -XOverloadedStrings

1886+-- >>> import qualified Data.Text as T

Data/Text/Array.hs+4−1

@@ -7,7 +7,6 @@

7--

8-- License : BSD-style

9-- Maintainer : bos@serpentine.com

10−-- Stability : experimental

11-- Portability : portable

10-- Portability : portable

12--

11--

13-- Packed, unboxed, heap-resident arrays. Suitable for performance

12-- Packed, unboxed, heap-resident arrays. Suitable for performance

@@ -80,6 +79,8 @@ import GHC.Word (Word16(..))

80import Prelude hiding (length, read)

79import Prelude hiding (length, read)

81

80

82-- | Immutable array type.

81-- | Immutable array type.

82+--

83+-- The 'Array' constructor is exposed since @text-1.1.1.3@

83data Array = Array {

84data Array = Array {

84 aBA :: ByteArray#

85 aBA :: ByteArray#

85#if defined(ASSERTS)

86#if defined(ASSERTS)

@@ -88,6 +89,8 @@ data Array = Array {

88 }

89 }

89

90

90-- | Mutable array type, for use in the ST monad.

91-- | Mutable array type, for use in the ST monad.

92+--

93+-- The 'MArray' constructor is exposed since @text-1.1.1.3@

91data MArray s = MArray {

94data MArray s = MArray {

92 maBA :: MutableByteArray# s

95 maBA :: MutableByteArray# s

93#if defined(ASSERTS)

96#if defined(ASSERTS)

Data/Text/Encoding.hs+67−11

@@ -11,14 +11,13 @@

11--

12-- License : BSD-style

13-- Maintainer : bos@serpentine.com

14−-- Stability : experimental

15-- Portability : portable

14-- Portability : portable

16--

15--

17-- Functions for converting 'Text' values to and from 'ByteString',

16-- Functions for converting 'Text' values to and from 'ByteString',

18-- using several standard encodings.

17-- using several standard encodings.

19--

18--

20-- To gain access to a much larger family of encodings, use the

19-- To gain access to a much larger family of encodings, use the

21−-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

20+-- <http://hackage.haskell.org/package/text-icu text-icu package>.

22

21

23module Data.Text.Encoding

22module Data.Text.Encoding

24 (

23 (

@@ -66,7 +65,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

66import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

65import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

67#endif

66#endif

68

67

69−import Control.Exception (evaluate, try)

68+import Control.Exception (evaluate, try, throwIO, ErrorCall(ErrorCall))

70import Control.Monad.ST (runST)

69import Control.Monad.ST (runST)

71import Data.Bits ((.&.))

70import Data.Bits ((.&.))

72import Data.ByteString as B

71import Data.ByteString as B

@@ -79,7 +78,11 @@ import Data.Text.Internal.Unsafe.Shift (shiftR)

79import Data.Text.Show ()

78import Data.Text.Show ()

80import Data.Text.Unsafe (unsafeDupablePerformIO)

79import Data.Text.Unsafe (unsafeDupablePerformIO)

81import Data.Word (Word8, Word32)

80import Data.Word (Word8, Word32)

81+#if __GLASGOW_HASKELL__ >= 703

82−import Foreign.C.Types (CSize(..))

82+import Foreign.C.Types (CSize(CSize))

83+#else

84+import Foreign.C.Types (CSize)

85+#endif

83import Foreign.ForeignPtr (withForeignPtr)

86import Foreign.ForeignPtr (withForeignPtr)

84import Foreign.Marshal.Utils (with)

87import Foreign.Marshal.Utils (with)

85import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

88import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

@@ -128,6 +131,13 @@ decodeLatin1 (PS fp off len) = text a 0 len

128 return dest

131 return dest

129

132

130-- | Decode a 'ByteString' containing UTF-8 encoded text.

133-- | Decode a 'ByteString' containing UTF-8 encoded text.

134+--

135+-- __NOTE__: The replacement character returned by 'OnDecodeError'

136+-- MUST be within the BMP plane; surrogate code points will

137+-- automatically be remapped to the replacement char @U+FFFD@

138+-- (/since 0.11.3.0/), whereas code points beyond the BMP will throw an

139+-- 'error' (/since 1.2.3.1/); For earlier versions of @text@ using

140+-- those unsupported code points would result in undefined behavior.

131decodeUtf8With :: OnDecodeError -> ByteString -> Text

141decodeUtf8With :: OnDecodeError -> ByteString -> Text

132decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

142decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

133 let go dest = withForeignPtr fp $ \ptr ->

143 let go dest = withForeignPtr fp $ \ptr ->

@@ -143,16 +153,52 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

143 x <- peek curPtr'

153 x <- peek curPtr'

144 case onErr desc (Just x) of

154 case onErr desc (Just x) of

145 Nothing -> loop $ curPtr' `plusPtr` 1

155 Nothing -> loop $ curPtr' `plusPtr` 1

146− Just c -> do

156+ Just c

157+ | c > '\xFFFF' -> throwUnsupportedReplChar

158+ | otherwise -> do

147− destOff <- peek destOffPtr

159+ destOff <- peek destOffPtr

148− w <- unsafeSTToIO $

160+ w <- unsafeSTToIO $

149− unsafeWrite dest (fromIntegral destOff) (safe c)

161+ unsafeWrite dest (fromIntegral destOff)

162+ (safe c)

150− poke destOffPtr (destOff + fromIntegral w)

163+ poke destOffPtr (destOff + fromIntegral w)

151− loop $ curPtr' `plusPtr` 1

164+ loop $ curPtr' `plusPtr` 1

152 loop (ptr `plusPtr` off)

165 loop (ptr `plusPtr` off)

153 (unsafeIOToST . go) =<< A.new len

166 (unsafeIOToST . go) =<< A.new len

154 where

167 where

155 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"

168 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"

169+

170+ throwUnsupportedReplChar = throwIO $

171+ ErrorCall "decodeUtf8With: non-BMP replacement characters not supported"

172+ -- TODO: The code currently assumes that the transcoded UTF-16

173+ -- stream is at most twice as long (in bytes) as the input UTF-8

174+ -- stream. To justify this assumption one has to assume that the

175+ -- error handler replacement character also satisfies this

176+ -- invariant, by emitting at most one UTF16 code unit.

177+ --

178+ -- One easy way to support the full range of code-points for

179+ -- replacement characters in the error handler is to simply change

180+ -- the (over-)allocation to `A.new (2*len)` and then shrink back the

181+ -- `ByteArray#` to the real size (recent GHCs have a cheap

182+ -- `ByteArray#` resize-primop for that which allow the GC to reclaim

183+ -- the overallocation). However, this would require 4 times as much

184+ -- (temporary) storage as the original UTF-8 required.

185+ --

186+ -- Another strategy would be to optimistically assume that

187+ -- replacement characters are within the BMP, and if the case of a

188+ -- non-BMP replacement occurs reallocate the target buffer (or throw

189+ -- an exception, and fallback to a pessimistic codepath, like e.g.

190+ -- `decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)`)

191+ --

192+ -- Alternatively, `OnDecodeError` could become a datastructure which

193+ -- statically encodes the replacement-character range,

194+ -- e.g. something isomorphic to

195+ --

196+ -- Either (... -> Maybe Word16) (... -> Maybe Char)

197+ --

198+ -- And allow to statically switch between the BMP/non-BMP

199+ -- replacement-character codepaths. There's multiple ways to address

200+ -- this with different tradeoffs; but ideally we should optimise for

201+ -- the optimistic/error-free case.

156{- INLINE[0] decodeUtf8With #-}

202{- INLINE[0] decodeUtf8With #-}

157

203

158-- $stream

204-- $stream

@@ -209,6 +255,8 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

209-- or continuation where it is encountered.

255-- or continuation where it is encountered.

210

256

211-- | A stream oriented decoding result.

257-- | A stream oriented decoding result.

258+--

259+-- @since 1.0.0.0

212data Decoding = Some Text ByteString (ByteString -> Decoding)

260data Decoding = Some Text ByteString (ByteString -> Decoding)

213

261

214instance Show Decoding where

262instance Show Decoding where

@@ -228,11 +276,15 @@ newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)

228-- thrown (either by this function or a continuation) that cannot be

276-- thrown (either by this function or a continuation) that cannot be

229-- caught in pure code. For more control over the handling of invalid

277-- caught in pure code. For more control over the handling of invalid

230-- data, use 'streamDecodeUtf8With'.

278-- data, use 'streamDecodeUtf8With'.

279+--

280+-- @since 1.0.0.0

231streamDecodeUtf8 :: ByteString -> Decoding

281streamDecodeUtf8 :: ByteString -> Decoding

232streamDecodeUtf8 = streamDecodeUtf8With strictDecode

282streamDecodeUtf8 = streamDecodeUtf8With strictDecode

233

283

234-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8

284-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8

235-- encoded text.

285-- encoded text.

286+--

287+-- @since 1.0.0.0

236streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding

288streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding

237streamDecodeUtf8With onErr = decodeChunk B.empty 0 0

289streamDecodeUtf8With onErr = decodeChunk B.empty 0 0

238 where

290 where

@@ -308,6 +360,8 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDec

308{-# INLINE decodeUtf8' #-}

360{-# INLINE decodeUtf8' #-}

309

361

310-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

362-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

363+--

364+-- @since 1.1.0.0

311encodeUtf8Builder :: Text -> B.Builder

365encodeUtf8Builder :: Text -> B.Builder

312encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

366encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

313

367

@@ -316,6 +370,8 @@ encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

316--

370--

317-- Use this function is to implement efficient encoders for text-based formats

371-- Use this function is to implement efficient encoders for text-based formats

318-- like JSON or HTML.

372-- like JSON or HTML.

373+--

374+-- @since 1.1.0.0

319{-# INLINE encodeUtf8BuilderEscaped #-}

375{-# INLINE encodeUtf8BuilderEscaped #-}

320-- TODO: Extend documentation with references to source code in @blaze-html@

376-- TODO: Extend documentation with references to source code in @blaze-html@

321-- or @aeson@ that uses this function.

377-- or @aeson@ that uses this function.

@@ -373,7 +429,7 @@ encodeUtf8 :: Text -> ByteString

373encodeUtf8 (Text arr off len)

429encodeUtf8 (Text arr off len)

374 | len == 0 = B.empty

430 | len == 0 = B.empty

375 | otherwise = unsafeDupablePerformIO $ do

431 | otherwise = unsafeDupablePerformIO $ do

376− fp <- mallocByteString (len*4)

432+ fp <- mallocByteString (len*3) -- see https://github.com/haskell/text/issues/194 for why len*3 is enough

377 withForeignPtr fp $ \ptr ->

433 withForeignPtr fp $ \ptr ->

378 with ptr $ \destPtr -> do

434 with ptr $ \destPtr -> do

379 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

435 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

Data/Text/Encoding/Error.hs−1

@@ -10,7 +10,6 @@

10--

11-- License : BSD-style

12-- Maintainer : bos@serpentine.com

13−-- Stability : experimental

14-- Portability : GHC

13-- Portability : GHC

15--

14--

16-- Types and functions for dealing with encoding and decoding errors

15-- Types and functions for dealing with encoding and decoding errors

Data/Text/Foreign.hs+5−2

@@ -5,7 +5,6 @@

5--

6-- License : BSD-style

7-- Maintainer : bos@serpentine.com

8−-- Stability : experimental

9-- Portability : GHC

8-- Portability : GHC

10--

9--

11-- Support for using 'Text' data with native code via the Haskell

10-- Support for using 'Text' data with native code via the Haskell

@@ -108,7 +107,7 @@ takeWord16 (I16 n) t@(Text arr off len)

108 | n >= len || m >= len = t

107 | n >= len || m >= len = t

109 | otherwise = Text arr off m

108 | otherwise = Text arr off m

110 where

109 where

111− m | w < 0xDB00 || w > 0xD8FF = n

110+ m | w < 0xD800 || w > 0xDBFF = n

112 | otherwise = n+1

111 | otherwise = n+1

113 w = A.unsafeIndex arr (off+n-1)

112 w = A.unsafeIndex arr (off+n-1)

114

113

@@ -157,6 +156,8 @@ asForeignPtr t@(Text _arr _off len) = do

157-- | /O(n)/ Decode a C string with explicit length, which is assumed

156-- | /O(n)/ Decode a C string with explicit length, which is assumed

158-- to have been encoded as UTF-8. If decoding fails, a

157-- to have been encoded as UTF-8. If decoding fails, a

159-- 'UnicodeException' is thrown.

158-- 'UnicodeException' is thrown.

159+--

160+-- @since 1.0.0.0

160peekCStringLen :: CStringLen -> IO Text

161peekCStringLen :: CStringLen -> IO Text

161peekCStringLen cs = do

162peekCStringLen cs = do

162 bs <- unsafePackCStringLen cs

163 bs <- unsafePackCStringLen cs

@@ -169,5 +170,7 @@ peekCStringLen cs = do

169-- The temporary storage is freed when the subcomputation terminates

170-- The temporary storage is freed when the subcomputation terminates

170-- (either normally or via an exception), so the pointer to the

171-- (either normally or via an exception), so the pointer to the

171-- temporary storage must /not/ be used after this function returns.

172-- temporary storage must /not/ be used after this function returns.

173+--

174+-- @since 1.0.0.0

172withCStringLen :: Text -> (CStringLen -> IO a) -> IO a

175withCStringLen :: Text -> (CStringLen -> IO a) -> IO a

173withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act

176withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act

Data/Text/IO.hs+4−1

@@ -8,7 +8,6 @@

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11−-- Stability : experimental

12-- Portability : GHC

11-- Portability : GHC

13--

12--

14-- Efficient locale-sensitive support for text I\/O.

13-- Efficient locale-sensitive support for text I\/O.

@@ -336,3 +335,7 @@ putStrLn = hPutStrLn stdout

336-- >

335-- >

337-- > putStr_Utf16LE :: Text -> IO ()

336-- > putStr_Utf16LE :: Text -> IO ()

338-- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)

337-- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)

338+--

339+-- On transcoding errors, an 'IOError' exception is thrown. You can

340+-- use the API in "Data.Text.Encoding" if you need more control over

341+-- error handling or transcoding.

Data/Text/Internal/Fusion/CaseMapping.hs+164−4

@@ -1,10 +1,10 @@

1{-# LANGUAGE Rank2Types #-}

2-- AUTOMATICALLY GENERATED - DO NOT EDIT

3-- Generated by scripts/CaseMapping.hs

4−-- CaseFolding-8.0.0.txt

4+-- CaseFolding-9.0.0.txt

5−-- Date: 2015-01-13, 18:16:36 GMT [MD]

5+-- Date: 2016-03-02, 18:54:54 GMT

6−-- SpecialCasing-8.0.0.txt

6+-- SpecialCasing-9.0.0.txt

7−-- Date: 2014-12-16, 23:08:04 GMT [MD]

7+-- Date: 2016-03-02, 18:55:13 GMT

8

9module Data.Text.Internal.Fusion.CaseMapping where

10import Data.Char

@@ -371,6 +371,24 @@ foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')

371foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')

372-- CHEROKEE SMALL LETTER MV

373foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')

374+-- CYRILLIC SMALL LETTER ROUNDED VE

375+foldMapping '\x1c80' s = Yield '\x0432' (CC s '\x0000' '\x0000')

376+-- CYRILLIC SMALL LETTER LONG-LEGGED DE

377+foldMapping '\x1c81' s = Yield '\x0434' (CC s '\x0000' '\x0000')

378+-- CYRILLIC SMALL LETTER NARROW O

379+foldMapping '\x1c82' s = Yield '\x043e' (CC s '\x0000' '\x0000')

380+-- CYRILLIC SMALL LETTER WIDE ES

381+foldMapping '\x1c83' s = Yield '\x0441' (CC s '\x0000' '\x0000')

382+-- CYRILLIC SMALL LETTER TALL TE

383+foldMapping '\x1c84' s = Yield '\x0442' (CC s '\x0000' '\x0000')

384+-- CYRILLIC SMALL LETTER THREE-LEGGED TE

385+foldMapping '\x1c85' s = Yield '\x0442' (CC s '\x0000' '\x0000')

386+-- CYRILLIC SMALL LETTER TALL HARD SIGN

387+foldMapping '\x1c86' s = Yield '\x044a' (CC s '\x0000' '\x0000')

388+-- CYRILLIC SMALL LETTER TALL YAT

389+foldMapping '\x1c87' s = Yield '\x0463' (CC s '\x0000' '\x0000')

390+-- CYRILLIC SMALL LETTER UNBLENDED UK

391+foldMapping '\x1c88' s = Yield '\xa64b' (CC s '\x0000' '\x0000')

374-- LATIN SMALL LETTER H WITH LINE BELOW

392-- LATIN SMALL LETTER H WITH LINE BELOW

375foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

393foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

376-- LATIN SMALL LETTER T WITH DIAERESIS

394-- LATIN SMALL LETTER T WITH DIAERESIS

@@ -545,6 +563,8 @@ foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')

545foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

563foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

546-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

564-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

547foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

565foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

566+-- LATIN CAPITAL LETTER SMALL CAPITAL I

567+foldMapping '\xa7ae' s = Yield '\x026a' (CC s '\x0000' '\x0000')

548-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

568-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

549foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

569foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

550-- LATIN CAPITAL LETTER CHI

570-- LATIN CAPITAL LETTER CHI

@@ -737,6 +757,78 @@ foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')

737foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

757foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

738-- ARMENIAN SMALL LIGATURE MEN XEH

758-- ARMENIAN SMALL LIGATURE MEN XEH

739foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

759foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

760+-- OSAGE CAPITAL LETTER A

761+foldMapping '\x104b0' s = Yield '\x104d8' (CC s '\x0000' '\x0000')

762+-- OSAGE CAPITAL LETTER AI

763+foldMapping '\x104b1' s = Yield '\x104d9' (CC s '\x0000' '\x0000')

764+-- OSAGE CAPITAL LETTER AIN

765+foldMapping '\x104b2' s = Yield '\x104da' (CC s '\x0000' '\x0000')

766+-- OSAGE CAPITAL LETTER AH

767+foldMapping '\x104b3' s = Yield '\x104db' (CC s '\x0000' '\x0000')

768+-- OSAGE CAPITAL LETTER BRA

769+foldMapping '\x104b4' s = Yield '\x104dc' (CC s '\x0000' '\x0000')

770+-- OSAGE CAPITAL LETTER CHA

771+foldMapping '\x104b5' s = Yield '\x104dd' (CC s '\x0000' '\x0000')

772+-- OSAGE CAPITAL LETTER EHCHA

773+foldMapping '\x104b6' s = Yield '\x104de' (CC s '\x0000' '\x0000')

774+-- OSAGE CAPITAL LETTER E

775+foldMapping '\x104b7' s = Yield '\x104df' (CC s '\x0000' '\x0000')

776+-- OSAGE CAPITAL LETTER EIN

777+foldMapping '\x104b8' s = Yield '\x104e0' (CC s '\x0000' '\x0000')

778+-- OSAGE CAPITAL LETTER HA

779+foldMapping '\x104b9' s = Yield '\x104e1' (CC s '\x0000' '\x0000')

780+-- OSAGE CAPITAL LETTER HYA

781+foldMapping '\x104ba' s = Yield '\x104e2' (CC s '\x0000' '\x0000')

782+-- OSAGE CAPITAL LETTER I

783+foldMapping '\x104bb' s = Yield '\x104e3' (CC s '\x0000' '\x0000')

784+-- OSAGE CAPITAL LETTER KA

785+foldMapping '\x104bc' s = Yield '\x104e4' (CC s '\x0000' '\x0000')

786+-- OSAGE CAPITAL LETTER EHKA

787+foldMapping '\x104bd' s = Yield '\x104e5' (CC s '\x0000' '\x0000')

788+-- OSAGE CAPITAL LETTER KYA

789+foldMapping '\x104be' s = Yield '\x104e6' (CC s '\x0000' '\x0000')

790+-- OSAGE CAPITAL LETTER LA

791+foldMapping '\x104bf' s = Yield '\x104e7' (CC s '\x0000' '\x0000')

792+-- OSAGE CAPITAL LETTER MA

793+foldMapping '\x104c0' s = Yield '\x104e8' (CC s '\x0000' '\x0000')

794+-- OSAGE CAPITAL LETTER NA

795+foldMapping '\x104c1' s = Yield '\x104e9' (CC s '\x0000' '\x0000')

796+-- OSAGE CAPITAL LETTER O

797+foldMapping '\x104c2' s = Yield '\x104ea' (CC s '\x0000' '\x0000')

798+-- OSAGE CAPITAL LETTER OIN

799+foldMapping '\x104c3' s = Yield '\x104eb' (CC s '\x0000' '\x0000')

800+-- OSAGE CAPITAL LETTER PA

801+foldMapping '\x104c4' s = Yield '\x104ec' (CC s '\x0000' '\x0000')

802+-- OSAGE CAPITAL LETTER EHPA

803+foldMapping '\x104c5' s = Yield '\x104ed' (CC s '\x0000' '\x0000')

804+-- OSAGE CAPITAL LETTER SA

805+foldMapping '\x104c6' s = Yield '\x104ee' (CC s '\x0000' '\x0000')

806+-- OSAGE CAPITAL LETTER SHA

807+foldMapping '\x104c7' s = Yield '\x104ef' (CC s '\x0000' '\x0000')

808+-- OSAGE CAPITAL LETTER TA

809+foldMapping '\x104c8' s = Yield '\x104f0' (CC s '\x0000' '\x0000')

810+-- OSAGE CAPITAL LETTER EHTA

811+foldMapping '\x104c9' s = Yield '\x104f1' (CC s '\x0000' '\x0000')

812+-- OSAGE CAPITAL LETTER TSA

813+foldMapping '\x104ca' s = Yield '\x104f2' (CC s '\x0000' '\x0000')

814+-- OSAGE CAPITAL LETTER EHTSA

815+foldMapping '\x104cb' s = Yield '\x104f3' (CC s '\x0000' '\x0000')

816+-- OSAGE CAPITAL LETTER TSHA

817+foldMapping '\x104cc' s = Yield '\x104f4' (CC s '\x0000' '\x0000')

818+-- OSAGE CAPITAL LETTER DHA

819+foldMapping '\x104cd' s = Yield '\x104f5' (CC s '\x0000' '\x0000')

820+-- OSAGE CAPITAL LETTER U

821+foldMapping '\x104ce' s = Yield '\x104f6' (CC s '\x0000' '\x0000')

822+-- OSAGE CAPITAL LETTER WA

823+foldMapping '\x104cf' s = Yield '\x104f7' (CC s '\x0000' '\x0000')

824+-- OSAGE CAPITAL LETTER KHA

825+foldMapping '\x104d0' s = Yield '\x104f8' (CC s '\x0000' '\x0000')

826+-- OSAGE CAPITAL LETTER GHA

827+foldMapping '\x104d1' s = Yield '\x104f9' (CC s '\x0000' '\x0000')

828+-- OSAGE CAPITAL LETTER ZA

829+foldMapping '\x104d2' s = Yield '\x104fa' (CC s '\x0000' '\x0000')

830+-- OSAGE CAPITAL LETTER ZHA

831+foldMapping '\x104d3' s = Yield '\x104fb' (CC s '\x0000' '\x0000')

740-- OLD HUNGARIAN CAPITAL LETTER A

832-- OLD HUNGARIAN CAPITAL LETTER A

741foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

833foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

742-- OLD HUNGARIAN CAPITAL LETTER AA

834-- OLD HUNGARIAN CAPITAL LETTER AA

@@ -839,4 +931,72 @@ foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')

839foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

931foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

840-- OLD HUNGARIAN CAPITAL LETTER US

932-- OLD HUNGARIAN CAPITAL LETTER US

841foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

933foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

934+-- ADLAM CAPITAL LETTER ALIF

935+foldMapping '\x1e900' s = Yield '\x1e922' (CC s '\x0000' '\x0000')

936+-- ADLAM CAPITAL LETTER DAALI

937+foldMapping '\x1e901' s = Yield '\x1e923' (CC s '\x0000' '\x0000')

938+-- ADLAM CAPITAL LETTER LAAM

939+foldMapping '\x1e902' s = Yield '\x1e924' (CC s '\x0000' '\x0000')

940+-- ADLAM CAPITAL LETTER MIIM

941+foldMapping '\x1e903' s = Yield '\x1e925' (CC s '\x0000' '\x0000')

942+-- ADLAM CAPITAL LETTER BA

943+foldMapping '\x1e904' s = Yield '\x1e926' (CC s '\x0000' '\x0000')

944+-- ADLAM CAPITAL LETTER SINNYIIYHE

945+foldMapping '\x1e905' s = Yield '\x1e927' (CC s '\x0000' '\x0000')

946+-- ADLAM CAPITAL LETTER PE

947+foldMapping '\x1e906' s = Yield '\x1e928' (CC s '\x0000' '\x0000')

948+-- ADLAM CAPITAL LETTER BHE

949+foldMapping '\x1e907' s = Yield '\x1e929' (CC s '\x0000' '\x0000')

950+-- ADLAM CAPITAL LETTER RA

951+foldMapping '\x1e908' s = Yield '\x1e92a' (CC s '\x0000' '\x0000')

952+-- ADLAM CAPITAL LETTER E

953+foldMapping '\x1e909' s = Yield '\x1e92b' (CC s '\x0000' '\x0000')

954+-- ADLAM CAPITAL LETTER FA

955+foldMapping '\x1e90a' s = Yield '\x1e92c' (CC s '\x0000' '\x0000')

956+-- ADLAM CAPITAL LETTER I

957+foldMapping '\x1e90b' s = Yield '\x1e92d' (CC s '\x0000' '\x0000')

958+-- ADLAM CAPITAL LETTER O

959+foldMapping '\x1e90c' s = Yield '\x1e92e' (CC s '\x0000' '\x0000')

960+-- ADLAM CAPITAL LETTER DHA

961+foldMapping '\x1e90d' s = Yield '\x1e92f' (CC s '\x0000' '\x0000')

962+-- ADLAM CAPITAL LETTER YHE

963+foldMapping '\x1e90e' s = Yield '\x1e930' (CC s '\x0000' '\x0000')

964+-- ADLAM CAPITAL LETTER WAW

965+foldMapping '\x1e90f' s = Yield '\x1e931' (CC s '\x0000' '\x0000')

966+-- ADLAM CAPITAL LETTER NUN

967+foldMapping '\x1e910' s = Yield '\x1e932' (CC s '\x0000' '\x0000')

968+-- ADLAM CAPITAL LETTER KAF

969+foldMapping '\x1e911' s = Yield '\x1e933' (CC s '\x0000' '\x0000')

970+-- ADLAM CAPITAL LETTER YA

971+foldMapping '\x1e912' s = Yield '\x1e934' (CC s '\x0000' '\x0000')

972+-- ADLAM CAPITAL LETTER U

973+foldMapping '\x1e913' s = Yield '\x1e935' (CC s '\x0000' '\x0000')

974+-- ADLAM CAPITAL LETTER JIIM

975+foldMapping '\x1e914' s = Yield '\x1e936' (CC s '\x0000' '\x0000')

976+-- ADLAM CAPITAL LETTER CHI

977+foldMapping '\x1e915' s = Yield '\x1e937' (CC s '\x0000' '\x0000')

978+-- ADLAM CAPITAL LETTER HA

979+foldMapping '\x1e916' s = Yield '\x1e938' (CC s '\x0000' '\x0000')

980+-- ADLAM CAPITAL LETTER QAAF

981+foldMapping '\x1e917' s = Yield '\x1e939' (CC s '\x0000' '\x0000')

982+-- ADLAM CAPITAL LETTER GA

983+foldMapping '\x1e918' s = Yield '\x1e93a' (CC s '\x0000' '\x0000')

984+-- ADLAM CAPITAL LETTER NYA

985+foldMapping '\x1e919' s = Yield '\x1e93b' (CC s '\x0000' '\x0000')

986+-- ADLAM CAPITAL LETTER TU

987+foldMapping '\x1e91a' s = Yield '\x1e93c' (CC s '\x0000' '\x0000')

988+-- ADLAM CAPITAL LETTER NHA

989+foldMapping '\x1e91b' s = Yield '\x1e93d' (CC s '\x0000' '\x0000')

990+-- ADLAM CAPITAL LETTER VA

991+foldMapping '\x1e91c' s = Yield '\x1e93e' (CC s '\x0000' '\x0000')

992+-- ADLAM CAPITAL LETTER KHA

993+foldMapping '\x1e91d' s = Yield '\x1e93f' (CC s '\x0000' '\x0000')

994+-- ADLAM CAPITAL LETTER GBE

995+foldMapping '\x1e91e' s = Yield '\x1e940' (CC s '\x0000' '\x0000')

996+-- ADLAM CAPITAL LETTER ZAL

997+foldMapping '\x1e91f' s = Yield '\x1e941' (CC s '\x0000' '\x0000')

998+-- ADLAM CAPITAL LETTER KPO

999+foldMapping '\x1e920' s = Yield '\x1e942' (CC s '\x0000' '\x0000')

1000+-- ADLAM CAPITAL LETTER SHA

1001+foldMapping '\x1e921' s = Yield '\x1e943' (CC s '\x0000' '\x0000')

842foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

1002foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

Data/Text/Internal/Fusion/Common.hs+44−37

@@ -1,4 +1,4 @@

1−{-# LANGUAGE BangPatterns, MagicHash, Rank2Types #-}

1+{-# LANGUAGE PatternGuards, BangPatterns, MagicHash, Rank2Types #-}

2-- |

3-- Module : Data.Text.Internal.Fusion.Common

4-- Copyright : (c) Bryan O'Sullivan 2009, 2012

@@ -107,7 +107,7 @@ import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..),

107import qualified Data.List as L

108import qualified Prelude as P

109import Data.Bits (shiftL)

110−import Data.Char (isLetter)

110+import Data.Char (isLetter, isSpace)

111import Data.Int (Int64)

112import Data.Text.Internal.Fusion.Types

113import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,

@@ -117,7 +117,7 @@ import GHC.Prim (Addr#, chr#, indexCharOffAddr#, ord#)

117import GHC.Types (Char(..), Int(..))

118

119singleton :: Char -> Stream Char

120−singleton c = Stream next False 1

120+singleton c = Stream next False (codePointsSize 1)

121 where next False = Yield c True

122 next True = Done

123{-# INLINE [0] singleton #-}

@@ -175,7 +175,7 @@ data C s = C0 !s

175

176-- | /O(n)/ Adds a character to the front of a Stream Char.

177cons :: Char -> Stream Char -> Stream Char

178−cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len+1)

178+cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len + codePointsSize 1)

179 where

180 next (C1 s) = Yield w (C0 s)

181 next (C0 s) = case next0 s of

@@ -189,7 +189,7 @@ data Snoc a = N

189

190-- | /O(n)/ Adds a character to the end of a stream.

191snoc :: Stream Char -> Char -> Stream Char

192−snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len+1)

192+snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len + codePointsSize 1)

193 where

194 next (J xs) = case next0 xs of

195 Done -> Yield w N

@@ -237,7 +237,7 @@ uncons :: Stream Char -> Maybe (Char, Stream Char)

237uncons (Stream next s0 len) = loop_uncons s0

238 where

239 loop_uncons !s = case next s of

240− Yield x s1 -> Just (x, Stream next s1 (len-1))

240+ Yield x s1 -> Just (x, Stream next s1 (len - codePointsSize 1))

241 Skip s' -> loop_uncons s'

242 Done -> Nothing

243{-# INLINE [0] uncons #-}

@@ -260,7 +260,7 @@ last (Stream next s0 _len) = loop0_last s0

260-- | /O(1)/ Returns all characters after the head of a Stream Char, which must

261-- be non-empty.

262tail :: Stream Char -> Stream Char

263−tail (Stream next0 s0 len) = Stream next (C0 s0) (len-1)

263+tail (Stream next0 s0 len) = Stream next (C0 s0) (len - codePointsSize 1)

264 where

265 next (C0 s) = case next0 s of

266 Done -> emptyError "tail"

@@ -278,7 +278,7 @@ data Init s = Init0 !s

278-- | /O(1)/ Returns all but the last character of a Stream Char, which

279-- must be non-empty.

280init :: Stream Char -> Stream Char

281−init (Stream next0 s0 len) = Stream next (Init0 s0) (len-1)

281+init (Stream next0 s0 len) = Stream next (Init0 s0) (len - codePointsSize 1)

282 where

283 next (Init0 s) = case next0 s of

284 Done -> emptyError "init"

@@ -318,11 +318,14 @@ lengthI (Stream next s0 _len) = loop_length 0 s0

318-- greater than the number or if the stream can't possibly be as long

319-- as the number supplied, and hence be more efficient.

320compareLengthI :: Integral a => Stream Char -> a -> Ordering

321−compareLengthI (Stream next s0 len) n =

321+compareLengthI (Stream next s0 len) n

322− case compareSize len (fromIntegral n) of

322+ -- Note that @len@ tracks code units whereas we want to compare the length

323− Just o -> o

323+ -- in code points. Specifically, a stream with hint @len@ may consist of

324+ -- anywhere from @len/2@ to @len@ code points.

325+ | Just r <- compareSize len n' = r

324− Nothing -> loop_cmp 0 s0

326+ | otherwise = loop_cmp 0 s0

325 where

327 where

328+ n' = codePointsSize $ fromIntegral n

326 loop_cmp !z s = case next s of

329 loop_cmp !z s = case next s of

327 Done -> compare z n

330 Done -> compare z n

328 Skip s' -> loop_cmp z s'

331 Skip s' -> loop_cmp z s'

@@ -368,7 +371,7 @@ data I s = I1 !s

368-- | /O(n)/ Take a character and place it between each of the

371-- | /O(n)/ Take a character and place it between each of the

369-- characters of a 'Stream Char'.

372-- characters of a 'Stream Char'.

370intersperse :: Char -> Stream Char -> Stream Char

373intersperse :: Char -> Stream Char -> Stream Char

371−intersperse c (Stream next0 s0 len) = Stream next (I1 s0) len

374+intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize)

372 where

375 where

373 next (I1 s) = case next0 s of

376 next (I1 s) = case next0 s of

374 Done -> Done

377 Done -> Done

@@ -393,9 +396,11 @@ intersperse c (Stream next0 s0 len) = Stream next (I1 s0) len

393-- functions may map one input character to two or three output

396-- functions may map one input character to two or three output

394-- characters.

397-- characters.

395

398

399+-- | Map a 'Stream' through the given case-mapping function.

396caseConvert :: (forall s. Char -> s -> Step (CC s) Char)

400caseConvert :: (forall s. Char -> s -> Step (CC s) Char)

397 -> Stream Char -> Stream Char

401 -> Stream Char -> Stream Char

398−caseConvert remap (Stream next0 s0 len) = Stream next (CC s0 '\0' '\0') len

402+caseConvert remap (Stream next0 s0 len) =

403+ Stream next (CC s0 '\0' '\0') (len `unionSize` 3*len)

399 where

404 where

400 next (CC s '\0' _) =

405 next (CC s '\0' _) =

401 case next0 s of

406 case next0 s of

@@ -458,19 +463,20 @@ toLower = caseConvert lowerMapping

458-- Fox\" is correctly title cased—but this function will

463-- Fox\" is correctly title cased—but this function will

459-- capitalize /every/ word.

464-- capitalize /every/ word.

460toTitle :: Stream Char -> Stream Char

465toTitle :: Stream Char -> Stream Char

461−toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') len

466+toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') (len + unknownSize)

462 where

467 where

463 next (CC (letter :*: s) '\0' _) =

468 next (CC (letter :*: s) '\0' _) =

464 case next0 s of

469 case next0 s of

465− Done -> Done

470+ Done -> Done

466− Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

471+ Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

467 Yield c s'

472 Yield c s'

468− | letter' -> if letter

473+ | nonSpace -> if letter

469− then lowerMapping c (letter' :*: s')

474+ then lowerMapping c (nonSpace :*: s')

470− else titleMapping c (letter' :*: s')

475+ else titleMapping c (letter' :*: s')

471− | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

476+ | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

477+ where nonSpace = P.not (isSpace c)

472− where letter' = isLetter c

478+ letter' = isLetter c

473− next (CC s a b) = Yield a (CC s b '\0')

479+ next (CC s a b) = Yield a (CC s b '\0')

474{-# INLINE [0] toTitle #-}

480{-# INLINE [0] toTitle #-}

475

481

476data Justify i s = Just1 !i !s

482data Justify i s = Just1 !i !s

@@ -478,7 +484,7 @@ data Justify i s = Just1 !i !s

478

484

479justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char

485justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char

480justifyLeftI k c (Stream next0 s0 len) =

486justifyLeftI k c (Stream next0 s0 len) =

481− Stream next (Just1 0 s0) (larger (fromIntegral k) len)

487+ Stream next (Just1 0 s0) (larger (fromIntegral k * charSize c + len) len)

482 where

488 where

483 next (Just1 n s) =

489 next (Just1 n s) =

484 case next0 s of

490 case next0 s of

@@ -698,7 +704,7 @@ replicateI n (Stream next0 s0 len) =

698-- (a,b), in which case, a is the next Char in the string, and b is

704-- (a,b), in which case, a is the next Char in the string, and b is

699-- the seed value for further production.

705-- the seed value for further production.

700unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char

706unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char

701−unfoldr f s0 = Stream next s0 1 -- HINT maybe too low

707+unfoldr f s0 = Stream next s0 unknownSize

702 where

708 where

703 {-# INLINE next #-}

709 {-# INLINE next #-}

704 next !s = case f s of

710 next !s = case f s of

@@ -712,7 +718,7 @@ unfoldr f s0 = Stream next s0 1 -- HINT maybe too low

712-- 'unfoldr' when the length of the result is known.

718-- 'unfoldr' when the length of the result is known.

713unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char

719unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char

714unfoldrNI n f s0 | n < 0 = empty

720unfoldrNI n f s0 | n < 0 = empty

715− | otherwise = Stream next (0 :*: s0) (fromIntegral (n*2)) -- HINT maybe too high

721+ | otherwise = Stream next (0 :*: s0) (maxSize $ fromIntegral (n*2))

716 where

722 where

717 {-# INLINE next #-}

723 {-# INLINE next #-}

718 next (z :*: s) = case f s of

724 next (z :*: s) = case f s of

@@ -724,12 +730,12 @@ unfoldrNI n f s0 | n < 0 = empty

724-------------------------------------------------------------------------------

730-------------------------------------------------------------------------------

725-- * Substreams

731-- * Substreams

726

732

727−-- | /O(n)/ take n, applied to a stream, returns the prefix of the

733+-- | /O(n)/ @'take' n@, applied to a stream, returns the prefix of the

728-- stream of length @n@, or the stream itself if @n@ is greater than the

734-- stream of length @n@, or the stream itself if @n@ is greater than the

729-- length of the stream.

735-- length of the stream.

730take :: Integral a => a -> Stream Char -> Stream Char

736take :: Integral a => a -> Stream Char -> Stream Char

731take n0 (Stream next0 s0 len) =

737take n0 (Stream next0 s0 len) =

732− Stream next (n0 :*: s0) (smaller len (fromIntegral (max 0 n0)))

738+ Stream next (n0 :*: s0) (smaller len (codePointsSize $ fromIntegral n0))

733 where

739 where

734 {-# INLINE next #-}

740 {-# INLINE next #-}

735 next (n :*: s) | n <= 0 = Done

741 next (n :*: s) | n <= 0 = Done

@@ -742,12 +748,12 @@ take n0 (Stream next0 s0 len) =

742data Drop a s = NS !s

748data Drop a s = NS !s

743 | JS !a !s

749 | JS !a !s

744

750

745−-- | /O(n)/ drop n, applied to a stream, returns the suffix of the

751+-- | /O(n)/ @'drop' n@, applied to a stream, returns the suffix of the

746-- stream after the first @n@ characters, or the empty stream if @n@

752-- stream after the first @n@ characters, or the empty stream if @n@

747-- is greater than the length of the stream.

753-- is greater than the length of the stream.

748drop :: Integral a => a -> Stream Char -> Stream Char

754drop :: Integral a => a -> Stream Char -> Stream Char

749drop n0 (Stream next0 s0 len) =

755drop n0 (Stream next0 s0 len) =

750− Stream next (JS n0 s0) (len - fromIntegral (max 0 n0))

756+ Stream next (JS n0 s0) (len - codePointsSize (fromIntegral n0))

751 where

757 where

752 {-# INLINE next #-}

758 {-# INLINE next #-}

753 next (JS n s)

759 next (JS n s)

@@ -762,10 +768,10 @@ drop n0 (Stream next0 s0 len) =

762 Yield x s' -> Yield x (NS s')

768 Yield x s' -> Yield x (NS s')

763{-# INLINE [0] drop #-}

769{-# INLINE [0] drop #-}

764

770

765−-- | takeWhile, applied to a predicate @p@ and a stream, returns the

771+-- | 'takeWhile', applied to a predicate @p@ and a stream, returns the

766−-- longest prefix (possibly empty) of elements that satisfy p.

772+-- longest prefix (possibly empty) of elements that satisfy @p@.

767takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char

773takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char

768−takeWhile p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high

774+takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize)

769 where

775 where

770 {-# INLINE next #-}

776 {-# INLINE next #-}

771 next !s = case next0 s of

777 next !s = case next0 s of

@@ -775,9 +781,9 @@ takeWhile p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high

775 | otherwise -> Done

781 | otherwise -> Done

776{-# INLINE [0] takeWhile #-}

782{-# INLINE [0] takeWhile #-}

777

783

778−-- | dropWhile @p @xs returns the suffix remaining after takeWhile @p @xs.

784+-- | @'dropWhile' p xs@ returns the suffix remaining after @'takeWhile' p xs@.

779dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char

785dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char

780−dropWhile p (Stream next0 s0 len) = Stream next (L s0) len -- HINT maybe too high

786+dropWhile p (Stream next0 s0 len) = Stream next (L s0) (len - unknownSize)

781 where

787 where

782 {-# INLINE next #-}

788 {-# INLINE next #-}

783 next (L s) = case next0 s of

789 next (L s) = case next0 s of

@@ -811,7 +817,7 @@ isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)

811-------------------------------------------------------------------------------

817-------------------------------------------------------------------------------

812-- ** Searching by equality

818-- ** Searching by equality

813

819

814−-- | /O(n)/ elem is the stream membership predicate.

820+-- | /O(n)/ 'elem' is the stream membership predicate.

815elem :: Char -> Stream Char -> Bool

821elem :: Char -> Stream Char -> Bool

816elem w (Stream next s0 _len) = loop_elem s0

822elem w (Stream next s0 _len) = loop_elem s0

817 where

823 where

@@ -856,7 +862,8 @@ indexI (Stream next s0 _len) n0

856-- returns a stream containing those characters that satisfy the

862-- returns a stream containing those characters that satisfy the

857-- predicate.

863-- predicate.

858filter :: (Char -> Bool) -> Stream Char -> Stream Char

864filter :: (Char -> Bool) -> Stream Char -> Stream Char

865+filter p (Stream next0 s0 len) =

859−filter p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high

866+ Stream next s0 (len - unknownSize) -- HINT maybe too high

860 where

867 where

861 next !s = case next0 s of

868 next !s = case next0 s of

862 Done -> Done

869 Done -> Done

Data/Text/Internal/Fusion/Size.hs+33−7

@@ -1,4 +1,4 @@

1−{-# LANGUAGE CPP #-}

1+{-# LANGUAGE CPP, PatternGuards #-}

2{-# OPTIONS_GHC -fno-warn-missing-methods #-}

3-- |

4-- Module : Data.Text.Internal.Fusion.Internal

@@ -19,11 +19,16 @@

19module Data.Text.Internal.Fusion.Size

20 (

21 Size

22− , exactly

22+ -- * Sizes

23 , exactSize

24 , maxSize

25 , betweenSize

26 , unknownSize

27+ , unionSize

28+ , charSize

29+ , codePointsSize

30+ -- * Querying sizes

31+ , exactly

27 , smaller

32 , smaller

28 , larger

33 , larger

29 , upperBound

34 , upperBound

@@ -32,11 +37,13 @@ module Data.Text.Internal.Fusion.Size

32 , isEmpty

37 , isEmpty

33 ) where

38 ) where

34

39

40+import Data.Char (ord)

35import Data.Text.Internal (mul)

41import Data.Text.Internal (mul)

36#if defined(ASSERTS)

42#if defined(ASSERTS)

37import Control.Exception (assert)

43import Control.Exception (assert)

38#endif

44#endif

39

45

46+-- | A size in UTF-16 code units.

40data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.

47data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.

41 | Unknown -- ^ Unknown size.

48 | Unknown -- ^ Unknown size.

42 deriving (Eq, Show)

49 deriving (Eq, Show)

@@ -46,6 +53,17 @@ exactly (Between na nb) | na == nb = Just na

46exactly _ = Nothing

53exactly _ = Nothing

47{-# INLINE exactly #-}

54{-# INLINE exactly #-}

48

55

56+-- | The 'Size' of the given code point.

57+charSize :: Char -> Size

58+charSize c

59+ | ord c < 0x10000 = exactSize 1

60+ | otherwise = exactSize 2

61+

62+-- | The 'Size' of @n@ code points.

63+codePointsSize :: Int -> Size

64+codePointsSize n = Between n (2*n)

65+{-# INLINE codePointsSize #-}

66+

49exactSize :: Int -> Size

67exactSize :: Int -> Size

50exactSize n =

68exactSize n =

51#if defined(ASSERTS)

69#if defined(ASSERTS)

@@ -71,6 +89,10 @@ betweenSize m n =

71 Between m n

89 Between m n

72{-# INLINE betweenSize #-}

90{-# INLINE betweenSize #-}

73

91

92+unionSize :: Size -> Size -> Size

93+unionSize (Between a b) (Between c d) = Between (min a c) (max b d)

94+unionSize _ _ = Unknown

95+

74unknownSize :: Size

96unknownSize :: Size

75unknownSize = Unknown

97unknownSize = Unknown

76{-# INLINE unknownSize #-}

98{-# INLINE unknownSize #-}

@@ -140,11 +162,15 @@ lowerBound _ (Between n _) = n

140lowerBound k _ = k

162lowerBound k _ = k

141{-# INLINE lowerBound #-}

163{-# INLINE lowerBound #-}

142

164

165+-- | Determine the ordering relationship between two 'Size's, or 'Nothing' in

166+-- the indeterminate case.

143−compareSize :: Size -> Int -> Maybe Ordering

167+compareSize :: Size -> Size -> Maybe Ordering

144−compareSize (Between ma mb) n

168+compareSize (Between ma mb) (Between na nb)

145− | mb < n = Just LT

169+ | mb < na = Just LT

146− | ma > n = Just GT

170+ | ma > nb = Just GT

171+ | ma == mb

172+ , ma == na

147− | ma == n && mb == n = Just EQ

173+ , ma == nb = Just EQ

148compareSize _ _ = Nothing

174compareSize _ _ = Nothing

149

175

150

176

Data/Text/Internal/Fusion/Types.hs+4−4

@@ -75,16 +75,16 @@ instance (Ord a) => Ord (Stream a) where

75-- unstreaming functions must be able to cope with the hint being too

76-- small or too large.

77--

78−-- The size hint tries to track the UTF-16 code points in a stream,

78+-- The size hint tries to track the UTF-16 code units in a stream,

79−-- but often counts the number of characters instead. It can easily

79+-- but often counts the number of code points instead. It can easily

80-- undercount if, for instance, a transformed stream contains astral

81−-- plane characters (those above 0x10000).

81+-- plane code points (those above 0x10000).

82

83data Stream a =

84 forall s. Stream

85 (s -> Step s a) -- stepper function

86 !s -- current state

87− !Size -- size hint

87+ !Size -- size hint in code units

88

89-- | /O(n)/ Determines if two streams are equal.

90eq :: (Eq a) => Stream a -> Stream a -> Bool

Data/Text/Lazy.hs+54−20

@@ -13,7 +13,6 @@

13--

14-- License : BSD-style

15-- Maintainer : bos@serpentine.com

16−-- Stability : experimental

17-- Portability : GHC

16-- Portability : GHC

18--

17--

19-- A time and space-efficient implementation of Unicode text using

18-- A time and space-efficient implementation of Unicode text using

@@ -68,6 +67,7 @@ module Data.Text.Lazy

68 , snoc

67 , snoc

69 , append

68 , append

70 , uncons

69 , uncons

70+ , unsnoc

71 , head

72 , last

73 , tail

@@ -273,14 +273,16 @@ import Text.Printf (PrintfArg, formatArg, formatString)

273-- $replacement

274--

275-- A 'Text' value is a sequence of Unicode scalar values, as defined

276+-- in

276−-- in §3.9, definition D76 of the Unicode 5.2 standard:

277+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.

277−-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

278−-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

278+-- As such, a 'Text' cannot contain values in the range U+D800 to

279−-- inclusive. Haskell implementations admit all Unicode code points

279+-- U+DFFF inclusive. Haskell implementations admit all Unicode code

280+-- points

281+-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)

280−-- (§3.4, definition D10) as 'Char' values, including code points

282+-- as 'Char' values, including code points from this invalid range.

281−-- from this invalid range. This means that there are some 'Char'

283+-- This means that there are some 'Char' values that are not valid

282−-- values that are not valid Unicode scalar values, and the functions

284+-- Unicode scalar values, and the functions in this module must handle

283−-- in this module must handle those cases.

285+-- those cases.

284--

286--

285-- Within this module, many functions construct a 'Text' from one or

287-- Within this module, many functions construct a 'Text' from one or

286-- more 'Char' values. Those functions will substitute 'Char' values

288-- more 'Char' values. Those functions will substitute 'Char' values

@@ -294,8 +296,8 @@ import Text.Printf (PrintfArg, formatArg, formatString)

294-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

296-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

295-- code points, and so cannot be represented. The functions replace

297-- code points, and so cannot be represented. The functions replace

296-- invalid scalar values, instead of dropping them, as a security

298-- invalid scalar values, instead of dropping them, as a security

297−-- measure. For details, see Unicode Technical Report 36, §3.5:

299+-- measure. For details, see

298−-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)

300+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)

299

301

300equal :: Text -> Text -> Bool

302equal :: Text -> Text -> Bool

301equal Empty Empty = True

303equal Empty Empty = True

@@ -342,9 +344,13 @@ instance Read Text where

342 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

344 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

343

345

344#if MIN_VERSION_base(4,9,0)

346#if MIN_VERSION_base(4,9,0)

347+-- | Non-orphan 'Semigroup' instance only defined for

345−-- Semigroup orphan instances for older GHCs are provided by

348+-- @base-4.9.0.0@ and later; orphan instances for older GHCs are

349+-- provided by

350+-- the [semigroups](http://hackage.haskell.org/package/semigroups)

346−-- 'semigroups` package

351+-- package

347−

352+--

353+-- @since 1.2.2.0

348instance Semigroup Text where

354instance Semigroup Text where

349 (<>) = append

355 (<>) = append

350#endif

356#endif

@@ -362,6 +368,7 @@ instance IsString Text where

362 fromString = pack

368 fromString = pack

363

369

364#if __GLASGOW_HASKELL__ >= 708

370#if __GLASGOW_HASKELL__ >= 708

371+-- | @since 1.2.0.0

365instance Exts.IsList Text where

372instance Exts.IsList Text where

366 type Item Text = Char

373 type Item Text = Char

367 fromList = pack

374 fromList = pack

@@ -374,6 +381,7 @@ instance NFData Text where

374 rnf (Chunk _ ts) = rnf ts

381 rnf (Chunk _ ts) = rnf ts

375#endif

382#endif

376

383

384+-- | @since 1.2.1.0

377instance Binary Text where

385instance Binary Text where

378 put t = put (encodeUtf8 t)

386 put t = put (encodeUtf8 t)

379 get = do

387 get = do

@@ -397,6 +405,8 @@ instance Data Text where

397

405

398#if MIN_VERSION_base(4,7,0)

406#if MIN_VERSION_base(4,7,0)

399-- | Only defined for @base-4.7.0.0@ and later

407-- | Only defined for @base-4.7.0.0@ and later

408+--

409+-- @since 1.2.2.0

400instance PrintfArg Text where

410instance PrintfArg Text where

401 formatArg txt = formatString $ unpack txt

411 formatArg txt = formatString $ unpack txt

402#endif

412#endif

@@ -475,9 +485,7 @@ fromStrict t = chunk t Empty

475-- -----------------------------------------------------------------------------

485-- -----------------------------------------------------------------------------

476-- * Basic functions

486-- * Basic functions

477

487

478−-- | /O(n)/ Adds a character to the front of a 'Text'. This function

488+-- | /O(1)/ Adds a character to the front of a 'Text'. Subject to fusion.

479−-- is more costly than its 'List' counterpart because it requires

480−-- copying a new array. Subject to fusion.

481cons :: Char -> Text -> Text

489cons :: Char -> Text -> Text

482cons c t = Chunk (T.singleton c) t

490cons c t = Chunk (T.singleton c) t

483{-# INLINE [1] cons #-}

491{-# INLINE [1] cons #-}

@@ -561,6 +569,17 @@ init Empty = emptyError "init"

561 unstream (S.init (stream t)) = init t

569 unstream (S.init (stream t)) = init t

562 #-}

570 #-}

563

571

572+-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if

573+-- empty.

574+--

575+-- * It is no faster than using 'init' and 'last'.

576+--

577+-- @since 1.2.3.0

578+unsnoc :: Text -> Maybe (Text, Char)

579+unsnoc Empty = Nothing

580+unsnoc ts@(Chunk _ _) = Just (init ts, last ts)

581+{-# INLINE unsnoc #-}

582+

564-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

583-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

565-- fusion.

584-- fusion.

566null :: Text -> Bool

585null :: Text -> Bool

@@ -820,6 +839,8 @@ toUpper t = unstream (S.toUpper (stream t))

820-- guides disagree on whether the book name \"The Hill of the Red

839-- guides disagree on whether the book name \"The Hill of the Red

821-- Fox\" is correctly title cased—but this function will

840-- Fox\" is correctly title cased—but this function will

822-- capitalize /every/ word.

841-- capitalize /every/ word.

842+--

843+-- @since 1.0.0.0

823toTitle :: Text -> Text

844toTitle :: Text -> Text

824toTitle t = unstream (S.toTitle (stream t))

845toTitle t = unstream (S.toTitle (stream t))

825{-# INLINE toTitle #-}

846{-# INLINE toTitle #-}

@@ -972,6 +993,8 @@ mapAccumR f = go

972

993

973-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every

994-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every

974-- element.

995-- element.

996+--

997+-- @since 1.2.0.5

975repeat :: Char -> Text

998repeat :: Char -> Text

976repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t

999repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t

977 in t

1000 in t

@@ -989,6 +1012,8 @@ replicate n t

989

1012

990-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or

1013-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or

991-- equivalently, the infinite repetition of the original 'Text'.

1014-- equivalently, the infinite repetition of the original 'Text'.

1015+--

1016+-- @since 1.2.0.5

992cycle :: Text -> Text

1017cycle :: Text -> Text

993cycle Empty = emptyError "cycle"

1018cycle Empty = emptyError "cycle"

994cycle t = let t' = foldrChunks Chunk t' t

1019cycle t = let t' = foldrChunks Chunk t' t

@@ -998,6 +1023,8 @@ cycle t = let t' = foldrChunks Chunk t' t

998-- of @f@ to @x@:

1023-- of @f@ to @x@:

999--

1024--

1000-- > iterate f x == [x, f x, f (f x), ...]

1025-- > iterate f x == [x, f x, f (f x), ...]

1026+--

1027+-- @since 1.2.0.5

1001iterate :: (Char -> Char) -> Char -> Text

1028iterate :: (Char -> Char) -> Char -> Text

1002iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))

1029iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))

1003 in t c

1030 in t c

@@ -1061,6 +1088,8 @@ take i t0 = take' i t0

1061-- Examples:

1088-- Examples:

1062--

1089--

1063-- > takeEnd 3 "foobar" == "bar"

1090-- > takeEnd 3 "foobar" == "bar"

1091+--

1092+-- @since 1.1.1.0

1064takeEnd :: Int64 -> Text -> Text

1093takeEnd :: Int64 -> Text -> Text

1065takeEnd n t0

1094takeEnd n t0

1066 | n <= 0 = empty

1095 | n <= 0 = empty

@@ -1099,6 +1128,8 @@ drop i t0

1099-- Examples:

1128-- Examples:

1100--

1129--

1101-- > dropEnd 3 "foobar" == "foo"

1130-- > dropEnd 3 "foobar" == "foo"

1131+--

1132+-- @since 1.1.1.0

1102dropEnd :: Int64 -> Text -> Text

1133dropEnd :: Int64 -> Text -> Text

1103dropEnd n t0

1134dropEnd n t0

1104 | n <= 0 = t0

1135 | n <= 0 = t0

@@ -1150,12 +1181,15 @@ takeWhile p t0 = takeWhile' t0

1150-- Examples:

1181-- Examples:

1151--

1182--

1152-- > takeWhileEnd (=='o') "foo" == "oo"

1183-- > takeWhileEnd (=='o') "foo" == "oo"

1184+--

1185+-- @since 1.2.2.0

1153takeWhileEnd :: (Char -> Bool) -> Text -> Text

1186takeWhileEnd :: (Char -> Bool) -> Text -> Text

1154takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1187takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1155 where takeChunk acc [] = acc

1188 where takeChunk acc [] = acc

1156− takeChunk acc (t:ts) = if T.length t' < T.length t

1189+ takeChunk acc (t:ts)

1157− then (Chunk t' acc)

1190+ | T.lengthWord16 t' < T.lengthWord16 t

1191+ = chunk t' acc

1158− else takeChunk (Chunk t' acc) ts

1192+ | otherwise = takeChunk (Chunk t' acc) ts

1159 where t' = T.takeWhileEnd p t

1193 where t' = T.takeWhileEnd p t

1160{-# INLINE takeWhileEnd #-}

1194{-# INLINE takeWhileEnd #-}

1161

1195

Data/Text/Lazy/Builder.hs−1

@@ -11,7 +11,6 @@

11-- License : BSD-style (see LICENSE)

12--

13-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

14−-- Stability : experimental

15-- Portability : portable to Hugs and GHC

14-- Portability : portable to Hugs and GHC

16--

15--

17-- Efficient construction of lazy @Text@ values. The principal

16-- Efficient construction of lazy @Text@ values. The principal

Data/Text/Lazy/Builder/Int.hs+3−1

@@ -9,7 +9,6 @@

10-- License: BSD-style

11-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

12−-- Stability: experimental

13-- Portability: portable

12-- Portability: portable

14--

13--

15-- Efficiently write an integral value to a 'Builder'.

14-- Efficiently write an integral value to a 'Builder'.

@@ -32,6 +31,9 @@ import GHC.Base (quotInt, remInt)

32import GHC.Num (quotRemInteger)

31import GHC.Num (quotRemInteger)

33import GHC.Types (Int(..))

32import GHC.Types (Int(..))

34import Control.Monad.ST

33import Control.Monad.ST

34+#if MIN_VERSION_base(4,11,0)

35+import Prelude hiding ((<>))

36+#endif

35

37

36#ifdef __GLASGOW_HASKELL__

38#ifdef __GLASGOW_HASKELL__

37# if defined(INTEGER_GMP)

39# if defined(INTEGER_GMP)

Data/Text/Lazy/Builder/RealFloat.hs+3

@@ -24,6 +24,9 @@ import Data.Text.Lazy.Builder.Int (decimal)

24import Data.Text.Internal.Builder.RealFloat.Functions (roundTo)

25import Data.Text.Lazy.Builder

26import qualified Data.Text as T

27+#if MIN_VERSION_base(4,11,0)

28+import Prelude hiding ((<>))

29+#endif

27

30

28-- | Control the rendering of floating point numbers.

31-- | Control the rendering of floating point numbers.

29data FPFormat = Exponent

32data FPFormat = Exponent

Data/Text/Lazy/Encoding.hs+13−3

@@ -8,14 +8,13 @@

8--

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11−-- Stability : experimental

12-- Portability : portable

11-- Portability : portable

13--

12--

14-- Functions for converting lazy 'Text' values to and from lazy

13-- Functions for converting lazy 'Text' values to and from lazy

15-- 'ByteString', using several standard encodings.

14-- 'ByteString', using several standard encodings.

16--

15--

17−-- To gain access to a much larger variety of encodings, use the

16+-- To gain access to a much larger family of encodings, use the

18−-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

17+-- <http://hackage.haskell.org/package/text-icu text-icu package>.

19

18

20module Data.Text.Lazy.Encoding

19module Data.Text.Lazy.Encoding

21 (

20 (

@@ -141,6 +140,7 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

141 rnf (Chunk _ ts) = rnf ts

140 rnf (Chunk _ ts) = rnf ts

142{-# INLINE decodeUtf8' #-}

141{-# INLINE decodeUtf8' #-}

143

142

143+-- | Encode text using UTF-8 encoding.

144encodeUtf8 :: Text -> B.ByteString

145encodeUtf8 Empty = B.empty

146encodeUtf8 lt@(Chunk t _) =

@@ -154,10 +154,20 @@ encodeUtf8 lt@(Chunk t _) =

154 firstChunkSize = min B.smallChunkSize (4 * (T.length t + 1))

155 strategy = B.safeStrategy firstChunkSize B.defaultChunkSize

156

157+-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

158+--

159+-- @since 1.1.0.0

157encodeUtf8Builder :: Text -> B.Builder

160encodeUtf8Builder :: Text -> B.Builder

158encodeUtf8Builder =

161encodeUtf8Builder =

159 foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

162 foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

160

163

164+-- | Encode text using UTF-8 encoding and escape the ASCII characters using

165+-- a 'BP.BoundedPrim'.

166+--

167+-- Use this function is to implement efficient encoders for text-based formats

168+-- like JSON or HTML.

169+--

170+-- @since 1.1.0.0

161{-# INLINE encodeUtf8BuilderEscaped #-}

171{-# INLINE encodeUtf8BuilderEscaped #-}

162encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

172encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

163encodeUtf8BuilderEscaped prim =

173encodeUtf8BuilderEscaped prim =

Data/Text/Lazy/IO.hs−1

@@ -8,7 +8,6 @@

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11−-- Stability : experimental

12-- Portability : GHC

11-- Portability : GHC

13--

12--

14-- Efficient locale-sensitive support for lazy text I\/O.

13-- Efficient locale-sensitive support for lazy text I\/O.

Data/Text/Lazy/Read.hs+3−4

@@ -11,7 +11,6 @@

11--

12-- License : BSD-style

13-- Maintainer : bos@serpentine.com

14−-- Stability : experimental

15-- Portability : GHC

14-- Portability : GHC

16--

15--

17-- Functions used frequently when reading textual data.

16-- Functions used frequently when reading textual data.

@@ -68,9 +67,9 @@ decimal txt

68 go n d = (n * 10 + fromIntegral (digitToInt d))

67 go n d = (n * 10 + fromIntegral (digitToInt d))

69

68

70-- | Read a hexadecimal integer, consisting of an optional leading

69-- | Read a hexadecimal integer, consisting of an optional leading

71−-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

70+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

72−-- until a non-hex-digit or end of string is reached. This function

71+-- consumed until a non-hex-digit or end of string is reached.

73−-- is case insensitive.

72+-- This function is case insensitive.

74--

73--

75-- This function does not handle leading sign characters. If you need

74-- This function does not handle leading sign characters. If you need

76-- to handle signed input, use @'signed' 'hexadecimal'@.

75-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Read.hs+3−4

@@ -9,7 +9,6 @@

9--

10-- License : BSD-style

11-- Maintainer : bos@serpentine.com

12−-- Stability : experimental

13-- Portability : GHC

12-- Portability : GHC

14--

13--

15-- Functions used frequently when reading textual data.

14-- Functions used frequently when reading textual data.

@@ -67,9 +66,9 @@ decimal txt

67 go n d = (n * 10 + fromIntegral (digitToInt d))

66 go n d = (n * 10 + fromIntegral (digitToInt d))

68

67

69-- | Read a hexadecimal integer, consisting of an optional leading

68-- | Read a hexadecimal integer, consisting of an optional leading

70−-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

69+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

71−-- until a non-hex-digit or end of string is reached. This function

70+-- consumed until a non-hex-digit or end of string is reached.

72−-- is case insensitive.

71+-- This function is case insensitive.

73--

72--

74-- This function does not handle leading sign characters. If you need

73-- This function does not handle leading sign characters. If you need

75-- to handle signed input, use @'signed' 'hexadecimal'@.

74-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Show.hs+2

@@ -46,6 +46,8 @@ unpack = S.unstreamList . stream

46-- fusion.

47--

48-- This is exposed solely for people writing GHC rewrite rules.

49+--

50+-- @since 1.2.1.1

49unpackCString# :: Addr# -> Text

51unpackCString# :: Addr# -> Text

50unpackCString# addr# = unstream (S.streamCString# addr#)

52unpackCString# addr# = unstream (S.streamCString# addr#)

51{-# NOINLINE unpackCString# #-}

53{-# NOINLINE unpackCString# #-}

Data/Text/Unsafe.hs+2−1

@@ -4,7 +4,6 @@

5-- License : BSD-style

6-- Maintainer : bos@serpentine.com

7−-- Stability : experimental

8-- Portability : portable

7-- Portability : portable

9--

8--

10-- A module containing unsafe 'Text' operations, for very very careful

9-- A module containing unsafe 'Text' operations, for very very careful

@@ -98,6 +97,8 @@ reverseIter (Text arr off _len) i

98-- | /O(1)/ Iterate one step backwards through a UTF-16 array,

97-- | /O(1)/ Iterate one step backwards through a UTF-16 array,

99-- returning the delta to add (i.e. a negative number) to give the

98-- returning the delta to add (i.e. a negative number) to give the

100-- next offset to iterate at.

99-- next offset to iterate at.

100+--

101+-- @since 1.1.1.0

101reverseIter_ :: Text -> Int -> Int

102reverseIter_ :: Text -> Int -> Int

102reverseIter_ (Text arr off _len) i

103reverseIter_ (Text arr off _len) i

103 | m < 0xDC00 || m > 0xDFFF = -1

104 | m < 0xDC00 || m > 0xDFFF = -1

README.markdown+5−16

@@ -1,29 +1,18 @@

1−# Text: Fast, packed Unicode strings, using stream fusion

1+# `text`: Fast, packed Unicode strings, using stream fusion

2

3This package provides the Data.Text library, a library for the space-

4and time-efficient manipulation of Unicode text in Haskell.

5

6−

7−# Normalization, conversion, and collation, oh my!

8−

9−This library intentionally provides a simple API based on the

10−Haskell prelude's list manipulation functions. For more complicated

11−real-world tasks, such as Unicode normalization, conversion to and

12−from a larger variety of encodings, and collation, use the [text-icu

13−package](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/text-icu).

6+**Please refer to the [package description on Hackage](https://hackage.haskell.org/package/text#description) for more information.**

14−

15−That library uses the well-respected and liberally licensed ICU

16−library to provide these facilities.

17−

18

7

19# Get involved!

8# Get involved!

20

9

21Please report bugs via the

10Please report bugs via the

22−[github issue tracker](https://github.com/bos/text/issues).

11+[github issue tracker](https://github.com/haskell/text/issues).

23

12

24−Master [git repository](https://github.com/bos/text):

13+Master [git repository](https://github.com/haskell/text):

25

14

26−* `git clone git://github.com/bos/text.git`

15+* `git clone git://github.com/haskell/text.git`

27

16

28There's also a [Mercurial mirror](https://bitbucket.org/bos/text):

17There's also a [Mercurial mirror](https://bitbucket.org/bos/text):

29

18

benchmarks/.gitignore+4−1

@@ -1 +1,4 @@

1+/dist/

1−dist

2+/dist-newstyle/

3+/.ghc.environment.*

4+/cabal.project.local

benchmarks/cabal.projectnew file+2

@@ -0,0 +1,2 @@

1+-- NB: we use a separate project

2+packages: .

benchmarks/haskell/Benchmarks.hs+2

@@ -10,6 +10,7 @@ import System.FilePath ((</>))

10import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8)

11

12import qualified Benchmarks.Builder as Builder

13+import qualified Benchmarks.Concat as Concat

13import qualified Benchmarks.DecodeUtf8 as DecodeUtf8

14import qualified Benchmarks.DecodeUtf8 as DecodeUtf8

14import qualified Benchmarks.EncodeUtf8 as EncodeUtf8

15import qualified Benchmarks.EncodeUtf8 as EncodeUtf8

15import qualified Benchmarks.Equality as Equality

16import qualified Benchmarks.Equality as Equality

@@ -41,6 +42,7 @@ benchmarks = do

41 -- Traditional benchmarks

42 -- Traditional benchmarks

42 bs <- sequence

43 bs <- sequence

43 [ Builder.benchmark

44 [ Builder.benchmark

45+ , Concat.benchmark

44 , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")

46 , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")

45 , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")

47 , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")

46 , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")

48 , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")

benchmarks/haskell/Benchmarks/Concat.hsnew file+25

@@ -0,0 +1,25 @@

1+{-# LANGUAGE OverloadedStrings #-}

2+

3+module Benchmarks.Concat (benchmark) where

4+

5+import Control.Monad.Trans.Writer

6+import Criterion (Benchmark, bgroup, bench, whnf)

7+import Data.Text as T

8+

9+benchmark :: IO Benchmark

10+benchmark = return $ bgroup "Concat"

11+ [ bench "append" $ whnf (append4 "Text 1" "Text 2" "Text 3") "Text 4"

12+ , bench "concat" $ whnf (concat4 "Text 1" "Text 2" "Text 3") "Text 4"

13+ , bench "write" $ whnf (write4 "Text 1" "Text 2" "Text 3") "Text 4"

14+ ]

15+

16+append4, concat4, write4 :: Text -> Text -> Text -> Text -> Text

17+

18+{-# NOINLINE append4 #-}

19+append4 x1 x2 x3 x4 = x1 `append` x2 `append` x3 `append` x4

20+

21+{-# NOINLINE concat4 #-}

22+concat4 x1 x2 x3 x4 = T.concat [x1, x2, x3, x4]

23+

24+{-# NOINLINE write4 #-}

25+write4 x1 x2 x3 x4 = execWriter $ tell x1 >> tell x2 >> tell x3 >> tell x4

benchmarks/text-benchmarks.cabal+82−8

@@ -12,8 +12,7 @@ author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

12maintainer: jaspervdj@gmail.com

13category: Text

14build-type: Simple

15−

16−cabal-version: >=1.2

15+cabal-version: >=1.8

17

16

18flag bytestring-builder

17flag bytestring-builder

19 description: Depend on the bytestring-builder package for backwards compatibility.

18 description: Depend on the bytestring-builder package for backwards compatibility.

@@ -26,16 +25,12 @@ flag llvm

26 manual: True

25 manual: True

27

26

28executable text-benchmarks

27executable text-benchmarks

29− hs-source-dirs: haskell ..

30− c-sources: ../cbits/cbits.c

31− cbits/time_iconv.c

32− include-dirs: ../include

33− main-is: Benchmarks.hs

34 ghc-options: -Wall -O2 -rtsopts

28 ghc-options: -Wall -O2 -rtsopts

35 if flag(llvm)

29 if flag(llvm)

36 ghc-options: -fllvm

30 ghc-options: -fllvm

37 cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP

31 cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP

38− build-depends: base == 4.*,

32+ build-depends: array,

33+ base == 4.*,

39 binary,

34 binary,

40 blaze-builder,

35 blaze-builder,

41 bytestring-lexing >= 0.5.0,

36 bytestring-lexing >= 0.5.0,

@@ -47,6 +42,7 @@ executable text-benchmarks

47 ghc-prim,

42 ghc-prim,

48 integer-gmp,

43 integer-gmp,

49 stringsearch,

44 stringsearch,

45+ transformers,

50 utf8-string,

46 utf8-string,

51 vector

47 vector

52

48

@@ -56,9 +52,87 @@ executable text-benchmarks

56 else

52 else

57 build-depends: bytestring >= 0.10.4

53 build-depends: bytestring >= 0.10.4

58

54

55+ -- modules for benchmark proper

56+ c-sources: cbits/time_iconv.c

57+ hs-source-dirs: haskell

58+ main-is: Benchmarks.hs

59+ other-modules:

60+ Benchmarks.Builder

61+ Benchmarks.Concat

62+ Benchmarks.DecodeUtf8

63+ Benchmarks.EncodeUtf8

64+ Benchmarks.Equality

65+ Benchmarks.FileRead

66+ Benchmarks.FoldLines

67+ Benchmarks.Mul

68+ Benchmarks.Programs.BigTable

69+ Benchmarks.Programs.Cut

70+ Benchmarks.Programs.Fold

71+ Benchmarks.Programs.Sort

72+ Benchmarks.Programs.StripTags

73+ Benchmarks.Programs.Throughput

74+ Benchmarks.Pure

75+ Benchmarks.ReadNumbers

76+ Benchmarks.Replace

77+ Benchmarks.Search

78+ Benchmarks.Stream

79+ Benchmarks.WordFrequencies

80+

81+ -- Source code for IUT (implementation under test)

82+ -- "borrowed" from parent folder

83+ include-dirs: ../include

84+ c-sources: ../cbits/cbits.c

85+ hs-source-dirs: ..

86+ other-modules:

87+ Data.Text

88+ Data.Text.Array

89+ Data.Text.Encoding

90+ Data.Text.Encoding.Error

91+ Data.Text.Foreign

92+ Data.Text.IO

93+ Data.Text.Internal

94+ Data.Text.Internal.Builder

95+ Data.Text.Internal.Builder.Functions

96+ Data.Text.Internal.Builder.Int.Digits

97+ Data.Text.Internal.Builder.RealFloat.Functions

98+ Data.Text.Internal.Encoding.Fusion

99+ Data.Text.Internal.Encoding.Fusion.Common

100+ Data.Text.Internal.Encoding.Utf16

101+ Data.Text.Internal.Encoding.Utf32

102+ Data.Text.Internal.Encoding.Utf8

103+ Data.Text.Internal.Functions

104+ Data.Text.Internal.Fusion

105+ Data.Text.Internal.Fusion.CaseMapping

106+ Data.Text.Internal.Fusion.Common

107+ Data.Text.Internal.Fusion.Size

108+ Data.Text.Internal.Fusion.Types

109+ Data.Text.Internal.IO

110+ Data.Text.Internal.Lazy

111+ Data.Text.Internal.Lazy.Encoding.Fusion

112+ Data.Text.Internal.Lazy.Fusion

113+ Data.Text.Internal.Lazy.Search

114+ Data.Text.Internal.Private

115+ Data.Text.Internal.Read

116+ Data.Text.Internal.Search

117+ Data.Text.Internal.Unsafe

118+ Data.Text.Internal.Unsafe.Char

119+ Data.Text.Internal.Unsafe.Shift

120+ Data.Text.Lazy

121+ Data.Text.Lazy.Builder

122+ Data.Text.Lazy.Builder.Int

123+ Data.Text.Lazy.Builder.RealFloat

124+ Data.Text.Lazy.Encoding

125+ Data.Text.Lazy.IO

126+ Data.Text.Lazy.Internal

127+ Data.Text.Lazy.Read

128+ Data.Text.Read

129+ Data.Text.Unsafe

130+ Data.Text.Show

131+

59executable text-multilang

132executable text-multilang

60 hs-source-dirs: haskell

133 hs-source-dirs: haskell

61 main-is: Multilang.hs

134 main-is: Multilang.hs

135+ other-modules: Timer

62 ghc-options: -Wall -O2

136 ghc-options: -Wall -O2

63 build-depends: base == 4.*,

137 build-depends: base == 4.*,

64 bytestring,

138 bytestring,

cabal.projectnew file+3

@@ -0,0 +1,3 @@

1+-- See http://cabal.readthedocs.io/en/latest/nix-local-build-overview.html

2+packages: .

3+tests: True

cbits/cbits.c+6−11

@@ -206,11 +206,9 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

206 const uint8_t *srcend,

207 uint32_t *codepoint0, uint32_t *state0)

208{

209− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,

209+ _hs_text_decode_utf8_int(dest, destoff, src, srcend, codepoint0, state0);

210− codepoint0, state0);

210+

211− if (*state0 == UTF8_REJECT)

212− ret -=1;

213− return ret;

211+ return *src;

214}

212}

215

213

216/*

214/*

@@ -222,12 +220,9 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

222{

220{

223 uint32_t codepoint;

221 uint32_t codepoint;

224 uint32_t state = UTF8_ACCEPT;

222 uint32_t state = UTF8_ACCEPT;

225− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

223+ _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

226− &codepoint, &state);

224+ &codepoint, &state);

227− /* Back up if we have an incomplete or invalid encoding */

228− if (state != UTF8_ACCEPT)

229− ret -= 1;

230− return ret;

225+ return src;

231}

226}

232

227

233void

228void

changelog.md+61−19

@@ -1,15 +1,57 @@

1+### 1.2.3.1 TBD

2+

3+* Make `decodeUtf8With` fail explicitly for unsupported non-BMP

4+ replacement characters instead silent undefined behaviour (gh-213)

5+

6+### 1.2.3.0

7+

8+* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec

9+ (updated from 8.0).

10+

11+* Bug fix: the lazy `takeWhileEnd` function violated the

12+ [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)

13+ (gh-184).

14+

15+* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).

16+

17+* New function: `unsnoc` (gh-173).

18+

19+* Reduce memory overhead in `encodeUTF8` (gh-194).

20+

21+* Improve UTF-8 decoder error-recovery (gh-182).

22+

23+* Minor documentation improvements (`@since` annotations, more

24+ examples, clarifications).

25+

26+#### 1.2.2.2

27+

28+* The `toTitle` function now correctly handles letters that

29+ immediately follow punctuation. Before, `"there's"` would turn into

30+ `"There'S"`. Now, it becomes `"There's"`.

31+

32+* The implementation of unstreaming is faster, resulting in operations

33+ such as `map` and `intersperse` speeding up by up to 30%, with

34+ smaller code generated.

35+

36+* The optimised length comparison function is now more likely to be

37+ used after some rewrite rule tweaking.

38+

39+* Bug fix: an off-by-one bug in `takeEnd` is fixed.

40+

41+* Bug fix: a logic error in `takeWord16` is fixed.

42+

1−1.2.2.1

43+#### 1.2.2.1

2

44

3* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

45* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

4 The build flag has been renamed accordingly. Your army of diligent

46 The build flag has been renamed accordingly. Your army of diligent

5 maintainers apologizes for the churn.

47 maintainers apologizes for the churn.

6

48

7−* Spec compliance: toCaseFold now follows the Unicode 8.0 spec

49+* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec

8 (updated from 7.0)

50 (updated from 7.0)

9

51

10* An STG lint error has been fixed

52* An STG lint error has been fixed

11

53

12−1.2.2.0

54+### 1.2.2.0

13

55

14* The `integer-simple` package, upon which this package optionally

56* The `integer-simple` package, upon which this package optionally

15 depended, has been replaced with `integer-pure`. The build flag has

57 depended, has been replaced with `integer-pure`. The build flag has

@@ -25,7 +67,7 @@

25 * if `base` >= 4.7: `PrintfArg`

67 * if `base` >= 4.7: `PrintfArg`

26 * if `base` >= 4.9: `Semigroup`

68 * if `base` >= 4.9: `Semigroup`

27

69

28−1.2.1.3

70+#### 1.2.1.3

29

71

30* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

72* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

31 phase 2 does not prevent competition with the `unpack` rule, which is

73 phase 2 does not prevent competition with the `unpack` rule, which is

@@ -33,7 +75,7 @@

33 test environment mistake. Moving literal rules back to phase 1 finally

75 test environment mistake. Moving literal rules back to phase 1 finally

34 fixes GHC Trac #10528 correctly.

76 fixes GHC Trac #10528 correctly.

35

77

36−1.2.1.2

78+#### 1.2.1.2

37

79

38* Bug fix: Run literal rewrite rules in simplifier phase 2.

80* Bug fix: Run literal rewrite rules in simplifier phase 2.

39 The behavior of the simplifier changed in GHC 7.10.2,

81 The behavior of the simplifier changed in GHC 7.10.2,

@@ -41,21 +83,21 @@

41 and long compilation times. See

83 and long compilation times. See

42 [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

84 [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

43

85

44−1.2.1.1

86+#### 1.2.1.1

45

87

46* Expose unpackCString#, which you should never use.

88* Expose unpackCString#, which you should never use.

47

89

48−1.2.1.0

90+### 1.2.1.0

49

91

50* Added Binary instances for both Text types. (If you have previously

92* Added Binary instances for both Text types. (If you have previously

51 been using the text-binary package to get a Binary instance, it is

93 been using the text-binary package to get a Binary instance, it is

52 now obsolete.)

94 now obsolete.)

53

95

54−1.2.0.6

96+#### 1.2.0.6

55

97

56* Fixed a space leak in UTF-8 decoding

98* Fixed a space leak in UTF-8 decoding

57

99

58−1.2.0.5

100+#### 1.2.0.5

59

101

60* Feature parity: repeat, cycle, iterate are now implemented for lazy

102* Feature parity: repeat, cycle, iterate are now implemented for lazy

61 Text, and the Data instance is more complete

103 Text, and the Data instance is more complete

@@ -75,26 +117,26 @@

75* Spec compliance: toCaseFold now follows the Unicode 7.0 spec

117* Spec compliance: toCaseFold now follows the Unicode 7.0 spec

76 (updated from 6.3)

118 (updated from 6.3)

77

119

78−1.2.0.4

120+#### 1.2.0.4

79

121

80* Fixed an incompatibility with base < 4.5

122* Fixed an incompatibility with base < 4.5

81

123

82−1.2.0.3

124+#### 1.2.0.3

83

125

84* Update formatRealFloat to correspond to the definition in versions

126* Update formatRealFloat to correspond to the definition in versions

85 of base newer than 4.5 (https://github.com/bos/text/issues/105)

127 of base newer than 4.5 (https://github.com/bos/text/issues/105)

86

128

87−1.2.0.2

129+#### 1.2.0.2

88

130

89* Bumped lower bound on deepseq to 1.4 for compatibility with the

131* Bumped lower bound on deepseq to 1.4 for compatibility with the

90 upcoming GHC 7.10

132 upcoming GHC 7.10

91

133

92−1.2.0.1

134+#### 1.2.0.1

93

135

94* Fixed a buffer overflow in rendering of large Integers

136* Fixed a buffer overflow in rendering of large Integers

95 (https://github.com/bos/text/issues/99)

137 (https://github.com/bos/text/issues/99)

96

138

97−1.2.0.0

139+## 1.2.0.0

98

140

99* Fixed an integer overflow in the replace function

141* Fixed an integer overflow in the replace function

100 (https://github.com/bos/text/issues/81)

142 (https://github.com/bos/text/issues/81)

@@ -107,7 +149,7 @@

107

149

108* Added an instance of IsList for GHC 7.8 and above

150* Added an instance of IsList for GHC 7.8 and above

109

151

110−1.1.1.0

152+### 1.1.1.0

111

153

112* The Data.Data instance now allows gunfold to work, via a virtual

154* The Data.Data instance now allows gunfold to work, via a virtual

113 pack constructor

155 pack constructor

@@ -117,12 +159,12 @@

117* Comparing the length of a Text against a number can now

159* Comparing the length of a Text against a number can now

118 short-circuit in more cases

160 short-circuit in more cases

119

161

120−1.1.0.1

162+#### 1.1.0.1

121

163

122* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes

164* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes

123 in single-byte chunks

165 in single-byte chunks

124

166

125−1.1.0.0

167+## 1.1.0.0

126

168

127* encodeUtf8: Performance is improved by up to 4x.

169* encodeUtf8: Performance is improved by up to 4x.

128

170

@@ -141,12 +183,12 @@

141 use at your own risk - there are no API stability guarantees for

183 use at your own risk - there are no API stability guarantees for

142 internal modules!

184 internal modules!

143

185

144−1.0.0.1

186+#### 1.0.0.1

145

187

146* decodeUtf8: Fixed a regression that caused us to incorrectly

188* decodeUtf8: Fixed a regression that caused us to incorrectly

147 identify truncated UTF-8 as valid (gh-61)

189 identify truncated UTF-8 as valid (gh-61)

148

190

149−1.0.0.0

191+# 1.0.0.0

150

192

151* Added support for Unicode 6.3.0 to case conversion functions

193* Added support for Unicode 6.3.0 to case conversion functions

152

194

tests-and-benchmarks.markdown+13−8

@@ -13,10 +13,13 @@ you can clone from one of the following locations:

13* Git mirror repository:

14 [github.com/bos/text-test-data](https://github.com/bos/text-test-data)

15

16−You should clone that repository into the `tests` subdirectory (your

16+You can clone either repository into the `tests` subdirectory using

17−clone must be named `text-test-data` locally), then run `make -C

17+

18−tests/text-test-data` to uncompress the test files. Many tests and

18+ cd tests/

19+ make text-test-data # to clone from mercurial, OR

20+ make VCS=git text-test-data # to clone from git

21+

19−benchmarks will fail if the test files are missing.

22+Many tests and benchmarks will fail if the test files are missing.

20

23

21Functional tests

24Functional tests

22----------------

25----------------

@@ -54,10 +57,12 @@ To compile the benchmarks, navigate to the `benchmarks` subdirectory and run

54

57

55 ./dist/build/text-benchmarks/text-benchmarks

58 ./dist/build/text-benchmarks/text-benchmarks

56

59

57−However, since there's quite a lot of benchmarks, you usually don't want to

60+Or if you have a recent enough `cabal`, you can build and run the

58−run them all. Instead, use the `-l` flag to get a list of benchmarks:

61+benchmarks via

59

62

60− ./dist/build/text-benchmarks/text-benchmarks

63+ cabal new-run exe:text-benchmarks -- --help

61

64

65+However, since there's quite a lot of benchmarks, you usually don't want to

66+run them all. Instead, use the `-l` flag to get a list of benchmarks

62−And run the ones you want to inspect. If you want to configure the benchmarks

67+and run the ones you want to inspect. If you want to configure the benchmarks

63further, the exact parameters can be changed in `Benchmarks.hs`.

68further, the exact parameters can be changed in `Benchmarks.hs`.

tests/Makefile+6−1

@@ -1,3 +1,4 @@

1+VCS = hg

1count = 1000

2count = 1000

2

3

3all: coverage literal-rule-test

4all: coverage literal-rule-test

@@ -12,7 +13,11 @@ build: text-test-data

12 cabal build

13 cabal build

13

14

14text-test-data:

15text-test-data:

16+ifeq ($(VCS),git)

17+ git clone https://github.com/bos/text-test-data.git

18+else

15 hg clone https://bitbucket.org/bos/text-test-data

19 hg clone https://bitbucket.org/bos/text-test-data

20+endif

16 $(MAKE) -C text-test-data

21 $(MAKE) -C text-test-data

17

22

18coverage/text-tests.tix:

23coverage/text-tests.tix:

@@ -37,4 +42,4 @@ coverage/hpc_index.html: coverage/coverage.tix

37clean:

42clean:

38 rm -rf dist coverage .hpc

43 rm -rf dist coverage .hpc

39

44

40−.PHONY: build coverage all literal-rule-test

45+.PHONY: all build clean coverage literal-rule-test

tests/Tests/Properties.hs+94−21

@@ -9,9 +9,9 @@ module Tests.Properties

9 ) where

10

11import Control.Applicative ((<$>), (<*>))

12−import Control.Arrow ((***), second)

12+import Control.Arrow ((***), first, second)

13import Data.Bits ((.&.))

14−import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)

14+import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord)

15import Data.Int (Int8, Int16, Int32, Int64)

16import Data.Monoid (Monoid(..))

17import Data.String (IsString(fromString))

@@ -23,6 +23,7 @@ import Data.Text.Internal.Search (indices)

23import Data.Text.Lazy.Read as TL

24import Data.Text.Read as T

25import Data.Word (Word, Word8, Word16, Word32, Word64)

26+import Data.Maybe (mapMaybe)

26import Numeric (showEFloat, showFFloat, showGFloat, showHex)

27import Numeric (showEFloat, showFFloat, showGFloat, showHex)

27import Prelude hiding (replicate)

28import Prelude hiding (replicate)

28import Test.Framework (Test, testGroup)

29import Test.Framework (Test, testGroup)

@@ -30,6 +31,7 @@ import Test.Framework.Providers.QuickCheck2 (testProperty)

30import Test.QuickCheck hiding ((.&.))

31import Test.QuickCheck hiding ((.&.))

31import Test.QuickCheck.Monadic

32import Test.QuickCheck.Monadic

32import Test.QuickCheck.Property (Property(..))

33import Test.QuickCheck.Property (Property(..))

34+import Test.QuickCheck.Unicode (char)

33import Tests.QuickCheckUtils

35import Tests.QuickCheckUtils

34import Tests.Utils

36import Tests.Utils

35import Text.Show.Functions ()

37import Text.Show.Functions ()

@@ -37,6 +39,7 @@ import qualified Control.Exception as Exception

37import qualified Data.Bits as Bits (shiftL, shiftR)

39import qualified Data.Bits as Bits (shiftL, shiftR)

38import qualified Data.ByteString as B

40import qualified Data.ByteString as B

39import qualified Data.ByteString.Lazy as BL

41import qualified Data.ByteString.Lazy as BL

42+import qualified Data.Char as C

40import qualified Data.List as L

43import qualified Data.List as L

41import qualified Data.Text as T

44import qualified Data.Text as T

42import qualified Data.Text.Encoding as E

45import qualified Data.Text.Encoding as E

@@ -60,10 +63,10 @@ t_pack_unpack = (T.unpack . T.pack) `eq` id

60tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

63tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

61t_stream_unstream = (S.unstream . S.stream) `eq` id

64t_stream_unstream = (S.unstream . S.stream) `eq` id

62tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

65tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

63−t_reverse_stream t = (S.reverse . S.reverseStream) t == t

66+t_reverse_stream t = (S.reverse . S.reverseStream) t === t

64−t_singleton c = [c] == (T.unpack . T.singleton) c

67+t_singleton c = [c] === (T.unpack . T.singleton) c

65−tl_singleton c = [c] == (TL.unpack . TL.singleton) c

68+tl_singleton c = [c] === (TL.unpack . TL.singleton) c

66−tl_unstreamChunks x = f 11 x == f 1000 x

69+tl_unstreamChunks x = f 11 x === f 1000 x

67 where f n = SL.unstreamChunks n . S.streamList

70 where f n = SL.unstreamChunks n . S.streamList

68tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

71tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

69tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

72tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

@@ -74,13 +77,13 @@ encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

74encodeLazyL1 :: TL.Text -> BL.ByteString

77encodeLazyL1 :: TL.Text -> BL.ByteString

75encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

78encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

76

79

77−t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

80+t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a

78 where a = T.map (\c -> chr (ord c `mod` 128)) t

81 where a = T.map (\c -> chr (ord c `mod` 128)) t

79−tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

82+tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a

80 where a = TL.map (\c -> chr (ord c `mod` 128)) t

83 where a = TL.map (\c -> chr (ord c `mod` 128)) t

81−t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

84+t_latin1 t = E.decodeLatin1 (encodeL1 a) === a

82 where a = T.map (\c -> chr (ord c `mod` 256)) t

85 where a = T.map (\c -> chr (ord c `mod` 256)) t

83−tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

86+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a

84 where a = TL.map (\c -> chr (ord c `mod` 256)) t

87 where a = TL.map (\c -> chr (ord c `mod` 256)) t

85t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

88t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

86t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

89t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

@@ -111,7 +114,7 @@ t_utf8_undecoded = forAll genUnicode $ \t ->

111 let b = E.encodeUtf8 t

114 let b = E.encodeUtf8 t

112 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

115 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

113 leftover = (++ [B.empty]) . init . tail . B.inits

116 leftover = (++ [B.empty]) . init . tail . B.inits

114− in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b == ls

117+ in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls

115

118

116data Badness = Solo | Leading | Trailing

119data Badness = Solo | Leading | Trailing

117 deriving (Eq, Show)

120 deriving (Eq, Show)

@@ -177,6 +180,33 @@ genInvalidUTF8 = B.pack <$> oneof [

177 k <- choose (0,n)

180 k <- choose (0,n)

178 vectorOf k gen

181 vectorOf k gen

179

182

183+-- See http://unicode.org/faq/utf_bom.html#gen8

184+-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...

185+-- When faced with this illegal byte sequence ... a UTF-8 conformant process

186+-- must treat the first byte 110xxxxx2 as an illegal termination error

187+-- (e.g. filter it out or replace by 0xFFFD) ...

188+-- ... and continue processing at the second byte 0xxxxxxx2

189+t_decode_with_error2 =

190+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"

191+t_decode_with_error3 =

192+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"

193+t_decode_with_error4 =

194+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"

195+

196+t_decode_with_error2' =

197+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of

198+ E.Some x _ _ -> x === "xa"

199+t_decode_with_error3' =

200+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of

201+ E.Some x _ _ -> x === "xaa"

202+t_decode_with_error4' =

203+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of

204+ E.Some x _ _ -> x === "xaaa"

205+

206+t_infix_concat bs1 text bs2 rep =

207+ text `T.isInfixOf`

208+ E.decodeUtf8With (\_ _ -> rep) (B.concat [bs1, E.encodeUtf8 text, bs2])

209+

180s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

210s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

181 where _types = s :: String

211 where _types = s :: String

182sf_Eq p s =

212sf_Eq p s =

@@ -201,8 +231,8 @@ t_mconcat = unsquare $

201 mconcat `eq` (unpackS . mconcat . L.map T.pack)

231 mconcat `eq` (unpackS . mconcat . L.map T.pack)

202tl_mconcat = unsquare $

232tl_mconcat = unsquare $

203 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

233 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

204−t_mempty = mempty == (unpackS (mempty :: T.Text))

234+t_mempty = mempty === (unpackS (mempty :: T.Text))

205−tl_mempty = mempty == (unpackS (mempty :: TL.Text))

235+tl_mempty = mempty === (unpackS (mempty :: TL.Text))

206t_IsString = fromString `eqP` (T.unpack . fromString)

236t_IsString = fromString `eqP` (T.unpack . fromString)

207tl_IsString = fromString `eqP` (TL.unpack . fromString)

237tl_IsString = fromString `eqP` (TL.unpack . fromString)

208

238

@@ -229,6 +259,13 @@ sf_uncons p = (uncons . L.filter p) `eqP`

229 (fmap (second unpackS) . S.uncons . S.filter p)

259 (fmap (second unpackS) . S.uncons . S.filter p)

230t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

260t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

231tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

261tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

262+

263+unsnoc xs@(_:_) = Just (init xs, last xs)

264+unsnoc [] = Nothing

265+

266+t_unsnoc = unsnoc `eqP` (fmap (first unpackS) . T.unsnoc)

267+tl_unsnoc = unsnoc `eqP` (fmap (first unpackS) . TL.unsnoc)

268+

232s_head = head `eqP` S.head

269s_head = head `eqP` S.head

233sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

270sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

234t_head = head `eqP` T.head

271t_head = head `eqP` T.head

@@ -326,6 +363,19 @@ t_toUpper_upper t = p (T.toUpper t) >= p t

326 where p = T.length . T.filter isUpper

363 where p = T.length . T.filter isUpper

327tl_toUpper_upper t = p (TL.toUpper t) >= p t

364tl_toUpper_upper t = p (TL.toUpper t) >= p t

328 where p = TL.length . TL.filter isUpper

365 where p = TL.length . TL.filter isUpper

366+t_toTitle_title t = all (<= 1) (caps w)

367+ where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle

368+ -- TIL: there exist uppercase-only letters

369+ w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t

370+t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable

371+ where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words

372+ -- Surprise! The Spanish/Portuguese ordinal indicators changed

373+ -- from category Ll (letter, lowercase) to Lo (letter, other)

374+ -- in Unicode 7.0

375+ -- Oh, and there exist lowercase-only letters (see previous test)

376+ stable c = if isLower c

377+ then C.toUpper c /= c

378+ else c /= '\170' && c /= '\186'

329

379

330justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

380justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

331justifyRight m n xs = L.replicate (m - length xs) n ++ xs

381justifyRight m n xs = L.replicate (m - length xs) n ++ xs

@@ -514,12 +564,20 @@ s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p)

514s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

564s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

515sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

565sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

516 (unpackS . S.takeWhile p . S.filter q)

566 (unpackS . S.takeWhile p . S.filter q)

567+noMatch = do

568+ c <- char

569+ d <- suchThat char (/= c)

570+ return (c,d)

517t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

571t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

518tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

572tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

519t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

573t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

520 (unpackS . T.takeWhileEnd p)

574 (unpackS . T.takeWhileEnd p)

575+t_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> T.null $

576+ T.takeWhileEnd (==d) (T.snoc t c)

521tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

577tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

522 (unpackS . TL.takeWhileEnd p)

578 (unpackS . TL.takeWhileEnd p)

579+tl_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> TL.null $

580+ TL.takeWhileEnd (==d) (TL.snoc t c)

523s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

581s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

524s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

582s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

525sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

583sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

@@ -720,7 +778,7 @@ tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s

720 conc = T.concat . TL.toChunks

778 conc = T.concat . TL.toChunks

721t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

779t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

722 let s = T.intercalate t ts

780 let s = T.intercalate t ts

723− in Slow.indices t s == indices t s

781+ in Slow.indices t s === indices t s

724

782

725-- Bit shifts.

783-- Bit shifts.

726shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

784shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

@@ -812,18 +870,18 @@ tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a

812-- Reading.

870-- Reading.

813

871

814t_decimal (n::Int) s =

872t_decimal (n::Int) s =

815− T.signed T.decimal (T.pack (show n) `T.append` t) == Right (n,t)

873+ T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)

816 where t = T.dropWhile isDigit s

874 where t = T.dropWhile isDigit s

817tl_decimal (n::Int) s =

875tl_decimal (n::Int) s =

818− TL.signed TL.decimal (TL.pack (show n) `TL.append` t) == Right (n,t)

876+ TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)

819 where t = TL.dropWhile isDigit s

877 where t = TL.dropWhile isDigit s

820t_hexadecimal m s ox =

878t_hexadecimal m s ox =

821− T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) == Right (n,t)

879+ T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)

822 where t = T.dropWhile isHexDigit s

880 where t = T.dropWhile isHexDigit s

823 p = if ox then "0x" else ""

881 p = if ox then "0x" else ""

824 n = getPositive m :: Int

882 n = getPositive m :: Int

825tl_hexadecimal m s ox =

883tl_hexadecimal m s ox =

826− TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) == Right (n,t)

884+ TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)

827 where t = TL.dropWhile isHexDigit s

885 where t = TL.dropWhile isHexDigit s

828 p = if ox then "0x" else ""

886 p = if ox then "0x" else ""

829 n = getPositive m :: Int

887 n = getPositive m :: Int

@@ -867,11 +925,11 @@ tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn

867

925

868t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

926t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

869t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

927t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

870−t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) == t

928+t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t

871 where n = small m

929 where n = small m

872t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

930t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

873

931

874−t_copy t = T.copy t == t

932+t_copy t = T.copy t === t

875

933

876-- Regression tests.

934-- Regression tests.

877s_filter_eq s = S.filter p t == S.streamList (filter p s)

935s_filter_eq s = S.filter p t == S.streamList (filter p s)

@@ -924,6 +982,15 @@ tests =

924 testGroup "errors" [

982 testGroup "errors" [

925 testProperty "t_utf8_err" t_utf8_err,

983 testProperty "t_utf8_err" t_utf8_err,

926 testProperty "t_utf8_err'" t_utf8_err'

984 testProperty "t_utf8_err'" t_utf8_err'

985+ ],

986+ testGroup "error recovery" [

987+ testProperty "t_decode_with_error2" t_decode_with_error2,

988+ testProperty "t_decode_with_error3" t_decode_with_error3,

989+ testProperty "t_decode_with_error4" t_decode_with_error4,

990+ testProperty "t_decode_with_error2'" t_decode_with_error2',

991+ testProperty "t_decode_with_error3'" t_decode_with_error3',

992+ testProperty "t_decode_with_error4'" t_decode_with_error4',

993+ testProperty "t_infix_concat" t_infix_concat

927 ]

994 ]

928 ],

995 ],

929

996

@@ -967,6 +1034,8 @@ tests =

967 testProperty "sf_uncons" sf_uncons,

1034 testProperty "sf_uncons" sf_uncons,

968 testProperty "t_uncons" t_uncons,

1035 testProperty "t_uncons" t_uncons,

969 testProperty "tl_uncons" tl_uncons,

1036 testProperty "tl_uncons" tl_uncons,

1037+ testProperty "t_unsnoc" t_unsnoc,

1038+ testProperty "tl_unsnoc" tl_unsnoc,

970 testProperty "s_head" s_head,

1039 testProperty "s_head" s_head,

971 testProperty "sf_head" sf_head,

1040 testProperty "sf_head" sf_head,

972 testProperty "t_head" t_head,

1041 testProperty "t_head" t_head,

@@ -1030,7 +1099,9 @@ tests =

1030 testProperty "tl_toLower_lower" tl_toLower_lower,

1099 testProperty "tl_toLower_lower" tl_toLower_lower,

1031 testProperty "t_toUpper_length" t_toUpper_length,

1100 testProperty "t_toUpper_length" t_toUpper_length,

1032 testProperty "t_toUpper_upper" t_toUpper_upper,

1101 testProperty "t_toUpper_upper" t_toUpper_upper,

1033− testProperty "tl_toUpper_upper" tl_toUpper_upper

1102+ testProperty "tl_toUpper_upper" tl_toUpper_upper,

1103+ testProperty "t_toTitle_title" t_toTitle_title,

1104+ testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower

1034 ],

1105 ],

1035

1106

1036 testGroup "justification" [

1107 testGroup "justification" [

@@ -1147,7 +1218,9 @@ tests =

1147 testProperty "t_takeWhile" t_takeWhile,

1218 testProperty "t_takeWhile" t_takeWhile,

1148 testProperty "tl_takeWhile" tl_takeWhile,

1219 testProperty "tl_takeWhile" tl_takeWhile,

1149 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1220 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1221+ testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null,

1150 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1222 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1223+ testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null,

1151 testProperty "sf_dropWhile" sf_dropWhile,

1224 testProperty "sf_dropWhile" sf_dropWhile,

1152 testProperty "s_dropWhile" s_dropWhile,

1225 testProperty "s_dropWhile" s_dropWhile,

1153 testProperty "s_dropWhile_s" s_dropWhile_s,

1226 testProperty "s_dropWhile_s" s_dropWhile_s,

tests/Tests/QuickCheckUtils.hs+1−1

@@ -106,7 +106,7 @@ smallArbitrary = sized $ \n -> resize (smallish n) arbitrary

106 where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs

107

108instance Arbitrary T.Text where

109− arbitrary = T.pack `fmap` arbitrary

109+ arbitrary = T.pack `fmap` string

110 shrink = map T.pack . shrink . T.unpack

111

112instance Arbitrary TL.Text where

tests/Tests/Regressions.hs+11

@@ -72,6 +72,16 @@ mapAccumL_resize = do

72 assertEqual "mapAccumL should correctly size buffers for two-word results"

73 (count * 2) (T.lengthWord16 (snd val))

74

75+-- See GitHub #197

76+t197 :: IO ()

77+t197 =

78+ assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")

79+ where

80+ currencyParser x = cond == 1

81+ where

82+ cond = length fltr

83+ fltr = filter (== ',') x

84+

75tests :: F.Test

85tests :: F.Test

76tests = F.testGroup "Regressions"

86tests = F.testGroup "Regressions"

77 [ F.testCase "hGetContents_crash" hGetContents_crash

87 [ F.testCase "hGetContents_crash" hGetContents_crash

@@ -79,4 +89,5 @@ tests = F.testGroup "Regressions"

79 , F.testCase "mapAccumL_resize" mapAccumL_resize

89 , F.testCase "mapAccumL_resize" mapAccumL_resize

80 , F.testCase "replicate_crash" replicate_crash

90 , F.testCase "replicate_crash" replicate_crash

81 , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe

91 , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe

92+ , F.testCase "t197" t197

82 ]

93 ]

tests/text-tests.cabal+18−3

@@ -20,6 +20,11 @@ flag hpc

20 default: False

21 manual: True

22

23+flag bytestring-builder

24+ description: Depend on the bytestring-builder package for backwards compatibility.

25+ default: False

26+ manual: False

27+

23executable text-tests

28executable text-tests

24 main-is: Tests.hs

29 main-is: Tests.hs

25

30

@@ -48,16 +53,21 @@ executable text-tests

48 HUnit >= 1.2,

53 HUnit >= 1.2,

49 QuickCheck >= 2.7,

54 QuickCheck >= 2.7,

50 base == 4.*,

55 base == 4.*,

51− bytestring,

52 deepseq,

56 deepseq,

53 directory,

57 directory,

54− quickcheck-unicode,

58+ quickcheck-unicode >= 1.0.1.0,

55 random,

59 random,

56 test-framework >= 0.4,

60 test-framework >= 0.4,

57 test-framework-hunit >= 0.2,

61 test-framework-hunit >= 0.2,

58 test-framework-quickcheck2 >= 0.2,

62 test-framework-quickcheck2 >= 0.2,

59 text-tests

63 text-tests

60

64

65+ if flag(bytestring-builder)

66+ build-depends: bytestring >= 0.9 && < 0.10.4,

67+ bytestring-builder >= 0.10.4

68+ else

69+ build-depends: bytestring >= 0.10.4

70+

61executable text-tests-stdio

71executable text-tests-stdio

62 main-is: Tests/IO.hs

72 main-is: Tests/IO.hs

63

73

@@ -137,7 +147,12 @@ library

137 array,

147 array,

138 base == 4.*,

148 base == 4.*,

139 binary,

149 binary,

140− bytestring,

141 deepseq,

150 deepseq,

142 ghc-prim,

151 ghc-prim,

143 integer-gmp

152 integer-gmp

153+

154+ if flag(bytestring-builder)

155+ build-depends: bytestring >= 0.9 && < 0.10.4,

156+ bytestring-builder >= 0.10.4

157+ else

158+ build-depends: bytestring >= 0.10.4

text.cabal+89−17

@@ -1,7 +1,7 @@

1name: text

2−version: 1.2.2.1

2+version: 1.2.3.0

3−homepage: https://github.com/bos/text

3+homepage: https://github.com/haskell/text

4−bug-reports: https://github.com/bos/text/issues

4+bug-reports: https://github.com/haskell/text/issues

5synopsis: An efficient packed Unicode text type.

6description:

7 .

@@ -14,11 +14,13 @@ description:

14 in terms of large data quantities and high speed.

15 .

16 The 'Text' type provides character-encoding, type-safe case

17− conversion via whole-string case conversion functions. It also

17+ conversion via whole-string case conversion functions (see "Data.Text").

18− provides a range of functions for converting 'Text' values to and from

18+ It also provides a range of functions for converting 'Text' values to

19− 'ByteStrings', using several standard encodings.

19+ and from 'ByteStrings', using several standard encodings

20+ (see "Data.Text.Encoding").

20 .

21 .

21− Efficient locale-sensitive support for text IO is also supported.

22+ Efficient locale-sensitive support for text IO is also supported

23+ (see "Data.Text.IO").

22 .

24 .

23 These modules are intended to be imported qualified, to avoid name

25 These modules are intended to be imported qualified, to avoid name

24 clashes with Prelude functions, e.g.

26 clashes with Prelude functions, e.g.

@@ -28,8 +30,9 @@ description:

28 To use an extended and very rich family of functions for working

30 To use an extended and very rich family of functions for working

29 with Unicode text (including normalization, regular expressions,

31 with Unicode text (including normalization, regular expressions,

30 non-standard encodings, text breaking, and locales), see

32 non-standard encodings, text breaking, and locales), see

31− the @text-icu@ package:

32− <http://hackage.haskell.org/package/text-icu>

33+ the [text-icu package](https://hackage.haskell.org/package/text-icu)

34+ based on the well-respected and liberally

35+ licensed [ICU library](http://site.icu-project.org/).

33

36

34license: BSD2

37license: BSD2

35license-file: LICENSE

38license-file: LICENSE

39category: Data, Text

42category: Data, Text

40build-type: Simple

43build-type: Simple

41cabal-version: >= 1.8

44cabal-version: >= 1.8

45+tested-with: GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4,

46+ GHC==7.6.3, GHC==7.4.2, GHC==7.2.2, GHC==7.0.4

42extra-source-files:

47extra-source-files:

43 -- scripts/CaseFolding.txt

48 -- scripts/CaseFolding.txt

44 -- scripts/SpecialCasing.txt

49 -- scripts/SpecialCasing.txt

@@ -65,7 +70,9 @@ extra-source-files:

65 tests/text-tests.cabal

70 tests/text-tests.cabal

66

71

67flag bytestring-builder

72flag bytestring-builder

73+ description:

74+ Depend on the [bytestring-builder](https://hackage.haskell.org/package/bytestring-builder)

68− description: Depend on the bytestring-builder package for backwards compatibility.

75+ package for backwards compatibility.

69 default: False

76 default: False

70 manual: False

77 manual: False

71

78

@@ -75,7 +82,9 @@ flag developer

75 manual: True

82 manual: True

76

83

77flag integer-simple

84flag integer-simple

85+ description:

78− description: Use the simple integer library instead of GMP

86+ Use the [simple integer library](http://hackage.haskell.org/package/integer-simple)

87+ instead of [integer-gmp](http://hackage.haskell.org/package/integer-gmp)

79 default: False

88 default: False

80 manual: False

89 manual: False

81

90

@@ -160,27 +169,90 @@ library

160

169

161test-suite tests

170test-suite tests

162 type: exitcode-stdio-1.0

171 type: exitcode-stdio-1.0

163− hs-source-dirs: tests .

164− main-is: Tests.hs

165 c-sources: cbits/cbits.c

172 c-sources: cbits/cbits.c

166 include-dirs: include

173 include-dirs: include

167

174

168 ghc-options:

175 ghc-options:

169− -Wall -threaded -O0 -rtsopts

176+ -Wall -threaded -rtsopts

170

177

171 cpp-options:

178 cpp-options:

172 -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE

179 -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE

173

180

181+ -- modules specific to test-suite

182+ hs-source-dirs: tests

183+ main-is: Tests.hs

184+ other-modules:

185+ Tests.Properties

186+ Tests.Properties.Mul

187+ Tests.QuickCheckUtils

188+ Tests.Regressions

189+ Tests.SlowFunctions

190+ Tests.Utils

191+

192+ -- Same as in `library` stanza; this is needed by cabal for accurate

193+ -- file-monitoring as well as to avoid `-Wmissing-home-modules`

194+ -- warnings We can't use an inter-package library dependency because

195+ -- of different `ghc-options`/`cpp-options` (as a side-benefitt,

196+ -- this enables per-component build parallelism in `cabal

197+ -- new-build`!); We could, however, use cabal-version:2.2's `common`

198+ -- blocks at some point in the future to reduce the duplication.

199+ hs-source-dirs: .

200+ other-modules:

201+ Data.Text

202+ Data.Text.Array

203+ Data.Text.Encoding

204+ Data.Text.Encoding.Error

205+ Data.Text.Foreign

206+ Data.Text.IO

207+ Data.Text.Internal

208+ Data.Text.Internal.Builder

209+ Data.Text.Internal.Builder.Functions

210+ Data.Text.Internal.Builder.Int.Digits

211+ Data.Text.Internal.Builder.RealFloat.Functions

212+ Data.Text.Internal.Encoding.Fusion

213+ Data.Text.Internal.Encoding.Fusion.Common

214+ Data.Text.Internal.Encoding.Utf16

215+ Data.Text.Internal.Encoding.Utf32

216+ Data.Text.Internal.Encoding.Utf8

217+ Data.Text.Internal.Functions

218+ Data.Text.Internal.Fusion

219+ Data.Text.Internal.Fusion.CaseMapping

220+ Data.Text.Internal.Fusion.Common

221+ Data.Text.Internal.Fusion.Size

222+ Data.Text.Internal.Fusion.Types

223+ Data.Text.Internal.IO

224+ Data.Text.Internal.Lazy

225+ Data.Text.Internal.Lazy.Encoding.Fusion

226+ Data.Text.Internal.Lazy.Fusion

227+ Data.Text.Internal.Lazy.Search

228+ Data.Text.Internal.Private

229+ Data.Text.Internal.Read

230+ Data.Text.Internal.Search

231+ Data.Text.Internal.Unsafe

232+ Data.Text.Internal.Unsafe.Char

233+ Data.Text.Internal.Unsafe.Shift

234+ Data.Text.Lazy

235+ Data.Text.Lazy.Builder

236+ Data.Text.Lazy.Builder.Int

237+ Data.Text.Lazy.Builder.RealFloat

238+ Data.Text.Lazy.Encoding

239+ Data.Text.Lazy.IO

240+ Data.Text.Lazy.Internal

241+ Data.Text.Lazy.Read

242+ Data.Text.Read

243+ Data.Text.Unsafe

244+ Data.Text.Show

245+

174 build-depends:

246 build-depends:

175 HUnit >= 1.2,

247 HUnit >= 1.2,

176− QuickCheck >= 2.7,

248+ QuickCheck >= 2.7 && < 2.10,

177 array,

249 array,

178 base,

250 base,

179 binary,

251 binary,

180 deepseq,

252 deepseq,

181 directory,

253 directory,

182 ghc-prim,

254 ghc-prim,

183− quickcheck-unicode,

255+ quickcheck-unicode >= 1.0.1.0,

184 random,

256 random,

185 test-framework >= 0.4,

257 test-framework >= 0.4,

186 test-framework-hunit >= 0.2,

258 test-framework-hunit >= 0.2,

@@ -201,7 +273,7 @@ test-suite tests

201

273

202source-repository head

274source-repository head

203 type: git

275 type: git

204− location: https://github.com/bos/text

276+ location: https://github.com/haskell/text

205

277

206source-repository head

278source-repository head

207 type: mercurial

279 type: mercurial