e949bda · text · git.iko.soy

.gitignore+10−3

compare with master

unifiedsplit

@@ -1,3 +1,10 @@

1−dist

1+/dist/

2+/dist-boot/

3+/dist-install/

4+/dist-newstyle/

2−cabal-dev

5+/cabal-dev/

3−cabal.sandbox.config

6+/cabal.sandbox.config

7+/ghc.mk

8+/GNUmakefile

9+/.ghc.environment.*

10+/cabal.project.local

.hgtags+1

@@ -64,3 +64,4 @@ f8a60279237c1c28e094868740d9188dd94fcc9e 1.2.0.0

64631fd055e27fa10983e17e9ad36d1b285b85c661 1.2.1.3

658221a68ed5ec37028357df9c6c25f52320e54ad9 1.2.2.0

66ce1c5ac65eb9f5580114b4e380092b05a485e95d 1.2.2.1

67+d72c634a91ffe27ec5cd03921dd53efc8129d8f6 1.2.2.2

.travis.ymlnew file+111

@@ -0,0 +1,111 @@

1+# This Travis job script has been generated by a script via

2+#

3+# make_travis_yml_2.hs '-o' '.travis.yml' 'text.cabal'

4+#

5+# For more information, see https://github.com/hvr/multi-ghc-travis

6+#

7+language: c

8+sudo: false

9+

10+# Bryan threatens to turn off Travis-CI if he gets any email :-)

11+notifications:

12+ email: false

13+

14+cache:

15+ directories:

16+ - $HOME/.cabal/packages

17+ - $HOME/.cabal/store

18+

19+before_cache:

20+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/build-reports.log

21+ # remove files that are regenerated by 'cabal update'

22+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/00-index.*

23+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/*.json

24+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.cache

25+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.tar

26+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.tar.idx

27+

28+matrix:

29+ include:

30+ - compiler: "ghc-7.0.4"

31+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

32+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.0.4], sources: [hvr-ghc]}}

33+ - compiler: "ghc-7.2.2"

34+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

35+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.2.2], sources: [hvr-ghc]}}

36+ - compiler: "ghc-7.4.2"

37+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

38+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.4.2], sources: [hvr-ghc]}}

39+ - compiler: "ghc-7.6.3"

40+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

41+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.6.3], sources: [hvr-ghc]}}

42+ - compiler: "ghc-7.8.4"

43+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

44+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.8.4], sources: [hvr-ghc]}}

45+ - compiler: "ghc-7.10.3"

46+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

47+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.10.3], sources: [hvr-ghc]}}

48+ - compiler: "ghc-8.0.2"

49+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

50+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-8.0.2], sources: [hvr-ghc]}}

51+ - compiler: "ghc-8.2.2"

52+ # env: TEST=--disable-tests BENCH=--disable-benchmarks

53+ addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-8.2.2], sources: [hvr-ghc]}}

54+

55+before_install:

56+ - HC=${CC}

57+ - HCPKG=${HC/ghc/ghc-pkg}

58+ - unset CC

59+ - PATH=/opt/ghc/bin:/opt/ghc-ppa-tools/bin:$PATH

60+ - PKGNAME='text'

61+

62+install:

63+ - cabal --version

64+ - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"

65+ - BENCH=${BENCH---enable-benchmarks}

66+ - TEST=${TEST---enable-tests}

67+ - HADDOCK=${HADDOCK-true}

68+ - INSTALLED=${INSTALLED-true}

69+ - travis_retry cabal update -v

70+ - sed -i.bak 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config

71+ - rm -fv cabal.project.local

72+ - "echo 'packages: .' > cabal.project"

73+ - rm -f cabal.project.freeze

74+ - cabal new-build -w ${HC} ${TEST} ${BENCH} --dep -j2 all

75+ - cabal new-build -w ${HC} --disable-tests --disable-benchmarks --dep -j2 all

76+

77+# Here starts the actual work to be performed for the package under test;

78+# any command which exits with a non-zero exit code causes the build to fail.

79+script:

80+ - if [ -f configure.ac ]; then autoreconf -i; fi

81+ - rm -rf .ghc.environment.* dist/

82+ - cabal sdist # test that a source-distribution can be generated

83+ - cd dist/

84+ - SRCTAR=(${PKGNAME}-*.tar.gz)

85+ - SRC_BASENAME="${SRCTAR/%.tar.gz}"

86+ - tar -xvf "./$SRC_BASENAME.tar.gz"

87+ - cd "$SRC_BASENAME/"

88+## from here on, CWD is inside the extracted source-tarball

89+ - rm -fv cabal.project.local

90+ - "echo 'packages: .' > cabal.project"

91+ # this builds all libraries and executables (without tests/benchmarks)

92+ - rm -f cabal.project.freeze

93+ - cabal new-build -w ${HC} --disable-tests --disable-benchmarks all

94+ # this builds all libraries and executables (including tests/benchmarks)

95+ # - rm -rf ./dist-newstyle

96+

97+ # Build with installed constraints for packages in global-db

98+ - if $INSTALLED; then

99+ echo cabal new-build -w ${HC} --disable-tests --disable-benchmarks $(${HCPKG} list --global --simple-output --names-only | sed 's/$[a-zA-Z0-9-]\{1,\}$ */--constraint="\1 installed" /g') all | sh;

100+ else echo "Not building with installed constraints"; fi

101+

102+ # build & run tests, build benchmarks

103+ - cabal new-build -w ${HC} ${TEST} ${BENCH} all

104+ - if [ "x$TEST" = "x--enable-tests" ]; then cabal new-test -w ${HC} ${TEST} all; fi

105+

106+ # haddock

107+ - rm -rf ./dist-newstyle

108+ - if $HADDOCK; then cabal new-haddock -w ${HC} --disable-tests --disable-benchmarks all; else echo "Skipping haddock generation";fi

109+

110+# REGENDATA ["-o",".travis.yml","text.cabal"]

111+# EOF

Data/Text.hs+174−45

@@ -15,7 +15,6 @@

15--

16-- License : BSD-style

17-- Maintainer : bos@serpentine.com

18−-- Stability : experimental

19-- Portability : GHC

18-- Portability : GHC

20--

19--

21-- A time and space-efficient implementation of Unicode text.

20-- A time and space-efficient implementation of Unicode text.

@@ -32,8 +31,9 @@

32--

31--

33-- To use an extended and very rich family of functions for working

32-- To use an extended and very rich family of functions for working

34-- with Unicode text (including normalization, regular expressions,

33-- with Unicode text (including normalization, regular expressions,

35−-- non-standard encodings, text breaking, and locales), see

34+-- non-standard encodings, text breaking, and locales), see the

36−-- <http://hackage.haskell.org/package/text-icu the text-icu package >.

35+-- <http://hackage.haskell.org/package/text-icu text-icu package >.

36+--

37

38module Data.Text

39 (

@@ -43,6 +43,9 @@ module Data.Text

43 -- * Acceptable data

44 -- $replacement

45

46+ -- * Definition of character

47+ -- $character_definition

48+

46 -- * Fusion

49 -- * Fusion

47 -- $fusion

50 -- $fusion

48

51

@@ -60,6 +63,7 @@ module Data.Text

60 , snoc

63 , snoc

61 , append

64 , append

62 , uncons

65 , uncons

66+ , unsnoc

63 , head

67 , head

64 , last

68 , last

65 , tail

69 , tail

@@ -298,6 +302,17 @@ unpackCString# addr# = Text $ SJS.unstream (SJS.streamCString# addr#)

298 = singleton a #-}

302 = singleton a #-}

299

303

300---------------------------------------------------------------------

304---------------------------------------------------------------------

305+-- $character_definition

306+--

307+-- This package uses the term /character/ to denote Unicode /code points/.

308+--

309+-- Note that this is not the same thing as a grapheme (e.g. a

310+-- composition of code points that form one visual symbol). For

311+-- instance, consider the grapheme \"ä\". This symbol has two

312+-- Unicode representations: a single code-point representation

313+-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),

314+-- and a two code point representation @U+0061@ (the \"@A@\" code

315+-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).

301

316

302-- $strict

317-- $strict

303--

318--

@@ -392,9 +407,13 @@ instance Read Text where

392 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

407 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

393

408

394#if MIN_VERSION_base(4,9,0)

409#if MIN_VERSION_base(4,9,0)

410+-- | Non-orphan 'Semigroup' instance only defined for

395−-- Semigroup orphan instances for older GHCs are provided by

411+-- @base-4.9.0.0@ and later; orphan instances for older GHCs are

412+-- provided by

413+-- the [semigroups](http://hackage.haskell.org/package/semigroups)

396−-- 'semigroups` package

414+-- package

397−

415+--

416+-- @since 1.2.2.0

398instance Semigroup Text where

417instance Semigroup Text where

399 (<>) = append

418 (<>) = append

400#endif

419#endif

@@ -412,6 +431,7 @@ instance IsString Text where

412 fromString = pack

431 fromString = pack

413

432

414#if __GLASGOW_HASKELL__ >= 708

433#if __GLASGOW_HASKELL__ >= 708

434+-- | @since 1.2.0.0

415instance Exts.IsList Text where

435instance Exts.IsList Text where

416 type Item Text = Char

436 type Item Text = Char

417 fromList = pack

437 fromList = pack

@@ -422,6 +442,7 @@ instance Exts.IsList Text where

422instance NFData Text where rnf !_ = ()

442instance NFData Text where rnf !_ = ()

423#endif

443#endif

424

444

445+-- | @since 1.2.1.0

425instance Binary Text where

446instance Binary Text where

426 put t = put (encodeUtf8 t)

447 put t = put (encodeUtf8 t)

427 get = do

448 get = do

@@ -455,6 +476,8 @@ instance Data Text where

455

476

456#if MIN_VERSION_base(4,7,0)

477#if MIN_VERSION_base(4,7,0)

457-- | Only defined for @base-4.7.0.0@ and later

478-- | Only defined for @base-4.7.0.0@ and later

479+--

480+-- @since 1.2.2.0

458instance PrintfArg Text where

481instance PrintfArg Text where

459 formatArg t = formatString $ unpack t

482 formatArg t = formatString $ unpack t

460#endif

483#endif

@@ -612,6 +635,19 @@ init = coerce JSS.init

612 unstream (S.init (stream t)) = init t

635 unstream (S.init (stream t)) = init t

613 #-}

636 #-}

614

637

638+-- | /O(1)/ Returns all but the last character and the last character of a

639+-- 'Text', or 'Nothing' if empty.

640+--

641+-- @since 1.2.3.0

642+unsnoc :: Text -> Maybe (Text, Char)

643+unsnoc (Text arr off len)

644+ | len <= 0 = Nothing

645+ | n < 0xDC00 || n > 0xDFFF = Just (text arr off (len-1), unsafeChr n)

646+ | otherwise = Just (text arr off (len-2), U16.chr2 n0 n)

647+ where n = A.unsafeIndex arr (off+len-1)

648+ n0 = A.unsafeIndex arr (off+len-2)

649+{-# INLINE [1] unsnoc #-}

650+

615-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

651-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

616-- fusion.

652-- fusion.

617null :: Text -> Bool

653null :: Text -> Bool

@@ -693,8 +729,15 @@ compareLength t n = S.compareLengthI (stream t) n

693-- -----------------------------------------------------------------------------

729-- -----------------------------------------------------------------------------

694-- * Transformations

730-- * Transformations

695-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

731-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

732+-- each element of @t@.

733+--

734+-- Example:

735+--

736+-- >>> let message = pack "I am not angry. Not at all."

737+-- >>> T.map (\c -> if c == '.' then '!' else c) message

738+-- "I am not angry! Not at all!"

739+--

696−-- each element of @t@. Subject to fusion. Performs replacement on

740+-- Subject to fusion. Performs replacement on invalid scalar values.

697−-- invalid scalar values.

698map :: (Char -> Char) -> Text -> Text

741map :: (Char -> Char) -> Text -> Text

699map = coerce JSS.map

742map = coerce JSS.map

700-- map f t = unstream (S.map (safe . f) (stream t))

743-- map f t = unstream (S.map (safe . f) (stream t))

@@ -703,20 +746,38 @@ map = coerce JSS.map

703-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

746-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

704-- 'Text's and concatenates the list after interspersing the first

747-- 'Text's and concatenates the list after interspersing the first

705-- argument between each element of the list.

748-- argument between each element of the list.

749+--

750+-- Example:

751+--

752+-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]

753+-- "WeNI!seekNI!theNI!HolyNI!Grail"

706intercalate :: Text -> [Text] -> Text

754intercalate :: Text -> [Text] -> Text

707intercalate = coerce JSS.intercalate

755intercalate = coerce JSS.intercalate

708-- intercalate t = concat . (F.intersperse t)

756-- intercalate t = concat . (F.intersperse t)

709{-# INLINE intercalate #-}

757{-# INLINE intercalate #-}

710

758

711-- | /O(n)/ The 'intersperse' function takes a character and places it

759-- | /O(n)/ The 'intersperse' function takes a character and places it

712−-- between the characters of a 'Text'. Subject to fusion. Performs

760+-- between the characters of a 'Text'.

761+--

762+-- Example:

763+--

764+-- >>> T.intersperse '.' "SHIELD"

765+-- "S.H.I.E.L.D"

766+--

713−-- replacement on invalid scalar values.

767+-- Subject to fusion. Performs replacement on invalid scalar values.

714intersperse :: Char -> Text -> Text

768intersperse :: Char -> Text -> Text

715intersperse = coerce JSS.intersperse

769intersperse = coerce JSS.intersperse

716-- intersperse c t = unstream (S.intersperse (safe c) (stream t))

770-- intersperse c t = unstream (S.intersperse (safe c) (stream t))

717{-# INLINE intersperse #-}

771{-# INLINE intersperse #-}

718

772

719−-- | /O(n)/ Reverse the characters of a string. Subject to fusion.

773+-- | /O(n)/ Reverse the characters of a string.

774+--

775+-- Example:

776+--

777+-- >>> T.reverse "desrever"

778+-- "reversed"

779+--

780+-- Subject to fusion.

720reverse :: Text -> Text

781reverse :: Text -> Text

721reverse = coerce JSS.reverse

782reverse = coerce JSS.reverse

722-- reverse t = S.reverse (stream t)

783-- reverse t = S.reverse (stream t)

@@ -736,12 +797,14 @@ reverse = coerce JSS.reverse

736-- @needle@ occurs in @replacement@, that occurrence will /not/ itself

797-- @needle@ occurs in @replacement@, that occurrence will /not/ itself

737-- be replaced recursively:

798-- be replaced recursively:

738--

799--

739−-- > replace "oo" "foo" "oo" == "foo"

800+-- >>> replace "oo" "foo" "oo"

801+-- "foo"

740--

802--

741-- In cases where several instances of @needle@ overlap, only the

803-- In cases where several instances of @needle@ overlap, only the

742-- first one will be replaced:

804-- first one will be replaced:

743--

805--

744−-- > replace "ofo" "bar" "ofofo" == "barfo"

806+-- >>> replace "ofo" "bar" "ofofo"

807+-- "barfo"

745--

808--

746-- In (unlikely) bad cases, this function's time complexity degrades

809-- In (unlikely) bad cases, this function's time complexity degrades

747-- towards /O(n*m)/.

810-- towards /O(n*m)/.

@@ -859,6 +922,8 @@ toUpper = coerce JSS.toUpper

859-- guides disagree on whether the book name \"The Hill of the Red

922-- guides disagree on whether the book name \"The Hill of the Red

860-- Fox\" is correctly title cased—but this function will

923-- Fox\" is correctly title cased—but this function will

861-- capitalize /every/ word.

924-- capitalize /every/ word.

925+--

926+-- @since 1.0.0.0

862toTitle :: Text -> Text

927toTitle :: Text -> Text

863toTitle = coerce JSS.toTitle

928toTitle = coerce JSS.toTitle

864-- toTitle t = unstream (S.toTitle (stream t))

929-- toTitle t = unstream (S.toTitle (stream t))

@@ -870,8 +935,11 @@ toTitle = coerce JSS.toTitle

870--

935--

871-- Examples:

936-- Examples:

872--

937--

873−-- > justifyLeft 7 'x' "foo" == "fooxxxx"

938+-- >>> justifyLeft 7 'x' "foo"

939+-- "fooxxxx"

940+--

874−-- > justifyLeft 3 'x' "foobar" == "foobar"

941+-- >>> justifyLeft 3 'x' "foobar"

942+-- "foobar"

875justifyLeft :: Int -> Char -> Text -> Text

943justifyLeft :: Int -> Char -> Text -> Text

876justifyLeft = coerce JSS.justifyLeft

944justifyLeft = coerce JSS.justifyLeft

877-- justifyLeft k c t

945-- justifyLeft k c t

@@ -893,8 +961,11 @@ justifyLeft = coerce JSS.justifyLeft

893--

961--

894-- Examples:

962-- Examples:

895--

963--

896−-- > justifyRight 7 'x' "bar" == "xxxxbar"

964+-- >>> justifyRight 7 'x' "bar"

965+-- "xxxxbar"

966+--

897−-- > justifyRight 3 'x' "foobar" == "foobar"

967+-- >>> justifyRight 3 'x' "foobar"

968+-- "foobar"

898justifyRight :: Int -> Char -> Text -> Text

969justifyRight :: Int -> Char -> Text -> Text

899justifyRight = coerce JSS.justifyRight

970justifyRight = coerce JSS.justifyRight

900-- justifyRight k c t

971-- justifyRight k c t

@@ -909,7 +980,8 @@ justifyRight = coerce JSS.justifyRight

909--

980--

910-- Examples:

981-- Examples:

911--

982--

912−-- > center 8 'x' "HS" = "xxxHSxxx"

983+-- >>> center 8 'x' "HS"

984+-- "xxxHSxxx"

913center :: Int -> Char -> Text -> Text

985center :: Int -> Char -> Text -> Text

914center = coerce JSS.center

986center = coerce JSS.center

915-- center k c t

987-- center k c t

@@ -925,6 +997,14 @@ center = coerce JSS.center

925-- of its 'Text' argument. Note that this function uses 'pack',

997-- of its 'Text' argument. Note that this function uses 'pack',

926-- 'unpack', and the list version of transpose, and is thus not very

998-- 'unpack', and the list version of transpose, and is thus not very

927-- efficient.

999-- efficient.

1000+--

1001+-- Examples:

1002+--

1003+-- >>> transpose ["green","orange"]

1004+-- ["go","rr","ea","en","ng","e"]

1005+--

1006+-- >>> transpose ["blue","red"]

1007+-- ["br","le","ud","e"]

928transpose :: [Text] -> [Text]

1008transpose :: [Text] -> [Text]

929transpose = coerce JSS.transpose

1009transpose = coerce JSS.transpose

930-- transpose ts = P.map pack (L.transpose (P.map unpack ts))

1010-- transpose ts = P.map pack (L.transpose (P.map unpack ts))

@@ -1207,7 +1287,10 @@ take = coerce JSS.take

1207--

1287--

1208-- Examples:

1288-- Examples:

1209--

1289--

1210−-- > takeEnd 3 "foobar" == "bar"

1290+-- >>> takeEnd 3 "foobar"

1291+-- "bar"

1292+--

1293+-- @since 1.1.1.0

1211takeEnd :: Int -> Text -> Text

1294takeEnd :: Int -> Text -> Text

1212takeEnd = coerce JSS.takeEnd

1295takeEnd = coerce JSS.takeEnd

1213-- takeEnd n t@(Text arr off len)

1296-- takeEnd n t@(Text arr off len)

@@ -1248,7 +1331,10 @@ drop = coerce JSS.drop

1248--

1331--

1249-- Examples:

1332-- Examples:

1250--

1333--

1251−-- > dropEnd 3 "foobar" == "foo"

1334+-- >>> dropEnd 3 "foobar"

1335+-- "foo"

1336+--

1337+-- @since 1.1.1.0

1252dropEnd :: Int -> Text -> Text

1338dropEnd :: Int -> Text -> Text

1253dropEnd = coerce JSS.dropEnd

1339dropEnd = coerce JSS.dropEnd

1254-- dropEnd n t@(Text arr off len)

1340-- dropEnd n t@(Text arr off len)

@@ -1280,7 +1366,10 @@ takeWhile = coerce JSS.takeWhile

1280-- satisfy @p@. Subject to fusion.

1366-- satisfy @p@. Subject to fusion.

1281-- Examples:

1367-- Examples:

1282--

1368--

1283−-- > takeWhileEnd (=='o') "foo" == "oo"

1369+-- >>> takeWhileEnd (=='o') "foo"

1370+-- "oo"

1371+--

1372+-- @since 1.2.2.0

1284takeWhileEnd :: (Char -> Bool) -> Text -> Text

1373takeWhileEnd :: (Char -> Bool) -> Text -> Text

1285#ifndef __GHCJS__

1374#ifndef __GHCJS__

1286takeWhileEnd p t@(Text arr off len) = loop (len-1) len

1375takeWhileEnd p t@(Text arr off len) = loop (len-1) len

@@ -1329,7 +1418,8 @@ dropWhile = coerce JSS.dropWhile

1329--

1418--

1330-- Examples:

1419-- Examples:

1331--

1420--

1332−-- > dropWhileEnd (=='.') "foo..." == "foo"

1421+-- >>> dropWhileEnd (=='.') "foo..."

1422+-- "foo"

1333dropWhileEnd :: (Char -> Bool) -> Text -> Text

1423dropWhileEnd :: (Char -> Bool) -> Text -> Text

1334dropWhileEnd = coerce JSS.dropWhileEnd

1424dropWhileEnd = coerce JSS.dropWhileEnd

1335-- dropWhileEnd p t@(Text arr off len) = loop (len-1) len

1425-- dropWhileEnd p t@(Text arr off len) = loop (len-1) len

@@ -1454,9 +1544,14 @@ tails = coerce JSS.tails

1454--

1544--

1455-- Examples:

1545-- Examples:

1456--

1546--

1457−-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1547+-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"

1548+-- ["a","b","d","e"]

1549+--

1458−-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1550+-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa"

1551+-- ["","X","X","X",""]

1552+--

1459−-- > splitOn "x" "x" == ["",""]

1553+-- >>> splitOn "x" "x"

1554+-- ["",""]

1460--

1555--

1461-- and

1556-- and

1462--

1557--

@@ -1493,8 +1588,11 @@ splitOn = coerce JSS.splitOn

1493-- resulting components do not contain the separators. Two adjacent

1588-- resulting components do not contain the separators. Two adjacent

1494-- separators result in an empty component in the output. eg.

1589-- separators result in an empty component in the output. eg.

1495--

1590--

1496−-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1591+-- >>> split (=='a') "aabbaca"

1592+-- ["","","bb","c",""]

1593+--

1497−-- > split (=='a') "" == [""]

1594+-- >>> split (=='a') ""

1595+-- [""]

1498split :: (Char -> Bool) -> Text -> [Text]

1596split :: (Char -> Bool) -> Text -> [Text]

1499split = coerce JSS.split

1597split = coerce JSS.split

1500-- split _ t@(Text _off _arr 0) = [t]

1598-- split _ t@(Text _off _arr 0) = [t]

@@ -1508,8 +1606,11 @@ split = coerce JSS.split

1508-- element may be shorter than the other chunks, depending on the

1606-- element may be shorter than the other chunks, depending on the

1509-- length of the input. Examples:

1607-- length of the input. Examples:

1510--

1608--

1511−-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"]

1609+-- >>> chunksOf 3 "foobarbaz"

1610+-- ["foo","bar","baz"]

1611+--

1512−-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]

1612+-- >>> chunksOf 4 "haskell.org"

1613+-- ["hask","ell.","org"]

1513chunksOf :: Int -> Text -> [Text]

1614chunksOf :: Int -> Text -> [Text]

1514chunksOf = coerce JSS.chunksOf

1615chunksOf = coerce JSS.chunksOf

1515-- chunksOf k = go

1616-- chunksOf k = go

@@ -1558,8 +1659,11 @@ filter = coerce JSS.filter

1558--

1659--

1559-- Examples:

1660-- Examples:

1560--

1661--

1561−-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1662+-- >>> breakOn "::" "a::b::c"

1663+-- ("a","::b::c")

1664+--

1562−-- > breakOn "/" "foobar" ==> ("foobar", "")

1665+-- >>> breakOn "/" "foobar"

1666+-- ("foobar","")

1563--

1667--

1564-- Laws:

1668-- Laws:

1565--

1669--

@@ -1588,7 +1692,8 @@ breakOn = coerce JSS.breakOn

1588-- up to and including the last match of @needle@. The second is the

1692-- up to and including the last match of @needle@. The second is the

1589-- remainder of @haystack@, following the match.

1693-- remainder of @haystack@, following the match.

1590--

1694--

1591−-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1695+-- >>> breakOnEnd "::" "a::b::c"

1696+-- ("a::b::","c")

1592breakOnEnd :: Text -> Text -> (Text, Text)

1697breakOnEnd :: Text -> Text -> (Text, Text)

1593breakOnEnd = coerce JSS.breakOnEnd

1698breakOnEnd = coerce JSS.breakOnEnd

1594-- breakOnEnd pat src = (reverse b, reverse a)

1699-- breakOnEnd pat src = (reverse b, reverse a)

@@ -1604,10 +1709,11 @@ breakOnEnd = coerce JSS.breakOnEnd

1604--

1709--

1605-- Examples:

1710-- Examples:

1606--

1711--

1607−-- > breakOnAll "::" ""

1712+-- >>> breakOnAll "::" ""

1608−-- > ==> []

1713+-- []

1714+--

1609−-- > breakOnAll "/" "a/b/c/"

1715+-- >>> breakOnAll "/" "a/b/c/"

1610−-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]

1716+-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]

1611--

1717--

1612-- In (unlikely) bad cases, this function's time complexity degrades

1718-- In (unlikely) bad cases, this function's time complexity degrades

1613-- towards /O(n*m)/.

1719-- towards /O(n*m)/.

@@ -1821,9 +1927,14 @@ isInfixOf = coerce JSS.isInfixOf

1821--

1927--

1822-- Examples:

1928-- Examples:

1823--

1929--

1824−-- > stripPrefix "foo" "foobar" == Just "bar"

1930+-- >>> stripPrefix "foo" "foobar"

1931+-- Just "bar"

1932+--

1825−-- > stripPrefix "" "baz" == Just "baz"

1933+-- >>> stripPrefix "" "baz"

1934+-- Just "baz"

1935+--

1826−-- > stripPrefix "foo" "quux" == Nothing

1936+-- >>> stripPrefix "foo" "quux"

1937+-- Nothing

1827--

1938--

1828-- This is particularly useful with the @ViewPatterns@ extension to

1939-- This is particularly useful with the @ViewPatterns@ extension to

1829-- GHC, as follows:

1940-- GHC, as follows:

@@ -1849,9 +1960,14 @@ stripPrefix = coerce JSS.stripPrefix

1849--

1960--

1850-- Examples:

1961-- Examples:

1851--

1962--

1852−-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")

1963+-- >>> commonPrefixes "foobar" "fooquux"

1964+-- Just ("foo","bar","quux")

1965+--

1853−-- > commonPrefixes "veeble" "fetzer" == Nothing

1966+-- >>> commonPrefixes "veeble" "fetzer"

1967+-- Nothing

1968+--

1854−-- > commonPrefixes "" "baz" == Nothing

1969+-- >>> commonPrefixes "" "baz"

1970+-- Nothing

1855commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)

1971commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)

1856commonPrefixes = coerce JSS.commonPrefixes

1972commonPrefixes = coerce JSS.commonPrefixes

1857-- commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1973-- commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

@@ -1869,9 +1985,14 @@ commonPrefixes = coerce JSS.commonPrefixes

1869--

1985--

1870-- Examples:

1986-- Examples:

1871--

1987--

1872−-- > stripSuffix "bar" "foobar" == Just "foo"

1988+-- >>> stripSuffix "bar" "foobar"

1989+-- Just "foo"

1990+--

1873−-- > stripSuffix "" "baz" == Just "baz"

1991+-- >>> stripSuffix "" "baz"

1992+-- Just "baz"

1993+--

1874−-- > stripSuffix "foo" "quux" == Nothing

1994+-- >>> stripSuffix "foo" "quux"

1995+-- Nothing

1875--

1996--

1876-- This is particularly useful with the @ViewPatterns@ extension to

1997-- This is particularly useful with the @ViewPatterns@ extension to

1877-- GHC, as follows:

1998-- GHC, as follows:

@@ -1923,3 +2044,11 @@ foreign import javascript unsafe

1923 "h$jsstringReplicateChar" js_replicateChar :: Int -> Char -> JSString

2044 "h$jsstringReplicateChar" js_replicateChar :: Int -> Char -> JSString

1924foreign import javascript unsafe

2045foreign import javascript unsafe

1925 "$2.substr($1)" js_substr1 :: Int -> JSString -> JSString

2046 "$2.substr($1)" js_substr1 :: Int -> JSString -> JSString

2047+

2048+-------------------------------------------------

2049+-- NOTE: the named chunk below used by doctest;

2050+-- verify the doctests via `doctest -fobject-code Data/Text.hs`

2051+

2052+-- $setup

2053+-- >>> :set -XOverloadedStrings

2054+-- >>> import qualified Data.Text as T

Data/Text/Array.hs+4−1

@@ -7,7 +7,6 @@

7--

8-- License : BSD-style

9-- Maintainer : bos@serpentine.com

10−-- Stability : experimental

11-- Portability : portable

10-- Portability : portable

12--

11--

13-- Packed, unboxed, heap-resident arrays. Suitable for performance

12-- Packed, unboxed, heap-resident arrays. Suitable for performance

@@ -80,6 +79,8 @@ import GHC.Word (Word16(..))

80import Prelude hiding (length, read)

79import Prelude hiding (length, read)

81

80

82-- | Immutable array type.

81-- | Immutable array type.

82+--

83+-- The 'Array' constructor is exposed since @text-1.1.1.3@

83data Array = Array {

84data Array = Array {

84 aBA :: ByteArray#

85 aBA :: ByteArray#

85#if defined(ASSERTS)

86#if defined(ASSERTS)

@@ -88,6 +89,8 @@ data Array = Array {

88 }

89 }

89

90

90-- | Mutable array type, for use in the ST monad.

91-- | Mutable array type, for use in the ST monad.

92+--

93+-- The 'MArray' constructor is exposed since @text-1.1.1.3@

91data MArray s = MArray {

94data MArray s = MArray {

92 maBA :: MutableByteArray# s

95 maBA :: MutableByteArray# s

93#if defined(ASSERTS)

96#if defined(ASSERTS)

Data/Text/Encoding.hs+67−11

@@ -12,14 +12,13 @@

12--

13-- License : BSD-style

14-- Maintainer : bos@serpentine.com

15−-- Stability : experimental

16-- Portability : portable

15-- Portability : portable

17--

16--

18-- Functions for converting 'Text' values to and from 'ByteString',

17-- Functions for converting 'Text' values to and from 'ByteString',

19-- using several standard encodings.

18-- using several standard encodings.

20--

19--

21-- To gain access to a much larger family of encodings, use the

20-- To gain access to a much larger family of encodings, use the

22−-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

21+-- <http://hackage.haskell.org/package/text-icu text-icu package>.

23

22

24module Data.Text.Encoding

23module Data.Text.Encoding

25 (

24 (

@@ -67,7 +66,7 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

67import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

66import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

68#endif

67#endif

69

68

70−import Control.Exception (evaluate, try)

69+import Control.Exception (evaluate, try, throwIO, ErrorCall(ErrorCall))

71import Control.Monad.ST (runST)

70import Control.Monad.ST (runST)

72import Data.Bits ((.&.))

71import Data.Bits ((.&.))

73import Data.ByteString as B

72import Data.ByteString as B

@@ -82,7 +81,11 @@ import Data.Text.Show ()

82#endif

81#endif

83import Data.Text.Unsafe (unsafeDupablePerformIO)

82import Data.Text.Unsafe (unsafeDupablePerformIO)

84import Data.Word (Word8, Word32)

83import Data.Word (Word8, Word32)

84+#if __GLASGOW_HASKELL__ >= 703

85−import Foreign.C.Types (CSize(..))

85+import Foreign.C.Types (CSize(CSize))

86+#else

87+import Foreign.C.Types (CSize)

88+#endif

86import Foreign.ForeignPtr (withForeignPtr)

89import Foreign.ForeignPtr (withForeignPtr)

87import Foreign.Marshal.Utils (with)

90import Foreign.Marshal.Utils (with)

88import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

91import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

@@ -137,6 +140,13 @@ decodeLatin1 (PS fp off len) = text a 0 len

137 return dest

140 return dest

138

141

139-- | Decode a 'ByteString' containing UTF-8 encoded text.

142-- | Decode a 'ByteString' containing UTF-8 encoded text.

143+--

144+-- __NOTE__: The replacement character returned by 'OnDecodeError'

145+-- MUST be within the BMP plane; surrogate code points will

146+-- automatically be remapped to the replacement char @U+FFFD@

147+-- (/since 0.11.3.0/), whereas code points beyond the BMP will throw an

148+-- 'error' (/since 1.2.3.1/); For earlier versions of @text@ using

149+-- those unsupported code points would result in undefined behavior.

140decodeUtf8With :: OnDecodeError -> ByteString -> Text

150decodeUtf8With :: OnDecodeError -> ByteString -> Text

141decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

151decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

142 let go dest = withForeignPtr fp $ \ptr ->

152 let go dest = withForeignPtr fp $ \ptr ->

@@ -152,16 +162,52 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

152 x <- peek curPtr'

162 x <- peek curPtr'

153 case onErr desc (Just x) of

163 case onErr desc (Just x) of

154 Nothing -> loop $ curPtr' `plusPtr` 1

164 Nothing -> loop $ curPtr' `plusPtr` 1

155− Just c -> do

165+ Just c

166+ | c > '\xFFFF' -> throwUnsupportedReplChar

167+ | otherwise -> do

156− destOff <- peek destOffPtr

168+ destOff <- peek destOffPtr

157− w <- unsafeSTToIO $

169+ w <- unsafeSTToIO $

158− unsafeWrite dest (fromIntegral destOff) (safe c)

170+ unsafeWrite dest (fromIntegral destOff)

171+ (safe c)

159− poke destOffPtr (destOff + fromIntegral w)

172+ poke destOffPtr (destOff + fromIntegral w)

160− loop $ curPtr' `plusPtr` 1

173+ loop $ curPtr' `plusPtr` 1

161 loop (ptr `plusPtr` off)

174 loop (ptr `plusPtr` off)

162 (unsafeIOToST . go) =<< A.new len

175 (unsafeIOToST . go) =<< A.new len

163 where

176 where

164 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"

177 desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"

178+

179+ throwUnsupportedReplChar = throwIO $

180+ ErrorCall "decodeUtf8With: non-BMP replacement characters not supported"

181+ -- TODO: The code currently assumes that the transcoded UTF-16

182+ -- stream is at most twice as long (in bytes) as the input UTF-8

183+ -- stream. To justify this assumption one has to assume that the

184+ -- error handler replacement character also satisfies this

185+ -- invariant, by emitting at most one UTF16 code unit.

186+ --

187+ -- One easy way to support the full range of code-points for

188+ -- replacement characters in the error handler is to simply change

189+ -- the (over-)allocation to `A.new (2*len)` and then shrink back the

190+ -- `ByteArray#` to the real size (recent GHCs have a cheap

191+ -- `ByteArray#` resize-primop for that which allow the GC to reclaim

192+ -- the overallocation). However, this would require 4 times as much

193+ -- (temporary) storage as the original UTF-8 required.

194+ --

195+ -- Another strategy would be to optimistically assume that

196+ -- replacement characters are within the BMP, and if the case of a

197+ -- non-BMP replacement occurs reallocate the target buffer (or throw

198+ -- an exception, and fallback to a pessimistic codepath, like e.g.

199+ -- `decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)`)

200+ --

201+ -- Alternatively, `OnDecodeError` could become a datastructure which

202+ -- statically encodes the replacement-character range,

203+ -- e.g. something isomorphic to

204+ --

205+ -- Either (... -> Maybe Word16) (... -> Maybe Char)

206+ --

207+ -- And allow to statically switch between the BMP/non-BMP

208+ -- replacement-character codepaths. There's multiple ways to address

209+ -- this with different tradeoffs; but ideally we should optimise for

210+ -- the optimistic/error-free case.

165{- INLINE[0] decodeUtf8With #-}

211{- INLINE[0] decodeUtf8With #-}

166

212

167-- $stream

213-- $stream

@@ -218,6 +264,8 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

218-- or continuation where it is encountered.

264-- or continuation where it is encountered.

219

265

220-- | A stream oriented decoding result.

266-- | A stream oriented decoding result.

267+--

268+-- @since 1.0.0.0

221data Decoding = Some Text ByteString (ByteString -> Decoding)

269data Decoding = Some Text ByteString (ByteString -> Decoding)

222

270

223instance Show Decoding where

271instance Show Decoding where

@@ -237,11 +285,15 @@ newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)

237-- thrown (either by this function or a continuation) that cannot be

285-- thrown (either by this function or a continuation) that cannot be

238-- caught in pure code. For more control over the handling of invalid

286-- caught in pure code. For more control over the handling of invalid

239-- data, use 'streamDecodeUtf8With'.

287-- data, use 'streamDecodeUtf8With'.

288+--

289+-- @since 1.0.0.0

240streamDecodeUtf8 :: ByteString -> Decoding

290streamDecodeUtf8 :: ByteString -> Decoding

241streamDecodeUtf8 = streamDecodeUtf8With strictDecode

291streamDecodeUtf8 = streamDecodeUtf8With strictDecode

242

292

243-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8

293-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8

244-- encoded text.

294-- encoded text.

295+--

296+-- @since 1.0.0.0

245streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding

297streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding

246streamDecodeUtf8With onErr = decodeChunk B.empty 0 0

298streamDecodeUtf8With onErr = decodeChunk B.empty 0 0

247 where

299 where

@@ -317,6 +369,8 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDec

317{-# INLINE decodeUtf8' #-}

369{-# INLINE decodeUtf8' #-}

318

370

319-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

371-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

372+--

373+-- @since 1.1.0.0

320encodeUtf8Builder :: Text -> B.Builder

374encodeUtf8Builder :: Text -> B.Builder

321encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

375encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

322

376

@@ -325,6 +379,8 @@ encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

325--

379--

326-- Use this function is to implement efficient encoders for text-based formats

380-- Use this function is to implement efficient encoders for text-based formats

327-- like JSON or HTML.

381-- like JSON or HTML.

382+--

383+-- @since 1.1.0.0

328{-# INLINE encodeUtf8BuilderEscaped #-}

384{-# INLINE encodeUtf8BuilderEscaped #-}

329-- TODO: Extend documentation with references to source code in @blaze-html@

385-- TODO: Extend documentation with references to source code in @blaze-html@

330-- or @aeson@ that uses this function.

386-- or @aeson@ that uses this function.

@@ -392,7 +448,7 @@ encodeUtf8 :: Text -> ByteString

392encodeUtf8 (Text arr off len)

448encodeUtf8 (Text arr off len)

393 | len == 0 = B.empty

449 | len == 0 = B.empty

394 | otherwise = unsafeDupablePerformIO $ do

450 | otherwise = unsafeDupablePerformIO $ do

395− fp <- mallocByteString (len*4)

451+ fp <- mallocByteString (len*3) -- see https://github.com/haskell/text/issues/194 for why len*3 is enough

396 withForeignPtr fp $ \ptr ->

452 withForeignPtr fp $ \ptr ->

397 with ptr $ \destPtr -> do

453 with ptr $ \destPtr -> do

398 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

454 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

Data/Text/Encoding/Error.hs−1

@@ -10,7 +10,6 @@

10--

11-- License : BSD-style

12-- Maintainer : bos@serpentine.com

13−-- Stability : experimental

14-- Portability : GHC

13-- Portability : GHC

15--

14--

16-- Types and functions for dealing with encoding and decoding errors

15-- Types and functions for dealing with encoding and decoding errors

Data/Text/Foreign.hs+5−2

@@ -5,7 +5,6 @@

5--

6-- License : BSD-style

7-- Maintainer : bos@serpentine.com

8−-- Stability : experimental

9-- Portability : GHC

8-- Portability : GHC

10--

9--

11-- Support for using 'Text' data with native code via the Haskell

10-- Support for using 'Text' data with native code via the Haskell

@@ -108,7 +107,7 @@ takeWord16 (I16 n) t@(Text arr off len)

108 | n >= len || m >= len = t

107 | n >= len || m >= len = t

109 | otherwise = Text arr off m

108 | otherwise = Text arr off m

110 where

109 where

111− m | w < 0xDB00 || w > 0xD8FF = n

110+ m | w < 0xD800 || w > 0xDBFF = n

112 | otherwise = n+1

111 | otherwise = n+1

113 w = A.unsafeIndex arr (off+n-1)

112 w = A.unsafeIndex arr (off+n-1)

114

113

@@ -157,6 +156,8 @@ asForeignPtr t@(Text _arr _off len) = do

157-- | /O(n)/ Decode a C string with explicit length, which is assumed

156-- | /O(n)/ Decode a C string with explicit length, which is assumed

158-- to have been encoded as UTF-8. If decoding fails, a

157-- to have been encoded as UTF-8. If decoding fails, a

159-- 'UnicodeException' is thrown.

158-- 'UnicodeException' is thrown.

159+--

160+-- @since 1.0.0.0

160peekCStringLen :: CStringLen -> IO Text

161peekCStringLen :: CStringLen -> IO Text

161peekCStringLen cs = do

162peekCStringLen cs = do

162 bs <- unsafePackCStringLen cs

163 bs <- unsafePackCStringLen cs

@@ -169,5 +170,7 @@ peekCStringLen cs = do

169-- The temporary storage is freed when the subcomputation terminates

170-- The temporary storage is freed when the subcomputation terminates

170-- (either normally or via an exception), so the pointer to the

171-- (either normally or via an exception), so the pointer to the

171-- temporary storage must /not/ be used after this function returns.

172-- temporary storage must /not/ be used after this function returns.

173+--

174+-- @since 1.0.0.0

172withCStringLen :: Text -> (CStringLen -> IO a) -> IO a

175withCStringLen :: Text -> (CStringLen -> IO a) -> IO a

173withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act

176withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act

Data/Text/IO.hs+4−1

@@ -8,7 +8,6 @@

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11−-- Stability : experimental

12-- Portability : GHC

11-- Portability : GHC

13--

12--

14-- Efficient locale-sensitive support for text I\/O.

13-- Efficient locale-sensitive support for text I\/O.

@@ -352,3 +351,7 @@ putStrLn = hPutStrLn stdout

352-- >

351-- >

353-- > putStr_Utf16LE :: Text -> IO ()

352-- > putStr_Utf16LE :: Text -> IO ()

354-- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)

353-- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)

354+--

355+-- On transcoding errors, an 'IOError' exception is thrown. You can

356+-- use the API in "Data.Text.Encoding" if you need more control over

357+-- error handling or transcoding.

Data/Text/Internal/Fusion/CaseMapping.hs+164−4

@@ -1,10 +1,10 @@

1{-# LANGUAGE Rank2Types #-}

2-- AUTOMATICALLY GENERATED - DO NOT EDIT

3-- Generated by scripts/CaseMapping.hs

4−-- CaseFolding-8.0.0.txt

4+-- CaseFolding-9.0.0.txt

5−-- Date: 2015-01-13, 18:16:36 GMT [MD]

5+-- Date: 2016-03-02, 18:54:54 GMT

6−-- SpecialCasing-8.0.0.txt

6+-- SpecialCasing-9.0.0.txt

7−-- Date: 2014-12-16, 23:08:04 GMT [MD]

7+-- Date: 2016-03-02, 18:55:13 GMT

8

9module Data.Text.Internal.Fusion.CaseMapping where

10import Data.Char

@@ -371,6 +371,24 @@ foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')

371foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')

372-- CHEROKEE SMALL LETTER MV

373foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')

374+-- CYRILLIC SMALL LETTER ROUNDED VE

375+foldMapping '\x1c80' s = Yield '\x0432' (CC s '\x0000' '\x0000')

376+-- CYRILLIC SMALL LETTER LONG-LEGGED DE

377+foldMapping '\x1c81' s = Yield '\x0434' (CC s '\x0000' '\x0000')

378+-- CYRILLIC SMALL LETTER NARROW O

379+foldMapping '\x1c82' s = Yield '\x043e' (CC s '\x0000' '\x0000')

380+-- CYRILLIC SMALL LETTER WIDE ES

381+foldMapping '\x1c83' s = Yield '\x0441' (CC s '\x0000' '\x0000')

382+-- CYRILLIC SMALL LETTER TALL TE

383+foldMapping '\x1c84' s = Yield '\x0442' (CC s '\x0000' '\x0000')

384+-- CYRILLIC SMALL LETTER THREE-LEGGED TE

385+foldMapping '\x1c85' s = Yield '\x0442' (CC s '\x0000' '\x0000')

386+-- CYRILLIC SMALL LETTER TALL HARD SIGN

387+foldMapping '\x1c86' s = Yield '\x044a' (CC s '\x0000' '\x0000')

388+-- CYRILLIC SMALL LETTER TALL YAT

389+foldMapping '\x1c87' s = Yield '\x0463' (CC s '\x0000' '\x0000')

390+-- CYRILLIC SMALL LETTER UNBLENDED UK

391+foldMapping '\x1c88' s = Yield '\xa64b' (CC s '\x0000' '\x0000')

374-- LATIN SMALL LETTER H WITH LINE BELOW

392-- LATIN SMALL LETTER H WITH LINE BELOW

375foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

393foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

376-- LATIN SMALL LETTER T WITH DIAERESIS

394-- LATIN SMALL LETTER T WITH DIAERESIS

@@ -545,6 +563,8 @@ foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')

545foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

563foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

546-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

564-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

547foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

565foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

566+-- LATIN CAPITAL LETTER SMALL CAPITAL I

567+foldMapping '\xa7ae' s = Yield '\x026a' (CC s '\x0000' '\x0000')

548-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

568-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

549foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

569foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

550-- LATIN CAPITAL LETTER CHI

570-- LATIN CAPITAL LETTER CHI

@@ -737,6 +757,78 @@ foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')

737foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

757foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

738-- ARMENIAN SMALL LIGATURE MEN XEH

758-- ARMENIAN SMALL LIGATURE MEN XEH

739foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

759foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

760+-- OSAGE CAPITAL LETTER A

761+foldMapping '\x104b0' s = Yield '\x104d8' (CC s '\x0000' '\x0000')

762+-- OSAGE CAPITAL LETTER AI

763+foldMapping '\x104b1' s = Yield '\x104d9' (CC s '\x0000' '\x0000')

764+-- OSAGE CAPITAL LETTER AIN

765+foldMapping '\x104b2' s = Yield '\x104da' (CC s '\x0000' '\x0000')

766+-- OSAGE CAPITAL LETTER AH

767+foldMapping '\x104b3' s = Yield '\x104db' (CC s '\x0000' '\x0000')

768+-- OSAGE CAPITAL LETTER BRA

769+foldMapping '\x104b4' s = Yield '\x104dc' (CC s '\x0000' '\x0000')

770+-- OSAGE CAPITAL LETTER CHA

771+foldMapping '\x104b5' s = Yield '\x104dd' (CC s '\x0000' '\x0000')

772+-- OSAGE CAPITAL LETTER EHCHA

773+foldMapping '\x104b6' s = Yield '\x104de' (CC s '\x0000' '\x0000')

774+-- OSAGE CAPITAL LETTER E

775+foldMapping '\x104b7' s = Yield '\x104df' (CC s '\x0000' '\x0000')

776+-- OSAGE CAPITAL LETTER EIN

777+foldMapping '\x104b8' s = Yield '\x104e0' (CC s '\x0000' '\x0000')

778+-- OSAGE CAPITAL LETTER HA

779+foldMapping '\x104b9' s = Yield '\x104e1' (CC s '\x0000' '\x0000')

780+-- OSAGE CAPITAL LETTER HYA

781+foldMapping '\x104ba' s = Yield '\x104e2' (CC s '\x0000' '\x0000')

782+-- OSAGE CAPITAL LETTER I

783+foldMapping '\x104bb' s = Yield '\x104e3' (CC s '\x0000' '\x0000')

784+-- OSAGE CAPITAL LETTER KA

785+foldMapping '\x104bc' s = Yield '\x104e4' (CC s '\x0000' '\x0000')

786+-- OSAGE CAPITAL LETTER EHKA

787+foldMapping '\x104bd' s = Yield '\x104e5' (CC s '\x0000' '\x0000')

788+-- OSAGE CAPITAL LETTER KYA

789+foldMapping '\x104be' s = Yield '\x104e6' (CC s '\x0000' '\x0000')

790+-- OSAGE CAPITAL LETTER LA

791+foldMapping '\x104bf' s = Yield '\x104e7' (CC s '\x0000' '\x0000')

792+-- OSAGE CAPITAL LETTER MA

793+foldMapping '\x104c0' s = Yield '\x104e8' (CC s '\x0000' '\x0000')

794+-- OSAGE CAPITAL LETTER NA

795+foldMapping '\x104c1' s = Yield '\x104e9' (CC s '\x0000' '\x0000')

796+-- OSAGE CAPITAL LETTER O

797+foldMapping '\x104c2' s = Yield '\x104ea' (CC s '\x0000' '\x0000')

798+-- OSAGE CAPITAL LETTER OIN

799+foldMapping '\x104c3' s = Yield '\x104eb' (CC s '\x0000' '\x0000')

800+-- OSAGE CAPITAL LETTER PA

801+foldMapping '\x104c4' s = Yield '\x104ec' (CC s '\x0000' '\x0000')

802+-- OSAGE CAPITAL LETTER EHPA

803+foldMapping '\x104c5' s = Yield '\x104ed' (CC s '\x0000' '\x0000')

804+-- OSAGE CAPITAL LETTER SA

805+foldMapping '\x104c6' s = Yield '\x104ee' (CC s '\x0000' '\x0000')

806+-- OSAGE CAPITAL LETTER SHA

807+foldMapping '\x104c7' s = Yield '\x104ef' (CC s '\x0000' '\x0000')

808+-- OSAGE CAPITAL LETTER TA

809+foldMapping '\x104c8' s = Yield '\x104f0' (CC s '\x0000' '\x0000')

810+-- OSAGE CAPITAL LETTER EHTA

811+foldMapping '\x104c9' s = Yield '\x104f1' (CC s '\x0000' '\x0000')

812+-- OSAGE CAPITAL LETTER TSA

813+foldMapping '\x104ca' s = Yield '\x104f2' (CC s '\x0000' '\x0000')

814+-- OSAGE CAPITAL LETTER EHTSA

815+foldMapping '\x104cb' s = Yield '\x104f3' (CC s '\x0000' '\x0000')

816+-- OSAGE CAPITAL LETTER TSHA

817+foldMapping '\x104cc' s = Yield '\x104f4' (CC s '\x0000' '\x0000')

818+-- OSAGE CAPITAL LETTER DHA

819+foldMapping '\x104cd' s = Yield '\x104f5' (CC s '\x0000' '\x0000')

820+-- OSAGE CAPITAL LETTER U

821+foldMapping '\x104ce' s = Yield '\x104f6' (CC s '\x0000' '\x0000')

822+-- OSAGE CAPITAL LETTER WA

823+foldMapping '\x104cf' s = Yield '\x104f7' (CC s '\x0000' '\x0000')

824+-- OSAGE CAPITAL LETTER KHA

825+foldMapping '\x104d0' s = Yield '\x104f8' (CC s '\x0000' '\x0000')

826+-- OSAGE CAPITAL LETTER GHA

827+foldMapping '\x104d1' s = Yield '\x104f9' (CC s '\x0000' '\x0000')

828+-- OSAGE CAPITAL LETTER ZA

829+foldMapping '\x104d2' s = Yield '\x104fa' (CC s '\x0000' '\x0000')

830+-- OSAGE CAPITAL LETTER ZHA

831+foldMapping '\x104d3' s = Yield '\x104fb' (CC s '\x0000' '\x0000')

740-- OLD HUNGARIAN CAPITAL LETTER A

832-- OLD HUNGARIAN CAPITAL LETTER A

741foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

833foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

742-- OLD HUNGARIAN CAPITAL LETTER AA

834-- OLD HUNGARIAN CAPITAL LETTER AA

@@ -839,4 +931,72 @@ foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')

839foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

931foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

840-- OLD HUNGARIAN CAPITAL LETTER US

932-- OLD HUNGARIAN CAPITAL LETTER US

841foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

933foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

934+-- ADLAM CAPITAL LETTER ALIF

935+foldMapping '\x1e900' s = Yield '\x1e922' (CC s '\x0000' '\x0000')

936+-- ADLAM CAPITAL LETTER DAALI

937+foldMapping '\x1e901' s = Yield '\x1e923' (CC s '\x0000' '\x0000')

938+-- ADLAM CAPITAL LETTER LAAM

939+foldMapping '\x1e902' s = Yield '\x1e924' (CC s '\x0000' '\x0000')

940+-- ADLAM CAPITAL LETTER MIIM

941+foldMapping '\x1e903' s = Yield '\x1e925' (CC s '\x0000' '\x0000')

942+-- ADLAM CAPITAL LETTER BA

943+foldMapping '\x1e904' s = Yield '\x1e926' (CC s '\x0000' '\x0000')

944+-- ADLAM CAPITAL LETTER SINNYIIYHE

945+foldMapping '\x1e905' s = Yield '\x1e927' (CC s '\x0000' '\x0000')

946+-- ADLAM CAPITAL LETTER PE

947+foldMapping '\x1e906' s = Yield '\x1e928' (CC s '\x0000' '\x0000')

948+-- ADLAM CAPITAL LETTER BHE

949+foldMapping '\x1e907' s = Yield '\x1e929' (CC s '\x0000' '\x0000')

950+-- ADLAM CAPITAL LETTER RA

951+foldMapping '\x1e908' s = Yield '\x1e92a' (CC s '\x0000' '\x0000')

952+-- ADLAM CAPITAL LETTER E

953+foldMapping '\x1e909' s = Yield '\x1e92b' (CC s '\x0000' '\x0000')

954+-- ADLAM CAPITAL LETTER FA

955+foldMapping '\x1e90a' s = Yield '\x1e92c' (CC s '\x0000' '\x0000')

956+-- ADLAM CAPITAL LETTER I

957+foldMapping '\x1e90b' s = Yield '\x1e92d' (CC s '\x0000' '\x0000')

958+-- ADLAM CAPITAL LETTER O

959+foldMapping '\x1e90c' s = Yield '\x1e92e' (CC s '\x0000' '\x0000')

960+-- ADLAM CAPITAL LETTER DHA

961+foldMapping '\x1e90d' s = Yield '\x1e92f' (CC s '\x0000' '\x0000')

962+-- ADLAM CAPITAL LETTER YHE

963+foldMapping '\x1e90e' s = Yield '\x1e930' (CC s '\x0000' '\x0000')

964+-- ADLAM CAPITAL LETTER WAW

965+foldMapping '\x1e90f' s = Yield '\x1e931' (CC s '\x0000' '\x0000')

966+-- ADLAM CAPITAL LETTER NUN

967+foldMapping '\x1e910' s = Yield '\x1e932' (CC s '\x0000' '\x0000')

968+-- ADLAM CAPITAL LETTER KAF

969+foldMapping '\x1e911' s = Yield '\x1e933' (CC s '\x0000' '\x0000')

970+-- ADLAM CAPITAL LETTER YA

971+foldMapping '\x1e912' s = Yield '\x1e934' (CC s '\x0000' '\x0000')

972+-- ADLAM CAPITAL LETTER U

973+foldMapping '\x1e913' s = Yield '\x1e935' (CC s '\x0000' '\x0000')

974+-- ADLAM CAPITAL LETTER JIIM

975+foldMapping '\x1e914' s = Yield '\x1e936' (CC s '\x0000' '\x0000')

976+-- ADLAM CAPITAL LETTER CHI

977+foldMapping '\x1e915' s = Yield '\x1e937' (CC s '\x0000' '\x0000')

978+-- ADLAM CAPITAL LETTER HA

979+foldMapping '\x1e916' s = Yield '\x1e938' (CC s '\x0000' '\x0000')

980+-- ADLAM CAPITAL LETTER QAAF

981+foldMapping '\x1e917' s = Yield '\x1e939' (CC s '\x0000' '\x0000')

982+-- ADLAM CAPITAL LETTER GA

983+foldMapping '\x1e918' s = Yield '\x1e93a' (CC s '\x0000' '\x0000')

984+-- ADLAM CAPITAL LETTER NYA

985+foldMapping '\x1e919' s = Yield '\x1e93b' (CC s '\x0000' '\x0000')

986+-- ADLAM CAPITAL LETTER TU

987+foldMapping '\x1e91a' s = Yield '\x1e93c' (CC s '\x0000' '\x0000')

988+-- ADLAM CAPITAL LETTER NHA

989+foldMapping '\x1e91b' s = Yield '\x1e93d' (CC s '\x0000' '\x0000')

990+-- ADLAM CAPITAL LETTER VA

991+foldMapping '\x1e91c' s = Yield '\x1e93e' (CC s '\x0000' '\x0000')

992+-- ADLAM CAPITAL LETTER KHA

993+foldMapping '\x1e91d' s = Yield '\x1e93f' (CC s '\x0000' '\x0000')

994+-- ADLAM CAPITAL LETTER GBE

995+foldMapping '\x1e91e' s = Yield '\x1e940' (CC s '\x0000' '\x0000')

996+-- ADLAM CAPITAL LETTER ZAL

997+foldMapping '\x1e91f' s = Yield '\x1e941' (CC s '\x0000' '\x0000')

998+-- ADLAM CAPITAL LETTER KPO

999+foldMapping '\x1e920' s = Yield '\x1e942' (CC s '\x0000' '\x0000')

1000+-- ADLAM CAPITAL LETTER SHA

1001+foldMapping '\x1e921' s = Yield '\x1e943' (CC s '\x0000' '\x0000')

842foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

1002foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

Data/Text/Internal/Fusion/Size.hs+33−7

@@ -1,4 +1,4 @@

1−{-# LANGUAGE CPP #-}

1+{-# LANGUAGE CPP, PatternGuards #-}

2{-# OPTIONS_GHC -fno-warn-missing-methods #-}

3-- |

4-- Module : Data.Text.Internal.Fusion.Internal

@@ -19,11 +19,16 @@

19module Data.Text.Internal.Fusion.Size

20 (

21 Size

22− , exactly

22+ -- * Sizes

23 , exactSize

24 , maxSize

25 , betweenSize

26 , unknownSize

27+ , unionSize

28+ , charSize

29+ , codePointsSize

30+ -- * Querying sizes

31+ , exactly

27 , smaller

32 , smaller

28 , larger

33 , larger

29 , upperBound

34 , upperBound

@@ -32,11 +37,13 @@ module Data.Text.Internal.Fusion.Size

32 , isEmpty

37 , isEmpty

33 ) where

38 ) where

34

39

40+import Data.Char (ord)

35import Data.Text.Internal (mul)

41import Data.Text.Internal (mul)

36#if defined(ASSERTS)

42#if defined(ASSERTS)

37import Control.Exception (assert)

43import Control.Exception (assert)

38#endif

44#endif

39

45

46+-- | A size in UTF-16 code units.

40data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.

47data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.

41 | Unknown -- ^ Unknown size.

48 | Unknown -- ^ Unknown size.

42 deriving (Eq, Show)

49 deriving (Eq, Show)

@@ -46,6 +53,17 @@ exactly (Between na nb) | na == nb = Just na

46exactly _ = Nothing

53exactly _ = Nothing

47{-# INLINE exactly #-}

54{-# INLINE exactly #-}

48

55

56+-- | The 'Size' of the given code point.

57+charSize :: Char -> Size

58+charSize c

59+ | ord c < 0x10000 = exactSize 1

60+ | otherwise = exactSize 2

61+

62+-- | The 'Size' of @n@ code points.

63+codePointsSize :: Int -> Size

64+codePointsSize n = Between n (2*n)

65+{-# INLINE codePointsSize #-}

66+

49exactSize :: Int -> Size

67exactSize :: Int -> Size

50exactSize n =

68exactSize n =

51#if defined(ASSERTS)

69#if defined(ASSERTS)

@@ -71,6 +89,10 @@ betweenSize m n =

71 Between m n

89 Between m n

72{-# INLINE betweenSize #-}

90{-# INLINE betweenSize #-}

73

91

92+unionSize :: Size -> Size -> Size

93+unionSize (Between a b) (Between c d) = Between (min a c) (max b d)

94+unionSize _ _ = Unknown

95+

74unknownSize :: Size

96unknownSize :: Size

75unknownSize = Unknown

97unknownSize = Unknown

76{-# INLINE unknownSize #-}

98{-# INLINE unknownSize #-}

@@ -140,11 +162,15 @@ lowerBound _ (Between n _) = n

140lowerBound k _ = k

162lowerBound k _ = k

141{-# INLINE lowerBound #-}

163{-# INLINE lowerBound #-}

142

164

165+-- | Determine the ordering relationship between two 'Size's, or 'Nothing' in

166+-- the indeterminate case.

143−compareSize :: Size -> Int -> Maybe Ordering

167+compareSize :: Size -> Size -> Maybe Ordering

144−compareSize (Between ma mb) n

168+compareSize (Between ma mb) (Between na nb)

145− | mb < n = Just LT

169+ | mb < na = Just LT

146− | ma > n = Just GT

170+ | ma > nb = Just GT

171+ | ma == mb

172+ , ma == na

147− | ma == n && mb == n = Just EQ

173+ , ma == nb = Just EQ

148compareSize _ _ = Nothing

174compareSize _ _ = Nothing

149

175

150

176

Data/Text/Internal/Fusion/Types.hs+4−4

@@ -78,16 +78,16 @@ import Data.JSString.Internal.Fusion.Types

78-- -- unstreaming functions must be able to cope with the hint being too

79-- -- small or too large.

80-- --

81−-- -- The size hint tries to track the UTF-16 code points in a stream,

81+-- -- The size hint tries to track the UTF-16 code units in a stream,

82−-- -- but often counts the number of characters instead. It can easily

82+-- -- but often counts the number of code points instead. It can easily

83-- -- undercount if, for instance, a transformed stream contains astral

84−-- -- plane characters (those above 0x10000).

84+-- -- plane code points (those above 0x10000).

85

86-- data Stream a =

87-- forall s. Stream

88-- (s -> Step s a) -- stepper function

89-- !s -- current state

90−-- !Size -- size hint

90+-- !Size -- size hint in code units

91

92-- -- | /O(n)/ Determines if two streams are equal.

93-- eq :: (Eq a) => Stream a -> Stream a -> Bool

Data/Text/Lazy.hs+54−20

@@ -15,7 +15,6 @@

15--

16-- License : BSD-style

17-- Maintainer : bos@serpentine.com

18−-- Stability : experimental

19-- Portability : GHC

18-- Portability : GHC

20--

19--

21-- A time and space-efficient implementation of Unicode text using

20-- A time and space-efficient implementation of Unicode text using

@@ -70,6 +69,7 @@ module Data.Text.Lazy

70 , snoc

69 , snoc

71 , append

70 , append

72 , uncons

71 , uncons

72+ , unsnoc

73 , head

74 , last

75 , tail

@@ -278,14 +278,16 @@ import Data.JSString (JSString)

278-- $replacement

279--

280-- A 'Text' value is a sequence of Unicode scalar values, as defined

281+-- in

281−-- in §3.9, definition D76 of the Unicode 5.2 standard:

282+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.

282−-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

283−-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

283+-- As such, a 'Text' cannot contain values in the range U+D800 to

284−-- inclusive. Haskell implementations admit all Unicode code points

284+-- U+DFFF inclusive. Haskell implementations admit all Unicode code

285+-- points

286+-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)

285−-- (§3.4, definition D10) as 'Char' values, including code points

287+-- as 'Char' values, including code points from this invalid range.

286−-- from this invalid range. This means that there are some 'Char'

288+-- This means that there are some 'Char' values that are not valid

287−-- values that are not valid Unicode scalar values, and the functions

289+-- Unicode scalar values, and the functions in this module must handle

288−-- in this module must handle those cases.

290+-- those cases.

289--

291--

290-- Within this module, many functions construct a 'Text' from one or

292-- Within this module, many functions construct a 'Text' from one or

291-- more 'Char' values. Those functions will substitute 'Char' values

293-- more 'Char' values. Those functions will substitute 'Char' values

@@ -299,8 +301,8 @@ import Data.JSString (JSString)

299-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

301-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

300-- code points, and so cannot be represented. The functions replace

302-- code points, and so cannot be represented. The functions replace

301-- invalid scalar values, instead of dropping them, as a security

303-- invalid scalar values, instead of dropping them, as a security

302−-- measure. For details, see Unicode Technical Report 36, §3.5:

304+-- measure. For details, see

303−-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)

305+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)

304

306

305equal :: Text -> Text -> Bool

307equal :: Text -> Text -> Bool

306equal Empty Empty = True

308equal Empty Empty = True

@@ -362,9 +364,13 @@ instance Read Text where

362 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

364 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

363

365

364#if MIN_VERSION_base(4,9,0)

366#if MIN_VERSION_base(4,9,0)

367+-- | Non-orphan 'Semigroup' instance only defined for

365−-- Semigroup orphan instances for older GHCs are provided by

368+-- @base-4.9.0.0@ and later; orphan instances for older GHCs are

369+-- provided by

370+-- the [semigroups](http://hackage.haskell.org/package/semigroups)

366−-- 'semigroups` package

371+-- package

367−

372+--

373+-- @since 1.2.2.0

368instance Semigroup Text where

374instance Semigroup Text where

369 (<>) = append

375 (<>) = append

370#endif

376#endif

@@ -382,6 +388,7 @@ instance IsString Text where

382 fromString = pack

388 fromString = pack

383

389

384#if __GLASGOW_HASKELL__ >= 708

390#if __GLASGOW_HASKELL__ >= 708

391+-- | @since 1.2.0.0

385instance Exts.IsList Text where

392instance Exts.IsList Text where

386 type Item Text = Char

393 type Item Text = Char

387 fromList = pack

394 fromList = pack

@@ -394,6 +401,7 @@ instance NFData Text where

394 rnf (Chunk _ ts) = rnf ts

401 rnf (Chunk _ ts) = rnf ts

395#endif

402#endif

396

403

404+-- | @since 1.2.1.0

397instance Binary Text where

405instance Binary Text where

398 put t = put (encodeUtf8 t)

406 put t = put (encodeUtf8 t)

399 get = do

407 get = do

@@ -417,6 +425,8 @@ instance Data Text where

417

425

418#if MIN_VERSION_base(4,7,0)

426#if MIN_VERSION_base(4,7,0)

419-- | Only defined for @base-4.7.0.0@ and later

427-- | Only defined for @base-4.7.0.0@ and later

428+--

429+-- @since 1.2.2.0

420instance PrintfArg Text where

430instance PrintfArg Text where

421 formatArg txt = formatString $ unpack txt

431 formatArg txt = formatString $ unpack txt

422#endif

432#endif

@@ -495,9 +505,7 @@ fromStrict t = chunk t Empty

495-- -----------------------------------------------------------------------------

505-- -----------------------------------------------------------------------------

496-- * Basic functions

506-- * Basic functions

497

507

498−-- | /O(n)/ Adds a character to the front of a 'Text'. This function

508+-- | /O(1)/ Adds a character to the front of a 'Text'. Subject to fusion.

499−-- is more costly than its 'List' counterpart because it requires

500−-- copying a new array. Subject to fusion.

501cons :: Char -> Text -> Text

509cons :: Char -> Text -> Text

502cons c t = Chunk (T.singleton c) t

510cons c t = Chunk (T.singleton c) t

503{-# INLINE [1] cons #-}

511{-# INLINE [1] cons #-}

@@ -581,6 +589,17 @@ init Empty = emptyError "init"

581 unstream (S.init (stream t)) = init t

589 unstream (S.init (stream t)) = init t

582 #-}

590 #-}

583

591

592+-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if

593+-- empty.

594+--

595+-- * It is no faster than using 'init' and 'last'.

596+--

597+-- @since 1.2.3.0

598+unsnoc :: Text -> Maybe (Text, Char)

599+unsnoc Empty = Nothing

600+unsnoc ts@(Chunk _ _) = Just (init ts, last ts)

601+{-# INLINE unsnoc #-}

602+

584-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

603-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

585-- fusion.

604-- fusion.

586null :: Text -> Bool

605null :: Text -> Bool

@@ -840,6 +859,8 @@ toUpper t = unstream (S.toUpper (stream t))

840-- guides disagree on whether the book name \"The Hill of the Red

859-- guides disagree on whether the book name \"The Hill of the Red

841-- Fox\" is correctly title cased—but this function will

860-- Fox\" is correctly title cased—but this function will

842-- capitalize /every/ word.

861-- capitalize /every/ word.

862+--

863+-- @since 1.0.0.0

843toTitle :: Text -> Text

864toTitle :: Text -> Text

844toTitle t = unstream (S.toTitle (stream t))

865toTitle t = unstream (S.toTitle (stream t))

845{-# INLINE toTitle #-}

866{-# INLINE toTitle #-}

@@ -992,6 +1013,8 @@ mapAccumR f = go

992

1013

993-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every

1014-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every

994-- element.

1015-- element.

1016+--

1017+-- @since 1.2.0.5

995repeat :: Char -> Text

1018repeat :: Char -> Text

996repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t

1019repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t

997 in t

1020 in t

@@ -1009,6 +1032,8 @@ replicate n t

1009

1032

1010-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or

1033-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or

1011-- equivalently, the infinite repetition of the original 'Text'.

1034-- equivalently, the infinite repetition of the original 'Text'.

1035+--

1036+-- @since 1.2.0.5

1012cycle :: Text -> Text

1037cycle :: Text -> Text

1013cycle Empty = emptyError "cycle"

1038cycle Empty = emptyError "cycle"

1014cycle t = let t' = foldrChunks Chunk t' t

1039cycle t = let t' = foldrChunks Chunk t' t

@@ -1018,6 +1043,8 @@ cycle t = let t' = foldrChunks Chunk t' t

1018-- of @f@ to @x@:

1043-- of @f@ to @x@:

1019--

1044--

1020-- > iterate f x == [x, f x, f (f x), ...]

1045-- > iterate f x == [x, f x, f (f x), ...]

1046+--

1047+-- @since 1.2.0.5

1021iterate :: (Char -> Char) -> Char -> Text

1048iterate :: (Char -> Char) -> Char -> Text

1022iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))

1049iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))

1023 in t c

1050 in t c

@@ -1081,6 +1108,8 @@ take i t0 = take' i t0

1081-- Examples:

1108-- Examples:

1082--

1109--

1083-- > takeEnd 3 "foobar" == "bar"

1110-- > takeEnd 3 "foobar" == "bar"

1111+--

1112+-- @since 1.1.1.0

1084takeEnd :: Int64 -> Text -> Text

1113takeEnd :: Int64 -> Text -> Text

1085takeEnd n t0

1114takeEnd n t0

1086 | n <= 0 = empty

1115 | n <= 0 = empty

@@ -1119,6 +1148,8 @@ drop i t0

1119-- Examples:

1148-- Examples:

1120--

1149--

1121-- > dropEnd 3 "foobar" == "foo"

1150-- > dropEnd 3 "foobar" == "foo"

1151+--

1152+-- @since 1.1.1.0

1122dropEnd :: Int64 -> Text -> Text

1153dropEnd :: Int64 -> Text -> Text

1123dropEnd n t0

1154dropEnd n t0

1124 | n <= 0 = t0

1155 | n <= 0 = t0

@@ -1179,12 +1210,15 @@ takeWhile p t0 = takeWhile' t0

1179-- Examples:

1210-- Examples:

1180--

1211--

1181-- > takeWhileEnd (=='o') "foo" == "oo"

1212-- > takeWhileEnd (=='o') "foo" == "oo"

1213+--

1214+-- @since 1.2.2.0

1182takeWhileEnd :: (Char -> Bool) -> Text -> Text

1215takeWhileEnd :: (Char -> Bool) -> Text -> Text

1183takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1216takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1184 where takeChunk acc [] = acc

1217 where takeChunk acc [] = acc

1185− takeChunk acc (t:ts) = if T.length t' < T.length t

1218+ takeChunk acc (t:ts)

1186− then (Chunk t' acc)

1219+ | T.lengthWord16 t' < T.lengthWord16 t

1220+ = chunk t' acc

1187− else takeChunk (Chunk t' acc) ts

1221+ | otherwise = takeChunk (Chunk t' acc) ts

1188 where t' = T.takeWhileEnd p t

1222 where t' = T.takeWhileEnd p t

1189{-# INLINE takeWhileEnd #-}

1223{-# INLINE takeWhileEnd #-}

1190

1224

Data/Text/Lazy/Builder.hs−1

@@ -11,7 +11,6 @@

11-- License : BSD-style (see LICENSE)

12--

13-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

14−-- Stability : experimental

15-- Portability : portable to Hugs and GHC

14-- Portability : portable to Hugs and GHC

16--

15--

17-- Efficient construction of lazy @Text@ values. The principal

16-- Efficient construction of lazy @Text@ values. The principal

Data/Text/Lazy/Builder/Int.hs+46−2

@@ -1,4 +1,17 @@

1+-- Module: Data.Text.Lazy.Builder.Int

4+-- License: BSD-style

5+-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

6+-- Portability: portable

7+--

8+-- Efficiently write an integral value to a 'Builder'.

9+

1−module Data.Text.Lazy.Builder.Int where

10+module Data.Text.Lazy.Builder.Int

11+ (

12+ decimal

13+ , hexadecimal

14+ ) where

2

15

3-- import Data.Text.Lazy (Text)

16-- import Data.Text.Lazy (Text)

4import qualified Data.Text.Lazy as L

17import qualified Data.Text.Lazy as L

@@ -6,7 +19,38 @@ import qualified Data.Text as T

6import qualified Data.JSString.Int as JSS

19import qualified Data.JSString.Int as JSS

7import Data.Text.Internal (Text (..))

20import Data.Text.Internal (Text (..))

8import Data.Text.Lazy.Builder

21import Data.Text.Lazy.Builder

22+

23+import Data.Int (Int8, Int16, Int32, Int64)

9−import Data.Monoid

24+import Data.Monoid (mempty)

25+import qualified Data.ByteString.Unsafe as B

26+import Data.Text.Internal.Builder.Functions ((<>), i2d)

27+import Data.Text.Internal.Builder

28+import Data.Text.Internal.Builder.Int.Digits (digits)

29+import Data.Text.Array

30+import Data.Word (Word, Word8, Word16, Word32, Word64)

31+import GHC.Base (quotInt, remInt)

32+import GHC.Num (quotRemInteger)

33+import GHC.Types (Int(..))

34+import Control.Monad.ST

35+#if MIN_VERSION_base(4,11,0)

36+import Prelude hiding ((<>))

37+#endif

38+

39+#ifdef __GLASGOW_HASKELL__

40+# if defined(INTEGER_GMP)

41+import GHC.Integer.GMP.Internals (Integer(S#))

42+# elif defined(INTEGER_SIMPLE)

43+import GHC.Integer

44+# else

45+# error "You need to use either GMP or integer-simple."

46+# endif

47+#endif

48+

49+#if defined(INTEGER_GMP) || defined(INTEGER_SIMPLE)

50+# define PAIR(a,b) (# a,b #)

51+#else

52+# define PAIR(a,b) (a,b)

53+#endif

10

54

11decimal :: Integral a => a -> Builder

55decimal :: Integral a => a -> Builder

12decimal n = fromText $ Text $ JSS.decimal n

56decimal n = fromText $ Text $ JSS.decimal n

Data/Text/Lazy/Builder/RealFloat.hs+3

@@ -24,6 +24,9 @@ import Data.Text.Lazy.Builder.Int (decimal)

24import Data.Text.Internal.Builder.RealFloat.Functions (roundTo)

25import Data.Text.Lazy.Builder

26import qualified Data.Text as T

27+#if MIN_VERSION_base(4,11,0)

28+import Prelude hiding ((<>))

29+#endif

27

30

28

31

29-- | Control the rendering of floating point numbers.

32-- | Control the rendering of floating point numbers.

Data/Text/Lazy/Encoding.hs+13−3

@@ -8,14 +8,13 @@

8--

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11−-- Stability : experimental

12-- Portability : portable

11-- Portability : portable

13--

12--

14-- Functions for converting lazy 'Text' values to and from lazy

13-- Functions for converting lazy 'Text' values to and from lazy

15-- 'ByteString', using several standard encodings.

14-- 'ByteString', using several standard encodings.

16--

15--

17−-- To gain access to a much larger variety of encodings, use the

16+-- To gain access to a much larger family of encodings, use the

18−-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

17+-- <http://hackage.haskell.org/package/text-icu text-icu package>.

19

18

20module Data.Text.Lazy.Encoding

19module Data.Text.Lazy.Encoding

21 (

20 (

@@ -141,6 +140,7 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

141 rnf (Chunk _ ts) = rnf ts

140 rnf (Chunk _ ts) = rnf ts

142{-# INLINE decodeUtf8' #-}

141{-# INLINE decodeUtf8' #-}

143

142

143+-- | Encode text using UTF-8 encoding.

144encodeUtf8 :: Text -> B.ByteString

145encodeUtf8 Empty = B.empty

146encodeUtf8 lt@(Chunk t _) =

@@ -154,10 +154,20 @@ encodeUtf8 lt@(Chunk t _) =

154 firstChunkSize = min B.smallChunkSize (4 * (T.length t + 1))

155 strategy = B.safeStrategy firstChunkSize B.defaultChunkSize

156

157+-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

158+--

159+-- @since 1.1.0.0

157encodeUtf8Builder :: Text -> B.Builder

160encodeUtf8Builder :: Text -> B.Builder

158encodeUtf8Builder =

161encodeUtf8Builder =

159 foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

162 foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

160

163

164+-- | Encode text using UTF-8 encoding and escape the ASCII characters using

165+-- a 'BP.BoundedPrim'.

166+--

167+-- Use this function is to implement efficient encoders for text-based formats

168+-- like JSON or HTML.

169+--

170+-- @since 1.1.0.0

161{-# INLINE encodeUtf8BuilderEscaped #-}

171{-# INLINE encodeUtf8BuilderEscaped #-}

162encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

172encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

163encodeUtf8BuilderEscaped prim =

173encodeUtf8BuilderEscaped prim =

Data/Text/Lazy/IO.hs−1

@@ -8,7 +8,6 @@

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11−-- Stability : experimental

12-- Portability : GHC

11-- Portability : GHC

13--

12--

14-- Efficient locale-sensitive support for lazy text I\/O.

13-- Efficient locale-sensitive support for lazy text I\/O.

Data/Text/Lazy/Read.hs+3−4

@@ -11,7 +11,6 @@

11--

12-- License : BSD-style

13-- Maintainer : bos@serpentine.com

14−-- Stability : experimental

15-- Portability : GHC

14-- Portability : GHC

16--

15--

17-- Functions used frequently when reading textual data.

16-- Functions used frequently when reading textual data.

@@ -68,9 +67,9 @@ decimal txt

68 go n d = (n * 10 + fromIntegral (digitToInt d))

67 go n d = (n * 10 + fromIntegral (digitToInt d))

69

68

70-- | Read a hexadecimal integer, consisting of an optional leading

69-- | Read a hexadecimal integer, consisting of an optional leading

71−-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

70+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

72−-- until a non-hex-digit or end of string is reached. This function

71+-- consumed until a non-hex-digit or end of string is reached.

73−-- is case insensitive.

72+-- This function is case insensitive.

74--

73--

75-- This function does not handle leading sign characters. If you need

74-- This function does not handle leading sign characters. If you need

76-- to handle signed input, use @'signed' 'hexadecimal'@.

75-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Read.hs+3−4

@@ -9,7 +9,6 @@

9--

10-- License : BSD-style

11-- Maintainer : bos@serpentine.com

12−-- Stability : experimental

13-- Portability : GHC

12-- Portability : GHC

14--

13--

15-- Functions used frequently when reading textual data.

14-- Functions used frequently when reading textual data.

@@ -67,9 +66,9 @@ decimal txt

67 go n d = (n * 10 + fromIntegral (digitToInt d))

66 go n d = (n * 10 + fromIntegral (digitToInt d))

68

67

69-- | Read a hexadecimal integer, consisting of an optional leading

68-- | Read a hexadecimal integer, consisting of an optional leading

70−-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

69+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

71−-- until a non-hex-digit or end of string is reached. This function

70+-- consumed until a non-hex-digit or end of string is reached.

72−-- is case insensitive.

71+-- This function is case insensitive.

73--

72--

74-- This function does not handle leading sign characters. If you need

73-- This function does not handle leading sign characters. If you need

75-- to handle signed input, use @'signed' 'hexadecimal'@.

74-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Show.hs+2

@@ -46,6 +46,8 @@ unpack = S.unstreamList . stream

46-- fusion.

47--

48-- This is exposed solely for people writing GHC rewrite rules.

49+--

50+-- @since 1.2.1.1

49unpackCString# :: Addr# -> Text

51unpackCString# :: Addr# -> Text

50unpackCString# addr# = unstream (S.streamCString# addr#)

52unpackCString# addr# = unstream (S.streamCString# addr#)

51{-# NOINLINE unpackCString# #-}

53{-# NOINLINE unpackCString# #-}

Data/Text/Unsafe.hs+2−1

@@ -7,7 +7,6 @@

8-- License : BSD-style

9-- Maintainer : bos@serpentine.com

10−-- Stability : experimental

11-- Portability : portable

10-- Portability : portable

12--

11--

13-- A module containing unsafe 'Text' operations, for very very careful

12-- A module containing unsafe 'Text' operations, for very very careful

@@ -150,6 +149,8 @@ reverseIter (Text txt) i

150-- | /O(1)/ Iterate one step backwards through a UTF-16 array,

149-- | /O(1)/ Iterate one step backwards through a UTF-16 array,

151-- returning the delta to add (i.e. a negative number) to give the

150-- returning the delta to add (i.e. a negative number) to give the

152-- next offset to iterate at.

151-- next offset to iterate at.

152+--

153+-- @since 1.1.1.0

153reverseIter_ :: Text -> Int -> Int

154reverseIter_ :: Text -> Int -> Int

154#ifndef __GHCJS__

155#ifndef __GHCJS__

155reverseIter_ (Text arr off _len) i

156reverseIter_ (Text arr off _len) i

README.markdown+5−16

@@ -1,29 +1,18 @@

1−# Text: Fast, packed Unicode strings, using stream fusion

1+# `text`: Fast, packed Unicode strings, using stream fusion

2

3This package provides the Data.Text library, a library for the space-

4and time-efficient manipulation of Unicode text in Haskell.

5

6−

7−# Normalization, conversion, and collation, oh my!

8−

9−This library intentionally provides a simple API based on the

10−Haskell prelude's list manipulation functions. For more complicated

11−real-world tasks, such as Unicode normalization, conversion to and

12−from a larger variety of encodings, and collation, use the [text-icu

13−package](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/text-icu).

6+**Please refer to the [package description on Hackage](https://hackage.haskell.org/package/text#description) for more information.**

14−

15−That library uses the well-respected and liberally licensed ICU

16−library to provide these facilities.

17−

18

7

19# Get involved!

8# Get involved!

20

9

21Please report bugs via the

10Please report bugs via the

22−[github issue tracker](https://github.com/bos/text/issues).

11+[github issue tracker](https://github.com/haskell/text/issues).

23

12

24−Master [git repository](https://github.com/bos/text):

13+Master [git repository](https://github.com/haskell/text):

25

14

26−* `git clone git://github.com/bos/text.git`

15+* `git clone git://github.com/haskell/text.git`

27

16

28There's also a [Mercurial mirror](https://bitbucket.org/bos/text):

17There's also a [Mercurial mirror](https://bitbucket.org/bos/text):

29

18

benchmarks/.gitignore+4−1

@@ -1 +1,4 @@

1+/dist/

1−dist

2+/dist-newstyle/

3+/.ghc.environment.*

4+/cabal.project.local

benchmarks/cabal.projectnew file+2

@@ -0,0 +1,2 @@

1+-- NB: we use a separate project

2+packages: .

benchmarks/haskell/Benchmarks.hs+2

@@ -10,6 +10,7 @@ import System.FilePath ((</>))

10import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8)

11

12import qualified Benchmarks.Builder as Builder

13+import qualified Benchmarks.Concat as Concat

13import qualified Benchmarks.DecodeUtf8 as DecodeUtf8

14import qualified Benchmarks.DecodeUtf8 as DecodeUtf8

14import qualified Benchmarks.EncodeUtf8 as EncodeUtf8

15import qualified Benchmarks.EncodeUtf8 as EncodeUtf8

15import qualified Benchmarks.Equality as Equality

16import qualified Benchmarks.Equality as Equality

@@ -41,6 +42,7 @@ benchmarks = do

41 -- Traditional benchmarks

42 -- Traditional benchmarks

42 bs <- sequence

43 bs <- sequence

43 [ Builder.benchmark

44 [ Builder.benchmark

45+ , Concat.benchmark

44 , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")

46 , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")

45 , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")

47 , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")

46 , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")

48 , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")

benchmarks/haskell/Benchmarks/Concat.hsnew file+25

@@ -0,0 +1,25 @@

1+{-# LANGUAGE OverloadedStrings #-}

2+

3+module Benchmarks.Concat (benchmark) where

4+

5+import Control.Monad.Trans.Writer

6+import Criterion (Benchmark, bgroup, bench, whnf)

7+import Data.Text as T

8+

9+benchmark :: IO Benchmark

10+benchmark = return $ bgroup "Concat"

11+ [ bench "append" $ whnf (append4 "Text 1" "Text 2" "Text 3") "Text 4"

12+ , bench "concat" $ whnf (concat4 "Text 1" "Text 2" "Text 3") "Text 4"

13+ , bench "write" $ whnf (write4 "Text 1" "Text 2" "Text 3") "Text 4"

14+ ]

15+

16+append4, concat4, write4 :: Text -> Text -> Text -> Text -> Text

17+

18+{-# NOINLINE append4 #-}

19+append4 x1 x2 x3 x4 = x1 `append` x2 `append` x3 `append` x4

20+

21+{-# NOINLINE concat4 #-}

22+concat4 x1 x2 x3 x4 = T.concat [x1, x2, x3, x4]

23+

24+{-# NOINLINE write4 #-}

25+write4 x1 x2 x3 x4 = execWriter $ tell x1 >> tell x2 >> tell x3 >> tell x4

benchmarks/text-benchmarks.cabal+82−8

@@ -12,8 +12,7 @@ author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

12maintainer: jaspervdj@gmail.com

13category: Text

14build-type: Simple

15−

16−cabal-version: >=1.2

15+cabal-version: >=1.8

17

16

18flag bytestring-builder

17flag bytestring-builder

19 description: Depend on the bytestring-builder package for backwards compatibility.

18 description: Depend on the bytestring-builder package for backwards compatibility.

@@ -26,16 +25,12 @@ flag llvm

26 manual: True

25 manual: True

27

26

28executable text-benchmarks

27executable text-benchmarks

29− hs-source-dirs: haskell ..

30− c-sources: ../cbits/cbits.c

31− cbits/time_iconv.c

32− include-dirs: ../include

33− main-is: Benchmarks.hs

34 ghc-options: -Wall -O2 -rtsopts

28 ghc-options: -Wall -O2 -rtsopts

35 if flag(llvm)

29 if flag(llvm)

36 ghc-options: -fllvm

30 ghc-options: -fllvm

37 cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP

31 cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP

38− build-depends: base == 4.*,

32+ build-depends: array,

33+ base == 4.*,

39 binary,

34 binary,

40 blaze-builder,

35 blaze-builder,

41 bytestring-lexing >= 0.5.0,

36 bytestring-lexing >= 0.5.0,

@@ -47,6 +42,7 @@ executable text-benchmarks

47 ghc-prim,

42 ghc-prim,

48 integer-gmp,

43 integer-gmp,

49 stringsearch,

44 stringsearch,

45+ transformers,

50 utf8-string,

46 utf8-string,

51 vector

47 vector

52

48

@@ -56,9 +52,87 @@ executable text-benchmarks

56 else

52 else

57 build-depends: bytestring >= 0.10.4

53 build-depends: bytestring >= 0.10.4

58

54

55+ -- modules for benchmark proper

56+ c-sources: cbits/time_iconv.c

57+ hs-source-dirs: haskell

58+ main-is: Benchmarks.hs

59+ other-modules:

60+ Benchmarks.Builder

61+ Benchmarks.Concat

62+ Benchmarks.DecodeUtf8

63+ Benchmarks.EncodeUtf8

64+ Benchmarks.Equality

65+ Benchmarks.FileRead

66+ Benchmarks.FoldLines

67+ Benchmarks.Mul

68+ Benchmarks.Programs.BigTable

69+ Benchmarks.Programs.Cut

70+ Benchmarks.Programs.Fold

71+ Benchmarks.Programs.Sort

72+ Benchmarks.Programs.StripTags

73+ Benchmarks.Programs.Throughput

74+ Benchmarks.Pure

75+ Benchmarks.ReadNumbers

76+ Benchmarks.Replace

77+ Benchmarks.Search

78+ Benchmarks.Stream

79+ Benchmarks.WordFrequencies

80+

81+ -- Source code for IUT (implementation under test)

82+ -- "borrowed" from parent folder

83+ include-dirs: ../include

84+ c-sources: ../cbits/cbits.c

85+ hs-source-dirs: ..

86+ other-modules:

87+ Data.Text

88+ Data.Text.Array

89+ Data.Text.Encoding

90+ Data.Text.Encoding.Error

91+ Data.Text.Foreign

92+ Data.Text.IO

93+ Data.Text.Internal

94+ Data.Text.Internal.Builder

95+ Data.Text.Internal.Builder.Functions

96+ Data.Text.Internal.Builder.Int.Digits

97+ Data.Text.Internal.Builder.RealFloat.Functions

98+ Data.Text.Internal.Encoding.Fusion

99+ Data.Text.Internal.Encoding.Fusion.Common

100+ Data.Text.Internal.Encoding.Utf16

101+ Data.Text.Internal.Encoding.Utf32

102+ Data.Text.Internal.Encoding.Utf8

103+ Data.Text.Internal.Functions

104+ Data.Text.Internal.Fusion

105+ Data.Text.Internal.Fusion.CaseMapping

106+ Data.Text.Internal.Fusion.Common

107+ Data.Text.Internal.Fusion.Size

108+ Data.Text.Internal.Fusion.Types

109+ Data.Text.Internal.IO

110+ Data.Text.Internal.Lazy

111+ Data.Text.Internal.Lazy.Encoding.Fusion

112+ Data.Text.Internal.Lazy.Fusion

113+ Data.Text.Internal.Lazy.Search

114+ Data.Text.Internal.Private

115+ Data.Text.Internal.Read

116+ Data.Text.Internal.Search

117+ Data.Text.Internal.Unsafe

118+ Data.Text.Internal.Unsafe.Char

119+ Data.Text.Internal.Unsafe.Shift

120+ Data.Text.Lazy

121+ Data.Text.Lazy.Builder

122+ Data.Text.Lazy.Builder.Int

123+ Data.Text.Lazy.Builder.RealFloat

124+ Data.Text.Lazy.Encoding

125+ Data.Text.Lazy.IO

126+ Data.Text.Lazy.Internal

127+ Data.Text.Lazy.Read

128+ Data.Text.Read

129+ Data.Text.Unsafe

130+ Data.Text.Show

131+

59executable text-multilang

132executable text-multilang

60 hs-source-dirs: haskell

133 hs-source-dirs: haskell

61 main-is: Multilang.hs

134 main-is: Multilang.hs

135+ other-modules: Timer

62 ghc-options: -Wall -O2

136 ghc-options: -Wall -O2

63 build-depends: base == 4.*,

137 build-depends: base == 4.*,

64 bytestring,

138 bytestring,

cabal.projectnew file+3

@@ -0,0 +1,3 @@

1+-- See http://cabal.readthedocs.io/en/latest/nix-local-build-overview.html

2+packages: .

3+tests: True

cbits/cbits.c+6−11

@@ -206,11 +206,9 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

206 const uint8_t *srcend,

207 uint32_t *codepoint0, uint32_t *state0)

208{

209− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,

209+ _hs_text_decode_utf8_int(dest, destoff, src, srcend, codepoint0, state0);

210− codepoint0, state0);

210+

211− if (*state0 == UTF8_REJECT)

212− ret -=1;

213− return ret;

211+ return *src;

214}

212}

215

213

216/*

214/*

@@ -222,12 +220,9 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

222{

220{

223 uint32_t codepoint;

221 uint32_t codepoint;

224 uint32_t state = UTF8_ACCEPT;

222 uint32_t state = UTF8_ACCEPT;

225− uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

223+ _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

226− &codepoint, &state);

224+ &codepoint, &state);

227− /* Back up if we have an incomplete or invalid encoding */

228− if (state != UTF8_ACCEPT)

229− ret -= 1;

230− return ret;

225+ return src;

231}

226}

232

227

233void

228void

changelog.md+61−19

@@ -1,15 +1,57 @@

1+### 1.2.3.1 TBD

2+

3+* Make `decodeUtf8With` fail explicitly for unsupported non-BMP

4+ replacement characters instead silent undefined behaviour (gh-213)

5+

6+### 1.2.3.0

7+

8+* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec

9+ (updated from 8.0).

10+

11+* Bug fix: the lazy `takeWhileEnd` function violated the

12+ [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)

13+ (gh-184).

14+

15+* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).

16+

17+* New function: `unsnoc` (gh-173).

18+

19+* Reduce memory overhead in `encodeUTF8` (gh-194).

20+

21+* Improve UTF-8 decoder error-recovery (gh-182).

22+

23+* Minor documentation improvements (`@since` annotations, more

24+ examples, clarifications).

25+

26+#### 1.2.2.2

27+

28+* The `toTitle` function now correctly handles letters that

29+ immediately follow punctuation. Before, `"there's"` would turn into

30+ `"There'S"`. Now, it becomes `"There's"`.

31+

32+* The implementation of unstreaming is faster, resulting in operations

33+ such as `map` and `intersperse` speeding up by up to 30%, with

34+ smaller code generated.

35+

36+* The optimised length comparison function is now more likely to be

37+ used after some rewrite rule tweaking.

38+

39+* Bug fix: an off-by-one bug in `takeEnd` is fixed.

40+

41+* Bug fix: a logic error in `takeWord16` is fixed.

42+

1−1.2.2.1

43+#### 1.2.2.1

2

44

3* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

45* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

4 The build flag has been renamed accordingly. Your army of diligent

46 The build flag has been renamed accordingly. Your army of diligent

5 maintainers apologizes for the churn.

47 maintainers apologizes for the churn.

6

48

7−* Spec compliance: toCaseFold now follows the Unicode 8.0 spec

49+* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec

8 (updated from 7.0)

50 (updated from 7.0)

9

51

10* An STG lint error has been fixed

52* An STG lint error has been fixed

11

53

12−1.2.2.0

54+### 1.2.2.0

13

55

14* The `integer-simple` package, upon which this package optionally

56* The `integer-simple` package, upon which this package optionally

15 depended, has been replaced with `integer-pure`. The build flag has

57 depended, has been replaced with `integer-pure`. The build flag has

@@ -25,7 +67,7 @@

25 * if `base` >= 4.7: `PrintfArg`

67 * if `base` >= 4.7: `PrintfArg`

26 * if `base` >= 4.9: `Semigroup`

68 * if `base` >= 4.9: `Semigroup`

27

69

28−1.2.1.3

70+#### 1.2.1.3

29

71

30* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

72* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

31 phase 2 does not prevent competition with the `unpack` rule, which is

73 phase 2 does not prevent competition with the `unpack` rule, which is

@@ -33,7 +75,7 @@

33 test environment mistake. Moving literal rules back to phase 1 finally

75 test environment mistake. Moving literal rules back to phase 1 finally

34 fixes GHC Trac #10528 correctly.

76 fixes GHC Trac #10528 correctly.

35

77

36−1.2.1.2

78+#### 1.2.1.2

37

79

38* Bug fix: Run literal rewrite rules in simplifier phase 2.

80* Bug fix: Run literal rewrite rules in simplifier phase 2.

39 The behavior of the simplifier changed in GHC 7.10.2,

81 The behavior of the simplifier changed in GHC 7.10.2,

@@ -41,21 +83,21 @@

41 and long compilation times. See

83 and long compilation times. See

42 [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

84 [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

43

85

44−1.2.1.1

86+#### 1.2.1.1

45

87

46* Expose unpackCString#, which you should never use.

88* Expose unpackCString#, which you should never use.

47

89

48−1.2.1.0

90+### 1.2.1.0

49

91

50* Added Binary instances for both Text types. (If you have previously

92* Added Binary instances for both Text types. (If you have previously

51 been using the text-binary package to get a Binary instance, it is

93 been using the text-binary package to get a Binary instance, it is

52 now obsolete.)

94 now obsolete.)

53

95

54−1.2.0.6

96+#### 1.2.0.6

55

97

56* Fixed a space leak in UTF-8 decoding

98* Fixed a space leak in UTF-8 decoding

57

99

58−1.2.0.5

100+#### 1.2.0.5

59

101

60* Feature parity: repeat, cycle, iterate are now implemented for lazy

102* Feature parity: repeat, cycle, iterate are now implemented for lazy

61 Text, and the Data instance is more complete

103 Text, and the Data instance is more complete

@@ -75,26 +117,26 @@

75* Spec compliance: toCaseFold now follows the Unicode 7.0 spec

117* Spec compliance: toCaseFold now follows the Unicode 7.0 spec

76 (updated from 6.3)

118 (updated from 6.3)

77

119

78−1.2.0.4

120+#### 1.2.0.4

79

121

80* Fixed an incompatibility with base < 4.5

122* Fixed an incompatibility with base < 4.5

81

123

82−1.2.0.3

124+#### 1.2.0.3

83

125

84* Update formatRealFloat to correspond to the definition in versions

126* Update formatRealFloat to correspond to the definition in versions

85 of base newer than 4.5 (https://github.com/bos/text/issues/105)

127 of base newer than 4.5 (https://github.com/bos/text/issues/105)

86

128

87−1.2.0.2

129+#### 1.2.0.2

88

130

89* Bumped lower bound on deepseq to 1.4 for compatibility with the

131* Bumped lower bound on deepseq to 1.4 for compatibility with the

90 upcoming GHC 7.10

132 upcoming GHC 7.10

91

133

92−1.2.0.1

134+#### 1.2.0.1

93

135

94* Fixed a buffer overflow in rendering of large Integers

136* Fixed a buffer overflow in rendering of large Integers

95 (https://github.com/bos/text/issues/99)

137 (https://github.com/bos/text/issues/99)

96

138

97−1.2.0.0

139+## 1.2.0.0

98

140

99* Fixed an integer overflow in the replace function

141* Fixed an integer overflow in the replace function

100 (https://github.com/bos/text/issues/81)

142 (https://github.com/bos/text/issues/81)

@@ -107,7 +149,7 @@

107

149

108* Added an instance of IsList for GHC 7.8 and above

150* Added an instance of IsList for GHC 7.8 and above

109

151

110−1.1.1.0

152+### 1.1.1.0

111

153

112* The Data.Data instance now allows gunfold to work, via a virtual

154* The Data.Data instance now allows gunfold to work, via a virtual

113 pack constructor

155 pack constructor

@@ -117,12 +159,12 @@

117* Comparing the length of a Text against a number can now

159* Comparing the length of a Text against a number can now

118 short-circuit in more cases

160 short-circuit in more cases

119

161

120−1.1.0.1

162+#### 1.1.0.1

121

163

122* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes

164* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes

123 in single-byte chunks

165 in single-byte chunks

124

166

125−1.1.0.0

167+## 1.1.0.0

126

168

127* encodeUtf8: Performance is improved by up to 4x.

169* encodeUtf8: Performance is improved by up to 4x.

128

170

@@ -141,12 +183,12 @@

141 use at your own risk - there are no API stability guarantees for

183 use at your own risk - there are no API stability guarantees for

142 internal modules!

184 internal modules!

143

185

144−1.0.0.1

186+#### 1.0.0.1

145

187

146* decodeUtf8: Fixed a regression that caused us to incorrectly

188* decodeUtf8: Fixed a regression that caused us to incorrectly

147 identify truncated UTF-8 as valid (gh-61)

189 identify truncated UTF-8 as valid (gh-61)

148

190

149−1.0.0.0

191+# 1.0.0.0

150

192

151* Added support for Unicode 6.3.0 to case conversion functions

193* Added support for Unicode 6.3.0 to case conversion functions

152

194

tests-and-benchmarks.markdown+13−8

@@ -13,10 +13,13 @@ you can clone from one of the following locations:

13* Git mirror repository:

14 [github.com/bos/text-test-data](https://github.com/bos/text-test-data)

15

16−You should clone that repository into the `tests` subdirectory (your

16+You can clone either repository into the `tests` subdirectory using

17−clone must be named `text-test-data` locally), then run `make -C

17+

18−tests/text-test-data` to uncompress the test files. Many tests and

18+ cd tests/

19+ make text-test-data # to clone from mercurial, OR

20+ make VCS=git text-test-data # to clone from git

21+

19−benchmarks will fail if the test files are missing.

22+Many tests and benchmarks will fail if the test files are missing.

20

23

21Functional tests

24Functional tests

22----------------

25----------------

@@ -54,10 +57,12 @@ To compile the benchmarks, navigate to the `benchmarks` subdirectory and run

54

57

55 ./dist/build/text-benchmarks/text-benchmarks

58 ./dist/build/text-benchmarks/text-benchmarks

56

59

57−However, since there's quite a lot of benchmarks, you usually don't want to

60+Or if you have a recent enough `cabal`, you can build and run the

58−run them all. Instead, use the `-l` flag to get a list of benchmarks:

61+benchmarks via

59

62

60− ./dist/build/text-benchmarks/text-benchmarks

63+ cabal new-run exe:text-benchmarks -- --help

61

64

65+However, since there's quite a lot of benchmarks, you usually don't want to

66+run them all. Instead, use the `-l` flag to get a list of benchmarks

62−And run the ones you want to inspect. If you want to configure the benchmarks

67+and run the ones you want to inspect. If you want to configure the benchmarks

63further, the exact parameters can be changed in `Benchmarks.hs`.

68further, the exact parameters can be changed in `Benchmarks.hs`.

tests/Makefile+6−1

@@ -1,3 +1,4 @@

1+VCS = hg

1count = 1000

2count = 1000

2

3

3all: coverage literal-rule-test

4all: coverage literal-rule-test

@@ -12,7 +13,11 @@ build: text-test-data

12 cabal build

13 cabal build

13

14

14text-test-data:

15text-test-data:

16+ifeq ($(VCS),git)

17+ git clone https://github.com/bos/text-test-data.git

18+else

15 hg clone https://bitbucket.org/bos/text-test-data

19 hg clone https://bitbucket.org/bos/text-test-data

20+endif

16 $(MAKE) -C text-test-data

21 $(MAKE) -C text-test-data

17

22

18coverage/text-tests.tix:

23coverage/text-tests.tix:

@@ -37,4 +42,4 @@ coverage/hpc_index.html: coverage/coverage.tix

37clean:

42clean:

38 rm -rf dist coverage .hpc

43 rm -rf dist coverage .hpc

39

44

40−.PHONY: build coverage all literal-rule-test

45+.PHONY: all build clean coverage literal-rule-test

tests/Tests/Properties.hs+94−21

@@ -9,9 +9,9 @@ module Tests.Properties

9 ) where

10

11import Control.Applicative ((<$>), (<*>))

12−import Control.Arrow ((***), second)

12+import Control.Arrow ((***), first, second)

13import Data.Bits ((.&.))

14−import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)

14+import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord)

15import Data.Int (Int8, Int16, Int32, Int64)

16import Data.Monoid (Monoid(..))

17import Data.String (IsString(fromString))

@@ -23,6 +23,7 @@ import Data.Text.Internal.Search (indices)

23import Data.Text.Lazy.Read as TL

24import Data.Text.Read as T

25import Data.Word (Word, Word8, Word16, Word32, Word64)

26+import Data.Maybe (mapMaybe)

26import Numeric (showEFloat, showFFloat, showGFloat, showHex)

27import Numeric (showEFloat, showFFloat, showGFloat, showHex)

27import Prelude hiding (replicate)

28import Prelude hiding (replicate)

28import Test.Framework (Test, testGroup)

29import Test.Framework (Test, testGroup)

@@ -30,6 +31,7 @@ import Test.Framework.Providers.QuickCheck2 (testProperty)

30import Test.QuickCheck hiding ((.&.))

31import Test.QuickCheck hiding ((.&.))

31import Test.QuickCheck.Monadic

32import Test.QuickCheck.Monadic

32import Test.QuickCheck.Property (Property(..))

33import Test.QuickCheck.Property (Property(..))

34+import Test.QuickCheck.Unicode (char)

33import Tests.QuickCheckUtils

35import Tests.QuickCheckUtils

34import Tests.Utils

36import Tests.Utils

35import Text.Show.Functions ()

37import Text.Show.Functions ()

@@ -37,6 +39,7 @@ import qualified Control.Exception as Exception

37import qualified Data.Bits as Bits (shiftL, shiftR)

39import qualified Data.Bits as Bits (shiftL, shiftR)

38import qualified Data.ByteString as B

40import qualified Data.ByteString as B

39import qualified Data.ByteString.Lazy as BL

41import qualified Data.ByteString.Lazy as BL

42+import qualified Data.Char as C

40import qualified Data.List as L

43import qualified Data.List as L

41import qualified Data.Text as T

44import qualified Data.Text as T

42import qualified Data.Text.Encoding as E

45import qualified Data.Text.Encoding as E

@@ -60,10 +63,10 @@ t_pack_unpack = (T.unpack . T.pack) `eq` id

60tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

63tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

61t_stream_unstream = (S.unstream . S.stream) `eq` id

64t_stream_unstream = (S.unstream . S.stream) `eq` id

62tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

65tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

63−t_reverse_stream t = (S.reverse . S.reverseStream) t == t

66+t_reverse_stream t = (S.reverse . S.reverseStream) t === t

64−t_singleton c = [c] == (T.unpack . T.singleton) c

67+t_singleton c = [c] === (T.unpack . T.singleton) c

65−tl_singleton c = [c] == (TL.unpack . TL.singleton) c

68+tl_singleton c = [c] === (TL.unpack . TL.singleton) c

66−tl_unstreamChunks x = f 11 x == f 1000 x

69+tl_unstreamChunks x = f 11 x === f 1000 x

67 where f n = SL.unstreamChunks n . S.streamList

70 where f n = SL.unstreamChunks n . S.streamList

68tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

71tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

69tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

72tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

@@ -74,13 +77,13 @@ encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

74encodeLazyL1 :: TL.Text -> BL.ByteString

77encodeLazyL1 :: TL.Text -> BL.ByteString

75encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

78encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

76

79

77−t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

80+t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a

78 where a = T.map (\c -> chr (ord c `mod` 128)) t

81 where a = T.map (\c -> chr (ord c `mod` 128)) t

79−tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

82+tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a

80 where a = TL.map (\c -> chr (ord c `mod` 128)) t

83 where a = TL.map (\c -> chr (ord c `mod` 128)) t

81−t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

84+t_latin1 t = E.decodeLatin1 (encodeL1 a) === a

82 where a = T.map (\c -> chr (ord c `mod` 256)) t

85 where a = T.map (\c -> chr (ord c `mod` 256)) t

83−tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

86+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a

84 where a = TL.map (\c -> chr (ord c `mod` 256)) t

87 where a = TL.map (\c -> chr (ord c `mod` 256)) t

85t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

88t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

86t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

89t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

@@ -111,7 +114,7 @@ t_utf8_undecoded = forAll genUnicode $ \t ->

111 let b = E.encodeUtf8 t

114 let b = E.encodeUtf8 t

112 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

115 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

113 leftover = (++ [B.empty]) . init . tail . B.inits

116 leftover = (++ [B.empty]) . init . tail . B.inits

114− in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b == ls

117+ in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls

115

118

116data Badness = Solo | Leading | Trailing

119data Badness = Solo | Leading | Trailing

117 deriving (Eq, Show)

120 deriving (Eq, Show)

@@ -177,6 +180,33 @@ genInvalidUTF8 = B.pack <$> oneof [

177 k <- choose (0,n)

180 k <- choose (0,n)

178 vectorOf k gen

181 vectorOf k gen

179

182

183+-- See http://unicode.org/faq/utf_bom.html#gen8

184+-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...

185+-- When faced with this illegal byte sequence ... a UTF-8 conformant process

186+-- must treat the first byte 110xxxxx2 as an illegal termination error

187+-- (e.g. filter it out or replace by 0xFFFD) ...

188+-- ... and continue processing at the second byte 0xxxxxxx2

189+t_decode_with_error2 =

190+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"

191+t_decode_with_error3 =

192+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"

193+t_decode_with_error4 =

194+ E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"

195+

196+t_decode_with_error2' =

197+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of

198+ E.Some x _ _ -> x === "xa"

199+t_decode_with_error3' =

200+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of

201+ E.Some x _ _ -> x === "xaa"

202+t_decode_with_error4' =

203+ case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of

204+ E.Some x _ _ -> x === "xaaa"

205+

206+t_infix_concat bs1 text bs2 rep =

207+ text `T.isInfixOf`

208+ E.decodeUtf8With (\_ _ -> rep) (B.concat [bs1, E.encodeUtf8 text, bs2])

209+

180s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

210s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

181 where _types = s :: String

211 where _types = s :: String

182sf_Eq p s =

212sf_Eq p s =

@@ -201,8 +231,8 @@ t_mconcat = unsquare $

201 mconcat `eq` (unpackS . mconcat . L.map T.pack)

231 mconcat `eq` (unpackS . mconcat . L.map T.pack)

202tl_mconcat = unsquare $

232tl_mconcat = unsquare $

203 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

233 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

204−t_mempty = mempty == (unpackS (mempty :: T.Text))

234+t_mempty = mempty === (unpackS (mempty :: T.Text))

205−tl_mempty = mempty == (unpackS (mempty :: TL.Text))

235+tl_mempty = mempty === (unpackS (mempty :: TL.Text))

206t_IsString = fromString `eqP` (T.unpack . fromString)

236t_IsString = fromString `eqP` (T.unpack . fromString)

207tl_IsString = fromString `eqP` (TL.unpack . fromString)

237tl_IsString = fromString `eqP` (TL.unpack . fromString)

208

238

@@ -229,6 +259,13 @@ sf_uncons p = (uncons . L.filter p) `eqP`

229 (fmap (second unpackS) . S.uncons . S.filter p)

259 (fmap (second unpackS) . S.uncons . S.filter p)

230t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

260t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

231tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

261tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

262+

263+unsnoc xs@(_:_) = Just (init xs, last xs)

264+unsnoc [] = Nothing

265+

266+t_unsnoc = unsnoc `eqP` (fmap (first unpackS) . T.unsnoc)

267+tl_unsnoc = unsnoc `eqP` (fmap (first unpackS) . TL.unsnoc)

268+

232s_head = head `eqP` S.head

269s_head = head `eqP` S.head

233sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

270sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

234t_head = head `eqP` T.head

271t_head = head `eqP` T.head

@@ -326,6 +363,19 @@ t_toUpper_upper t = p (T.toUpper t) >= p t

326 where p = T.length . T.filter isUpper

363 where p = T.length . T.filter isUpper

327tl_toUpper_upper t = p (TL.toUpper t) >= p t

364tl_toUpper_upper t = p (TL.toUpper t) >= p t

328 where p = TL.length . TL.filter isUpper

365 where p = TL.length . TL.filter isUpper

366+t_toTitle_title t = all (<= 1) (caps w)

367+ where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle

368+ -- TIL: there exist uppercase-only letters

369+ w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t

370+t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable

371+ where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words

372+ -- Surprise! The Spanish/Portuguese ordinal indicators changed

373+ -- from category Ll (letter, lowercase) to Lo (letter, other)

374+ -- in Unicode 7.0

375+ -- Oh, and there exist lowercase-only letters (see previous test)

376+ stable c = if isLower c

377+ then C.toUpper c /= c

378+ else c /= '\170' && c /= '\186'

329

379

330justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

380justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

331justifyRight m n xs = L.replicate (m - length xs) n ++ xs

381justifyRight m n xs = L.replicate (m - length xs) n ++ xs

@@ -514,12 +564,20 @@ s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p)

514s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

564s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

515sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

565sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

516 (unpackS . S.takeWhile p . S.filter q)

566 (unpackS . S.takeWhile p . S.filter q)

567+noMatch = do

568+ c <- char

569+ d <- suchThat char (/= c)

570+ return (c,d)

517t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

571t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

518tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

572tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

519t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

573t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

520 (unpackS . T.takeWhileEnd p)

574 (unpackS . T.takeWhileEnd p)

575+t_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> T.null $

576+ T.takeWhileEnd (==d) (T.snoc t c)

521tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

577tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

522 (unpackS . TL.takeWhileEnd p)

578 (unpackS . TL.takeWhileEnd p)

579+tl_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> TL.null $

580+ TL.takeWhileEnd (==d) (TL.snoc t c)

523s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

581s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

524s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

582s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

525sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

583sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

@@ -720,7 +778,7 @@ tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s

720 conc = T.concat . TL.toChunks

778 conc = T.concat . TL.toChunks

721t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

779t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

722 let s = T.intercalate t ts

780 let s = T.intercalate t ts

723− in Slow.indices t s == indices t s

781+ in Slow.indices t s === indices t s

724

782

725-- Bit shifts.

783-- Bit shifts.

726shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

784shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

@@ -812,18 +870,18 @@ tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a

812-- Reading.

870-- Reading.

813

871

814t_decimal (n::Int) s =

872t_decimal (n::Int) s =

815− T.signed T.decimal (T.pack (show n) `T.append` t) == Right (n,t)

873+ T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)

816 where t = T.dropWhile isDigit s

874 where t = T.dropWhile isDigit s

817tl_decimal (n::Int) s =

875tl_decimal (n::Int) s =

818− TL.signed TL.decimal (TL.pack (show n) `TL.append` t) == Right (n,t)

876+ TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)

819 where t = TL.dropWhile isDigit s

877 where t = TL.dropWhile isDigit s

820t_hexadecimal m s ox =

878t_hexadecimal m s ox =

821− T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) == Right (n,t)

879+ T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)

822 where t = T.dropWhile isHexDigit s

880 where t = T.dropWhile isHexDigit s

823 p = if ox then "0x" else ""

881 p = if ox then "0x" else ""

824 n = getPositive m :: Int

882 n = getPositive m :: Int

825tl_hexadecimal m s ox =

883tl_hexadecimal m s ox =

826− TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) == Right (n,t)

884+ TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)

827 where t = TL.dropWhile isHexDigit s

885 where t = TL.dropWhile isHexDigit s

828 p = if ox then "0x" else ""

886 p = if ox then "0x" else ""

829 n = getPositive m :: Int

887 n = getPositive m :: Int

@@ -867,11 +925,11 @@ tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn

867

925

868t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

926t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

869t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

927t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

870−t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) == t

928+t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t

871 where n = small m

929 where n = small m

872t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

930t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

873

931

874−t_copy t = T.copy t == t

932+t_copy t = T.copy t === t

875

933

876-- Regression tests.

934-- Regression tests.

877s_filter_eq s = S.filter p t == S.streamList (filter p s)

935s_filter_eq s = S.filter p t == S.streamList (filter p s)

@@ -924,6 +982,15 @@ tests =

924 testGroup "errors" [

982 testGroup "errors" [

925 testProperty "t_utf8_err" t_utf8_err,

983 testProperty "t_utf8_err" t_utf8_err,

926 testProperty "t_utf8_err'" t_utf8_err'

984 testProperty "t_utf8_err'" t_utf8_err'

985+ ],

986+ testGroup "error recovery" [

987+ testProperty "t_decode_with_error2" t_decode_with_error2,

988+ testProperty "t_decode_with_error3" t_decode_with_error3,

989+ testProperty "t_decode_with_error4" t_decode_with_error4,

990+ testProperty "t_decode_with_error2'" t_decode_with_error2',

991+ testProperty "t_decode_with_error3'" t_decode_with_error3',

992+ testProperty "t_decode_with_error4'" t_decode_with_error4',

993+ testProperty "t_infix_concat" t_infix_concat

927 ]

994 ]

928 ],

995 ],

929

996

@@ -967,6 +1034,8 @@ tests =

967 testProperty "sf_uncons" sf_uncons,

1034 testProperty "sf_uncons" sf_uncons,

968 testProperty "t_uncons" t_uncons,

1035 testProperty "t_uncons" t_uncons,

969 testProperty "tl_uncons" tl_uncons,

1036 testProperty "tl_uncons" tl_uncons,

1037+ testProperty "t_unsnoc" t_unsnoc,

1038+ testProperty "tl_unsnoc" tl_unsnoc,

970 testProperty "s_head" s_head,

1039 testProperty "s_head" s_head,

971 testProperty "sf_head" sf_head,

1040 testProperty "sf_head" sf_head,

972 testProperty "t_head" t_head,

1041 testProperty "t_head" t_head,

@@ -1030,7 +1099,9 @@ tests =

1030 testProperty "tl_toLower_lower" tl_toLower_lower,

1099 testProperty "tl_toLower_lower" tl_toLower_lower,

1031 testProperty "t_toUpper_length" t_toUpper_length,

1100 testProperty "t_toUpper_length" t_toUpper_length,

1032 testProperty "t_toUpper_upper" t_toUpper_upper,

1101 testProperty "t_toUpper_upper" t_toUpper_upper,

1033− testProperty "tl_toUpper_upper" tl_toUpper_upper

1102+ testProperty "tl_toUpper_upper" tl_toUpper_upper,

1103+ testProperty "t_toTitle_title" t_toTitle_title,

1104+ testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower

1034 ],

1105 ],

1035

1106

1036 testGroup "justification" [

1107 testGroup "justification" [

@@ -1147,7 +1218,9 @@ tests =

1147 testProperty "t_takeWhile" t_takeWhile,

1218 testProperty "t_takeWhile" t_takeWhile,

1148 testProperty "tl_takeWhile" tl_takeWhile,

1219 testProperty "tl_takeWhile" tl_takeWhile,

1149 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1220 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1221+ testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null,

1150 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1222 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1223+ testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null,

1151 testProperty "sf_dropWhile" sf_dropWhile,

1224 testProperty "sf_dropWhile" sf_dropWhile,

1152 testProperty "s_dropWhile" s_dropWhile,

1225 testProperty "s_dropWhile" s_dropWhile,

1153 testProperty "s_dropWhile_s" s_dropWhile_s,

1226 testProperty "s_dropWhile_s" s_dropWhile_s,

tests/Tests/QuickCheckUtils.hs+1−1

@@ -106,7 +106,7 @@ smallArbitrary = sized $ \n -> resize (smallish n) arbitrary

106 where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs

107

108instance Arbitrary T.Text where

109− arbitrary = T.pack `fmap` arbitrary

109+ arbitrary = T.pack `fmap` string

110 shrink = map T.pack . shrink . T.unpack

111

112instance Arbitrary TL.Text where

tests/Tests/Regressions.hs+11

@@ -72,6 +72,16 @@ mapAccumL_resize = do

72 assertEqual "mapAccumL should correctly size buffers for two-word results"

73 (count * 2) (T.lengthWord16 (snd val))

74

75+-- See GitHub #197

76+t197 :: IO ()

77+t197 =

78+ assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")

79+ where

80+ currencyParser x = cond == 1

81+ where

82+ cond = length fltr

83+ fltr = filter (== ',') x

84+

75tests :: F.Test

85tests :: F.Test

76tests = F.testGroup "Regressions"

86tests = F.testGroup "Regressions"

77 [ F.testCase "hGetContents_crash" hGetContents_crash

87 [ F.testCase "hGetContents_crash" hGetContents_crash

@@ -79,4 +89,5 @@ tests = F.testGroup "Regressions"

79 , F.testCase "mapAccumL_resize" mapAccumL_resize

89 , F.testCase "mapAccumL_resize" mapAccumL_resize

80 , F.testCase "replicate_crash" replicate_crash

90 , F.testCase "replicate_crash" replicate_crash

81 , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe

91 , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe

92+ , F.testCase "t197" t197

82 ]

93 ]

tests/text-tests.cabal+18−3

@@ -20,6 +20,11 @@ flag hpc

20 default: False

21 manual: True

22

23+flag bytestring-builder

24+ description: Depend on the bytestring-builder package for backwards compatibility.

25+ default: False

26+ manual: False

27+

23executable text-tests

28executable text-tests

24 main-is: Tests.hs

29 main-is: Tests.hs

25

30

@@ -48,16 +53,21 @@ executable text-tests

48 HUnit >= 1.2,

53 HUnit >= 1.2,

49 QuickCheck >= 2.7,

54 QuickCheck >= 2.7,

50 base == 4.*,

55 base == 4.*,

51− bytestring,

52 deepseq,

56 deepseq,

53 directory,

57 directory,

54− quickcheck-unicode,

58+ quickcheck-unicode >= 1.0.1.0,

55 random,

59 random,

56 test-framework >= 0.4,

60 test-framework >= 0.4,

57 test-framework-hunit >= 0.2,

61 test-framework-hunit >= 0.2,

58 test-framework-quickcheck2 >= 0.2,

62 test-framework-quickcheck2 >= 0.2,

59 text-tests

63 text-tests

60

64

65+ if flag(bytestring-builder)

66+ build-depends: bytestring >= 0.9 && < 0.10.4,

67+ bytestring-builder >= 0.10.4

68+ else

69+ build-depends: bytestring >= 0.10.4

70+

61executable text-tests-stdio

71executable text-tests-stdio

62 main-is: Tests/IO.hs

72 main-is: Tests/IO.hs

63

73

@@ -137,7 +147,12 @@ library

137 array,

147 array,

138 base == 4.*,

148 base == 4.*,

139 binary,

149 binary,

140− bytestring,

141 deepseq,

150 deepseq,

142 ghc-prim,

151 ghc-prim,

143 integer-gmp

152 integer-gmp

153+

154+ if flag(bytestring-builder)

155+ build-depends: bytestring >= 0.9 && < 0.10.4,

156+ bytestring-builder >= 0.10.4

157+ else

158+ build-depends: bytestring >= 0.10.4

text.cabal+100−33

@@ -1,7 +1,7 @@

1name: text

2−version: 1.2.2.1

2+version: 1.2.3.0

3−homepage: https://github.com/bos/text

3+homepage: https://github.com/haskell/text

4−bug-reports: https://github.com/bos/text/issues

4+bug-reports: https://github.com/haskell/text/issues

5synopsis: An efficient packed Unicode text type.

6description:

7 .

@@ -14,11 +14,13 @@ description:

14 in terms of large data quantities and high speed.

15 .

16 The 'Text' type provides character-encoding, type-safe case

17− conversion via whole-string case conversion functions. It also

17+ conversion via whole-string case conversion functions (see "Data.Text").

18− provides a range of functions for converting 'Text' values to and from

18+ It also provides a range of functions for converting 'Text' values to

19− 'ByteStrings', using several standard encodings.

19+ and from 'ByteStrings', using several standard encodings

20+ (see "Data.Text.Encoding").

20 .

21 .

21− Efficient locale-sensitive support for text IO is also supported.

22+ Efficient locale-sensitive support for text IO is also supported

23+ (see "Data.Text.IO").

22 .

24 .

23 These modules are intended to be imported qualified, to avoid name

25 These modules are intended to be imported qualified, to avoid name

24 clashes with Prelude functions, e.g.

26 clashes with Prelude functions, e.g.

@@ -28,8 +30,9 @@ description:

28 To use an extended and very rich family of functions for working

30 To use an extended and very rich family of functions for working

29 with Unicode text (including normalization, regular expressions,

31 with Unicode text (including normalization, regular expressions,

30 non-standard encodings, text breaking, and locales), see

32 non-standard encodings, text breaking, and locales), see

31− the @text-icu@ package:

32− <http://hackage.haskell.org/package/text-icu>

33+ the [text-icu package](https://hackage.haskell.org/package/text-icu)

34+ based on the well-respected and liberally

35+ licensed [ICU library](http://site.icu-project.org/).

33

36

34license: BSD2

37license: BSD2

35license-file: LICENSE

38license-file: LICENSE

39category: Data, Text

42category: Data, Text

40build-type: Simple

43build-type: Simple

41cabal-version: >= 1.8

44cabal-version: >= 1.8

45+tested-with: GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4,

46+ GHC==7.6.3, GHC==7.4.2, GHC==7.2.2, GHC==7.0.4

42extra-source-files:

47extra-source-files:

43 -- scripts/CaseFolding.txt

48 -- scripts/CaseFolding.txt

44 -- scripts/SpecialCasing.txt

49 -- scripts/SpecialCasing.txt

@@ -65,7 +70,9 @@ extra-source-files:

65 tests/text-tests.cabal

70 tests/text-tests.cabal

66

71

67flag bytestring-builder

72flag bytestring-builder

73+ description:

74+ Depend on the [bytestring-builder](https://hackage.haskell.org/package/bytestring-builder)

68− description: Depend on the bytestring-builder package for backwards compatibility.

75+ package for backwards compatibility.

69 default: False

76 default: False

70 manual: False

77 manual: False

71

78

@@ -75,7 +82,9 @@ flag developer

75 manual: True

82 manual: True

76

83

77flag integer-simple

84flag integer-simple

85+ description:

78− description: Use the simple integer library instead of GMP

86+ Use the [simple integer library](http://hackage.haskell.org/package/integer-simple)

87+ instead of [integer-gmp](http://hackage.haskell.org/package/integer-gmp)

79 default: False

88 default: False

80 manual: False

89 manual: False

81

90

@@ -88,13 +97,13 @@ library

88 Data.Text.Array

97 Data.Text.Array

89 Data.Text.Encoding

98 Data.Text.Encoding

90 Data.Text.Encoding.Error

99 Data.Text.Encoding.Error

91− -- Data.Text.Foreign

100+ Data.Text.Foreign

92 Data.Text.IO

101 Data.Text.IO

93 Data.Text.Internal

102 Data.Text.Internal

94 Data.Text.Internal.Builder

103 Data.Text.Internal.Builder

95− -- Data.Text.Internal.Builder.Functions

104+ Data.Text.Internal.Builder.Functions

96− -- Data.Text.Internal.Builder.Int.Digits

105+ Data.Text.Internal.Builder.Int.Digits

97− -- Data.Text.Internal.Builder.RealFloat.Functions

106+ Data.Text.Internal.Builder.RealFloat.Functions

98 Data.Text.Internal.Encoding.Fusion

107 Data.Text.Internal.Encoding.Fusion

99 Data.Text.Internal.Encoding.Fusion.Common

108 Data.Text.Internal.Encoding.Fusion.Common

100 Data.Text.Internal.Encoding.Utf16

109 Data.Text.Internal.Encoding.Utf16

@@ -106,14 +115,14 @@ library

106 Data.Text.Internal.Fusion.Common

115 Data.Text.Internal.Fusion.Common

107 Data.Text.Internal.Fusion.Size

116 Data.Text.Internal.Fusion.Size

108 Data.Text.Internal.Fusion.Types

117 Data.Text.Internal.Fusion.Types

109− -- Data.Text.Internal.IO

118+ Data.Text.Internal.IO

110 Data.Text.Internal.Lazy

119 Data.Text.Internal.Lazy

111− -- Data.Text.Internal.Lazy.Encoding.Fusion

120+ Data.Text.Internal.Lazy.Encoding.Fusion

112 Data.Text.Internal.Lazy.Fusion

121 Data.Text.Internal.Lazy.Fusion

113 Data.Text.Internal.Lazy.Search

122 Data.Text.Internal.Lazy.Search

114 Data.Text.Internal.Private

123 Data.Text.Internal.Private

115 Data.Text.Internal.Read

124 Data.Text.Internal.Read

116− -- Data.Text.Internal.Search

125+ Data.Text.Internal.Search

117 Data.Text.Internal.Unsafe

126 Data.Text.Internal.Unsafe

118 Data.Text.Internal.Unsafe.Char

127 Data.Text.Internal.Unsafe.Char

119 Data.Text.Internal.Unsafe.Shift

128 Data.Text.Internal.Unsafe.Shift

@@ -124,23 +133,18 @@ library

124 Data.Text.Lazy.Encoding

133 Data.Text.Lazy.Encoding

125 Data.Text.Lazy.IO

134 Data.Text.Lazy.IO

126 Data.Text.Lazy.Internal

135 Data.Text.Lazy.Internal

127− -- Data.Text.Lazy.Read

136+ Data.Text.Lazy.Read

128 Data.Text.Read

137 Data.Text.Read

129 Data.Text.Unsafe

138 Data.Text.Unsafe

130

139

131− -- moved from ghcjs-base

132− -- reflex-dom depends in it

133− Data.JSString.Text

134− -- other-modules:

140+ other-modules:

135− -- Data.Text.Show

141+ Data.Text.Show

136

142

137 build-depends:

143 build-depends:

138− -- array >= 0.3,

144+ array >= 0.3,

139 base >= 4.2 && < 5,

145 base >= 4.2 && < 5,

140 binary,

146 binary,

141 deepseq >= 1.1.0.0,

147 deepseq >= 1.1.0.0,

142− ghcjs-base,

143− ghcjs-prim,

144 ghc-prim >= 0.2

148 ghc-prim >= 0.2

145

149

146 if flag(bytestring-builder)

150 if flag(bytestring-builder)

@@ -165,27 +169,90 @@ library

165

169

166-- test-suite tests

170-- test-suite tests

167-- type: exitcode-stdio-1.0

171-- type: exitcode-stdio-1.0

168−-- hs-source-dirs: tests .

169−-- main-is: Tests.hs

170-- c-sources: cbits/cbits.c

172-- c-sources: cbits/cbits.c

171-- include-dirs: include

173-- include-dirs: include

172

174

173-- ghc-options:

175-- ghc-options:

174−-- -Wall -threaded -O0 -rtsopts

176+-- -Wall -threaded -rtsopts

175

177

176-- cpp-options:

178-- cpp-options:

177-- -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE

179-- -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE

178

180

181+-- -- modules specific to test-suite

182+-- hs-source-dirs: tests

183+-- main-is: Tests.hs

184+-- other-modules:

185+-- Tests.Properties

186+-- Tests.Properties.Mul

187+-- Tests.QuickCheckUtils

188+-- Tests.Regressions

189+-- Tests.SlowFunctions

190+-- Tests.Utils

191+

192+-- -- Same as in `library` stanza; this is needed by cabal for accurate

193+-- -- file-monitoring as well as to avoid `-Wmissing-home-modules`

194+-- -- warnings We can't use an inter-package library dependency because

195+-- -- of different `ghc-options`/`cpp-options` (as a side-benefitt,

196+-- -- this enables per-component build parallelism in `cabal

197+-- -- new-build`!); We could, however, use cabal-version:2.2's `common`

198+-- -- blocks at some point in the future to reduce the duplication.

199+-- hs-source-dirs: .

200+-- other-modules:

201+-- Data.Text

202+-- Data.Text.Array

203+-- Data.Text.Encoding

204+-- Data.Text.Encoding.Error

205+-- Data.Text.Foreign

206+-- Data.Text.IO

207+-- Data.Text.Internal

208+-- Data.Text.Internal.Builder

209+-- Data.Text.Internal.Builder.Functions

210+-- Data.Text.Internal.Builder.Int.Digits

211+-- Data.Text.Internal.Builder.RealFloat.Functions

212+-- Data.Text.Internal.Encoding.Fusion

213+-- Data.Text.Internal.Encoding.Fusion.Common

214+-- Data.Text.Internal.Encoding.Utf16

215+-- Data.Text.Internal.Encoding.Utf32

216+-- Data.Text.Internal.Encoding.Utf8

217+-- Data.Text.Internal.Functions

218+-- Data.Text.Internal.Fusion

219+-- Data.Text.Internal.Fusion.CaseMapping

220+-- Data.Text.Internal.Fusion.Common

221+-- Data.Text.Internal.Fusion.Size

222+-- Data.Text.Internal.Fusion.Types

223+-- Data.Text.Internal.IO

224+-- Data.Text.Internal.Lazy

225+-- Data.Text.Internal.Lazy.Encoding.Fusion

226+-- Data.Text.Internal.Lazy.Fusion

227+-- Data.Text.Internal.Lazy.Search

228+-- Data.Text.Internal.Private

229+-- Data.Text.Internal.Read

230+-- Data.Text.Internal.Search

231+-- Data.Text.Internal.Unsafe

232+-- Data.Text.Internal.Unsafe.Char

233+-- Data.Text.Internal.Unsafe.Shift

234+-- Data.Text.Lazy

235+-- Data.Text.Lazy.Builder

236+-- Data.Text.Lazy.Builder.Int

237+-- Data.Text.Lazy.Builder.RealFloat

238+-- Data.Text.Lazy.Encoding

239+-- Data.Text.Lazy.IO

240+-- Data.Text.Lazy.Internal

241+-- Data.Text.Lazy.Read

242+-- Data.Text.Read

243+-- Data.Text.Unsafe

244+-- Data.Text.Show

245+

179-- build-depends:

246-- build-depends:

180-- HUnit >= 1.2,

247-- HUnit >= 1.2,

181−-- QuickCheck >= 2.7,

248+-- QuickCheck >= 2.7 && < 2.10,

182-- array,

249-- array,

183-- base,

250-- base,

184-- binary,

251-- binary,

185-- deepseq,

252-- deepseq,

186-- directory,

253-- directory,

187-- ghc-prim,

254-- ghc-prim,

188−-- quickcheck-unicode,

255+-- quickcheck-unicode >= 1.0.1.0,

189-- random,

256-- random,

190-- test-framework >= 0.4,

257-- test-framework >= 0.4,

191-- test-framework-hunit >= 0.2,

258-- test-framework-hunit >= 0.2,

@@ -206,7 +273,7 @@ library

206

273

207-- source-repository head

274-- source-repository head

208-- type: git

275-- type: git

209−-- location: https://github.com/bos/text

276+-- location: https://github.com/haskell/text

210

277

211-- source-repository head

278-- source-repository head

212-- type: mercurial

279-- type: mercurial