a02c2da..master · text

.gitignore+3−10

since splitsnapshots

unifiedsplit

@@ -1,10 +1,3 @@

1−/dist/

1+dist

2−/dist-boot/

3−/dist-install/

4−/dist-newstyle/

5−/cabal-dev/

2+cabal-dev

6−/cabal.sandbox.config

3+cabal.sandbox.config

7−/ghc.mk

8−/GNUmakefile

9−/.ghc.environment.*

10−/cabal.project.local

.hgtags−1

@@ -64,4 +64,3 @@ f8a60279237c1c28e094868740d9188dd94fcc9e 1.2.0.0

64631fd055e27fa10983e17e9ad36d1b285b85c661 1.2.1.3

658221a68ed5ec37028357df9c6c25f52320e54ad9 1.2.2.0

66ce1c5ac65eb9f5580114b4e380092b05a485e95d 1.2.2.1

67−d72c634a91ffe27ec5cd03921dd53efc8129d8f6 1.2.2.2

.travis.ymldeleted−111

@@ -1,111 +0,0 @@

1−# This Travis job script has been generated by a script via

2−#

3−# make_travis_yml_2.hs '-o' '.travis.yml' 'text.cabal'

4−#

5−# For more information, see https://github.com/hvr/multi-ghc-travis

6−#

7−language: c

8−sudo: false

9−

10−# Bryan threatens to turn off Travis-CI if he gets any email :-)

11−notifications:

12− email: false

13−

14−cache:

15− directories:

16− - $HOME/.cabal/packages

17− - $HOME/.cabal/store

18−

19−before_cache:

20− - rm -fv $HOME/.cabal/packages/hackage.haskell.org/build-reports.log

21− # remove files that are regenerated by 'cabal update'

22− - rm -fv $HOME/.cabal/packages/hackage.haskell.org/00-index.*

23− - rm -fv $HOME/.cabal/packages/hackage.haskell.org/*.json

24− - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.cache

25− - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.tar

26− - rm -fv $HOME/.cabal/packages/hackage.haskell.org/01-index.tar.idx

27−

28−matrix:

29− include:

30− - compiler: "ghc-7.0.4"

31− # env: TEST=--disable-tests BENCH=--disable-benchmarks

32− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.0.4], sources: [hvr-ghc]}}

33− - compiler: "ghc-7.2.2"

34− # env: TEST=--disable-tests BENCH=--disable-benchmarks

35− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.2.2], sources: [hvr-ghc]}}

36− - compiler: "ghc-7.4.2"

37− # env: TEST=--disable-tests BENCH=--disable-benchmarks

38− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.4.2], sources: [hvr-ghc]}}

39− - compiler: "ghc-7.6.3"

40− # env: TEST=--disable-tests BENCH=--disable-benchmarks

41− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.6.3], sources: [hvr-ghc]}}

42− - compiler: "ghc-7.8.4"

43− # env: TEST=--disable-tests BENCH=--disable-benchmarks

44− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.8.4], sources: [hvr-ghc]}}

45− - compiler: "ghc-7.10.3"

46− # env: TEST=--disable-tests BENCH=--disable-benchmarks

47− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-7.10.3], sources: [hvr-ghc]}}

48− - compiler: "ghc-8.0.2"

49− # env: TEST=--disable-tests BENCH=--disable-benchmarks

50− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-8.0.2], sources: [hvr-ghc]}}

51− - compiler: "ghc-8.2.2"

52− # env: TEST=--disable-tests BENCH=--disable-benchmarks

53− addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-8.2.2], sources: [hvr-ghc]}}

54−

55−before_install:

56− - HC=${CC}

57− - HCPKG=${HC/ghc/ghc-pkg}

58− - unset CC

59− - PATH=/opt/ghc/bin:/opt/ghc-ppa-tools/bin:$PATH

60− - PKGNAME='text'

61−

62−install:

63− - cabal --version

64− - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"

65− - BENCH=${BENCH---enable-benchmarks}

66− - TEST=${TEST---enable-tests}

67− - HADDOCK=${HADDOCK-true}

68− - INSTALLED=${INSTALLED-true}

69− - travis_retry cabal update -v

70− - sed -i.bak 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config

71− - rm -fv cabal.project.local

72− - "echo 'packages: .' > cabal.project"

73− - rm -f cabal.project.freeze

74− - cabal new-build -w ${HC} ${TEST} ${BENCH} --dep -j2 all

75− - cabal new-build -w ${HC} --disable-tests --disable-benchmarks --dep -j2 all

76−

77−# Here starts the actual work to be performed for the package under test;

78−# any command which exits with a non-zero exit code causes the build to fail.

79−script:

80− - if [ -f configure.ac ]; then autoreconf -i; fi

81− - rm -rf .ghc.environment.* dist/

82− - cabal sdist # test that a source-distribution can be generated

83− - cd dist/

84− - SRCTAR=(${PKGNAME}-*.tar.gz)

85− - SRC_BASENAME="${SRCTAR/%.tar.gz}"

86− - tar -xvf "./$SRC_BASENAME.tar.gz"

87− - cd "$SRC_BASENAME/"

88−## from here on, CWD is inside the extracted source-tarball

89− - rm -fv cabal.project.local

90− - "echo 'packages: .' > cabal.project"

91− # this builds all libraries and executables (without tests/benchmarks)

92− - rm -f cabal.project.freeze

93− - cabal new-build -w ${HC} --disable-tests --disable-benchmarks all

94− # this builds all libraries and executables (including tests/benchmarks)

95− # - rm -rf ./dist-newstyle

96−

97− # Build with installed constraints for packages in global-db

98− - if $INSTALLED; then

99− echo cabal new-build -w ${HC} --disable-tests --disable-benchmarks $(${HCPKG} list --global --simple-output --names-only | sed 's/$[a-zA-Z0-9-]\{1,\}$ */--constraint="\1 installed" /g') all | sh;

100− else echo "Not building with installed constraints"; fi

101−

102− # build & run tests, build benchmarks

103− - cabal new-build -w ${HC} ${TEST} ${BENCH} all

104− - if [ "x$TEST" = "x--enable-tests" ]; then cabal new-test -w ${HC} ${TEST} all; fi

105−

106− # haddock

107− - rm -rf ./dist-newstyle

108− - if $HADDOCK; then cabal new-haddock -w ${HC} --disable-tests --disable-benchmarks all; else echo "Skipping haddock generation";fi

109−

110−# REGENDATA ["-o",".travis.yml","text.cabal"]

111−# EOF

Data/Text.hs+46−177

@@ -15,6 +15,7 @@

15--

16-- License : BSD-style

17-- Maintainer : bos@serpentine.com

18+-- Stability : experimental

18-- Portability : GHC

19-- Portability : GHC

19--

20--

20-- A time and space-efficient implementation of Unicode text.

21-- A time and space-efficient implementation of Unicode text.

@@ -31,9 +32,8 @@

31--

32--

32-- To use an extended and very rich family of functions for working

33-- To use an extended and very rich family of functions for working

33-- with Unicode text (including normalization, regular expressions,

34-- with Unicode text (including normalization, regular expressions,

34−-- non-standard encodings, text breaking, and locales), see the

35+-- non-standard encodings, text breaking, and locales), see

35−-- <http://hackage.haskell.org/package/text-icu text-icu package >.

36+-- <http://hackage.haskell.org/package/text-icu the text-icu package >.

36−--

37

38module Data.Text

39 (

@@ -43,9 +43,6 @@ module Data.Text

43 -- * Acceptable data

44 -- $replacement

45

46− -- * Definition of character

47− -- $character_definition

48−

49 -- * Fusion

46 -- * Fusion

50 -- $fusion

47 -- $fusion

51

48

@@ -63,7 +60,6 @@ module Data.Text

63 , snoc

60 , snoc

64 , append

61 , append

65 , uncons

62 , uncons

66− , unsnoc

67 , head

63 , head

68 , last

64 , last

69 , tail

65 , tail

@@ -250,18 +246,6 @@ import qualified GHC.Exts as Exts

250import Text.Printf (PrintfArg, formatArg, formatString)

246import Text.Printf (PrintfArg, formatArg, formatString)

251#endif

247#endif

252

248

253−-- $character_definition

254−--

255−-- This package uses the term /character/ to denote Unicode /code points/.

256−--

257−-- Note that this is not the same thing as a grapheme (e.g. a

258−-- composition of code points that form one visual symbol). For

259−-- instance, consider the grapheme \"ä\". This symbol has two

260−-- Unicode representations: a single code-point representation

261−-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),

262−-- and a two code point representation @U+0061@ (the \"@A@\" code

263−-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).

264−

265-- $strict

249-- $strict

266--

250--

267-- This package provides both strict and lazy 'Text' types. The

251-- This package provides both strict and lazy 'Text' types. The

@@ -350,13 +334,9 @@ instance Read Text where

350 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

334 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

351

335

352#if MIN_VERSION_base(4,9,0)

336#if MIN_VERSION_base(4,9,0)

353−-- | Non-orphan 'Semigroup' instance only defined for

354−-- @base-4.9.0.0@ and later; orphan instances for older GHCs are

337+-- Semigroup orphan instances for older GHCs are provided by

355−-- provided by

356−-- the [semigroups](http://hackage.haskell.org/package/semigroups)

357−-- package

338+-- 'semigroups` package

358−--

339+

359−-- @since 1.2.2.0

360instance Semigroup Text where

340instance Semigroup Text where

361 (<>) = append

341 (<>) = append

362#endif

342#endif

@@ -374,7 +354,6 @@ instance IsString Text where

374 fromString = pack

354 fromString = pack

375

355

376#if __GLASGOW_HASKELL__ >= 708

356#if __GLASGOW_HASKELL__ >= 708

377−-- | @since 1.2.0.0

378instance Exts.IsList Text where

357instance Exts.IsList Text where

379 type Item Text = Char

358 type Item Text = Char

380 fromList = pack

359 fromList = pack

@@ -385,7 +364,6 @@ instance Exts.IsList Text where

385instance NFData Text where rnf !_ = ()

364instance NFData Text where rnf !_ = ()

386#endif

365#endif

387

366

388−-- | @since 1.2.1.0

389instance Binary Text where

367instance Binary Text where

390 put t = put (encodeUtf8 t)

368 put t = put (encodeUtf8 t)

391 get = do

369 get = do

@@ -419,8 +397,6 @@ instance Data Text where

419

397

420#if MIN_VERSION_base(4,7,0)

398#if MIN_VERSION_base(4,7,0)

421-- | Only defined for @base-4.7.0.0@ and later

399-- | Only defined for @base-4.7.0.0@ and later

422−--

423−-- @since 1.2.2.0

424instance PrintfArg Text where

400instance PrintfArg Text where

425 formatArg txt = formatString $ unpack txt

401 formatArg txt = formatString $ unpack txt

426#endif

402#endif

@@ -569,19 +545,6 @@ init (Text arr off len) | len <= 0 = emptyError "init"

569 unstream (S.init (stream t)) = init t

545 unstream (S.init (stream t)) = init t

570 #-}

546 #-}

571

547

572−-- | /O(1)/ Returns all but the last character and the last character of a

573−-- 'Text', or 'Nothing' if empty.

574−--

575−-- @since 1.2.3.0

576−unsnoc :: Text -> Maybe (Text, Char)

577−unsnoc (Text arr off len)

578− | len <= 0 = Nothing

579− | n < 0xDC00 || n > 0xDFFF = Just (text arr off (len-1), unsafeChr n)

580− | otherwise = Just (text arr off (len-2), U16.chr2 n0 n)

581− where n = A.unsafeIndex arr (off+len-1)

582− n0 = A.unsafeIndex arr (off+len-2)

583−{-# INLINE [1] unsnoc #-}

584−

585-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

548-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

586-- fusion.

549-- fusion.

587null :: Text -> Bool

550null :: Text -> Bool

@@ -661,15 +624,8 @@ compareLength t n = S.compareLengthI (stream t) n

661-- -----------------------------------------------------------------------------

624-- -----------------------------------------------------------------------------

662-- * Transformations

625-- * Transformations

663-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

626-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to

664−-- each element of @t@.

665−--

666−-- Example:

667−--

668−-- >>> let message = pack "I am not angry. Not at all."

669−-- >>> T.map (\c -> if c == '.' then '!' else c) message

670−-- "I am not angry! Not at all!"

671−--

672−-- Subject to fusion. Performs replacement on invalid scalar values.

627+-- each element of @t@. Subject to fusion. Performs replacement on

628+-- invalid scalar values.

673map :: (Char -> Char) -> Text -> Text

629map :: (Char -> Char) -> Text -> Text

674map f t = unstream (S.map (safe . f) (stream t))

630map f t = unstream (S.map (safe . f) (stream t))

675{-# INLINE [1] map #-}

631{-# INLINE [1] map #-}

@@ -677,36 +633,18 @@ map f t = unstream (S.map (safe . f) (stream t))

677-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

633-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of

678-- 'Text's and concatenates the list after interspersing the first

634-- 'Text's and concatenates the list after interspersing the first

679-- argument between each element of the list.

635-- argument between each element of the list.

680−--

681−-- Example:

682−--

683−-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]

684−-- "WeNI!seekNI!theNI!HolyNI!Grail"

685intercalate :: Text -> [Text] -> Text

636intercalate :: Text -> [Text] -> Text

686intercalate t = concat . (F.intersperse t)

637intercalate t = concat . (F.intersperse t)

687{-# INLINE intercalate #-}

638{-# INLINE intercalate #-}

688

639

689-- | /O(n)/ The 'intersperse' function takes a character and places it

640-- | /O(n)/ The 'intersperse' function takes a character and places it

690−-- between the characters of a 'Text'.

641+-- between the characters of a 'Text'. Subject to fusion. Performs

691−--

692−-- Example:

693−--

694−-- >>> T.intersperse '.' "SHIELD"

695−-- "S.H.I.E.L.D"

696−--

697−-- Subject to fusion. Performs replacement on invalid scalar values.

642+-- replacement on invalid scalar values.

698intersperse :: Char -> Text -> Text

643intersperse :: Char -> Text -> Text

699intersperse c t = unstream (S.intersperse (safe c) (stream t))

644intersperse c t = unstream (S.intersperse (safe c) (stream t))

700{-# INLINE intersperse #-}

645{-# INLINE intersperse #-}

701

646

702−-- | /O(n)/ Reverse the characters of a string.

647+-- | /O(n)/ Reverse the characters of a string. Subject to fusion.

703−--

704−-- Example:

705−--

706−-- >>> T.reverse "desrever"

707−-- "reversed"

708−--

709−-- Subject to fusion.

710reverse :: Text -> Text

648reverse :: Text -> Text

711reverse t = S.reverse (stream t)

649reverse t = S.reverse (stream t)

712{-# INLINE reverse #-}

650{-# INLINE reverse #-}

@@ -725,14 +663,12 @@ reverse t = S.reverse (stream t)

725-- @needle@ occurs in @replacement@, that occurrence will /not/ itself

663-- @needle@ occurs in @replacement@, that occurrence will /not/ itself

726-- be replaced recursively:

664-- be replaced recursively:

727--

665--

728−-- >>> replace "oo" "foo" "oo"

666+-- > replace "oo" "foo" "oo" == "foo"

729−-- "foo"

730--

667--

731-- In cases where several instances of @needle@ overlap, only the

668-- In cases where several instances of @needle@ overlap, only the

732-- first one will be replaced:

669-- first one will be replaced:

733--

670--

734−-- >>> replace "ofo" "bar" "ofofo"

671+-- > replace "ofo" "bar" "ofofo" == "barfo"

735−-- "barfo"

736--

672--

737-- In (unlikely) bad cases, this function's time complexity degrades

673-- In (unlikely) bad cases, this function's time complexity degrades

738-- towards /O(n*m)/.

674-- towards /O(n*m)/.

@@ -846,8 +782,6 @@ toUpper t = unstream (S.toUpper (stream t))

846-- guides disagree on whether the book name \"The Hill of the Red

782-- guides disagree on whether the book name \"The Hill of the Red

847-- Fox\" is correctly title cased—but this function will

783-- Fox\" is correctly title cased—but this function will

848-- capitalize /every/ word.

784-- capitalize /every/ word.

849−--

850−-- @since 1.0.0.0

851toTitle :: Text -> Text

785toTitle :: Text -> Text

852toTitle t = unstream (S.toTitle (stream t))

786toTitle t = unstream (S.toTitle (stream t))

853{-# INLINE toTitle #-}

787{-# INLINE toTitle #-}

@@ -858,11 +792,8 @@ toTitle t = unstream (S.toTitle (stream t))

858--

792--

859-- Examples:

793-- Examples:

860--

794--

861−-- >>> justifyLeft 7 'x' "foo"

795+-- > justifyLeft 7 'x' "foo" == "fooxxxx"

862−-- "fooxxxx"

863−--

864−-- >>> justifyLeft 3 'x' "foobar"

796+-- > justifyLeft 3 'x' "foobar" == "foobar"

865−-- "foobar"

866justifyLeft :: Int -> Char -> Text -> Text

797justifyLeft :: Int -> Char -> Text -> Text

867justifyLeft k c t

798justifyLeft k c t

868 | len >= k = t

799 | len >= k = t

@@ -883,11 +814,8 @@ justifyLeft k c t

883--

814--

884-- Examples:

815-- Examples:

885--

816--

886−-- >>> justifyRight 7 'x' "bar"

817+-- > justifyRight 7 'x' "bar" == "xxxxbar"

887−-- "xxxxbar"

888−--

889−-- >>> justifyRight 3 'x' "foobar"

818+-- > justifyRight 3 'x' "foobar" == "foobar"

890−-- "foobar"

891justifyRight :: Int -> Char -> Text -> Text

819justifyRight :: Int -> Char -> Text -> Text

892justifyRight k c t

820justifyRight k c t

893 | len >= k = t

821 | len >= k = t

@@ -901,8 +829,7 @@ justifyRight k c t

901--

829--

902-- Examples:

830-- Examples:

903--

831--

904−-- >>> center 8 'x' "HS"

832+-- > center 8 'x' "HS" = "xxxHSxxx"

905−-- "xxxHSxxx"

906center :: Int -> Char -> Text -> Text

833center :: Int -> Char -> Text -> Text

907center k c t

834center k c t

908 | len >= k = t

835 | len >= k = t

@@ -917,14 +844,6 @@ center k c t

917-- of its 'Text' argument. Note that this function uses 'pack',

844-- of its 'Text' argument. Note that this function uses 'pack',

918-- 'unpack', and the list version of transpose, and is thus not very

845-- 'unpack', and the list version of transpose, and is thus not very

919-- efficient.

846-- efficient.

920−--

921−-- Examples:

922−--

923−-- >>> transpose ["green","orange"]

924−-- ["go","rr","ea","en","ng","e"]

925−--

926−-- >>> transpose ["blue","red"]

927−-- ["br","le","ud","e"]

928transpose :: [Text] -> [Text]

847transpose :: [Text] -> [Text]

929transpose ts = P.map pack (L.transpose (P.map unpack ts))

848transpose ts = P.map pack (L.transpose (P.map unpack ts))

930

849

@@ -1169,10 +1088,7 @@ iterN n t@(Text _arr _off len) = loop 0 0

1169--

1088--

1170-- Examples:

1089-- Examples:

1171--

1090--

1172−-- >>> takeEnd 3 "foobar"

1091+-- > takeEnd 3 "foobar" == "bar"

1173−-- "bar"

1174−--

1175−-- @since 1.1.1.0

1176takeEnd :: Int -> Text -> Text

1092takeEnd :: Int -> Text -> Text

1177takeEnd n t@(Text arr off len)

1093takeEnd n t@(Text arr off len)

1178 | n <= 0 = empty

1094 | n <= 0 = empty

@@ -1183,8 +1099,8 @@ takeEnd n t@(Text arr off len)

1183iterNEnd :: Int -> Text -> Int

1099iterNEnd :: Int -> Text -> Int

1184iterNEnd n t@(Text _arr _off len) = loop (len-1) n

1100iterNEnd n t@(Text _arr _off len) = loop (len-1) n

1185 where loop i !m

1101 where loop i !m

1186− | m <= 0 = i+1

1187 | i <= 0 = 0

1102 | i <= 0 = 0

1103+ | m <= 1 = i

1188 | otherwise = loop (i+d) (m-1)

1104 | otherwise = loop (i+d) (m-1)

1189 where d = reverseIter_ t i

1105 where d = reverseIter_ t i

1190

1106

@@ -1211,10 +1127,7 @@ drop n t@(Text arr off len)

1211--

1127--

1212-- Examples:

1128-- Examples:

1213--

1129--

1214−-- >>> dropEnd 3 "foobar"

1130+-- > dropEnd 3 "foobar" == "foo"

1215−-- "foo"

1216−--

1217−-- @since 1.1.1.0

1218dropEnd :: Int -> Text -> Text

1131dropEnd :: Int -> Text -> Text

1219dropEnd n t@(Text arr off len)

1132dropEnd n t@(Text arr off len)

1220 | n <= 0 = t

1133 | n <= 0 = t

@@ -1244,10 +1157,7 @@ takeWhile p t@(Text arr off len) = loop 0

1244-- satisfy @p@. Subject to fusion.

1157-- satisfy @p@. Subject to fusion.

1245-- Examples:

1158-- Examples:

1246--

1159--

1247−-- >>> takeWhileEnd (=='o') "foo"

1160+-- > takeWhileEnd (=='o') "foo" == "oo"

1248−-- "oo"

1249−--

1250−-- @since 1.2.2.0

1251takeWhileEnd :: (Char -> Bool) -> Text -> Text

1161takeWhileEnd :: (Char -> Bool) -> Text -> Text

1252takeWhileEnd p t@(Text arr off len) = loop (len-1) len

1162takeWhileEnd p t@(Text arr off len) = loop (len-1) len

1253 where loop !i !l | l <= 0 = t

1163 where loop !i !l | l <= 0 = t

@@ -1286,8 +1196,7 @@ dropWhile p t@(Text arr off len) = loop 0 0

1286--

1196--

1287-- Examples:

1197-- Examples:

1288--

1198--

1289−-- >>> dropWhileEnd (=='.') "foo..."

1199+-- > dropWhileEnd (=='.') "foo..." == "foo"

1290−-- "foo"

1291dropWhileEnd :: (Char -> Bool) -> Text -> Text

1200dropWhileEnd :: (Char -> Bool) -> Text -> Text

1292dropWhileEnd p t@(Text arr off len) = loop (len-1) len

1201dropWhileEnd p t@(Text arr off len) = loop (len-1) len

1293 where loop !i !l | l <= 0 = empty

1202 where loop !i !l | l <= 0 = empty

@@ -1405,14 +1314,9 @@ tails t | null t = [empty]

1405--

1314--

1406-- Examples:

1315-- Examples:

1407--

1316--

1408−-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"

1317+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1409−-- ["a","b","d","e"]

1410−--

1411−-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa"

1318+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1412−-- ["","X","X","X",""]

1413−--

1414−-- >>> splitOn "x" "x"

1319+-- > splitOn "x" "x" == ["",""]

1415−-- ["",""]

1416--

1320--

1417-- and

1321-- and

1418--

1322--

@@ -1448,11 +1352,8 @@ splitOn pat@(Text _ _ l) src@(Text arr off len)

1448-- resulting components do not contain the separators. Two adjacent

1352-- resulting components do not contain the separators. Two adjacent

1449-- separators result in an empty component in the output. eg.

1353-- separators result in an empty component in the output. eg.

1450--

1354--

1451−-- >>> split (=='a') "aabbaca"

1355+-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1452−-- ["","","bb","c",""]

1453−--

1454−-- >>> split (=='a') ""

1356+-- > split (=='a') "" == [""]

1455−-- [""]

1456split :: (Char -> Bool) -> Text -> [Text]

1357split :: (Char -> Bool) -> Text -> [Text]

1457split _ t@(Text _off _arr 0) = [t]

1358split _ t@(Text _off _arr 0) = [t]

1458split p t = loop t

1359split p t = loop t

@@ -1465,11 +1366,8 @@ split p t = loop t

1465-- element may be shorter than the other chunks, depending on the

1366-- element may be shorter than the other chunks, depending on the

1466-- length of the input. Examples:

1367-- length of the input. Examples:

1467--

1368--

1468−-- >>> chunksOf 3 "foobarbaz"

1369+-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"]

1469−-- ["foo","bar","baz"]

1470−--

1471−-- >>> chunksOf 4 "haskell.org"

1370+-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]

1472−-- ["hask","ell.","org"]

1473chunksOf :: Int -> Text -> [Text]

1371chunksOf :: Int -> Text -> [Text]

1474chunksOf k = go

1372chunksOf k = go

1475 where

1373 where

@@ -1514,11 +1412,8 @@ filter p t = unstream (S.filter p (stream t))

1514--

1412--

1515-- Examples:

1413-- Examples:

1516--

1414--

1517−-- >>> breakOn "::" "a::b::c"

1415+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1518−-- ("a","::b::c")

1519−--

1520−-- >>> breakOn "/" "foobar"

1416+-- > breakOn "/" "foobar" ==> ("foobar", "")

1521−-- ("foobar","")

1522--

1417--

1523-- Laws:

1418-- Laws:

1524--

1419--

@@ -1546,8 +1441,7 @@ breakOn pat src@(Text arr off len)

1546-- up to and including the last match of @needle@. The second is the

1441-- up to and including the last match of @needle@. The second is the

1547-- remainder of @haystack@, following the match.

1442-- remainder of @haystack@, following the match.

1548--

1443--

1549−-- >>> breakOnEnd "::" "a::b::c"

1444+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1550−-- ("a::b::","c")

1551breakOnEnd :: Text -> Text -> (Text, Text)

1445breakOnEnd :: Text -> Text -> (Text, Text)

1552breakOnEnd pat src = (reverse b, reverse a)

1446breakOnEnd pat src = (reverse b, reverse a)

1553 where (a,b) = breakOn (reverse pat) (reverse src)

1447 where (a,b) = breakOn (reverse pat) (reverse src)

@@ -1562,11 +1456,10 @@ breakOnEnd pat src = (reverse b, reverse a)

1562--

1456--

1563-- Examples:

1457-- Examples:

1564--

1458--

1565−-- >>> breakOnAll "::" ""

1459+-- > breakOnAll "::" ""

1566−-- []

1460+-- > ==> []

1567−--

1568−-- >>> breakOnAll "/" "a/b/c/"

1461+-- > breakOnAll "/" "a/b/c/"

1569−-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]

1462+-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]

1570--

1463--

1571-- In (unlikely) bad cases, this function's time complexity degrades

1464-- In (unlikely) bad cases, this function's time complexity degrades

1572-- towards /O(n*m)/.

1465-- towards /O(n*m)/.

@@ -1765,14 +1658,9 @@ isInfixOf needle haystack

1765--

1658--

1766-- Examples:

1659-- Examples:

1767--

1660--

1768−-- >>> stripPrefix "foo" "foobar"

1661+-- > stripPrefix "foo" "foobar" == Just "bar"

1769−-- Just "bar"

1770−--

1771−-- >>> stripPrefix "" "baz"

1662+-- > stripPrefix "" "baz" == Just "baz"

1772−-- Just "baz"

1773−--

1774−-- >>> stripPrefix "foo" "quux"

1663+-- > stripPrefix "foo" "quux" == Nothing

1775−-- Nothing

1776--

1664--

1777-- This is particularly useful with the @ViewPatterns@ extension to

1665-- This is particularly useful with the @ViewPatterns@ extension to

1778-- GHC, as follows:

1666-- GHC, as follows:

@@ -1797,14 +1685,9 @@ stripPrefix p@(Text _arr _off plen) t@(Text arr off len)

1797--

1685--

1798-- Examples:

1686-- Examples:

1799--

1687--

1800−-- >>> commonPrefixes "foobar" "fooquux"

1688+-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")

1801−-- Just ("foo","bar","quux")

1802−--

1803−-- >>> commonPrefixes "veeble" "fetzer"

1689+-- > commonPrefixes "veeble" "fetzer" == Nothing

1804−-- Nothing

1805−--

1806−-- >>> commonPrefixes "" "baz"

1690+-- > commonPrefixes "" "baz" == Nothing

1807−-- Nothing

1808commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)

1691commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)

1809commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1692commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1810 where

1693 where

@@ -1821,14 +1704,9 @@ commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0

1821--

1704--

1822-- Examples:

1705-- Examples:

1823--

1706--

1824−-- >>> stripSuffix "bar" "foobar"

1707+-- > stripSuffix "bar" "foobar" == Just "foo"

1825−-- Just "foo"

1826−--

1827−-- >>> stripSuffix "" "baz"

1708+-- > stripSuffix "" "baz" == Just "baz"

1828−-- Just "baz"

1829−--

1830−-- >>> stripSuffix "foo" "quux"

1709+-- > stripSuffix "foo" "quux" == Nothing

1831−-- Nothing

1832--

1710--

1833-- This is particularly useful with the @ViewPatterns@ extension to

1711-- This is particularly useful with the @ViewPatterns@ extension to

1834-- GHC, as follows:

1712-- GHC, as follows:

@@ -1875,12 +1753,3 @@ copy (Text arr off len) = Text (A.run go) 0 len

1875 marr <- A.new len

1753 marr <- A.new len

1876 A.copyI marr 0 arr off len

1754 A.copyI marr 0 arr off len

1877 return marr

1755 return marr

1878−

1879−

1880−-------------------------------------------------

1881−-- NOTE: the named chunk below used by doctest;

1882−-- verify the doctests via `doctest -fobject-code Data/Text.hs`

1883−

1884−-- $setup

1885−-- >>> :set -XOverloadedStrings

1886−-- >>> import qualified Data.Text as T

Data/Text/Array.hs+1−4

@@ -7,6 +7,7 @@

7--

8-- License : BSD-style

9-- Maintainer : bos@serpentine.com

10+-- Stability : experimental

10-- Portability : portable

11-- Portability : portable

11--

12--

12-- Packed, unboxed, heap-resident arrays. Suitable for performance

13-- Packed, unboxed, heap-resident arrays. Suitable for performance

@@ -79,8 +80,6 @@ import GHC.Word (Word16(..))

79import Prelude hiding (length, read)

80import Prelude hiding (length, read)

80

81

81-- | Immutable array type.

82-- | Immutable array type.

82−--

83−-- The 'Array' constructor is exposed since @text-1.1.1.3@

84data Array = Array {

83data Array = Array {

85 aBA :: ByteArray#

84 aBA :: ByteArray#

86#if defined(ASSERTS)

85#if defined(ASSERTS)

@@ -89,8 +88,6 @@ data Array = Array {

89 }

88 }

90

89

91-- | Mutable array type, for use in the ST monad.

90-- | Mutable array type, for use in the ST monad.

92−--

93−-- The 'MArray' constructor is exposed since @text-1.1.1.3@

94data MArray s = MArray {

91data MArray s = MArray {

95 maBA :: MutableByteArray# s

92 maBA :: MutableByteArray# s

96#if defined(ASSERTS)

93#if defined(ASSERTS)

Data/Text/Encoding.hs+4−17

@@ -11,13 +11,14 @@

11--

12-- License : BSD-style

13-- Maintainer : bos@serpentine.com

14+-- Stability : experimental

14-- Portability : portable

15-- Portability : portable

15--

16--

16-- Functions for converting 'Text' values to and from 'ByteString',

17-- Functions for converting 'Text' values to and from 'ByteString',

17-- using several standard encodings.

18-- using several standard encodings.

18--

19--

19-- To gain access to a much larger family of encodings, use the

20-- To gain access to a much larger family of encodings, use the

20−-- <http://hackage.haskell.org/package/text-icu text-icu package>.

21+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

21

22

22module Data.Text.Encoding

23module Data.Text.Encoding

23 (

24 (

@@ -78,11 +79,7 @@ import Data.Text.Internal.Unsafe.Shift (shiftR)

78import Data.Text.Show ()

79import Data.Text.Show ()

79import Data.Text.Unsafe (unsafeDupablePerformIO)

80import Data.Text.Unsafe (unsafeDupablePerformIO)

80import Data.Word (Word8, Word32)

81import Data.Word (Word8, Word32)

81−#if __GLASGOW_HASKELL__ >= 703

82−import Foreign.C.Types (CSize(CSize))

82+import Foreign.C.Types (CSize(..))

83−#else

84−import Foreign.C.Types (CSize)

85−#endif

86import Foreign.ForeignPtr (withForeignPtr)

83import Foreign.ForeignPtr (withForeignPtr)

87import Foreign.Marshal.Utils (with)

84import Foreign.Marshal.Utils (with)

88import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

85import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

@@ -212,8 +209,6 @@ decodeUtf8With onErr (PS fp off len) = runText $ \done -> do

212-- or continuation where it is encountered.

209-- or continuation where it is encountered.

213

210

214-- | A stream oriented decoding result.

211-- | A stream oriented decoding result.

215−--

216−-- @since 1.0.0.0

217data Decoding = Some Text ByteString (ByteString -> Decoding)

212data Decoding = Some Text ByteString (ByteString -> Decoding)

218

213

219instance Show Decoding where

214instance Show Decoding where

@@ -233,15 +228,11 @@ newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)

233-- thrown (either by this function or a continuation) that cannot be

228-- thrown (either by this function or a continuation) that cannot be

234-- caught in pure code. For more control over the handling of invalid

229-- caught in pure code. For more control over the handling of invalid

235-- data, use 'streamDecodeUtf8With'.

230-- data, use 'streamDecodeUtf8With'.

236−--

237−-- @since 1.0.0.0

238streamDecodeUtf8 :: ByteString -> Decoding

231streamDecodeUtf8 :: ByteString -> Decoding

239streamDecodeUtf8 = streamDecodeUtf8With strictDecode

232streamDecodeUtf8 = streamDecodeUtf8With strictDecode

240

233

241-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8

234-- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8

242-- encoded text.

235-- encoded text.

243−--

244−-- @since 1.0.0.0

245streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding

236streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding

246streamDecodeUtf8With onErr = decodeChunk B.empty 0 0

237streamDecodeUtf8With onErr = decodeChunk B.empty 0 0

247 where

238 where

@@ -317,8 +308,6 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDec

317{-# INLINE decodeUtf8' #-}

308{-# INLINE decodeUtf8' #-}

318

309

319-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

310-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

320−--

321−-- @since 1.1.0.0

322encodeUtf8Builder :: Text -> B.Builder

311encodeUtf8Builder :: Text -> B.Builder

323encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

312encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

324

313

@@ -327,8 +316,6 @@ encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

327--

316--

328-- Use this function is to implement efficient encoders for text-based formats

317-- Use this function is to implement efficient encoders for text-based formats

329-- like JSON or HTML.

318-- like JSON or HTML.

330−--

331−-- @since 1.1.0.0

332{-# INLINE encodeUtf8BuilderEscaped #-}

319{-# INLINE encodeUtf8BuilderEscaped #-}

333-- TODO: Extend documentation with references to source code in @blaze-html@

320-- TODO: Extend documentation with references to source code in @blaze-html@

334-- or @aeson@ that uses this function.

321-- or @aeson@ that uses this function.

@@ -386,7 +373,7 @@ encodeUtf8 :: Text -> ByteString

386encodeUtf8 (Text arr off len)

373encodeUtf8 (Text arr off len)

387 | len == 0 = B.empty

374 | len == 0 = B.empty

388 | otherwise = unsafeDupablePerformIO $ do

375 | otherwise = unsafeDupablePerformIO $ do

389− fp <- mallocByteString (len*3) -- see https://github.com/haskell/text/issues/194 for why len*3 is enough

376+ fp <- mallocByteString (len*4)

390 withForeignPtr fp $ \ptr ->

377 withForeignPtr fp $ \ptr ->

391 with ptr $ \destPtr -> do

378 with ptr $ \destPtr -> do

392 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

379 c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)

Data/Text/Encoding/Error.hs+1

@@ -10,6 +10,7 @@

10--

11-- License : BSD-style

12-- Maintainer : bos@serpentine.com

13+-- Stability : experimental

13-- Portability : GHC

14-- Portability : GHC

14--

15--

15-- Types and functions for dealing with encoding and decoding errors

16-- Types and functions for dealing with encoding and decoding errors

Data/Text/Foreign.hs+2−5

@@ -5,6 +5,7 @@

5--

6-- License : BSD-style

7-- Maintainer : bos@serpentine.com

8+-- Stability : experimental

8-- Portability : GHC

9-- Portability : GHC

9--

10--

10-- Support for using 'Text' data with native code via the Haskell

11-- Support for using 'Text' data with native code via the Haskell

@@ -107,7 +108,7 @@ takeWord16 (I16 n) t@(Text arr off len)

107 | n >= len || m >= len = t

108 | n >= len || m >= len = t

108 | otherwise = Text arr off m

109 | otherwise = Text arr off m

109 where

110 where

110− m | w < 0xD800 || w > 0xDBFF = n

111+ m | w < 0xDB00 || w > 0xD8FF = n

111 | otherwise = n+1

112 | otherwise = n+1

112 w = A.unsafeIndex arr (off+n-1)

113 w = A.unsafeIndex arr (off+n-1)

113

114

@@ -156,8 +157,6 @@ asForeignPtr t@(Text _arr _off len) = do

156-- | /O(n)/ Decode a C string with explicit length, which is assumed

157-- | /O(n)/ Decode a C string with explicit length, which is assumed

157-- to have been encoded as UTF-8. If decoding fails, a

158-- to have been encoded as UTF-8. If decoding fails, a

158-- 'UnicodeException' is thrown.

159-- 'UnicodeException' is thrown.

159−--

160−-- @since 1.0.0.0

161peekCStringLen :: CStringLen -> IO Text

160peekCStringLen :: CStringLen -> IO Text

162peekCStringLen cs = do

161peekCStringLen cs = do

163 bs <- unsafePackCStringLen cs

162 bs <- unsafePackCStringLen cs

@@ -170,7 +169,5 @@ peekCStringLen cs = do

170-- The temporary storage is freed when the subcomputation terminates

169-- The temporary storage is freed when the subcomputation terminates

171-- (either normally or via an exception), so the pointer to the

170-- (either normally or via an exception), so the pointer to the

172-- temporary storage must /not/ be used after this function returns.

171-- temporary storage must /not/ be used after this function returns.

173−--

174−-- @since 1.0.0.0

175withCStringLen :: Text -> (CStringLen -> IO a) -> IO a

172withCStringLen :: Text -> (CStringLen -> IO a) -> IO a

176withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act

173withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act

Data/Text/IO.hs+1−4

@@ -8,6 +8,7 @@

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11+-- Stability : experimental

11-- Portability : GHC

12-- Portability : GHC

12--

13--

13-- Efficient locale-sensitive support for text I\/O.

14-- Efficient locale-sensitive support for text I\/O.

@@ -335,7 +336,3 @@ putStrLn = hPutStrLn stdout

335-- >

336-- >

336-- > putStr_Utf16LE :: Text -> IO ()

337-- > putStr_Utf16LE :: Text -> IO ()

337-- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)

338-- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)

338−--

339−-- On transcoding errors, an 'IOError' exception is thrown. You can

340−-- use the API in "Data.Text.Encoding" if you need more control over

341−-- error handling or transcoding.

Data/Text/Internal/Fusion/CaseMapping.hs+4−164

@@ -1,10 +1,10 @@

1{-# LANGUAGE Rank2Types #-}

2-- AUTOMATICALLY GENERATED - DO NOT EDIT

3-- Generated by scripts/CaseMapping.hs

4−-- CaseFolding-9.0.0.txt

4+-- CaseFolding-8.0.0.txt

5−-- Date: 2016-03-02, 18:54:54 GMT

5+-- Date: 2015-01-13, 18:16:36 GMT [MD]

6−-- SpecialCasing-9.0.0.txt

6+-- SpecialCasing-8.0.0.txt

7−-- Date: 2016-03-02, 18:55:13 GMT

7+-- Date: 2014-12-16, 23:08:04 GMT [MD]

8

9module Data.Text.Internal.Fusion.CaseMapping where

10import Data.Char

@@ -371,24 +371,6 @@ foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')

371foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')

372-- CHEROKEE SMALL LETTER MV

373foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')

374−-- CYRILLIC SMALL LETTER ROUNDED VE

375−foldMapping '\x1c80' s = Yield '\x0432' (CC s '\x0000' '\x0000')

376−-- CYRILLIC SMALL LETTER LONG-LEGGED DE

377−foldMapping '\x1c81' s = Yield '\x0434' (CC s '\x0000' '\x0000')

378−-- CYRILLIC SMALL LETTER NARROW O

379−foldMapping '\x1c82' s = Yield '\x043e' (CC s '\x0000' '\x0000')

380−-- CYRILLIC SMALL LETTER WIDE ES

381−foldMapping '\x1c83' s = Yield '\x0441' (CC s '\x0000' '\x0000')

382−-- CYRILLIC SMALL LETTER TALL TE

383−foldMapping '\x1c84' s = Yield '\x0442' (CC s '\x0000' '\x0000')

384−-- CYRILLIC SMALL LETTER THREE-LEGGED TE

385−foldMapping '\x1c85' s = Yield '\x0442' (CC s '\x0000' '\x0000')

386−-- CYRILLIC SMALL LETTER TALL HARD SIGN

387−foldMapping '\x1c86' s = Yield '\x044a' (CC s '\x0000' '\x0000')

388−-- CYRILLIC SMALL LETTER TALL YAT

389−foldMapping '\x1c87' s = Yield '\x0463' (CC s '\x0000' '\x0000')

390−-- CYRILLIC SMALL LETTER UNBLENDED UK

391−foldMapping '\x1c88' s = Yield '\xa64b' (CC s '\x0000' '\x0000')

392-- LATIN SMALL LETTER H WITH LINE BELOW

374-- LATIN SMALL LETTER H WITH LINE BELOW

393foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

375foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

394-- LATIN SMALL LETTER T WITH DIAERESIS

376-- LATIN SMALL LETTER T WITH DIAERESIS

@@ -563,8 +545,6 @@ foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')

563foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

545foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

564-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

546-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

565foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

547foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

566−-- LATIN CAPITAL LETTER SMALL CAPITAL I

567−foldMapping '\xa7ae' s = Yield '\x026a' (CC s '\x0000' '\x0000')

568-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

548-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

569foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

549foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

570-- LATIN CAPITAL LETTER CHI

550-- LATIN CAPITAL LETTER CHI

@@ -757,78 +737,6 @@ foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')

757foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

737foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

758-- ARMENIAN SMALL LIGATURE MEN XEH

738-- ARMENIAN SMALL LIGATURE MEN XEH

759foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

739foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

760−-- OSAGE CAPITAL LETTER A

761−foldMapping '\x104b0' s = Yield '\x104d8' (CC s '\x0000' '\x0000')

762−-- OSAGE CAPITAL LETTER AI

763−foldMapping '\x104b1' s = Yield '\x104d9' (CC s '\x0000' '\x0000')

764−-- OSAGE CAPITAL LETTER AIN

765−foldMapping '\x104b2' s = Yield '\x104da' (CC s '\x0000' '\x0000')

766−-- OSAGE CAPITAL LETTER AH

767−foldMapping '\x104b3' s = Yield '\x104db' (CC s '\x0000' '\x0000')

768−-- OSAGE CAPITAL LETTER BRA

769−foldMapping '\x104b4' s = Yield '\x104dc' (CC s '\x0000' '\x0000')

770−-- OSAGE CAPITAL LETTER CHA

771−foldMapping '\x104b5' s = Yield '\x104dd' (CC s '\x0000' '\x0000')

772−-- OSAGE CAPITAL LETTER EHCHA

773−foldMapping '\x104b6' s = Yield '\x104de' (CC s '\x0000' '\x0000')

774−-- OSAGE CAPITAL LETTER E

775−foldMapping '\x104b7' s = Yield '\x104df' (CC s '\x0000' '\x0000')

776−-- OSAGE CAPITAL LETTER EIN

777−foldMapping '\x104b8' s = Yield '\x104e0' (CC s '\x0000' '\x0000')

778−-- OSAGE CAPITAL LETTER HA

779−foldMapping '\x104b9' s = Yield '\x104e1' (CC s '\x0000' '\x0000')

780−-- OSAGE CAPITAL LETTER HYA

781−foldMapping '\x104ba' s = Yield '\x104e2' (CC s '\x0000' '\x0000')

782−-- OSAGE CAPITAL LETTER I

783−foldMapping '\x104bb' s = Yield '\x104e3' (CC s '\x0000' '\x0000')

784−-- OSAGE CAPITAL LETTER KA

785−foldMapping '\x104bc' s = Yield '\x104e4' (CC s '\x0000' '\x0000')

786−-- OSAGE CAPITAL LETTER EHKA

787−foldMapping '\x104bd' s = Yield '\x104e5' (CC s '\x0000' '\x0000')

788−-- OSAGE CAPITAL LETTER KYA

789−foldMapping '\x104be' s = Yield '\x104e6' (CC s '\x0000' '\x0000')

790−-- OSAGE CAPITAL LETTER LA

791−foldMapping '\x104bf' s = Yield '\x104e7' (CC s '\x0000' '\x0000')

792−-- OSAGE CAPITAL LETTER MA

793−foldMapping '\x104c0' s = Yield '\x104e8' (CC s '\x0000' '\x0000')

794−-- OSAGE CAPITAL LETTER NA

795−foldMapping '\x104c1' s = Yield '\x104e9' (CC s '\x0000' '\x0000')

796−-- OSAGE CAPITAL LETTER O

797−foldMapping '\x104c2' s = Yield '\x104ea' (CC s '\x0000' '\x0000')

798−-- OSAGE CAPITAL LETTER OIN

799−foldMapping '\x104c3' s = Yield '\x104eb' (CC s '\x0000' '\x0000')

800−-- OSAGE CAPITAL LETTER PA

801−foldMapping '\x104c4' s = Yield '\x104ec' (CC s '\x0000' '\x0000')

802−-- OSAGE CAPITAL LETTER EHPA

803−foldMapping '\x104c5' s = Yield '\x104ed' (CC s '\x0000' '\x0000')

804−-- OSAGE CAPITAL LETTER SA

805−foldMapping '\x104c6' s = Yield '\x104ee' (CC s '\x0000' '\x0000')

806−-- OSAGE CAPITAL LETTER SHA

807−foldMapping '\x104c7' s = Yield '\x104ef' (CC s '\x0000' '\x0000')

808−-- OSAGE CAPITAL LETTER TA

809−foldMapping '\x104c8' s = Yield '\x104f0' (CC s '\x0000' '\x0000')

810−-- OSAGE CAPITAL LETTER EHTA

811−foldMapping '\x104c9' s = Yield '\x104f1' (CC s '\x0000' '\x0000')

812−-- OSAGE CAPITAL LETTER TSA

813−foldMapping '\x104ca' s = Yield '\x104f2' (CC s '\x0000' '\x0000')

814−-- OSAGE CAPITAL LETTER EHTSA

815−foldMapping '\x104cb' s = Yield '\x104f3' (CC s '\x0000' '\x0000')

816−-- OSAGE CAPITAL LETTER TSHA

817−foldMapping '\x104cc' s = Yield '\x104f4' (CC s '\x0000' '\x0000')

818−-- OSAGE CAPITAL LETTER DHA

819−foldMapping '\x104cd' s = Yield '\x104f5' (CC s '\x0000' '\x0000')

820−-- OSAGE CAPITAL LETTER U

821−foldMapping '\x104ce' s = Yield '\x104f6' (CC s '\x0000' '\x0000')

822−-- OSAGE CAPITAL LETTER WA

823−foldMapping '\x104cf' s = Yield '\x104f7' (CC s '\x0000' '\x0000')

824−-- OSAGE CAPITAL LETTER KHA

825−foldMapping '\x104d0' s = Yield '\x104f8' (CC s '\x0000' '\x0000')

826−-- OSAGE CAPITAL LETTER GHA

827−foldMapping '\x104d1' s = Yield '\x104f9' (CC s '\x0000' '\x0000')

828−-- OSAGE CAPITAL LETTER ZA

829−foldMapping '\x104d2' s = Yield '\x104fa' (CC s '\x0000' '\x0000')

830−-- OSAGE CAPITAL LETTER ZHA

831−foldMapping '\x104d3' s = Yield '\x104fb' (CC s '\x0000' '\x0000')

832-- OLD HUNGARIAN CAPITAL LETTER A

740-- OLD HUNGARIAN CAPITAL LETTER A

833foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

741foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

834-- OLD HUNGARIAN CAPITAL LETTER AA

742-- OLD HUNGARIAN CAPITAL LETTER AA

@@ -931,72 +839,4 @@ foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')

931foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

839foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

932-- OLD HUNGARIAN CAPITAL LETTER US

840-- OLD HUNGARIAN CAPITAL LETTER US

933foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

841foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

934−-- ADLAM CAPITAL LETTER ALIF

935−foldMapping '\x1e900' s = Yield '\x1e922' (CC s '\x0000' '\x0000')

936−-- ADLAM CAPITAL LETTER DAALI

937−foldMapping '\x1e901' s = Yield '\x1e923' (CC s '\x0000' '\x0000')

938−-- ADLAM CAPITAL LETTER LAAM

939−foldMapping '\x1e902' s = Yield '\x1e924' (CC s '\x0000' '\x0000')

940−-- ADLAM CAPITAL LETTER MIIM

941−foldMapping '\x1e903' s = Yield '\x1e925' (CC s '\x0000' '\x0000')

942−-- ADLAM CAPITAL LETTER BA

943−foldMapping '\x1e904' s = Yield '\x1e926' (CC s '\x0000' '\x0000')

944−-- ADLAM CAPITAL LETTER SINNYIIYHE

945−foldMapping '\x1e905' s = Yield '\x1e927' (CC s '\x0000' '\x0000')

946−-- ADLAM CAPITAL LETTER PE

947−foldMapping '\x1e906' s = Yield '\x1e928' (CC s '\x0000' '\x0000')

948−-- ADLAM CAPITAL LETTER BHE

949−foldMapping '\x1e907' s = Yield '\x1e929' (CC s '\x0000' '\x0000')

950−-- ADLAM CAPITAL LETTER RA

951−foldMapping '\x1e908' s = Yield '\x1e92a' (CC s '\x0000' '\x0000')

952−-- ADLAM CAPITAL LETTER E

953−foldMapping '\x1e909' s = Yield '\x1e92b' (CC s '\x0000' '\x0000')

954−-- ADLAM CAPITAL LETTER FA

955−foldMapping '\x1e90a' s = Yield '\x1e92c' (CC s '\x0000' '\x0000')

956−-- ADLAM CAPITAL LETTER I

957−foldMapping '\x1e90b' s = Yield '\x1e92d' (CC s '\x0000' '\x0000')

958−-- ADLAM CAPITAL LETTER O

959−foldMapping '\x1e90c' s = Yield '\x1e92e' (CC s '\x0000' '\x0000')

960−-- ADLAM CAPITAL LETTER DHA

961−foldMapping '\x1e90d' s = Yield '\x1e92f' (CC s '\x0000' '\x0000')

962−-- ADLAM CAPITAL LETTER YHE

963−foldMapping '\x1e90e' s = Yield '\x1e930' (CC s '\x0000' '\x0000')

964−-- ADLAM CAPITAL LETTER WAW

965−foldMapping '\x1e90f' s = Yield '\x1e931' (CC s '\x0000' '\x0000')

966−-- ADLAM CAPITAL LETTER NUN

967−foldMapping '\x1e910' s = Yield '\x1e932' (CC s '\x0000' '\x0000')

968−-- ADLAM CAPITAL LETTER KAF

969−foldMapping '\x1e911' s = Yield '\x1e933' (CC s '\x0000' '\x0000')

970−-- ADLAM CAPITAL LETTER YA

971−foldMapping '\x1e912' s = Yield '\x1e934' (CC s '\x0000' '\x0000')

972−-- ADLAM CAPITAL LETTER U

973−foldMapping '\x1e913' s = Yield '\x1e935' (CC s '\x0000' '\x0000')

974−-- ADLAM CAPITAL LETTER JIIM

975−foldMapping '\x1e914' s = Yield '\x1e936' (CC s '\x0000' '\x0000')

976−-- ADLAM CAPITAL LETTER CHI

977−foldMapping '\x1e915' s = Yield '\x1e937' (CC s '\x0000' '\x0000')

978−-- ADLAM CAPITAL LETTER HA

979−foldMapping '\x1e916' s = Yield '\x1e938' (CC s '\x0000' '\x0000')

980−-- ADLAM CAPITAL LETTER QAAF

981−foldMapping '\x1e917' s = Yield '\x1e939' (CC s '\x0000' '\x0000')

982−-- ADLAM CAPITAL LETTER GA

983−foldMapping '\x1e918' s = Yield '\x1e93a' (CC s '\x0000' '\x0000')

984−-- ADLAM CAPITAL LETTER NYA

985−foldMapping '\x1e919' s = Yield '\x1e93b' (CC s '\x0000' '\x0000')

986−-- ADLAM CAPITAL LETTER TU

987−foldMapping '\x1e91a' s = Yield '\x1e93c' (CC s '\x0000' '\x0000')

988−-- ADLAM CAPITAL LETTER NHA

989−foldMapping '\x1e91b' s = Yield '\x1e93d' (CC s '\x0000' '\x0000')

990−-- ADLAM CAPITAL LETTER VA

991−foldMapping '\x1e91c' s = Yield '\x1e93e' (CC s '\x0000' '\x0000')

992−-- ADLAM CAPITAL LETTER KHA

993−foldMapping '\x1e91d' s = Yield '\x1e93f' (CC s '\x0000' '\x0000')

994−-- ADLAM CAPITAL LETTER GBE

995−foldMapping '\x1e91e' s = Yield '\x1e940' (CC s '\x0000' '\x0000')

996−-- ADLAM CAPITAL LETTER ZAL

997−foldMapping '\x1e91f' s = Yield '\x1e941' (CC s '\x0000' '\x0000')

998−-- ADLAM CAPITAL LETTER KPO

999−foldMapping '\x1e920' s = Yield '\x1e942' (CC s '\x0000' '\x0000')

1000−-- ADLAM CAPITAL LETTER SHA

1001−foldMapping '\x1e921' s = Yield '\x1e943' (CC s '\x0000' '\x0000')

1002foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

842foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

Data/Text/Internal/Fusion/Common.hs+37−44

@@ -1,4 +1,4 @@

1−{-# LANGUAGE PatternGuards, BangPatterns, MagicHash, Rank2Types #-}

1+{-# LANGUAGE BangPatterns, MagicHash, Rank2Types #-}

2-- |

3-- Module : Data.Text.Internal.Fusion.Common

4-- Copyright : (c) Bryan O'Sullivan 2009, 2012

@@ -107,7 +107,7 @@ import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..),

107import qualified Data.List as L

108import qualified Prelude as P

109import Data.Bits (shiftL)

110−import Data.Char (isLetter, isSpace)

110+import Data.Char (isLetter)

111import Data.Int (Int64)

112import Data.Text.Internal.Fusion.Types

113import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,

@@ -117,7 +117,7 @@ import GHC.Prim (Addr#, chr#, indexCharOffAddr#, ord#)

117import GHC.Types (Char(..), Int(..))

118

119singleton :: Char -> Stream Char

120−singleton c = Stream next False (codePointsSize 1)

120+singleton c = Stream next False 1

121 where next False = Yield c True

122 next True = Done

123{-# INLINE [0] singleton #-}

@@ -175,7 +175,7 @@ data C s = C0 !s

175

176-- | /O(n)/ Adds a character to the front of a Stream Char.

177cons :: Char -> Stream Char -> Stream Char

178−cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len + codePointsSize 1)

178+cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len+1)

179 where

180 next (C1 s) = Yield w (C0 s)

181 next (C0 s) = case next0 s of

@@ -189,7 +189,7 @@ data Snoc a = N

189

190-- | /O(n)/ Adds a character to the end of a stream.

191snoc :: Stream Char -> Char -> Stream Char

192−snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len + codePointsSize 1)

192+snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len+1)

193 where

194 next (J xs) = case next0 xs of

195 Done -> Yield w N

@@ -237,7 +237,7 @@ uncons :: Stream Char -> Maybe (Char, Stream Char)

237uncons (Stream next s0 len) = loop_uncons s0

238 where

239 loop_uncons !s = case next s of

240− Yield x s1 -> Just (x, Stream next s1 (len - codePointsSize 1))

240+ Yield x s1 -> Just (x, Stream next s1 (len-1))

241 Skip s' -> loop_uncons s'

242 Done -> Nothing

243{-# INLINE [0] uncons #-}

@@ -260,7 +260,7 @@ last (Stream next s0 _len) = loop0_last s0

260-- | /O(1)/ Returns all characters after the head of a Stream Char, which must

261-- be non-empty.

262tail :: Stream Char -> Stream Char

263−tail (Stream next0 s0 len) = Stream next (C0 s0) (len - codePointsSize 1)

263+tail (Stream next0 s0 len) = Stream next (C0 s0) (len-1)

264 where

265 next (C0 s) = case next0 s of

266 Done -> emptyError "tail"

@@ -278,7 +278,7 @@ data Init s = Init0 !s

278-- | /O(1)/ Returns all but the last character of a Stream Char, which

279-- must be non-empty.

280init :: Stream Char -> Stream Char

281−init (Stream next0 s0 len) = Stream next (Init0 s0) (len - codePointsSize 1)

281+init (Stream next0 s0 len) = Stream next (Init0 s0) (len-1)

282 where

283 next (Init0 s) = case next0 s of

284 Done -> emptyError "init"

@@ -318,14 +318,11 @@ lengthI (Stream next s0 _len) = loop_length 0 s0

318-- greater than the number or if the stream can't possibly be as long

319-- as the number supplied, and hence be more efficient.

320compareLengthI :: Integral a => Stream Char -> a -> Ordering

321−compareLengthI (Stream next s0 len) n

321+compareLengthI (Stream next s0 len) n =

322− -- Note that @len@ tracks code units whereas we want to compare the length

322+ case compareSize len (fromIntegral n) of

323− -- in code points. Specifically, a stream with hint @len@ may consist of

323+ Just o -> o

324− -- anywhere from @len/2@ to @len@ code points.

325− | Just r <- compareSize len n' = r

326− | otherwise = loop_cmp 0 s0

324+ Nothing -> loop_cmp 0 s0

327 where

325 where

328− n' = codePointsSize $ fromIntegral n

329 loop_cmp !z s = case next s of

326 loop_cmp !z s = case next s of

330 Done -> compare z n

327 Done -> compare z n

331 Skip s' -> loop_cmp z s'

328 Skip s' -> loop_cmp z s'

@@ -371,7 +368,7 @@ data I s = I1 !s

371-- | /O(n)/ Take a character and place it between each of the

368-- | /O(n)/ Take a character and place it between each of the

372-- characters of a 'Stream Char'.

369-- characters of a 'Stream Char'.

373intersperse :: Char -> Stream Char -> Stream Char

370intersperse :: Char -> Stream Char -> Stream Char

374−intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize)

371+intersperse c (Stream next0 s0 len) = Stream next (I1 s0) len

375 where

372 where

376 next (I1 s) = case next0 s of

373 next (I1 s) = case next0 s of

377 Done -> Done

374 Done -> Done

@@ -396,11 +393,9 @@ intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize)

396-- functions may map one input character to two or three output

393-- functions may map one input character to two or three output

397-- characters.

394-- characters.

398

395

399−-- | Map a 'Stream' through the given case-mapping function.

400caseConvert :: (forall s. Char -> s -> Step (CC s) Char)

396caseConvert :: (forall s. Char -> s -> Step (CC s) Char)

401 -> Stream Char -> Stream Char

397 -> Stream Char -> Stream Char

402−caseConvert remap (Stream next0 s0 len) =

398+caseConvert remap (Stream next0 s0 len) = Stream next (CC s0 '\0' '\0') len

403− Stream next (CC s0 '\0' '\0') (len `unionSize` 3*len)

404 where

399 where

405 next (CC s '\0' _) =

400 next (CC s '\0' _) =

406 case next0 s of

401 case next0 s of

@@ -463,20 +458,19 @@ toLower = caseConvert lowerMapping

463-- Fox\" is correctly title cased—but this function will

458-- Fox\" is correctly title cased—but this function will

464-- capitalize /every/ word.

459-- capitalize /every/ word.

465toTitle :: Stream Char -> Stream Char

460toTitle :: Stream Char -> Stream Char

466−toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') (len + unknownSize)

461+toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') len

467 where

462 where

468 next (CC (letter :*: s) '\0' _) =

463 next (CC (letter :*: s) '\0' _) =

469 case next0 s of

464 case next0 s of

470− Done -> Done

465+ Done -> Done

471− Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

466+ Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

472 Yield c s'

467 Yield c s'

473− | nonSpace -> if letter

468+ | letter' -> if letter

474− then lowerMapping c (nonSpace :*: s')

469+ then lowerMapping c (letter' :*: s')

475− else titleMapping c (letter' :*: s')

470+ else titleMapping c (letter' :*: s')

476− | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

471+ | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

477− where nonSpace = P.not (isSpace c)

478− letter' = isLetter c

472+ where letter' = isLetter c

479− next (CC s a b) = Yield a (CC s b '\0')

473+ next (CC s a b) = Yield a (CC s b '\0')

480{-# INLINE [0] toTitle #-}

474{-# INLINE [0] toTitle #-}

481

475

482data Justify i s = Just1 !i !s

476data Justify i s = Just1 !i !s

@@ -484,7 +478,7 @@ data Justify i s = Just1 !i !s

484

478

485justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char

479justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char

486justifyLeftI k c (Stream next0 s0 len) =

480justifyLeftI k c (Stream next0 s0 len) =

487− Stream next (Just1 0 s0) (larger (fromIntegral k * charSize c + len) len)

481+ Stream next (Just1 0 s0) (larger (fromIntegral k) len)

488 where

482 where

489 next (Just1 n s) =

483 next (Just1 n s) =

490 case next0 s of

484 case next0 s of

@@ -704,7 +698,7 @@ replicateI n (Stream next0 s0 len) =

704-- (a,b), in which case, a is the next Char in the string, and b is

698-- (a,b), in which case, a is the next Char in the string, and b is

705-- the seed value for further production.

699-- the seed value for further production.

706unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char

700unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char

707−unfoldr f s0 = Stream next s0 unknownSize

701+unfoldr f s0 = Stream next s0 1 -- HINT maybe too low

708 where

702 where

709 {-# INLINE next #-}

703 {-# INLINE next #-}

710 next !s = case f s of

704 next !s = case f s of

@@ -718,7 +712,7 @@ unfoldr f s0 = Stream next s0 unknownSize

718-- 'unfoldr' when the length of the result is known.

712-- 'unfoldr' when the length of the result is known.

719unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char

713unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char

720unfoldrNI n f s0 | n < 0 = empty

714unfoldrNI n f s0 | n < 0 = empty

721− | otherwise = Stream next (0 :*: s0) (maxSize $ fromIntegral (n*2))

715+ | otherwise = Stream next (0 :*: s0) (fromIntegral (n*2)) -- HINT maybe too high

722 where

716 where

723 {-# INLINE next #-}

717 {-# INLINE next #-}

724 next (z :*: s) = case f s of

718 next (z :*: s) = case f s of

@@ -730,12 +724,12 @@ unfoldrNI n f s0 | n < 0 = empty

730-------------------------------------------------------------------------------

724-------------------------------------------------------------------------------

731-- * Substreams

725-- * Substreams

732

726

733−-- | /O(n)/ @'take' n@, applied to a stream, returns the prefix of the

727+-- | /O(n)/ take n, applied to a stream, returns the prefix of the

734-- stream of length @n@, or the stream itself if @n@ is greater than the

728-- stream of length @n@, or the stream itself if @n@ is greater than the

735-- length of the stream.

729-- length of the stream.

736take :: Integral a => a -> Stream Char -> Stream Char

730take :: Integral a => a -> Stream Char -> Stream Char

737take n0 (Stream next0 s0 len) =

731take n0 (Stream next0 s0 len) =

738− Stream next (n0 :*: s0) (smaller len (codePointsSize $ fromIntegral n0))

732+ Stream next (n0 :*: s0) (smaller len (fromIntegral (max 0 n0)))

739 where

733 where

740 {-# INLINE next #-}

734 {-# INLINE next #-}

741 next (n :*: s) | n <= 0 = Done

735 next (n :*: s) | n <= 0 = Done

@@ -748,12 +742,12 @@ take n0 (Stream next0 s0 len) =

748data Drop a s = NS !s

742data Drop a s = NS !s

749 | JS !a !s

743 | JS !a !s

750

744

751−-- | /O(n)/ @'drop' n@, applied to a stream, returns the suffix of the

745+-- | /O(n)/ drop n, applied to a stream, returns the suffix of the

752-- stream after the first @n@ characters, or the empty stream if @n@

746-- stream after the first @n@ characters, or the empty stream if @n@

753-- is greater than the length of the stream.

747-- is greater than the length of the stream.

754drop :: Integral a => a -> Stream Char -> Stream Char

748drop :: Integral a => a -> Stream Char -> Stream Char

755drop n0 (Stream next0 s0 len) =

749drop n0 (Stream next0 s0 len) =

756− Stream next (JS n0 s0) (len - codePointsSize (fromIntegral n0))

750+ Stream next (JS n0 s0) (len - fromIntegral (max 0 n0))

757 where

751 where

758 {-# INLINE next #-}

752 {-# INLINE next #-}

759 next (JS n s)

753 next (JS n s)

@@ -768,10 +762,10 @@ drop n0 (Stream next0 s0 len) =

768 Yield x s' -> Yield x (NS s')

762 Yield x s' -> Yield x (NS s')

769{-# INLINE [0] drop #-}

763{-# INLINE [0] drop #-}

770

764

771−-- | 'takeWhile', applied to a predicate @p@ and a stream, returns the

765+-- | takeWhile, applied to a predicate @p@ and a stream, returns the

772−-- longest prefix (possibly empty) of elements that satisfy @p@.

766+-- longest prefix (possibly empty) of elements that satisfy p.

773takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char

767takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char

774−takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize)

768+takeWhile p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high

775 where

769 where

776 {-# INLINE next #-}

770 {-# INLINE next #-}

777 next !s = case next0 s of

771 next !s = case next0 s of

@@ -781,9 +775,9 @@ takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize)

781 | otherwise -> Done

775 | otherwise -> Done

782{-# INLINE [0] takeWhile #-}

776{-# INLINE [0] takeWhile #-}

783

777

784−-- | @'dropWhile' p xs@ returns the suffix remaining after @'takeWhile' p xs@.

778+-- | dropWhile @p @xs returns the suffix remaining after takeWhile @p @xs.

785dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char

779dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char

786−dropWhile p (Stream next0 s0 len) = Stream next (L s0) (len - unknownSize)

780+dropWhile p (Stream next0 s0 len) = Stream next (L s0) len -- HINT maybe too high

787 where

781 where

788 {-# INLINE next #-}

782 {-# INLINE next #-}

789 next (L s) = case next0 s of

783 next (L s) = case next0 s of

@@ -817,7 +811,7 @@ isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)

817-------------------------------------------------------------------------------

811-------------------------------------------------------------------------------

818-- ** Searching by equality

812-- ** Searching by equality

819

813

820−-- | /O(n)/ 'elem' is the stream membership predicate.

814+-- | /O(n)/ elem is the stream membership predicate.

821elem :: Char -> Stream Char -> Bool

815elem :: Char -> Stream Char -> Bool

822elem w (Stream next s0 _len) = loop_elem s0

816elem w (Stream next s0 _len) = loop_elem s0

823 where

817 where

@@ -862,8 +856,7 @@ indexI (Stream next s0 _len) n0

862-- returns a stream containing those characters that satisfy the

856-- returns a stream containing those characters that satisfy the

863-- predicate.

857-- predicate.

864filter :: (Char -> Bool) -> Stream Char -> Stream Char

858filter :: (Char -> Bool) -> Stream Char -> Stream Char

865−filter p (Stream next0 s0 len) =

866− Stream next s0 (len - unknownSize) -- HINT maybe too high

859+filter p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high

867 where

860 where

868 next !s = case next0 s of

861 next !s = case next0 s of

869 Done -> Done

862 Done -> Done

Data/Text/Internal/Fusion/Size.hs+7−33

@@ -1,4 +1,4 @@

1−{-# LANGUAGE CPP, PatternGuards #-}

1+{-# LANGUAGE CPP #-}

2{-# OPTIONS_GHC -fno-warn-missing-methods #-}

3-- |

4-- Module : Data.Text.Internal.Fusion.Internal

@@ -19,16 +19,11 @@

19module Data.Text.Internal.Fusion.Size

20 (

21 Size

22− -- * Sizes

22+ , exactly

23 , exactSize

24 , maxSize

25 , betweenSize

26 , unknownSize

27− , unionSize

28− , charSize

29− , codePointsSize

30− -- * Querying sizes

31− , exactly

32 , smaller

27 , smaller

33 , larger

28 , larger

34 , upperBound

29 , upperBound

@@ -37,13 +32,11 @@ module Data.Text.Internal.Fusion.Size

37 , isEmpty

32 , isEmpty

38 ) where

33 ) where

39

34

40−import Data.Char (ord)

41import Data.Text.Internal (mul)

35import Data.Text.Internal (mul)

42#if defined(ASSERTS)

36#if defined(ASSERTS)

43import Control.Exception (assert)

37import Control.Exception (assert)

44#endif

38#endif

45

39

46−-- | A size in UTF-16 code units.

47data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.

40data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.

48 | Unknown -- ^ Unknown size.

41 | Unknown -- ^ Unknown size.

49 deriving (Eq, Show)

42 deriving (Eq, Show)

@@ -53,17 +46,6 @@ exactly (Between na nb) | na == nb = Just na

53exactly _ = Nothing

46exactly _ = Nothing

54{-# INLINE exactly #-}

47{-# INLINE exactly #-}

55

48

56−-- | The 'Size' of the given code point.

57−charSize :: Char -> Size

58−charSize c

59− | ord c < 0x10000 = exactSize 1

60− | otherwise = exactSize 2

61−

62−-- | The 'Size' of @n@ code points.

63−codePointsSize :: Int -> Size

64−codePointsSize n = Between n (2*n)

65−{-# INLINE codePointsSize #-}

66−

67exactSize :: Int -> Size

49exactSize :: Int -> Size

68exactSize n =

50exactSize n =

69#if defined(ASSERTS)

51#if defined(ASSERTS)

@@ -89,10 +71,6 @@ betweenSize m n =

89 Between m n

71 Between m n

90{-# INLINE betweenSize #-}

72{-# INLINE betweenSize #-}

91

73

92−unionSize :: Size -> Size -> Size

93−unionSize (Between a b) (Between c d) = Between (min a c) (max b d)

94−unionSize _ _ = Unknown

95−

96unknownSize :: Size

74unknownSize :: Size

97unknownSize = Unknown

75unknownSize = Unknown

98{-# INLINE unknownSize #-}

76{-# INLINE unknownSize #-}

@@ -162,15 +140,11 @@ lowerBound _ (Between n _) = n

162lowerBound k _ = k

140lowerBound k _ = k

163{-# INLINE lowerBound #-}

141{-# INLINE lowerBound #-}

164

142

165−-- | Determine the ordering relationship between two 'Size's, or 'Nothing' in

166−-- the indeterminate case.

167−compareSize :: Size -> Size -> Maybe Ordering

143+compareSize :: Size -> Int -> Maybe Ordering

168−compareSize (Between ma mb) (Between na nb)

144+compareSize (Between ma mb) n

169− | mb < na = Just LT

145+ | mb < n = Just LT

170− | ma > nb = Just GT

146+ | ma > n = Just GT

171− | ma == mb

172− , ma == na

173− , ma == nb = Just EQ

147+ | ma == n && mb == n = Just EQ

174compareSize _ _ = Nothing

148compareSize _ _ = Nothing

175

149

176

150

Data/Text/Internal/Fusion/Types.hs+4−4

@@ -75,16 +75,16 @@ instance (Ord a) => Ord (Stream a) where

75-- unstreaming functions must be able to cope with the hint being too

76-- small or too large.

77--

78−-- The size hint tries to track the UTF-16 code units in a stream,

78+-- The size hint tries to track the UTF-16 code points in a stream,

79−-- but often counts the number of code points instead. It can easily

79+-- but often counts the number of characters instead. It can easily

80-- undercount if, for instance, a transformed stream contains astral

81−-- plane code points (those above 0x10000).

81+-- plane characters (those above 0x10000).

82

83data Stream a =

84 forall s. Stream

85 (s -> Step s a) -- stepper function

86 !s -- current state

87− !Size -- size hint in code units

87+ !Size -- size hint

88

89-- | /O(n)/ Determines if two streams are equal.

90eq :: (Eq a) => Stream a -> Stream a -> Bool

Data/Text/Lazy.hs+20−54

@@ -13,6 +13,7 @@

13--

14-- License : BSD-style

15-- Maintainer : bos@serpentine.com

16+-- Stability : experimental

16-- Portability : GHC

17-- Portability : GHC

17--

18--

18-- A time and space-efficient implementation of Unicode text using

19-- A time and space-efficient implementation of Unicode text using

@@ -67,7 +68,6 @@ module Data.Text.Lazy

67 , snoc

68 , snoc

68 , append

69 , append

69 , uncons

70 , uncons

70− , unsnoc

71 , head

72 , last

73 , tail

@@ -273,16 +273,14 @@ import Text.Printf (PrintfArg, formatArg, formatString)

273-- $replacement

274--

275-- A 'Text' value is a sequence of Unicode scalar values, as defined

276−-- in

277−-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.

276+-- in §3.9, definition D76 of the Unicode 5.2 standard:

277+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As

278−-- As such, a 'Text' cannot contain values in the range U+D800 to

278+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF

279−-- U+DFFF inclusive. Haskell implementations admit all Unicode code

279+-- inclusive. Haskell implementations admit all Unicode code points

280−-- points

281−-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)

282−-- as 'Char' values, including code points from this invalid range.

280+-- (§3.4, definition D10) as 'Char' values, including code points

283−-- This means that there are some 'Char' values that are not valid

281+-- from this invalid range. This means that there are some 'Char'

284−-- Unicode scalar values, and the functions in this module must handle

282+-- values that are not valid Unicode scalar values, and the functions

285−-- those cases.

283+-- in this module must handle those cases.

286--

284--

287-- Within this module, many functions construct a 'Text' from one or

285-- Within this module, many functions construct a 'Text' from one or

288-- more 'Char' values. Those functions will substitute 'Char' values

286-- more 'Char' values. Those functions will substitute 'Char' values

@@ -296,8 +294,8 @@ import Text.Printf (PrintfArg, formatArg, formatString)

296-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

294-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate

297-- code points, and so cannot be represented. The functions replace

295-- code points, and so cannot be represented. The functions replace

298-- invalid scalar values, instead of dropping them, as a security

296-- invalid scalar values, instead of dropping them, as a security

299−-- measure. For details, see

297+-- measure. For details, see Unicode Technical Report 36, §3.5:

300−-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)

298+-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)

301

299

302equal :: Text -> Text -> Bool

300equal :: Text -> Text -> Bool

303equal Empty Empty = True

301equal Empty Empty = True

@@ -344,13 +342,9 @@ instance Read Text where

344 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

342 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

345

343

346#if MIN_VERSION_base(4,9,0)

344#if MIN_VERSION_base(4,9,0)

347−-- | Non-orphan 'Semigroup' instance only defined for

348−-- @base-4.9.0.0@ and later; orphan instances for older GHCs are

345+-- Semigroup orphan instances for older GHCs are provided by

349−-- provided by

350−-- the [semigroups](http://hackage.haskell.org/package/semigroups)

351−-- package

346+-- 'semigroups` package

352−--

347+

353−-- @since 1.2.2.0

354instance Semigroup Text where

348instance Semigroup Text where

355 (<>) = append

349 (<>) = append

356#endif

350#endif

@@ -368,7 +362,6 @@ instance IsString Text where

368 fromString = pack

362 fromString = pack

369

363

370#if __GLASGOW_HASKELL__ >= 708

364#if __GLASGOW_HASKELL__ >= 708

371−-- | @since 1.2.0.0

372instance Exts.IsList Text where

365instance Exts.IsList Text where

373 type Item Text = Char

366 type Item Text = Char

374 fromList = pack

367 fromList = pack

@@ -381,7 +374,6 @@ instance NFData Text where

381 rnf (Chunk _ ts) = rnf ts

374 rnf (Chunk _ ts) = rnf ts

382#endif

375#endif

383

376

384−-- | @since 1.2.1.0

385instance Binary Text where

377instance Binary Text where

386 put t = put (encodeUtf8 t)

378 put t = put (encodeUtf8 t)

387 get = do

379 get = do

@@ -405,8 +397,6 @@ instance Data Text where

405

397

406#if MIN_VERSION_base(4,7,0)

398#if MIN_VERSION_base(4,7,0)

407-- | Only defined for @base-4.7.0.0@ and later

399-- | Only defined for @base-4.7.0.0@ and later

408−--

409−-- @since 1.2.2.0

410instance PrintfArg Text where

400instance PrintfArg Text where

411 formatArg txt = formatString $ unpack txt

401 formatArg txt = formatString $ unpack txt

412#endif

402#endif

@@ -485,7 +475,9 @@ fromStrict t = chunk t Empty

485-- -----------------------------------------------------------------------------

475-- -----------------------------------------------------------------------------

486-- * Basic functions

476-- * Basic functions

487

477

488−-- | /O(1)/ Adds a character to the front of a 'Text'. Subject to fusion.

478+-- | /O(n)/ Adds a character to the front of a 'Text'. This function

479+-- is more costly than its 'List' counterpart because it requires

480+-- copying a new array. Subject to fusion.

489cons :: Char -> Text -> Text

481cons :: Char -> Text -> Text

490cons c t = Chunk (T.singleton c) t

482cons c t = Chunk (T.singleton c) t

491{-# INLINE [1] cons #-}

483{-# INLINE [1] cons #-}

@@ -569,17 +561,6 @@ init Empty = emptyError "init"

569 unstream (S.init (stream t)) = init t

561 unstream (S.init (stream t)) = init t

570 #-}

562 #-}

571

563

572−-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if

573−-- empty.

574−--

575−-- * It is no faster than using 'init' and 'last'.

576−--

577−-- @since 1.2.3.0

578−unsnoc :: Text -> Maybe (Text, Char)

579−unsnoc Empty = Nothing

580−unsnoc ts@(Chunk _ _) = Just (init ts, last ts)

581−{-# INLINE unsnoc #-}

582−

583-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

564-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

584-- fusion.

565-- fusion.

585null :: Text -> Bool

566null :: Text -> Bool

@@ -839,8 +820,6 @@ toUpper t = unstream (S.toUpper (stream t))

839-- guides disagree on whether the book name \"The Hill of the Red

820-- guides disagree on whether the book name \"The Hill of the Red

840-- Fox\" is correctly title cased—but this function will

821-- Fox\" is correctly title cased—but this function will

841-- capitalize /every/ word.

822-- capitalize /every/ word.

842−--

843−-- @since 1.0.0.0

844toTitle :: Text -> Text

823toTitle :: Text -> Text

845toTitle t = unstream (S.toTitle (stream t))

824toTitle t = unstream (S.toTitle (stream t))

846{-# INLINE toTitle #-}

825{-# INLINE toTitle #-}

@@ -993,8 +972,6 @@ mapAccumR f = go

993

972

994-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every

973-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every

995-- element.

974-- element.

996−--

997−-- @since 1.2.0.5

998repeat :: Char -> Text

975repeat :: Char -> Text

999repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t

976repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t

1000 in t

977 in t

@@ -1012,8 +989,6 @@ replicate n t

1012

989

1013-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or

990-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or

1014-- equivalently, the infinite repetition of the original 'Text'.

991-- equivalently, the infinite repetition of the original 'Text'.

1015−--

1016−-- @since 1.2.0.5

1017cycle :: Text -> Text

992cycle :: Text -> Text

1018cycle Empty = emptyError "cycle"

993cycle Empty = emptyError "cycle"

1019cycle t = let t' = foldrChunks Chunk t' t

994cycle t = let t' = foldrChunks Chunk t' t

@@ -1023,8 +998,6 @@ cycle t = let t' = foldrChunks Chunk t' t

1023-- of @f@ to @x@:

998-- of @f@ to @x@:

1024--

999--

1025-- > iterate f x == [x, f x, f (f x), ...]

1000-- > iterate f x == [x, f x, f (f x), ...]

1026−--

1027−-- @since 1.2.0.5

1028iterate :: (Char -> Char) -> Char -> Text

1001iterate :: (Char -> Char) -> Char -> Text

1029iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))

1002iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))

1030 in t c

1003 in t c

@@ -1088,8 +1061,6 @@ take i t0 = take' i t0

1088-- Examples:

1061-- Examples:

1089--

1062--

1090-- > takeEnd 3 "foobar" == "bar"

1063-- > takeEnd 3 "foobar" == "bar"

1091−--

1092−-- @since 1.1.1.0

1093takeEnd :: Int64 -> Text -> Text

1064takeEnd :: Int64 -> Text -> Text

1094takeEnd n t0

1065takeEnd n t0

1095 | n <= 0 = empty

1066 | n <= 0 = empty

@@ -1128,8 +1099,6 @@ drop i t0

1128-- Examples:

1099-- Examples:

1129--

1100--

1130-- > dropEnd 3 "foobar" == "foo"

1101-- > dropEnd 3 "foobar" == "foo"

1131−--

1132−-- @since 1.1.1.0

1133dropEnd :: Int64 -> Text -> Text

1102dropEnd :: Int64 -> Text -> Text

1134dropEnd n t0

1103dropEnd n t0

1135 | n <= 0 = t0

1104 | n <= 0 = t0

@@ -1181,15 +1150,12 @@ takeWhile p t0 = takeWhile' t0

1181-- Examples:

1150-- Examples:

1182--

1151--

1183-- > takeWhileEnd (=='o') "foo" == "oo"

1152-- > takeWhileEnd (=='o') "foo" == "oo"

1184−--

1185−-- @since 1.2.2.0

1186takeWhileEnd :: (Char -> Bool) -> Text -> Text

1153takeWhileEnd :: (Char -> Bool) -> Text -> Text

1187takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1154takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1188 where takeChunk acc [] = acc

1155 where takeChunk acc [] = acc

1189− takeChunk acc (t:ts)

1156+ takeChunk acc (t:ts) = if T.length t' < T.length t

1190− | T.lengthWord16 t' < T.lengthWord16 t

1157+ then (Chunk t' acc)

1191− = chunk t' acc

1192− | otherwise = takeChunk (Chunk t' acc) ts

1158+ else takeChunk (Chunk t' acc) ts

1193 where t' = T.takeWhileEnd p t

1159 where t' = T.takeWhileEnd p t

1194{-# INLINE takeWhileEnd #-}

1160{-# INLINE takeWhileEnd #-}

1195

1161

Data/Text/Lazy/Builder.hs+1

@@ -11,6 +11,7 @@

11-- License : BSD-style (see LICENSE)

12--

13-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

14+-- Stability : experimental

14-- Portability : portable to Hugs and GHC

15-- Portability : portable to Hugs and GHC

15--

16--

16-- Efficient construction of lazy @Text@ values. The principal

17-- Efficient construction of lazy @Text@ values. The principal

Data/Text/Lazy/Builder/Int.hs+1−3

@@ -9,6 +9,7 @@

10-- License: BSD-style

11-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

12+-- Stability: experimental

12-- Portability: portable

13-- Portability: portable

13--

14--

14-- Efficiently write an integral value to a 'Builder'.

15-- Efficiently write an integral value to a 'Builder'.

@@ -31,9 +32,6 @@ import GHC.Base (quotInt, remInt)

31import GHC.Num (quotRemInteger)

32import GHC.Num (quotRemInteger)

32import GHC.Types (Int(..))

33import GHC.Types (Int(..))

33import Control.Monad.ST

34import Control.Monad.ST

34−#if MIN_VERSION_base(4,11,0)

35−import Prelude hiding ((<>))

36−#endif

37

35

38#ifdef __GLASGOW_HASKELL__

36#ifdef __GLASGOW_HASKELL__

39# if defined(INTEGER_GMP)

37# if defined(INTEGER_GMP)

Data/Text/Lazy/Builder/RealFloat.hs−3

@@ -24,9 +24,6 @@ import Data.Text.Lazy.Builder.Int (decimal)

24import Data.Text.Internal.Builder.RealFloat.Functions (roundTo)

25import Data.Text.Lazy.Builder

26import qualified Data.Text as T

27−#if MIN_VERSION_base(4,11,0)

28−import Prelude hiding ((<>))

29−#endif

30

27

31-- | Control the rendering of floating point numbers.

28-- | Control the rendering of floating point numbers.

32data FPFormat = Exponent

29data FPFormat = Exponent

Data/Text/Lazy/Encoding.hs+3−13

@@ -8,13 +8,14 @@

8--

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11+-- Stability : experimental

11-- Portability : portable

12-- Portability : portable

12--

13--

13-- Functions for converting lazy 'Text' values to and from lazy

14-- Functions for converting lazy 'Text' values to and from lazy

14-- 'ByteString', using several standard encodings.

15-- 'ByteString', using several standard encodings.

15--

16--

16−-- To gain access to a much larger family of encodings, use the

17+-- To gain access to a much larger variety of encodings, use the

17−-- <http://hackage.haskell.org/package/text-icu text-icu package>.

18+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

18

19

19module Data.Text.Lazy.Encoding

20module Data.Text.Lazy.Encoding

20 (

21 (

@@ -140,7 +141,6 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

140 rnf (Chunk _ ts) = rnf ts

141 rnf (Chunk _ ts) = rnf ts

141{-# INLINE decodeUtf8' #-}

142{-# INLINE decodeUtf8' #-}

142

143

143−-- | Encode text using UTF-8 encoding.

144encodeUtf8 :: Text -> B.ByteString

145encodeUtf8 Empty = B.empty

146encodeUtf8 lt@(Chunk t _) =

@@ -154,20 +154,10 @@ encodeUtf8 lt@(Chunk t _) =

154 firstChunkSize = min B.smallChunkSize (4 * (T.length t + 1))

155 strategy = B.safeStrategy firstChunkSize B.defaultChunkSize

156

157−-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

158−--

159−-- @since 1.1.0.0

160encodeUtf8Builder :: Text -> B.Builder

157encodeUtf8Builder :: Text -> B.Builder

161encodeUtf8Builder =

158encodeUtf8Builder =

162 foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

159 foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

163

160

164−-- | Encode text using UTF-8 encoding and escape the ASCII characters using

165−-- a 'BP.BoundedPrim'.

166−--

167−-- Use this function is to implement efficient encoders for text-based formats

168−-- like JSON or HTML.

169−--

170−-- @since 1.1.0.0

171{-# INLINE encodeUtf8BuilderEscaped #-}

161{-# INLINE encodeUtf8BuilderEscaped #-}

172encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

162encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

173encodeUtf8BuilderEscaped prim =

163encodeUtf8BuilderEscaped prim =

Data/Text/Lazy/IO.hs+1

@@ -8,6 +8,7 @@

9-- License : BSD-style

10-- Maintainer : bos@serpentine.com

11+-- Stability : experimental

11-- Portability : GHC

12-- Portability : GHC

12--

13--

13-- Efficient locale-sensitive support for lazy text I\/O.

14-- Efficient locale-sensitive support for lazy text I\/O.

Data/Text/Lazy/Read.hs+4−3

@@ -11,6 +11,7 @@

11--

12-- License : BSD-style

13-- Maintainer : bos@serpentine.com

14+-- Stability : experimental

14-- Portability : GHC

15-- Portability : GHC

15--

16--

16-- Functions used frequently when reading textual data.

17-- Functions used frequently when reading textual data.

@@ -67,9 +68,9 @@ decimal txt

67 go n d = (n * 10 + fromIntegral (digitToInt d))

68 go n d = (n * 10 + fromIntegral (digitToInt d))

68

69

69-- | Read a hexadecimal integer, consisting of an optional leading

70-- | Read a hexadecimal integer, consisting of an optional leading

70−-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

71+-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

71−-- consumed until a non-hex-digit or end of string is reached.

72+-- until a non-hex-digit or end of string is reached. This function

72−-- This function is case insensitive.

73+-- is case insensitive.

73--

74--

74-- This function does not handle leading sign characters. If you need

75-- This function does not handle leading sign characters. If you need

75-- to handle signed input, use @'signed' 'hexadecimal'@.

76-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Read.hs+4−3

@@ -9,6 +9,7 @@

9--

10-- License : BSD-style

11-- Maintainer : bos@serpentine.com

12+-- Stability : experimental

12-- Portability : GHC

13-- Portability : GHC

13--

14--

14-- Functions used frequently when reading textual data.

15-- Functions used frequently when reading textual data.

@@ -66,9 +67,9 @@ decimal txt

66 go n d = (n * 10 + fromIntegral (digitToInt d))

67 go n d = (n * 10 + fromIntegral (digitToInt d))

67

68

68-- | Read a hexadecimal integer, consisting of an optional leading

69-- | Read a hexadecimal integer, consisting of an optional leading

69−-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

70+-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

70−-- consumed until a non-hex-digit or end of string is reached.

71+-- until a non-hex-digit or end of string is reached. This function

71−-- This function is case insensitive.

72+-- is case insensitive.

72--

73--

73-- This function does not handle leading sign characters. If you need

74-- This function does not handle leading sign characters. If you need

74-- to handle signed input, use @'signed' 'hexadecimal'@.

75-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Show.hs−2

@@ -46,8 +46,6 @@ unpack = S.unstreamList . stream

46-- fusion.

47--

48-- This is exposed solely for people writing GHC rewrite rules.

49−--

50−-- @since 1.2.1.1

51unpackCString# :: Addr# -> Text

49unpackCString# :: Addr# -> Text

52unpackCString# addr# = unstream (S.streamCString# addr#)

50unpackCString# addr# = unstream (S.streamCString# addr#)

53{-# NOINLINE unpackCString# #-}

51{-# NOINLINE unpackCString# #-}

Data/Text/Unsafe.hs+1−2

@@ -4,6 +4,7 @@

5-- License : BSD-style

6-- Maintainer : bos@serpentine.com

7+-- Stability : experimental

7-- Portability : portable

8-- Portability : portable

8--

9--

9-- A module containing unsafe 'Text' operations, for very very careful

10-- A module containing unsafe 'Text' operations, for very very careful

@@ -97,8 +98,6 @@ reverseIter (Text arr off _len) i

97-- | /O(1)/ Iterate one step backwards through a UTF-16 array,

98-- | /O(1)/ Iterate one step backwards through a UTF-16 array,

98-- returning the delta to add (i.e. a negative number) to give the

99-- returning the delta to add (i.e. a negative number) to give the

99-- next offset to iterate at.

100-- next offset to iterate at.

100−--

101−-- @since 1.1.1.0

102reverseIter_ :: Text -> Int -> Int

101reverseIter_ :: Text -> Int -> Int

103reverseIter_ (Text arr off _len) i

102reverseIter_ (Text arr off _len) i

104 | m < 0xDC00 || m > 0xDFFF = -1

103 | m < 0xDC00 || m > 0xDFFF = -1

README.markdown+16−5

@@ -1,18 +1,29 @@

1−# `text`: Fast, packed Unicode strings, using stream fusion

1+# Text: Fast, packed Unicode strings, using stream fusion

2

3This package provides the Data.Text library, a library for the space-

4and time-efficient manipulation of Unicode text in Haskell.

5

6−**Please refer to the [package description on Hackage](https://hackage.haskell.org/package/text#description) for more information.**

6+

7+# Normalization, conversion, and collation, oh my!

8+

9+This library intentionally provides a simple API based on the

10+Haskell prelude's list manipulation functions. For more complicated

11+real-world tasks, such as Unicode normalization, conversion to and

12+from a larger variety of encodings, and collation, use the [text-icu

13+package](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/text-icu).

14+

15+That library uses the well-respected and liberally licensed ICU

16+library to provide these facilities.

17+

7

18

8# Get involved!

19# Get involved!

9

20

10Please report bugs via the

21Please report bugs via the

11−[github issue tracker](https://github.com/haskell/text/issues).

22+[github issue tracker](https://github.com/bos/text/issues).

12

23

13−Master [git repository](https://github.com/haskell/text):

24+Master [git repository](https://github.com/bos/text):

14

25

15−* `git clone git://github.com/haskell/text.git`

26+* `git clone git://github.com/bos/text.git`

16

27

17There's also a [Mercurial mirror](https://bitbucket.org/bos/text):

28There's also a [Mercurial mirror](https://bitbucket.org/bos/text):

18

29

benchmarks/.gitignore+1−4

@@ -1,4 +1 @@

1−/dist/

2−/dist-newstyle/

1+dist

3−/.ghc.environment.*

4−/cabal.project.local

benchmarks/cabal.projectdeleted−2

@@ -1,2 +0,0 @@

1−-- NB: we use a separate project

2−packages: .

benchmarks/haskell/Benchmarks.hs−2

@@ -10,7 +10,6 @@ import System.FilePath ((</>))

10import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8)

11

12import qualified Benchmarks.Builder as Builder

13−import qualified Benchmarks.Concat as Concat

14import qualified Benchmarks.DecodeUtf8 as DecodeUtf8

13import qualified Benchmarks.DecodeUtf8 as DecodeUtf8

15import qualified Benchmarks.EncodeUtf8 as EncodeUtf8

14import qualified Benchmarks.EncodeUtf8 as EncodeUtf8

16import qualified Benchmarks.Equality as Equality

15import qualified Benchmarks.Equality as Equality

@@ -42,7 +41,6 @@ benchmarks = do

42 -- Traditional benchmarks

41 -- Traditional benchmarks

43 bs <- sequence

42 bs <- sequence

44 [ Builder.benchmark

43 [ Builder.benchmark

45− , Concat.benchmark

46 , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")

44 , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")

47 , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")

45 , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")

48 , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")

46 , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")

benchmarks/haskell/Benchmarks/Concat.hsdeleted−25

@@ -1,25 +0,0 @@

1−{-# LANGUAGE OverloadedStrings #-}

2−

3−module Benchmarks.Concat (benchmark) where

4−

5−import Control.Monad.Trans.Writer

6−import Criterion (Benchmark, bgroup, bench, whnf)

7−import Data.Text as T

8−

9−benchmark :: IO Benchmark

10−benchmark = return $ bgroup "Concat"

11− [ bench "append" $ whnf (append4 "Text 1" "Text 2" "Text 3") "Text 4"

12− , bench "concat" $ whnf (concat4 "Text 1" "Text 2" "Text 3") "Text 4"

13− , bench "write" $ whnf (write4 "Text 1" "Text 2" "Text 3") "Text 4"

14− ]

15−

16−append4, concat4, write4 :: Text -> Text -> Text -> Text -> Text

17−

18−{-# NOINLINE append4 #-}

19−append4 x1 x2 x3 x4 = x1 `append` x2 `append` x3 `append` x4

20−

21−{-# NOINLINE concat4 #-}

22−concat4 x1 x2 x3 x4 = T.concat [x1, x2, x3, x4]

23−

24−{-# NOINLINE write4 #-}

25−write4 x1 x2 x3 x4 = execWriter $ tell x1 >> tell x2 >> tell x3 >> tell x4

benchmarks/text-benchmarks.cabal+8−82

@@ -12,7 +12,8 @@ author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

12maintainer: jaspervdj@gmail.com

13category: Text

14build-type: Simple

15+

15−cabal-version: >=1.8

16+cabal-version: >=1.2

16

17

17flag bytestring-builder

18flag bytestring-builder

18 description: Depend on the bytestring-builder package for backwards compatibility.

19 description: Depend on the bytestring-builder package for backwards compatibility.

@@ -25,12 +26,16 @@ flag llvm

25 manual: True

26 manual: True

26

27

27executable text-benchmarks

28executable text-benchmarks

29+ hs-source-dirs: haskell ..

30+ c-sources: ../cbits/cbits.c

31+ cbits/time_iconv.c

32+ include-dirs: ../include

33+ main-is: Benchmarks.hs

28 ghc-options: -Wall -O2 -rtsopts

34 ghc-options: -Wall -O2 -rtsopts

29 if flag(llvm)

35 if flag(llvm)

30 ghc-options: -fllvm

36 ghc-options: -fllvm

31 cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP

37 cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP

32− build-depends: array,

38+ build-depends: base == 4.*,

33− base == 4.*,

34 binary,

39 binary,

35 blaze-builder,

40 blaze-builder,

36 bytestring-lexing >= 0.5.0,

41 bytestring-lexing >= 0.5.0,

@@ -42,7 +47,6 @@ executable text-benchmarks

42 ghc-prim,

47 ghc-prim,

43 integer-gmp,

48 integer-gmp,

44 stringsearch,

49 stringsearch,

45− transformers,

46 utf8-string,

50 utf8-string,

47 vector

51 vector

48

52

@@ -52,87 +56,9 @@ executable text-benchmarks

52 else

56 else

53 build-depends: bytestring >= 0.10.4

57 build-depends: bytestring >= 0.10.4

54

58

55− -- modules for benchmark proper

56− c-sources: cbits/time_iconv.c

57− hs-source-dirs: haskell

58− main-is: Benchmarks.hs

59− other-modules:

60− Benchmarks.Builder

61− Benchmarks.Concat

62− Benchmarks.DecodeUtf8

63− Benchmarks.EncodeUtf8

64− Benchmarks.Equality

65− Benchmarks.FileRead

66− Benchmarks.FoldLines

67− Benchmarks.Mul

68− Benchmarks.Programs.BigTable

69− Benchmarks.Programs.Cut

70− Benchmarks.Programs.Fold

71− Benchmarks.Programs.Sort

72− Benchmarks.Programs.StripTags

73− Benchmarks.Programs.Throughput

74− Benchmarks.Pure

75− Benchmarks.ReadNumbers

76− Benchmarks.Replace

77− Benchmarks.Search

78− Benchmarks.Stream

79− Benchmarks.WordFrequencies

80−

81− -- Source code for IUT (implementation under test)

82− -- "borrowed" from parent folder

83− include-dirs: ../include

84− c-sources: ../cbits/cbits.c

85− hs-source-dirs: ..

86− other-modules:

87− Data.Text

88− Data.Text.Array

89− Data.Text.Encoding

90− Data.Text.Encoding.Error

91− Data.Text.Foreign

92− Data.Text.IO

93− Data.Text.Internal

94− Data.Text.Internal.Builder

95− Data.Text.Internal.Builder.Functions

96− Data.Text.Internal.Builder.Int.Digits

97− Data.Text.Internal.Builder.RealFloat.Functions

98− Data.Text.Internal.Encoding.Fusion

99− Data.Text.Internal.Encoding.Fusion.Common

100− Data.Text.Internal.Encoding.Utf16

101− Data.Text.Internal.Encoding.Utf32

102− Data.Text.Internal.Encoding.Utf8

103− Data.Text.Internal.Functions

104− Data.Text.Internal.Fusion

105− Data.Text.Internal.Fusion.CaseMapping

106− Data.Text.Internal.Fusion.Common

107− Data.Text.Internal.Fusion.Size

108− Data.Text.Internal.Fusion.Types

109− Data.Text.Internal.IO

110− Data.Text.Internal.Lazy

111− Data.Text.Internal.Lazy.Encoding.Fusion

112− Data.Text.Internal.Lazy.Fusion

113− Data.Text.Internal.Lazy.Search

114− Data.Text.Internal.Private

115− Data.Text.Internal.Read

116− Data.Text.Internal.Search

117− Data.Text.Internal.Unsafe

118− Data.Text.Internal.Unsafe.Char

119− Data.Text.Internal.Unsafe.Shift

120− Data.Text.Lazy

121− Data.Text.Lazy.Builder

122− Data.Text.Lazy.Builder.Int

123− Data.Text.Lazy.Builder.RealFloat

124− Data.Text.Lazy.Encoding

125− Data.Text.Lazy.IO

126− Data.Text.Lazy.Internal

127− Data.Text.Lazy.Read

128− Data.Text.Read

129− Data.Text.Unsafe

130− Data.Text.Show

131−

132executable text-multilang

59executable text-multilang

133 hs-source-dirs: haskell

60 hs-source-dirs: haskell

134 main-is: Multilang.hs

61 main-is: Multilang.hs

135− other-modules: Timer

136 ghc-options: -Wall -O2

62 ghc-options: -Wall -O2

137 build-depends: base == 4.*,

63 build-depends: base == 4.*,

138 bytestring,

64 bytestring,

cabal.projectdeleted−3

@@ -1,3 +0,0 @@

1−-- See http://cabal.readthedocs.io/en/latest/nix-local-build-overview.html

2−packages: .

3−tests: True

cbits/cbits.c+11−6

@@ -206,9 +206,11 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

206 const uint8_t *srcend,

207 uint32_t *codepoint0, uint32_t *state0)

208{

209− _hs_text_decode_utf8_int(dest, destoff, src, srcend, codepoint0, state0);

209+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,

210−

210+ codepoint0, state0);

211+ if (*state0 == UTF8_REJECT)

212+ ret -=1;

211− return *src;

213+ return ret;

212}

214}

213

215

214/*

216/*

@@ -220,9 +222,12 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

220{

222{

221 uint32_t codepoint;

223 uint32_t codepoint;

222 uint32_t state = UTF8_ACCEPT;

224 uint32_t state = UTF8_ACCEPT;

223− _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

225+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

224− &codepoint, &state);

226+ &codepoint, &state);

227+ /* Back up if we have an incomplete or invalid encoding */

228+ if (state != UTF8_ACCEPT)

229+ ret -= 1;

225− return src;

230+ return ret;

226}

231}

227

232

228void

233void

changelog.md+19−56

@@ -1,52 +1,15 @@

1−### 1.2.3.0

2−

3−* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec

4− (updated from 8.0).

5−

6−* Bug fix: the lazy `takeWhileEnd` function violated the

7− [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)

8− (gh-184).

9−

10−* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).

11−

12−* New function: `unsnoc` (gh-173).

13−

14−* Reduce memory overhead in `encodeUTF8` (gh-194).

15−

16−* Improve UTF-8 decoder error-recovery (gh-182).

17−

18−* Minor documentation improvements (`@since` annotations, more

19− examples, clarifications).

20−

21−#### 1.2.2.2

22−

23−* The `toTitle` function now correctly handles letters that

24− immediately follow punctuation. Before, `"there's"` would turn into

25− `"There'S"`. Now, it becomes `"There's"`.

26−

27−* The implementation of unstreaming is faster, resulting in operations

28− such as `map` and `intersperse` speeding up by up to 30%, with

29− smaller code generated.

30−

31−* The optimised length comparison function is now more likely to be

32− used after some rewrite rule tweaking.

33−

34−* Bug fix: an off-by-one bug in `takeEnd` is fixed.

35−

36−* Bug fix: a logic error in `takeWord16` is fixed.

37−

38−#### 1.2.2.1

1+1.2.2.1

39

2

40* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

3* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

41 The build flag has been renamed accordingly. Your army of diligent

4 The build flag has been renamed accordingly. Your army of diligent

42 maintainers apologizes for the churn.

5 maintainers apologizes for the churn.

43

6

44−* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec

7+* Spec compliance: toCaseFold now follows the Unicode 8.0 spec

45 (updated from 7.0)

8 (updated from 7.0)

46

9

47* An STG lint error has been fixed

10* An STG lint error has been fixed

48

11

49−### 1.2.2.0

12+1.2.2.0

50

13

51* The `integer-simple` package, upon which this package optionally

14* The `integer-simple` package, upon which this package optionally

52 depended, has been replaced with `integer-pure`. The build flag has

15 depended, has been replaced with `integer-pure`. The build flag has

@@ -62,7 +25,7 @@

62 * if `base` >= 4.7: `PrintfArg`

25 * if `base` >= 4.7: `PrintfArg`

63 * if `base` >= 4.9: `Semigroup`

26 * if `base` >= 4.9: `Semigroup`

64

27

65−#### 1.2.1.3

28+1.2.1.3

66

29

67* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

30* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

68 phase 2 does not prevent competition with the `unpack` rule, which is

31 phase 2 does not prevent competition with the `unpack` rule, which is

@@ -70,7 +33,7 @@

70 test environment mistake. Moving literal rules back to phase 1 finally

33 test environment mistake. Moving literal rules back to phase 1 finally

71 fixes GHC Trac #10528 correctly.

34 fixes GHC Trac #10528 correctly.

72

35

73−#### 1.2.1.2

36+1.2.1.2

74

37

75* Bug fix: Run literal rewrite rules in simplifier phase 2.

38* Bug fix: Run literal rewrite rules in simplifier phase 2.

76 The behavior of the simplifier changed in GHC 7.10.2,

39 The behavior of the simplifier changed in GHC 7.10.2,

@@ -78,21 +41,21 @@

78 and long compilation times. See

41 and long compilation times. See

79 [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

42 [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

80

43

81−#### 1.2.1.1

44+1.2.1.1

82

45

83* Expose unpackCString#, which you should never use.

46* Expose unpackCString#, which you should never use.

84

47

85−### 1.2.1.0

48+1.2.1.0

86

49

87* Added Binary instances for both Text types. (If you have previously

50* Added Binary instances for both Text types. (If you have previously

88 been using the text-binary package to get a Binary instance, it is

51 been using the text-binary package to get a Binary instance, it is

89 now obsolete.)

52 now obsolete.)

90

53

91−#### 1.2.0.6

54+1.2.0.6

92

55

93* Fixed a space leak in UTF-8 decoding

56* Fixed a space leak in UTF-8 decoding

94

57

95−#### 1.2.0.5

58+1.2.0.5

96

59

97* Feature parity: repeat, cycle, iterate are now implemented for lazy

60* Feature parity: repeat, cycle, iterate are now implemented for lazy

98 Text, and the Data instance is more complete

61 Text, and the Data instance is more complete

@@ -112,26 +75,26 @@

112* Spec compliance: toCaseFold now follows the Unicode 7.0 spec

75* Spec compliance: toCaseFold now follows the Unicode 7.0 spec

113 (updated from 6.3)

76 (updated from 6.3)

114

77

115−#### 1.2.0.4

78+1.2.0.4

116

79

117* Fixed an incompatibility with base < 4.5

80* Fixed an incompatibility with base < 4.5

118

81

119−#### 1.2.0.3

82+1.2.0.3

120

83

121* Update formatRealFloat to correspond to the definition in versions

84* Update formatRealFloat to correspond to the definition in versions

122 of base newer than 4.5 (https://github.com/bos/text/issues/105)

85 of base newer than 4.5 (https://github.com/bos/text/issues/105)

123

86

124−#### 1.2.0.2

87+1.2.0.2

125

88

126* Bumped lower bound on deepseq to 1.4 for compatibility with the

89* Bumped lower bound on deepseq to 1.4 for compatibility with the

127 upcoming GHC 7.10

90 upcoming GHC 7.10

128

91

129−#### 1.2.0.1

92+1.2.0.1

130

93

131* Fixed a buffer overflow in rendering of large Integers

94* Fixed a buffer overflow in rendering of large Integers

132 (https://github.com/bos/text/issues/99)

95 (https://github.com/bos/text/issues/99)

133

96

134−## 1.2.0.0

97+1.2.0.0

135

98

136* Fixed an integer overflow in the replace function

99* Fixed an integer overflow in the replace function

137 (https://github.com/bos/text/issues/81)

100 (https://github.com/bos/text/issues/81)

@@ -144,7 +107,7 @@

144

107

145* Added an instance of IsList for GHC 7.8 and above

108* Added an instance of IsList for GHC 7.8 and above

146

109

147−### 1.1.1.0

110+1.1.1.0

148

111

149* The Data.Data instance now allows gunfold to work, via a virtual

112* The Data.Data instance now allows gunfold to work, via a virtual

150 pack constructor

113 pack constructor

@@ -154,12 +117,12 @@

154* Comparing the length of a Text against a number can now

117* Comparing the length of a Text against a number can now

155 short-circuit in more cases

118 short-circuit in more cases

156

119

157−#### 1.1.0.1

120+1.1.0.1

158

121

159* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes

122* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes

160 in single-byte chunks

123 in single-byte chunks

161

124

162−## 1.1.0.0

125+1.1.0.0

163

126

164* encodeUtf8: Performance is improved by up to 4x.

127* encodeUtf8: Performance is improved by up to 4x.

165

128

@@ -178,12 +141,12 @@

178 use at your own risk - there are no API stability guarantees for

141 use at your own risk - there are no API stability guarantees for

179 internal modules!

142 internal modules!

180

143

181−#### 1.0.0.1

144+1.0.0.1

182

145

183* decodeUtf8: Fixed a regression that caused us to incorrectly

146* decodeUtf8: Fixed a regression that caused us to incorrectly

184 identify truncated UTF-8 as valid (gh-61)

147 identify truncated UTF-8 as valid (gh-61)

185

148

186−# 1.0.0.0

149+1.0.0.0

187

150

188* Added support for Unicode 6.3.0 to case conversion functions

151* Added support for Unicode 6.3.0 to case conversion functions

189

152

tests-and-benchmarks.markdown+8−13

@@ -13,13 +13,10 @@ you can clone from one of the following locations:

13* Git mirror repository:

14 [github.com/bos/text-test-data](https://github.com/bos/text-test-data)

15

16−You can clone either repository into the `tests` subdirectory using

16+You should clone that repository into the `tests` subdirectory (your

17−

17+clone must be named `text-test-data` locally), then run `make -C

18− cd tests/

18+tests/text-test-data` to uncompress the test files. Many tests and

19− make text-test-data # to clone from mercurial, OR

20− make VCS=git text-test-data # to clone from git

21−

22−Many tests and benchmarks will fail if the test files are missing.

19+benchmarks will fail if the test files are missing.

23

20

24Functional tests

21Functional tests

25----------------

22----------------

@@ -57,12 +54,10 @@ To compile the benchmarks, navigate to the `benchmarks` subdirectory and run

57

54

58 ./dist/build/text-benchmarks/text-benchmarks

55 ./dist/build/text-benchmarks/text-benchmarks

59

56

60−Or if you have a recent enough `cabal`, you can build and run the

57+However, since there's quite a lot of benchmarks, you usually don't want to

61−benchmarks via

58+run them all. Instead, use the `-l` flag to get a list of benchmarks:

62

59

63− cabal new-run exe:text-benchmarks -- --help

60+ ./dist/build/text-benchmarks/text-benchmarks

64

61

65−However, since there's quite a lot of benchmarks, you usually don't want to

66−run them all. Instead, use the `-l` flag to get a list of benchmarks

67−and run the ones you want to inspect. If you want to configure the benchmarks

62+And run the ones you want to inspect. If you want to configure the benchmarks

68further, the exact parameters can be changed in `Benchmarks.hs`.

63further, the exact parameters can be changed in `Benchmarks.hs`.

tests/Makefile+1−6

@@ -1,4 +1,3 @@

1−VCS = hg

2count = 1000

1count = 1000

3

2

4all: coverage literal-rule-test

3all: coverage literal-rule-test

@@ -13,11 +12,7 @@ build: text-test-data

13 cabal build

12 cabal build

14

13

15text-test-data:

14text-test-data:

16−ifeq ($(VCS),git)

17− git clone https://github.com/bos/text-test-data.git

18−else

19 hg clone https://bitbucket.org/bos/text-test-data

15 hg clone https://bitbucket.org/bos/text-test-data

20−endif

21 $(MAKE) -C text-test-data

16 $(MAKE) -C text-test-data

22

17

23coverage/text-tests.tix:

18coverage/text-tests.tix:

@@ -42,4 +37,4 @@ coverage/hpc_index.html: coverage/coverage.tix

42clean:

37clean:

43 rm -rf dist coverage .hpc

38 rm -rf dist coverage .hpc

44

39

45−.PHONY: all build clean coverage literal-rule-test

40+.PHONY: build coverage all literal-rule-test

tests/Tests/Properties.hs+21−94

@@ -9,9 +9,9 @@ module Tests.Properties

9 ) where

10

11import Control.Applicative ((<$>), (<*>))

12−import Control.Arrow ((***), first, second)

12+import Control.Arrow ((***), second)

13import Data.Bits ((.&.))

14−import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord)

14+import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)

15import Data.Int (Int8, Int16, Int32, Int64)

16import Data.Monoid (Monoid(..))

17import Data.String (IsString(fromString))

@@ -23,7 +23,6 @@ import Data.Text.Internal.Search (indices)

23import Data.Text.Lazy.Read as TL

24import Data.Text.Read as T

25import Data.Word (Word, Word8, Word16, Word32, Word64)

26−import Data.Maybe (mapMaybe)

27import Numeric (showEFloat, showFFloat, showGFloat, showHex)

26import Numeric (showEFloat, showFFloat, showGFloat, showHex)

28import Prelude hiding (replicate)

27import Prelude hiding (replicate)

29import Test.Framework (Test, testGroup)

28import Test.Framework (Test, testGroup)

@@ -31,7 +30,6 @@ import Test.Framework.Providers.QuickCheck2 (testProperty)

31import Test.QuickCheck hiding ((.&.))

30import Test.QuickCheck hiding ((.&.))

32import Test.QuickCheck.Monadic

31import Test.QuickCheck.Monadic

33import Test.QuickCheck.Property (Property(..))

32import Test.QuickCheck.Property (Property(..))

34−import Test.QuickCheck.Unicode (char)

35import Tests.QuickCheckUtils

33import Tests.QuickCheckUtils

36import Tests.Utils

34import Tests.Utils

37import Text.Show.Functions ()

35import Text.Show.Functions ()

@@ -39,7 +37,6 @@ import qualified Control.Exception as Exception

39import qualified Data.Bits as Bits (shiftL, shiftR)

37import qualified Data.Bits as Bits (shiftL, shiftR)

40import qualified Data.ByteString as B

38import qualified Data.ByteString as B

41import qualified Data.ByteString.Lazy as BL

39import qualified Data.ByteString.Lazy as BL

42−import qualified Data.Char as C

43import qualified Data.List as L

40import qualified Data.List as L

44import qualified Data.Text as T

41import qualified Data.Text as T

45import qualified Data.Text.Encoding as E

42import qualified Data.Text.Encoding as E

@@ -63,10 +60,10 @@ t_pack_unpack = (T.unpack . T.pack) `eq` id

63tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

60tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

64t_stream_unstream = (S.unstream . S.stream) `eq` id

61t_stream_unstream = (S.unstream . S.stream) `eq` id

65tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

62tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

66−t_reverse_stream t = (S.reverse . S.reverseStream) t === t

63+t_reverse_stream t = (S.reverse . S.reverseStream) t == t

67−t_singleton c = [c] === (T.unpack . T.singleton) c

64+t_singleton c = [c] == (T.unpack . T.singleton) c

68−tl_singleton c = [c] === (TL.unpack . TL.singleton) c

65+tl_singleton c = [c] == (TL.unpack . TL.singleton) c

69−tl_unstreamChunks x = f 11 x === f 1000 x

66+tl_unstreamChunks x = f 11 x == f 1000 x

70 where f n = SL.unstreamChunks n . S.streamList

67 where f n = SL.unstreamChunks n . S.streamList

71tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

68tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

72tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

69tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

@@ -77,13 +74,13 @@ encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

77encodeLazyL1 :: TL.Text -> BL.ByteString

74encodeLazyL1 :: TL.Text -> BL.ByteString

78encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

75encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

79

76

80−t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a

77+t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

81 where a = T.map (\c -> chr (ord c `mod` 128)) t

78 where a = T.map (\c -> chr (ord c `mod` 128)) t

82−tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a

79+tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

83 where a = TL.map (\c -> chr (ord c `mod` 128)) t

80 where a = TL.map (\c -> chr (ord c `mod` 128)) t

84−t_latin1 t = E.decodeLatin1 (encodeL1 a) === a

81+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

85 where a = T.map (\c -> chr (ord c `mod` 256)) t

82 where a = T.map (\c -> chr (ord c `mod` 256)) t

86−tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a

83+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

87 where a = TL.map (\c -> chr (ord c `mod` 256)) t

84 where a = TL.map (\c -> chr (ord c `mod` 256)) t

88t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

85t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

89t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

86t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

@@ -114,7 +111,7 @@ t_utf8_undecoded = forAll genUnicode $ \t ->

114 let b = E.encodeUtf8 t

111 let b = E.encodeUtf8 t

115 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

112 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

116 leftover = (++ [B.empty]) . init . tail . B.inits

113 leftover = (++ [B.empty]) . init . tail . B.inits

117− in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls

114+ in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b == ls

118

115

119data Badness = Solo | Leading | Trailing

116data Badness = Solo | Leading | Trailing

120 deriving (Eq, Show)

117 deriving (Eq, Show)

@@ -180,33 +177,6 @@ genInvalidUTF8 = B.pack <$> oneof [

180 k <- choose (0,n)

177 k <- choose (0,n)

181 vectorOf k gen

178 vectorOf k gen

182

179

183−-- See http://unicode.org/faq/utf_bom.html#gen8

184−-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...

185−-- When faced with this illegal byte sequence ... a UTF-8 conformant process

186−-- must treat the first byte 110xxxxx2 as an illegal termination error

187−-- (e.g. filter it out or replace by 0xFFFD) ...

188−-- ... and continue processing at the second byte 0xxxxxxx2

189−t_decode_with_error2 =

190− E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"

191−t_decode_with_error3 =

192− E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"

193−t_decode_with_error4 =

194− E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"

195−

196−t_decode_with_error2' =

197− case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of

198− E.Some x _ _ -> x === "xa"

199−t_decode_with_error3' =

200− case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of

201− E.Some x _ _ -> x === "xaa"

202−t_decode_with_error4' =

203− case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of

204− E.Some x _ _ -> x === "xaaa"

205−

206−t_infix_concat bs1 text bs2 rep =

207− text `T.isInfixOf`

208− E.decodeUtf8With (\_ _ -> rep) (B.concat [bs1, E.encodeUtf8 text, bs2])

209−

210s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

180s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

211 where _types = s :: String

181 where _types = s :: String

212sf_Eq p s =

182sf_Eq p s =

@@ -231,8 +201,8 @@ t_mconcat = unsquare $

231 mconcat `eq` (unpackS . mconcat . L.map T.pack)

201 mconcat `eq` (unpackS . mconcat . L.map T.pack)

232tl_mconcat = unsquare $

202tl_mconcat = unsquare $

233 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

203 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

234−t_mempty = mempty === (unpackS (mempty :: T.Text))

204+t_mempty = mempty == (unpackS (mempty :: T.Text))

235−tl_mempty = mempty === (unpackS (mempty :: TL.Text))

205+tl_mempty = mempty == (unpackS (mempty :: TL.Text))

236t_IsString = fromString `eqP` (T.unpack . fromString)

206t_IsString = fromString `eqP` (T.unpack . fromString)

237tl_IsString = fromString `eqP` (TL.unpack . fromString)

207tl_IsString = fromString `eqP` (TL.unpack . fromString)

238

208

@@ -259,13 +229,6 @@ sf_uncons p = (uncons . L.filter p) `eqP`

259 (fmap (second unpackS) . S.uncons . S.filter p)

229 (fmap (second unpackS) . S.uncons . S.filter p)

260t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

230t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

261tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

231tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

262−

263−unsnoc xs@(_:_) = Just (init xs, last xs)

264−unsnoc [] = Nothing

265−

266−t_unsnoc = unsnoc `eqP` (fmap (first unpackS) . T.unsnoc)

267−tl_unsnoc = unsnoc `eqP` (fmap (first unpackS) . TL.unsnoc)

268−

269s_head = head `eqP` S.head

232s_head = head `eqP` S.head

270sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

233sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

271t_head = head `eqP` T.head

234t_head = head `eqP` T.head

@@ -363,19 +326,6 @@ t_toUpper_upper t = p (T.toUpper t) >= p t

363 where p = T.length . T.filter isUpper

326 where p = T.length . T.filter isUpper

364tl_toUpper_upper t = p (TL.toUpper t) >= p t

327tl_toUpper_upper t = p (TL.toUpper t) >= p t

365 where p = TL.length . TL.filter isUpper

328 where p = TL.length . TL.filter isUpper

366−t_toTitle_title t = all (<= 1) (caps w)

367− where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle

368− -- TIL: there exist uppercase-only letters

369− w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t

370−t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable

371− where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words

372− -- Surprise! The Spanish/Portuguese ordinal indicators changed

373− -- from category Ll (letter, lowercase) to Lo (letter, other)

374− -- in Unicode 7.0

375− -- Oh, and there exist lowercase-only letters (see previous test)

376− stable c = if isLower c

377− then C.toUpper c /= c

378− else c /= '\170' && c /= '\186'

379

329

380justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

330justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

381justifyRight m n xs = L.replicate (m - length xs) n ++ xs

331justifyRight m n xs = L.replicate (m - length xs) n ++ xs

@@ -564,20 +514,12 @@ s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p)

564s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

514s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

565sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

515sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

566 (unpackS . S.takeWhile p . S.filter q)

516 (unpackS . S.takeWhile p . S.filter q)

567−noMatch = do

568− c <- char

569− d <- suchThat char (/= c)

570− return (c,d)

571t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

517t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

572tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

518tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

573t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

519t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

574 (unpackS . T.takeWhileEnd p)

520 (unpackS . T.takeWhileEnd p)

575−t_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> T.null $

576− T.takeWhileEnd (==d) (T.snoc t c)

577tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

521tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

578 (unpackS . TL.takeWhileEnd p)

522 (unpackS . TL.takeWhileEnd p)

579−tl_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> TL.null $

580− TL.takeWhileEnd (==d) (TL.snoc t c)

581s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

523s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

582s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

524s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

583sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

525sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

@@ -778,7 +720,7 @@ tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s

778 conc = T.concat . TL.toChunks

720 conc = T.concat . TL.toChunks

779t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

721t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

780 let s = T.intercalate t ts

722 let s = T.intercalate t ts

781− in Slow.indices t s === indices t s

723+ in Slow.indices t s == indices t s

782

724

783-- Bit shifts.

725-- Bit shifts.

784shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

726shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

@@ -870,18 +812,18 @@ tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a

870-- Reading.

812-- Reading.

871

813

872t_decimal (n::Int) s =

814t_decimal (n::Int) s =

873− T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)

815+ T.signed T.decimal (T.pack (show n) `T.append` t) == Right (n,t)

874 where t = T.dropWhile isDigit s

816 where t = T.dropWhile isDigit s

875tl_decimal (n::Int) s =

817tl_decimal (n::Int) s =

876− TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)

818+ TL.signed TL.decimal (TL.pack (show n) `TL.append` t) == Right (n,t)

877 where t = TL.dropWhile isDigit s

819 where t = TL.dropWhile isDigit s

878t_hexadecimal m s ox =

820t_hexadecimal m s ox =

879− T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)

821+ T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) == Right (n,t)

880 where t = T.dropWhile isHexDigit s

822 where t = T.dropWhile isHexDigit s

881 p = if ox then "0x" else ""

823 p = if ox then "0x" else ""

882 n = getPositive m :: Int

824 n = getPositive m :: Int

883tl_hexadecimal m s ox =

825tl_hexadecimal m s ox =

884− TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)

826+ TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) == Right (n,t)

885 where t = TL.dropWhile isHexDigit s

827 where t = TL.dropWhile isHexDigit s

886 p = if ox then "0x" else ""

828 p = if ox then "0x" else ""

887 n = getPositive m :: Int

829 n = getPositive m :: Int

@@ -925,11 +867,11 @@ tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn

925

867

926t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

868t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

927t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

869t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

928−t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t

870+t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) == t

929 where n = small m

871 where n = small m

930t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

872t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

931

873

932−t_copy t = T.copy t === t

874+t_copy t = T.copy t == t

933

875

934-- Regression tests.

876-- Regression tests.

935s_filter_eq s = S.filter p t == S.streamList (filter p s)

877s_filter_eq s = S.filter p t == S.streamList (filter p s)

@@ -982,15 +924,6 @@ tests =

982 testGroup "errors" [

924 testGroup "errors" [

983 testProperty "t_utf8_err" t_utf8_err,

925 testProperty "t_utf8_err" t_utf8_err,

984 testProperty "t_utf8_err'" t_utf8_err'

926 testProperty "t_utf8_err'" t_utf8_err'

985− ],

986− testGroup "error recovery" [

987− testProperty "t_decode_with_error2" t_decode_with_error2,

988− testProperty "t_decode_with_error3" t_decode_with_error3,

989− testProperty "t_decode_with_error4" t_decode_with_error4,

990− testProperty "t_decode_with_error2'" t_decode_with_error2',

991− testProperty "t_decode_with_error3'" t_decode_with_error3',

992− testProperty "t_decode_with_error4'" t_decode_with_error4',

993− testProperty "t_infix_concat" t_infix_concat

994 ]

927 ]

995 ],

928 ],

996

929

@@ -1034,8 +967,6 @@ tests =

1034 testProperty "sf_uncons" sf_uncons,

967 testProperty "sf_uncons" sf_uncons,

1035 testProperty "t_uncons" t_uncons,

968 testProperty "t_uncons" t_uncons,

1036 testProperty "tl_uncons" tl_uncons,

969 testProperty "tl_uncons" tl_uncons,

1037− testProperty "t_unsnoc" t_unsnoc,

1038− testProperty "tl_unsnoc" tl_unsnoc,

1039 testProperty "s_head" s_head,

970 testProperty "s_head" s_head,

1040 testProperty "sf_head" sf_head,

971 testProperty "sf_head" sf_head,

1041 testProperty "t_head" t_head,

972 testProperty "t_head" t_head,

@@ -1099,9 +1030,7 @@ tests =

1099 testProperty "tl_toLower_lower" tl_toLower_lower,

1030 testProperty "tl_toLower_lower" tl_toLower_lower,

1100 testProperty "t_toUpper_length" t_toUpper_length,

1031 testProperty "t_toUpper_length" t_toUpper_length,

1101 testProperty "t_toUpper_upper" t_toUpper_upper,

1032 testProperty "t_toUpper_upper" t_toUpper_upper,

1102− testProperty "tl_toUpper_upper" tl_toUpper_upper,

1033+ testProperty "tl_toUpper_upper" tl_toUpper_upper

1103− testProperty "t_toTitle_title" t_toTitle_title,

1104− testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower

1105 ],

1034 ],

1106

1035

1107 testGroup "justification" [

1036 testGroup "justification" [

@@ -1218,9 +1147,7 @@ tests =

1218 testProperty "t_takeWhile" t_takeWhile,

1147 testProperty "t_takeWhile" t_takeWhile,

1219 testProperty "tl_takeWhile" tl_takeWhile,

1148 testProperty "tl_takeWhile" tl_takeWhile,

1220 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1149 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1221− testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null,

1222 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1150 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1223− testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null,

1224 testProperty "sf_dropWhile" sf_dropWhile,

1151 testProperty "sf_dropWhile" sf_dropWhile,

1225 testProperty "s_dropWhile" s_dropWhile,

1152 testProperty "s_dropWhile" s_dropWhile,

1226 testProperty "s_dropWhile_s" s_dropWhile_s,

1153 testProperty "s_dropWhile_s" s_dropWhile_s,

tests/Tests/QuickCheckUtils.hs+1−1

@@ -106,7 +106,7 @@ smallArbitrary = sized $ \n -> resize (smallish n) arbitrary

106 where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs

107

108instance Arbitrary T.Text where

109− arbitrary = T.pack `fmap` string

109+ arbitrary = T.pack `fmap` arbitrary

110 shrink = map T.pack . shrink . T.unpack

111

112instance Arbitrary TL.Text where

tests/Tests/Regressions.hs−11

@@ -72,16 +72,6 @@ mapAccumL_resize = do

72 assertEqual "mapAccumL should correctly size buffers for two-word results"

73 (count * 2) (T.lengthWord16 (snd val))

74

75−-- See GitHub #197

76−t197 :: IO ()

77−t197 =

78− assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")

79− where

80− currencyParser x = cond == 1

81− where

82− cond = length fltr

83− fltr = filter (== ',') x

84−

85tests :: F.Test

75tests :: F.Test

86tests = F.testGroup "Regressions"

76tests = F.testGroup "Regressions"

87 [ F.testCase "hGetContents_crash" hGetContents_crash

77 [ F.testCase "hGetContents_crash" hGetContents_crash

@@ -89,5 +79,4 @@ tests = F.testGroup "Regressions"

89 , F.testCase "mapAccumL_resize" mapAccumL_resize

79 , F.testCase "mapAccumL_resize" mapAccumL_resize

90 , F.testCase "replicate_crash" replicate_crash

80 , F.testCase "replicate_crash" replicate_crash

91 , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe

81 , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe

92− , F.testCase "t197" t197

93 ]

82 ]

tests/text-tests.cabal+3−18

@@ -20,11 +20,6 @@ flag hpc

20 default: False

21 manual: True

22

23−flag bytestring-builder

24− description: Depend on the bytestring-builder package for backwards compatibility.

25− default: False

26− manual: False

27−

28executable text-tests

23executable text-tests

29 main-is: Tests.hs

24 main-is: Tests.hs

30

25

@@ -53,21 +48,16 @@ executable text-tests

53 HUnit >= 1.2,

48 HUnit >= 1.2,

54 QuickCheck >= 2.7,

49 QuickCheck >= 2.7,

55 base == 4.*,

50 base == 4.*,

51+ bytestring,

56 deepseq,

52 deepseq,

57 directory,

53 directory,

58− quickcheck-unicode >= 1.0.1.0,

54+ quickcheck-unicode,

59 random,

55 random,

60 test-framework >= 0.4,

56 test-framework >= 0.4,

61 test-framework-hunit >= 0.2,

57 test-framework-hunit >= 0.2,

62 test-framework-quickcheck2 >= 0.2,

58 test-framework-quickcheck2 >= 0.2,

63 text-tests

59 text-tests

64

60

65− if flag(bytestring-builder)

66− build-depends: bytestring >= 0.9 && < 0.10.4,

67− bytestring-builder >= 0.10.4

68− else

69− build-depends: bytestring >= 0.10.4

70−

71executable text-tests-stdio

61executable text-tests-stdio

72 main-is: Tests/IO.hs

62 main-is: Tests/IO.hs

73

63

@@ -147,12 +137,7 @@ library

147 array,

137 array,

148 base == 4.*,

138 base == 4.*,

149 binary,

139 binary,

140+ bytestring,

150 deepseq,

141 deepseq,

151 ghc-prim,

142 ghc-prim,

152 integer-gmp

143 integer-gmp

153−

154− if flag(bytestring-builder)

155− build-depends: bytestring >= 0.9 && < 0.10.4,

156− bytestring-builder >= 0.10.4

157− else

158− build-depends: bytestring >= 0.10.4

text.cabal+17−90

@@ -1,7 +1,7 @@

1name: text

2−version: 1.2.3.0

2+version: 1.2.2.1

3−homepage: https://github.com/haskell/text

3+homepage: https://github.com/bos/text

4−bug-reports: https://github.com/haskell/text/issues

4+bug-reports: https://github.com/bos/text/issues

5synopsis: An efficient packed Unicode text type.

6description:

7 .

@@ -14,13 +14,11 @@ description:

14 in terms of large data quantities and high speed.

15 .

16 The 'Text' type provides character-encoding, type-safe case

17− conversion via whole-string case conversion functions (see "Data.Text").

17+ conversion via whole-string case conversion functions. It also

18− It also provides a range of functions for converting 'Text' values to

18+ provides a range of functions for converting 'Text' values to and from

19− and from 'ByteStrings', using several standard encodings

19+ 'ByteStrings', using several standard encodings.

20− (see "Data.Text.Encoding").

21 .

20 .

22− Efficient locale-sensitive support for text IO is also supported

21+ Efficient locale-sensitive support for text IO is also supported.

23− (see "Data.Text.IO").

24 .

22 .

25 These modules are intended to be imported qualified, to avoid name

23 These modules are intended to be imported qualified, to avoid name

26 clashes with Prelude functions, e.g.

24 clashes with Prelude functions, e.g.

@@ -30,21 +28,17 @@ description:

30 To use an extended and very rich family of functions for working

28 To use an extended and very rich family of functions for working

31 with Unicode text (including normalization, regular expressions,

29 with Unicode text (including normalization, regular expressions,

32 non-standard encodings, text breaking, and locales), see

30 non-standard encodings, text breaking, and locales), see

31+ the @text-icu@ package:

33− the [text-icu package](https://hackage.haskell.org/package/text-icu)

32+ <http://hackage.haskell.org/package/text-icu>

34− based on the well-respected and liberally

35− licensed [ICU library](http://site.icu-project.org/).

36

33

37license: BSD2

34license: BSD2

38license-file: LICENSE

35license-file: LICENSE

39author: Bryan O'Sullivan <bos@serpentine.com>

36author: Bryan O'Sullivan <bos@serpentine.com>

40maintainer: Bryan O'Sullivan <bos@serpentine.com>

37maintainer: Bryan O'Sullivan <bos@serpentine.com>

41copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper

38copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper

42−bug-reports: https://github.com/haskell/text/issues

43category: Data, Text

39category: Data, Text

44build-type: Simple

40build-type: Simple

45cabal-version: >= 1.8

41cabal-version: >= 1.8

46−tested-with: GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4,

47− GHC==7.6.3, GHC==7.4.2, GHC==7.2.2, GHC==7.0.4

48extra-source-files:

42extra-source-files:

49 -- scripts/CaseFolding.txt

43 -- scripts/CaseFolding.txt

50 -- scripts/SpecialCasing.txt

44 -- scripts/SpecialCasing.txt

@@ -71,9 +65,7 @@ extra-source-files:

71 tests/text-tests.cabal

65 tests/text-tests.cabal

72

66

73flag bytestring-builder

67flag bytestring-builder

74− description:

75− Depend on the [bytestring-builder](https://hackage.haskell.org/package/bytestring-builder)

76− package for backwards compatibility.

68+ description: Depend on the bytestring-builder package for backwards compatibility.

77 default: False

69 default: False

78 manual: False

70 manual: False

79

71

@@ -83,9 +75,7 @@ flag developer

83 manual: True

75 manual: True

84

76

85flag integer-simple

77flag integer-simple

86− description:

87− Use the [simple integer library](http://hackage.haskell.org/package/integer-simple)

78+ description: Use the simple integer library instead of GMP

88− instead of [integer-gmp](http://hackage.haskell.org/package/integer-gmp)

89 default: False

79 default: False

90 manual: False

80 manual: False

91

81

@@ -170,90 +160,27 @@ library

170

160

171test-suite tests

161test-suite tests

172 type: exitcode-stdio-1.0

162 type: exitcode-stdio-1.0

163+ hs-source-dirs: tests .

164+ main-is: Tests.hs

173 c-sources: cbits/cbits.c

165 c-sources: cbits/cbits.c

174 include-dirs: include

166 include-dirs: include

175

167

176 ghc-options:

168 ghc-options:

177− -Wall -threaded -rtsopts

169+ -Wall -threaded -O0 -rtsopts

178

170

179 cpp-options:

171 cpp-options:

180 -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE

172 -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE

181

173

182− -- modules specific to test-suite

183− hs-source-dirs: tests

184− main-is: Tests.hs

185− other-modules:

186− Tests.Properties

187− Tests.Properties.Mul

188− Tests.QuickCheckUtils

189− Tests.Regressions

190− Tests.SlowFunctions

191− Tests.Utils

192−

193− -- Same as in `library` stanza; this is needed by cabal for accurate

194− -- file-monitoring as well as to avoid `-Wmissing-home-modules`

195− -- warnings We can't use an inter-package library dependency because

196− -- of different `ghc-options`/`cpp-options` (as a side-benefitt,

197− -- this enables per-component build parallelism in `cabal

198− -- new-build`!); We could, however, use cabal-version:2.2's `common`

199− -- blocks at some point in the future to reduce the duplication.

200− hs-source-dirs: .

201− other-modules:

202− Data.Text

203− Data.Text.Array

204− Data.Text.Encoding

205− Data.Text.Encoding.Error

206− Data.Text.Foreign

207− Data.Text.IO

208− Data.Text.Internal

209− Data.Text.Internal.Builder

210− Data.Text.Internal.Builder.Functions

211− Data.Text.Internal.Builder.Int.Digits

212− Data.Text.Internal.Builder.RealFloat.Functions

213− Data.Text.Internal.Encoding.Fusion

214− Data.Text.Internal.Encoding.Fusion.Common

215− Data.Text.Internal.Encoding.Utf16

216− Data.Text.Internal.Encoding.Utf32

217− Data.Text.Internal.Encoding.Utf8

218− Data.Text.Internal.Functions

219− Data.Text.Internal.Fusion

220− Data.Text.Internal.Fusion.CaseMapping

221− Data.Text.Internal.Fusion.Common

222− Data.Text.Internal.Fusion.Size

223− Data.Text.Internal.Fusion.Types

224− Data.Text.Internal.IO

225− Data.Text.Internal.Lazy

226− Data.Text.Internal.Lazy.Encoding.Fusion

227− Data.Text.Internal.Lazy.Fusion

228− Data.Text.Internal.Lazy.Search

229− Data.Text.Internal.Private

230− Data.Text.Internal.Read

231− Data.Text.Internal.Search

232− Data.Text.Internal.Unsafe

233− Data.Text.Internal.Unsafe.Char

234− Data.Text.Internal.Unsafe.Shift

235− Data.Text.Lazy

236− Data.Text.Lazy.Builder

237− Data.Text.Lazy.Builder.Int

238− Data.Text.Lazy.Builder.RealFloat

239− Data.Text.Lazy.Encoding

240− Data.Text.Lazy.IO

241− Data.Text.Lazy.Internal

242− Data.Text.Lazy.Read

243− Data.Text.Read

244− Data.Text.Unsafe

245− Data.Text.Show

246−

247 build-depends:

174 build-depends:

248 HUnit >= 1.2,

175 HUnit >= 1.2,

249− QuickCheck >= 2.7 && < 2.10,

176+ QuickCheck >= 2.7,

250 array,

177 array,

251 base,

178 base,

252 binary,

179 binary,

253 deepseq,

180 deepseq,

254 directory,

181 directory,

255 ghc-prim,

182 ghc-prim,

256− quickcheck-unicode >= 1.0.1.0,

183+ quickcheck-unicode,

257 random,

184 random,

258 test-framework >= 0.4,

185 test-framework >= 0.4,

259 test-framework-hunit >= 0.2,

186 test-framework-hunit >= 0.2,

@@ -274,7 +201,7 @@ test-suite tests

274

201

275source-repository head

202source-repository head

276 type: git

203 type: git

277− location: https://github.com/haskell/text

204+ location: https://github.com/bos/text

278

205

279source-repository head

206source-repository head

280 type: mercurial

207 type: mercurial