c976329..master · text

git.iko.soy/text

14 files changed+65−352

unifiedsplit

.hgtags−1 Data/Text.hs+1−13 Data/Text/Foreign.hs+1−1 Data/Text/Internal/Fusion/CaseMapping.hs+4−164 Data/Text/Internal/Fusion/Common.hs+9−10 Data/Text/Lazy.hs+6−15 Data/Text/Lazy/Read.hs+3−3 Data/Text/Read.hs+3−3 cbits/cbits.c+10−5 changelog.md+1−27 tests/Tests/Properties.hs+21−89 tests/Tests/QuickCheckUtils.hs+1−1 tests/text-tests.cabal+3−18 text.cabal+2−2

.hgtags−1

@@ -64,4 +64,3 @@ f8a60279237c1c28e094868740d9188dd94fcc9e 1.2.0.0

6464631fd055e27fa10983e17e9ad36d1b285b85c661 1.2.1.3

65658221a68ed5ec37028357df9c6c25f52320e54ad9 1.2.2.0

6666ce1c5ac65eb9f5580114b4e380092b05a485e95d 1.2.2.1

67−d72c634a91ffe27ec5cd03921dd53efc8129d8f6 1.2.2.2

Data/Text.hs+1−13

@@ -60,7 +60,6 @@ module Data.Text

6060 , snoc

6161 , append

6262 , uncons

63− , unsnoc

6463 , head

6564 , last

6665 , tail

@@ -546,17 +545,6 @@ init (Text arr off len) | len <= 0 = emptyError "init"

546545 unstream (S.init (stream t)) = init t

547546 #-}

548547

549−-- | /O(1)/ Returns all but the last character and the last character of a

550−-- 'Text', or 'Nothing' if empty.

551−unsnoc :: Text -> Maybe (Text, Char)

552−unsnoc (Text arr off len)

553− | len <= 0 = Nothing

554− | n < 0xDC00 || n > 0xDFFF = Just (text arr off (len-1), unsafeChr n)

555− | otherwise = Just (text arr off (len-2), U16.chr2 n0 n)

556− where n = A.unsafeIndex arr (off+len-1)

557− n0 = A.unsafeIndex arr (off+len-2)

558−{-# INLINE [1] unsnoc #-}

559−

560548-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

561549-- fusion.

562550null :: Text -> Bool

@@ -1111,8 +1099,8 @@ takeEnd n t@(Text arr off len)

11111099iterNEnd :: Int -> Text -> Int

11121100iterNEnd n t@(Text _arr _off len) = loop (len-1) n

11131101 where loop i !m

1114− | m <= 0 = i+1

11151102 | i <= 0 = 0

1103+ | m <= 1 = i

11161104 | otherwise = loop (i+d) (m-1)

11171105 where d = reverseIter_ t i

11181106

Data/Text/Foreign.hs+1−1

@@ -108,7 +108,7 @@ takeWord16 (I16 n) t@(Text arr off len)

108108 | n >= len || m >= len = t

109109 | otherwise = Text arr off m

110110 where

111− m | w < 0xD800 || w > 0xDBFF = n

111+ m | w < 0xDB00 || w > 0xD8FF = n

112112 | otherwise = n+1

113113 w = A.unsafeIndex arr (off+n-1)

114114

Data/Text/Internal/Fusion/CaseMapping.hs+4−164

@@ -1,10 +1,10 @@

11{-# LANGUAGE Rank2Types #-}

22-- AUTOMATICALLY GENERATED - DO NOT EDIT

33-- Generated by scripts/CaseMapping.hs

4−-- CaseFolding-9.0.0.txt

5−-- Date: 2016-03-02, 18:54:54 GMT

6−-- SpecialCasing-9.0.0.txt

7−-- Date: 2016-03-02, 18:55:13 GMT

4+-- CaseFolding-8.0.0.txt

5+-- Date: 2015-01-13, 18:16:36 GMT [MD]

6+-- SpecialCasing-8.0.0.txt

7+-- Date: 2014-12-16, 23:08:04 GMT [MD]

99module Data.Text.Internal.Fusion.CaseMapping where

1010import Data.Char

@@ -371,24 +371,6 @@ foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')

371371foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')

372372-- CHEROKEE SMALL LETTER MV

373373foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')

374−-- CYRILLIC SMALL LETTER ROUNDED VE

375−foldMapping '\x1c80' s = Yield '\x0432' (CC s '\x0000' '\x0000')

376−-- CYRILLIC SMALL LETTER LONG-LEGGED DE

377−foldMapping '\x1c81' s = Yield '\x0434' (CC s '\x0000' '\x0000')

378−-- CYRILLIC SMALL LETTER NARROW O

379−foldMapping '\x1c82' s = Yield '\x043e' (CC s '\x0000' '\x0000')

380−-- CYRILLIC SMALL LETTER WIDE ES

381−foldMapping '\x1c83' s = Yield '\x0441' (CC s '\x0000' '\x0000')

382−-- CYRILLIC SMALL LETTER TALL TE

383−foldMapping '\x1c84' s = Yield '\x0442' (CC s '\x0000' '\x0000')

384−-- CYRILLIC SMALL LETTER THREE-LEGGED TE

385−foldMapping '\x1c85' s = Yield '\x0442' (CC s '\x0000' '\x0000')

386−-- CYRILLIC SMALL LETTER TALL HARD SIGN

387−foldMapping '\x1c86' s = Yield '\x044a' (CC s '\x0000' '\x0000')

388−-- CYRILLIC SMALL LETTER TALL YAT

389−foldMapping '\x1c87' s = Yield '\x0463' (CC s '\x0000' '\x0000')

390−-- CYRILLIC SMALL LETTER UNBLENDED UK

391−foldMapping '\x1c88' s = Yield '\xa64b' (CC s '\x0000' '\x0000')

392374-- LATIN SMALL LETTER H WITH LINE BELOW

393375foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

394376-- LATIN SMALL LETTER T WITH DIAERESIS

@@ -563,8 +545,6 @@ foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')

563545foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

564546-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

565547foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

566−-- LATIN CAPITAL LETTER SMALL CAPITAL I

567−foldMapping '\xa7ae' s = Yield '\x026a' (CC s '\x0000' '\x0000')

568548-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

569549foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

570550-- LATIN CAPITAL LETTER CHI

@@ -757,78 +737,6 @@ foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')

757737foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

758738-- ARMENIAN SMALL LIGATURE MEN XEH

759739foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

760−-- OSAGE CAPITAL LETTER A

761−foldMapping '\x104b0' s = Yield '\x104d8' (CC s '\x0000' '\x0000')

762−-- OSAGE CAPITAL LETTER AI

763−foldMapping '\x104b1' s = Yield '\x104d9' (CC s '\x0000' '\x0000')

764−-- OSAGE CAPITAL LETTER AIN

765−foldMapping '\x104b2' s = Yield '\x104da' (CC s '\x0000' '\x0000')

766−-- OSAGE CAPITAL LETTER AH

767−foldMapping '\x104b3' s = Yield '\x104db' (CC s '\x0000' '\x0000')

768−-- OSAGE CAPITAL LETTER BRA

769−foldMapping '\x104b4' s = Yield '\x104dc' (CC s '\x0000' '\x0000')

770−-- OSAGE CAPITAL LETTER CHA

771−foldMapping '\x104b5' s = Yield '\x104dd' (CC s '\x0000' '\x0000')

772−-- OSAGE CAPITAL LETTER EHCHA

773−foldMapping '\x104b6' s = Yield '\x104de' (CC s '\x0000' '\x0000')

774−-- OSAGE CAPITAL LETTER E

775−foldMapping '\x104b7' s = Yield '\x104df' (CC s '\x0000' '\x0000')

776−-- OSAGE CAPITAL LETTER EIN

777−foldMapping '\x104b8' s = Yield '\x104e0' (CC s '\x0000' '\x0000')

778−-- OSAGE CAPITAL LETTER HA

779−foldMapping '\x104b9' s = Yield '\x104e1' (CC s '\x0000' '\x0000')

780−-- OSAGE CAPITAL LETTER HYA

781−foldMapping '\x104ba' s = Yield '\x104e2' (CC s '\x0000' '\x0000')

782−-- OSAGE CAPITAL LETTER I

783−foldMapping '\x104bb' s = Yield '\x104e3' (CC s '\x0000' '\x0000')

784−-- OSAGE CAPITAL LETTER KA

785−foldMapping '\x104bc' s = Yield '\x104e4' (CC s '\x0000' '\x0000')

786−-- OSAGE CAPITAL LETTER EHKA

787−foldMapping '\x104bd' s = Yield '\x104e5' (CC s '\x0000' '\x0000')

788−-- OSAGE CAPITAL LETTER KYA

789−foldMapping '\x104be' s = Yield '\x104e6' (CC s '\x0000' '\x0000')

790−-- OSAGE CAPITAL LETTER LA

791−foldMapping '\x104bf' s = Yield '\x104e7' (CC s '\x0000' '\x0000')

792−-- OSAGE CAPITAL LETTER MA

793−foldMapping '\x104c0' s = Yield '\x104e8' (CC s '\x0000' '\x0000')

794−-- OSAGE CAPITAL LETTER NA

795−foldMapping '\x104c1' s = Yield '\x104e9' (CC s '\x0000' '\x0000')

796−-- OSAGE CAPITAL LETTER O

797−foldMapping '\x104c2' s = Yield '\x104ea' (CC s '\x0000' '\x0000')

798−-- OSAGE CAPITAL LETTER OIN

799−foldMapping '\x104c3' s = Yield '\x104eb' (CC s '\x0000' '\x0000')

800−-- OSAGE CAPITAL LETTER PA

801−foldMapping '\x104c4' s = Yield '\x104ec' (CC s '\x0000' '\x0000')

802−-- OSAGE CAPITAL LETTER EHPA

803−foldMapping '\x104c5' s = Yield '\x104ed' (CC s '\x0000' '\x0000')

804−-- OSAGE CAPITAL LETTER SA

805−foldMapping '\x104c6' s = Yield '\x104ee' (CC s '\x0000' '\x0000')

806−-- OSAGE CAPITAL LETTER SHA

807−foldMapping '\x104c7' s = Yield '\x104ef' (CC s '\x0000' '\x0000')

808−-- OSAGE CAPITAL LETTER TA

809−foldMapping '\x104c8' s = Yield '\x104f0' (CC s '\x0000' '\x0000')

810−-- OSAGE CAPITAL LETTER EHTA

811−foldMapping '\x104c9' s = Yield '\x104f1' (CC s '\x0000' '\x0000')

812−-- OSAGE CAPITAL LETTER TSA

813−foldMapping '\x104ca' s = Yield '\x104f2' (CC s '\x0000' '\x0000')

814−-- OSAGE CAPITAL LETTER EHTSA

815−foldMapping '\x104cb' s = Yield '\x104f3' (CC s '\x0000' '\x0000')

816−-- OSAGE CAPITAL LETTER TSHA

817−foldMapping '\x104cc' s = Yield '\x104f4' (CC s '\x0000' '\x0000')

818−-- OSAGE CAPITAL LETTER DHA

819−foldMapping '\x104cd' s = Yield '\x104f5' (CC s '\x0000' '\x0000')

820−-- OSAGE CAPITAL LETTER U

821−foldMapping '\x104ce' s = Yield '\x104f6' (CC s '\x0000' '\x0000')

822−-- OSAGE CAPITAL LETTER WA

823−foldMapping '\x104cf' s = Yield '\x104f7' (CC s '\x0000' '\x0000')

824−-- OSAGE CAPITAL LETTER KHA

825−foldMapping '\x104d0' s = Yield '\x104f8' (CC s '\x0000' '\x0000')

826−-- OSAGE CAPITAL LETTER GHA

827−foldMapping '\x104d1' s = Yield '\x104f9' (CC s '\x0000' '\x0000')

828−-- OSAGE CAPITAL LETTER ZA

829−foldMapping '\x104d2' s = Yield '\x104fa' (CC s '\x0000' '\x0000')

830−-- OSAGE CAPITAL LETTER ZHA

831−foldMapping '\x104d3' s = Yield '\x104fb' (CC s '\x0000' '\x0000')

832740-- OLD HUNGARIAN CAPITAL LETTER A

833741foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

834742-- OLD HUNGARIAN CAPITAL LETTER AA

@@ -931,72 +839,4 @@ foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')

931839foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

932840-- OLD HUNGARIAN CAPITAL LETTER US

933841foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

934−-- ADLAM CAPITAL LETTER ALIF

935−foldMapping '\x1e900' s = Yield '\x1e922' (CC s '\x0000' '\x0000')

936−-- ADLAM CAPITAL LETTER DAALI

937−foldMapping '\x1e901' s = Yield '\x1e923' (CC s '\x0000' '\x0000')

938−-- ADLAM CAPITAL LETTER LAAM

939−foldMapping '\x1e902' s = Yield '\x1e924' (CC s '\x0000' '\x0000')

940−-- ADLAM CAPITAL LETTER MIIM

941−foldMapping '\x1e903' s = Yield '\x1e925' (CC s '\x0000' '\x0000')

942−-- ADLAM CAPITAL LETTER BA

943−foldMapping '\x1e904' s = Yield '\x1e926' (CC s '\x0000' '\x0000')

944−-- ADLAM CAPITAL LETTER SINNYIIYHE

945−foldMapping '\x1e905' s = Yield '\x1e927' (CC s '\x0000' '\x0000')

946−-- ADLAM CAPITAL LETTER PE

947−foldMapping '\x1e906' s = Yield '\x1e928' (CC s '\x0000' '\x0000')

948−-- ADLAM CAPITAL LETTER BHE

949−foldMapping '\x1e907' s = Yield '\x1e929' (CC s '\x0000' '\x0000')

950−-- ADLAM CAPITAL LETTER RA

951−foldMapping '\x1e908' s = Yield '\x1e92a' (CC s '\x0000' '\x0000')

952−-- ADLAM CAPITAL LETTER E

953−foldMapping '\x1e909' s = Yield '\x1e92b' (CC s '\x0000' '\x0000')

954−-- ADLAM CAPITAL LETTER FA

955−foldMapping '\x1e90a' s = Yield '\x1e92c' (CC s '\x0000' '\x0000')

956−-- ADLAM CAPITAL LETTER I

957−foldMapping '\x1e90b' s = Yield '\x1e92d' (CC s '\x0000' '\x0000')

958−-- ADLAM CAPITAL LETTER O

959−foldMapping '\x1e90c' s = Yield '\x1e92e' (CC s '\x0000' '\x0000')

960−-- ADLAM CAPITAL LETTER DHA

961−foldMapping '\x1e90d' s = Yield '\x1e92f' (CC s '\x0000' '\x0000')

962−-- ADLAM CAPITAL LETTER YHE

963−foldMapping '\x1e90e' s = Yield '\x1e930' (CC s '\x0000' '\x0000')

964−-- ADLAM CAPITAL LETTER WAW

965−foldMapping '\x1e90f' s = Yield '\x1e931' (CC s '\x0000' '\x0000')

966−-- ADLAM CAPITAL LETTER NUN

967−foldMapping '\x1e910' s = Yield '\x1e932' (CC s '\x0000' '\x0000')

968−-- ADLAM CAPITAL LETTER KAF

969−foldMapping '\x1e911' s = Yield '\x1e933' (CC s '\x0000' '\x0000')

970−-- ADLAM CAPITAL LETTER YA

971−foldMapping '\x1e912' s = Yield '\x1e934' (CC s '\x0000' '\x0000')

972−-- ADLAM CAPITAL LETTER U

973−foldMapping '\x1e913' s = Yield '\x1e935' (CC s '\x0000' '\x0000')

974−-- ADLAM CAPITAL LETTER JIIM

975−foldMapping '\x1e914' s = Yield '\x1e936' (CC s '\x0000' '\x0000')

976−-- ADLAM CAPITAL LETTER CHI

977−foldMapping '\x1e915' s = Yield '\x1e937' (CC s '\x0000' '\x0000')

978−-- ADLAM CAPITAL LETTER HA

979−foldMapping '\x1e916' s = Yield '\x1e938' (CC s '\x0000' '\x0000')

980−-- ADLAM CAPITAL LETTER QAAF

981−foldMapping '\x1e917' s = Yield '\x1e939' (CC s '\x0000' '\x0000')

982−-- ADLAM CAPITAL LETTER GA

983−foldMapping '\x1e918' s = Yield '\x1e93a' (CC s '\x0000' '\x0000')

984−-- ADLAM CAPITAL LETTER NYA

985−foldMapping '\x1e919' s = Yield '\x1e93b' (CC s '\x0000' '\x0000')

986−-- ADLAM CAPITAL LETTER TU

987−foldMapping '\x1e91a' s = Yield '\x1e93c' (CC s '\x0000' '\x0000')

988−-- ADLAM CAPITAL LETTER NHA

989−foldMapping '\x1e91b' s = Yield '\x1e93d' (CC s '\x0000' '\x0000')

990−-- ADLAM CAPITAL LETTER VA

991−foldMapping '\x1e91c' s = Yield '\x1e93e' (CC s '\x0000' '\x0000')

992−-- ADLAM CAPITAL LETTER KHA

993−foldMapping '\x1e91d' s = Yield '\x1e93f' (CC s '\x0000' '\x0000')

994−-- ADLAM CAPITAL LETTER GBE

995−foldMapping '\x1e91e' s = Yield '\x1e940' (CC s '\x0000' '\x0000')

996−-- ADLAM CAPITAL LETTER ZAL

997−foldMapping '\x1e91f' s = Yield '\x1e941' (CC s '\x0000' '\x0000')

998−-- ADLAM CAPITAL LETTER KPO

999−foldMapping '\x1e920' s = Yield '\x1e942' (CC s '\x0000' '\x0000')

1000−-- ADLAM CAPITAL LETTER SHA

1001−foldMapping '\x1e921' s = Yield '\x1e943' (CC s '\x0000' '\x0000')

1002842foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

Data/Text/Internal/Fusion/Common.hs+9−10

@@ -107,7 +107,7 @@ import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..),

107107import qualified Data.List as L

108108import qualified Prelude as P

109109import Data.Bits (shiftL)

110−import Data.Char (isLetter, isSpace)

110+import Data.Char (isLetter)

111111import Data.Int (Int64)

112112import Data.Text.Internal.Fusion.Types

113113import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,

@@ -462,16 +462,15 @@ toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') len

462462 where

463463 next (CC (letter :*: s) '\0' _) =

464464 case next0 s of

465− Done -> Done

466− Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

465+ Done -> Done

466+ Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

467467 Yield c s'

468− | nonSpace -> if letter

469− then lowerMapping c (nonSpace :*: s')

470− else titleMapping c (letter' :*: s')

471− | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

472− where nonSpace = P.not (isSpace c)

473− letter' = isLetter c

474− next (CC s a b) = Yield a (CC s b '\0')

468+ | letter' -> if letter

469+ then lowerMapping c (letter' :*: s')

470+ else titleMapping c (letter' :*: s')

471+ | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

472+ where letter' = isLetter c

473+ next (CC s a b) = Yield a (CC s b '\0')

475474{-# INLINE [0] toTitle #-}

476475

477476data Justify i s = Just1 !i !s

Data/Text/Lazy.hs+6−15

@@ -68,7 +68,6 @@ module Data.Text.Lazy

6868 , snoc

6969 , append

7070 , uncons

71− , unsnoc

7271 , head

7372 , last

7473 , tail

@@ -476,7 +475,9 @@ fromStrict t = chunk t Empty

476475-- -----------------------------------------------------------------------------

477476-- * Basic functions

478477

479−-- | /O(1)/ Adds a character to the front of a 'Text'. Subject to fusion.

478+-- | /O(n)/ Adds a character to the front of a 'Text'. This function

479+-- is more costly than its 'List' counterpart because it requires

480+-- copying a new array. Subject to fusion.

480481cons :: Char -> Text -> Text

481482cons c t = Chunk (T.singleton c) t

482483{-# INLINE [1] cons #-}

@@ -560,15 +561,6 @@ init Empty = emptyError "init"

560561 unstream (S.init (stream t)) = init t

561562 #-}

562563

563−-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if

564−-- empty.

565−--

566−-- * It is no faster than using 'init' and 'last'.

567−unsnoc :: Text -> Maybe (Text, Char)

568−unsnoc Empty = Nothing

569−unsnoc ts@(Chunk _ _) = Just (init ts, last ts)

570−{-# INLINE unsnoc #-}

571−

572564-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to

573565-- fusion.

574566null :: Text -> Bool

@@ -1161,10 +1153,9 @@ takeWhile p t0 = takeWhile' t0

11611153takeWhileEnd :: (Char -> Bool) -> Text -> Text

11621154takeWhileEnd p = takeChunk empty . L.reverse . toChunks

11631155 where takeChunk acc [] = acc

1164− takeChunk acc (t:ts)

1165− | T.lengthWord16 t' < T.lengthWord16 t

1166− = chunk t' acc

1167− | otherwise = takeChunk (Chunk t' acc) ts

1156+ takeChunk acc (t:ts) = if T.length t' < T.length t

1157+ then (Chunk t' acc)

1158+ else takeChunk (Chunk t' acc) ts

11681159 where t' = T.takeWhileEnd p t

11691160{-# INLINE takeWhileEnd #-}

11701161

Data/Text/Lazy/Read.hs+3−3

@@ -68,9 +68,9 @@ decimal txt

6868 go n d = (n * 10 + fromIntegral (digitToInt d))

6969

7070-- | Read a hexadecimal integer, consisting of an optional leading

71−-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

72−-- consumed until a non-hex-digit or end of string is reached.

73−-- This function is case insensitive.

71+-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

72+-- until a non-hex-digit or end of string is reached. This function

73+-- is case insensitive.

7474--

7575-- This function does not handle leading sign characters. If you need

7676-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Read.hs+3−3

@@ -67,9 +67,9 @@ decimal txt

6767 go n d = (n * 10 + fromIntegral (digitToInt d))

6868

6969-- | Read a hexadecimal integer, consisting of an optional leading

70−-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

71−-- consumed until a non-hex-digit or end of string is reached.

72−-- This function is case insensitive.

70+-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

71+-- until a non-hex-digit or end of string is reached. This function

72+-- is case insensitive.

7373--

7474-- This function does not handle leading sign characters. If you need

7575-- to handle signed input, use @'signed' 'hexadecimal'@.

cbits/cbits.c+10−5

@@ -207,8 +207,10 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,

207207 uint32_t *codepoint0, uint32_t *state0)

208208{

209209 uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,

210− codepoint0, state0);

211− return *src;

210+ codepoint0, state0);

211+ if (*state0 == UTF8_REJECT)

212+ ret -=1;

213+ return ret;

212214}

213215

214216/*

@@ -220,9 +222,12 @@ _hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,

220222{

221223 uint32_t codepoint;

222224 uint32_t state = UTF8_ACCEPT;

223− _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

224− &codepoint, &state);

225− return src;

225+ uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend,

226+ &codepoint, &state);

227+ /* Back up if we have an incomplete or invalid encoding */

228+ if (state != UTF8_ACCEPT)

229+ ret -= 1;

230+ return ret;

226231}

227232

228233void

changelog.md+1−27

@@ -1,36 +1,10 @@

1−1.2.3.0

2−

3−* Bug fix: the lazy `takeWhileEnd` function violated the

4− [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51).

5−

6−* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec

7− (updated from 8.0)

8−

9−

10−1.2.2.2

11−

12−* The `toTitle` function now correctly handles letters that

13− immediately follow punctuation. Before, `"there's"` would turn into

14− `"There'S"`. Now, it becomes `"There's"`.

15−

16−* The implementation of unstreaming is faster, resulting in operations

17− such as `map` and `intersperse` speeding up by up to 30%, with

18− smaller code generated.

19−

20−* The optimised length comparison function is now more likely to be

21− used after some rewrite rule tweaking.

22−

23−* Bug fix: an off-by-one bug in `takeEnd` is fixed.

24−

25−* Bug fix: a logic error in `takeWord16` is fixed.

26−

2711.2.2.1

282

293* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

304 The build flag has been renamed accordingly. Your army of diligent

315 maintainers apologizes for the churn.

326

33−* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec

7+* Spec compliance: toCaseFold now follows the Unicode 8.0 spec

348 (updated from 7.0)

359

3610* An STG lint error has been fixed

tests/Tests/Properties.hs+21−89

@@ -9,9 +9,9 @@ module Tests.Properties

99 ) where

1010

1111import Control.Applicative ((<$>), (<*>))

12−import Control.Arrow ((***), first, second)

12+import Control.Arrow ((***), second)

1313import Data.Bits ((.&.))

14−import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord)

14+import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)

1515import Data.Int (Int8, Int16, Int32, Int64)

1616import Data.Monoid (Monoid(..))

1717import Data.String (IsString(fromString))

@@ -23,7 +23,6 @@ import Data.Text.Internal.Search (indices)

2323import Data.Text.Lazy.Read as TL

2424import Data.Text.Read as T

2525import Data.Word (Word, Word8, Word16, Word32, Word64)

26−import Data.Maybe (mapMaybe)

2726import Numeric (showEFloat, showFFloat, showGFloat, showHex)

2827import Prelude hiding (replicate)

2928import Test.Framework (Test, testGroup)

@@ -31,7 +30,6 @@ import Test.Framework.Providers.QuickCheck2 (testProperty)

3130import Test.QuickCheck hiding ((.&.))

3231import Test.QuickCheck.Monadic

3332import Test.QuickCheck.Property (Property(..))

34−import Test.QuickCheck.Unicode (char)

3533import Tests.QuickCheckUtils

3634import Tests.Utils

3735import Text.Show.Functions ()

@@ -39,7 +37,6 @@ import qualified Control.Exception as Exception

3937import qualified Data.Bits as Bits (shiftL, shiftR)

4038import qualified Data.ByteString as B

4139import qualified Data.ByteString.Lazy as BL

42−import qualified Data.Char as C

4340import qualified Data.List as L

4441import qualified Data.Text as T

4542import qualified Data.Text.Encoding as E

@@ -63,10 +60,10 @@ t_pack_unpack = (T.unpack . T.pack) `eq` id

6360tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

6461t_stream_unstream = (S.unstream . S.stream) `eq` id

6562tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

66−t_reverse_stream t = (S.reverse . S.reverseStream) t === t

67−t_singleton c = [c] === (T.unpack . T.singleton) c

68−tl_singleton c = [c] === (TL.unpack . TL.singleton) c

69−tl_unstreamChunks x = f 11 x === f 1000 x

63+t_reverse_stream t = (S.reverse . S.reverseStream) t == t

64+t_singleton c = [c] == (T.unpack . T.singleton) c

65+tl_singleton c = [c] == (TL.unpack . TL.singleton) c

66+tl_unstreamChunks x = f 11 x == f 1000 x

7067 where f n = SL.unstreamChunks n . S.streamList

7168tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

7269tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

@@ -77,13 +74,13 @@ encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

7774encodeLazyL1 :: TL.Text -> BL.ByteString

7875encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

7976

80−t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a

77+t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

8178 where a = T.map (\c -> chr (ord c `mod` 128)) t

82−tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a

79+tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

8380 where a = TL.map (\c -> chr (ord c `mod` 128)) t

84−t_latin1 t = E.decodeLatin1 (encodeL1 a) === a

81+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

8582 where a = T.map (\c -> chr (ord c `mod` 256)) t

86−tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a

83+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

8784 where a = TL.map (\c -> chr (ord c `mod` 256)) t

8885t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

8986t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

@@ -114,7 +111,7 @@ t_utf8_undecoded = forAll genUnicode $ \t ->

114111 let b = E.encodeUtf8 t

115112 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

116113 leftover = (++ [B.empty]) . init . tail . B.inits

117− in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls

114+ in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b == ls

118115

119116data Badness = Solo | Leading | Trailing

120117 deriving (Eq, Show)

@@ -180,29 +177,6 @@ genInvalidUTF8 = B.pack <$> oneof [

180177 k <- choose (0,n)

181178 vectorOf k gen

182179

183−-- See http://unicode.org/faq/utf_bom.html#gen8

184−-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...

185−-- When faced with this illegal byte sequence ... a UTF-8 conformant process

186−-- must treat the first byte 110xxxxx2 as an illegal termination error

187−-- (e.g. filter it out or replace by 0xFFFD) ...

188−-- ... and continue processing at the second byte 0xxxxxxx2

189−t_decode_with_error2 =

190− E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"

191−t_decode_with_error3 =

192− E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"

193−t_decode_with_error4 =

194− E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"

195−

196−t_decode_with_error2' =

197− case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of

198− E.Some x _ _ -> x === "xa"

199−t_decode_with_error3' =

200− case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of

201− E.Some x _ _ -> x === "xaa"

202−t_decode_with_error4' =

203− case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of

204− E.Some x _ _ -> x === "xaaa"

205−

206180s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList)

207181 where _types = s :: String

208182sf_Eq p s =

@@ -227,8 +201,8 @@ t_mconcat = unsquare $

227201 mconcat `eq` (unpackS . mconcat . L.map T.pack)

228202tl_mconcat = unsquare $

229203 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

230−t_mempty = mempty === (unpackS (mempty :: T.Text))

231−tl_mempty = mempty === (unpackS (mempty :: TL.Text))

204+t_mempty = mempty == (unpackS (mempty :: T.Text))

205+tl_mempty = mempty == (unpackS (mempty :: TL.Text))

232206t_IsString = fromString `eqP` (T.unpack . fromString)

233207tl_IsString = fromString `eqP` (TL.unpack . fromString)

234208

@@ -255,13 +229,6 @@ sf_uncons p = (uncons . L.filter p) `eqP`

255229 (fmap (second unpackS) . S.uncons . S.filter p)

256230t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons)

257231tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons)

258−

259−unsnoc xs@(_:_) = Just (init xs, last xs)

260−unsnoc [] = Nothing

261−

262−t_unsnoc = unsnoc `eqP` (fmap (first unpackS) . T.unsnoc)

263−tl_unsnoc = unsnoc `eqP` (fmap (first unpackS) . TL.unsnoc)

264−

265232s_head = head `eqP` S.head

266233sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p)

267234t_head = head `eqP` T.head

@@ -359,19 +326,6 @@ t_toUpper_upper t = p (T.toUpper t) >= p t

359326 where p = T.length . T.filter isUpper

360327tl_toUpper_upper t = p (TL.toUpper t) >= p t

361328 where p = TL.length . TL.filter isUpper

362−t_toTitle_title t = all (<= 1) (caps w)

363− where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle

364− -- TIL: there exist uppercase-only letters

365− w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t

366−t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable

367− where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words

368− -- Surprise! The Spanish/Portuguese ordinal indicators changed

369− -- from category Ll (letter, lowercase) to Lo (letter, other)

370− -- in Unicode 7.0

371− -- Oh, and there exist lowercase-only letters (see previous test)

372− stable c = if isLower c

373− then C.toUpper c /= c

374− else c /= '\170' && c /= '\186'

375329

376330justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

377331justifyRight m n xs = L.replicate (m - length xs) n ++ xs

@@ -560,20 +514,12 @@ s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p)

560514s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)

561515sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

562516 (unpackS . S.takeWhile p . S.filter q)

563−noMatch = do

564− c <- char

565− d <- suchThat char (/= c)

566− return (c,d)

567517t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

568518tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

569519t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

570520 (unpackS . T.takeWhileEnd p)

571−t_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> T.null $

572− T.takeWhileEnd (==d) (T.snoc t c)

573521tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

574522 (unpackS . TL.takeWhileEnd p)

575−tl_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> TL.null $

576− TL.takeWhileEnd (==d) (TL.snoc t c)

577523s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

578524s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

579525sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

@@ -774,7 +720,7 @@ tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s

774720 conc = T.concat . TL.toChunks

775721t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

776722 let s = T.intercalate t ts

777− in Slow.indices t s === indices t s

723+ in Slow.indices t s == indices t s

778724

779725-- Bit shifts.

780726shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

@@ -866,18 +812,18 @@ tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a

866812-- Reading.

867813

868814t_decimal (n::Int) s =

869− T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)

815+ T.signed T.decimal (T.pack (show n) `T.append` t) == Right (n,t)

870816 where t = T.dropWhile isDigit s

871817tl_decimal (n::Int) s =

872− TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)

818+ TL.signed TL.decimal (TL.pack (show n) `TL.append` t) == Right (n,t)

873819 where t = TL.dropWhile isDigit s

874820t_hexadecimal m s ox =

875− T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)

821+ T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) == Right (n,t)

876822 where t = T.dropWhile isHexDigit s

877823 p = if ox then "0x" else ""

878824 n = getPositive m :: Int

879825tl_hexadecimal m s ox =

880− TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)

826+ TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) == Right (n,t)

881827 where t = TL.dropWhile isHexDigit s

882828 p = if ox then "0x" else ""

883829 n = getPositive m :: Int

@@ -921,11 +867,11 @@ tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn

921867

922868t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

923869t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

924−t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t

870+t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) == t

925871 where n = small m

926872t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

927873

928−t_copy t = T.copy t === t

874+t_copy t = T.copy t == t

929875

930876-- Regression tests.

931877s_filter_eq s = S.filter p t == S.streamList (filter p s)

@@ -978,14 +924,6 @@ tests =

978924 testGroup "errors" [

979925 testProperty "t_utf8_err" t_utf8_err,

980926 testProperty "t_utf8_err'" t_utf8_err'

981− ],

982− testGroup "error recovery" [

983− testProperty "t_decode_with_error2" t_decode_with_error2,

984− testProperty "t_decode_with_error3" t_decode_with_error3,

985− testProperty "t_decode_with_error4" t_decode_with_error4,

986− testProperty "t_decode_with_error2'" t_decode_with_error2',

987− testProperty "t_decode_with_error3'" t_decode_with_error3',

988− testProperty "t_decode_with_error4'" t_decode_with_error4'

989927 ]

990928 ],

991929

@@ -1029,8 +967,6 @@ tests =

1029967 testProperty "sf_uncons" sf_uncons,

1030968 testProperty "t_uncons" t_uncons,

1031969 testProperty "tl_uncons" tl_uncons,

1032− testProperty "t_unsnoc" t_unsnoc,

1033− testProperty "tl_unsnoc" tl_unsnoc,

1034970 testProperty "s_head" s_head,

1035971 testProperty "sf_head" sf_head,

1036972 testProperty "t_head" t_head,

@@ -1094,9 +1030,7 @@ tests =

10941030 testProperty "tl_toLower_lower" tl_toLower_lower,

10951031 testProperty "t_toUpper_length" t_toUpper_length,

10961032 testProperty "t_toUpper_upper" t_toUpper_upper,

1097− testProperty "tl_toUpper_upper" tl_toUpper_upper,

1098− testProperty "t_toTitle_title" t_toTitle_title,

1099− testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower

1033+ testProperty "tl_toUpper_upper" tl_toUpper_upper

11001034 ],

11011035

11021036 testGroup "justification" [

@@ -1213,9 +1147,7 @@ tests =

12131147 testProperty "t_takeWhile" t_takeWhile,

12141148 testProperty "tl_takeWhile" tl_takeWhile,

12151149 testProperty "t_takeWhileEnd" t_takeWhileEnd,

1216− testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null,

12171150 testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

1218− testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null,

12191151 testProperty "sf_dropWhile" sf_dropWhile,

12201152 testProperty "s_dropWhile" s_dropWhile,

12211153 testProperty "s_dropWhile_s" s_dropWhile_s,

tests/Tests/QuickCheckUtils.hs+1−1

@@ -106,7 +106,7 @@ smallArbitrary = sized $ \n -> resize (smallish n) arbitrary

106106 where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs

107107

108108instance Arbitrary T.Text where

109− arbitrary = T.pack `fmap` string

109+ arbitrary = T.pack `fmap` arbitrary

110110 shrink = map T.pack . shrink . T.unpack

111111

112112instance Arbitrary TL.Text where

tests/text-tests.cabal+3−18

@@ -20,11 +20,6 @@ flag hpc

2020 default: False

2121 manual: True

2222

23−flag bytestring-builder

24− description: Depend on the bytestring-builder package for backwards compatibility.

25− default: False

26− manual: False

27−

2823executable text-tests

2924 main-is: Tests.hs

3025

@@ -53,21 +48,16 @@ executable text-tests

5348 HUnit >= 1.2,

5449 QuickCheck >= 2.7,

5550 base == 4.*,

51+ bytestring,

5652 deepseq,

5753 directory,

58− quickcheck-unicode >= 1.0.1.0,

54+ quickcheck-unicode,

5955 random,

6056 test-framework >= 0.4,

6157 test-framework-hunit >= 0.2,

6258 test-framework-quickcheck2 >= 0.2,

6359 text-tests

6460

65− if flag(bytestring-builder)

66− build-depends: bytestring >= 0.9 && < 0.10.4,

67− bytestring-builder >= 0.10.4

68− else

69− build-depends: bytestring >= 0.10.4

70−

7161executable text-tests-stdio

7262 main-is: Tests/IO.hs

7363

@@ -147,12 +137,7 @@ library

147137 array,

148138 base == 4.*,

149139 binary,

140+ bytestring,

150141 deepseq,

151142 ghc-prim,

152143 integer-gmp

153−

154− if flag(bytestring-builder)

155− build-depends: bytestring >= 0.9 && < 0.10.4,

156− bytestring-builder >= 0.10.4

157− else

158− build-depends: bytestring >= 0.10.4

text.cabal+2−2

@@ -1,5 +1,5 @@

11name: text

2−version: 1.2.3.0

2+version: 1.2.2.1

33homepage: https://github.com/bos/text

44bug-reports: https://github.com/bos/text/issues

55synopsis: An efficient packed Unicode text type.

@@ -180,7 +180,7 @@ test-suite tests

180180 deepseq,

181181 directory,

182182 ghc-prim,

183− quickcheck-unicode >= 1.0.1.0,

183+ quickcheck-unicode,

184184 random,

185185 test-framework >= 0.4,

186186 test-framework-hunit >= 0.2,