de78a39..master · text

git.iko.soy/text

32 files changed+686−239

unifiedsplit

.gitignore+1 .hgtags+6 Data/Text.hs+71−16 Data/Text/Array.hs+5−4 Data/Text/Encoding.hs+7−18 Data/Text/Encoding/Error.hs+3−1 Data/Text/Internal/Builder.hs+17−2 Data/Text/Internal/Builder/Functions.hs+2−1 Data/Text/Internal/Builder/Int/Digits.hs+1−1 Data/Text/Internal/Fusion.hs+27−17 Data/Text/Internal/Fusion/CaseMapping.hs+286−112 Data/Text/Internal/Fusion/Common.hs+5−5 Data/Text/Internal/Read.hs+3−3 Data/Text/Lazy.hs+53−8 Data/Text/Lazy/Builder.hs+1−1 Data/Text/Lazy/Builder/Int.hs+4−4 Data/Text/Lazy/Encoding.hs+6−16 Data/Text/Lazy/Read.hs+4−2 Data/Text/Read.hs+1−1 Data/Text/Show.hs+9−5 benchmarks/haskell/Benchmarks.hs+1−1 benchmarks/haskell/Benchmarks/Pure.hs+7 benchmarks/haskell/Benchmarks/ReadNumbers.hs+2−5 benchmarks/haskell/Benchmarks/Stream.hs+16−5 benchmarks/text-benchmarks.cabal+13−3 changelog.md+53 tests/LiteralRuleTest.hs+21 tests/Makefile+6−1 tests/Tests/Properties.hs+6 tests/literal-rule-test.sh+29 tests/text-tests.cabal+1−1 text.cabal+19−6

.gitignore+1

@@ -1,2 +1,3 @@

11dist

22cabal-dev

3+cabal.sandbox.config

.hgtags+6

@@ -58,3 +58,9 @@ f8a60279237c1c28e094868740d9188dd94fcc9e 1.2.0.0

58580b064b0c69106bbcd13cb0099e72ae5fa2347bf0 1.2.0.4

59598bbb3e222a0037035d893f731b0a23be449f7346 1.2.0.5

606018c7f8e19872762b7e28ee1871e72f70ea54e3cd 1.2.0.6

61+42f751b87b4a187db93f595a8166d51746ff57d4 1.2.1.0

62+78227a84740b70dcd4cf7a65f3bcf603d87e48e2 1.2.1.1

63+40f268e6735749ef36f487e1d928eaa957bb431d 1.2.1.2

64+631fd055e27fa10983e17e9ad36d1b285b85c661 1.2.1.3

65+8221a68ed5ec37028357df9c6c25f52320e54ad9 1.2.2.0

66+ce1c5ac65eb9f5580114b4e380092b05a485e95d 1.2.2.1

Data/Text.hs+71−16

@@ -129,6 +129,7 @@ module Data.Text

129129 , drop

130130 , dropEnd

131131 , takeWhile

132+ , takeWhileEnd

132133 , dropWhile

133134 , dropWhileEnd

134135 , dropAround

@@ -191,6 +192,7 @@ module Data.Text

191192

192193 -- * Low level operations

193194 , copy

195+ , unpackCString#

194196 ) where

195197

196198import Prelude (Char, Bool(..), Int, Maybe(..), String,

@@ -213,14 +215,17 @@ import qualified Data.Text.Array as A

213215import qualified Data.List as L

214216import Data.Binary (Binary(get, put))

215217import Data.Monoid (Monoid(..))

218+#if MIN_VERSION_base(4,9,0)

219+import Data.Semigroup (Semigroup(..))

220+#endif

216221import Data.String (IsString(..))

217222import qualified Data.Text.Internal.Fusion as S

218223import qualified Data.Text.Internal.Fusion.Common as S

219−import Data.Text.Encoding (decodeUtf8, encodeUtf8)

224+import Data.Text.Encoding (decodeUtf8', encodeUtf8)

220225import Data.Text.Internal.Fusion (stream, reverseStream, unstream)

221226import Data.Text.Internal.Private (span_)

222227import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text)

223−import Data.Text.Show (singleton, unpack)

228+import Data.Text.Show (singleton, unpack, unpackCString#)

224229import qualified Prelude as P

225230import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,

226231 reverseIter_, unsafeHead, unsafeTail)

@@ -233,9 +238,13 @@ import Data.ByteString (ByteString)

233238import qualified Data.Text.Lazy as L

234239import Data.Int (Int64)

235240#endif

241+import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt)

236242#if __GLASGOW_HASKELL__ >= 708

237243import qualified GHC.Exts as Exts

238244#endif

245+#if MIN_VERSION_base(4,7,0)

246+import Text.Printf (PrintfArg, formatArg, formatString)

247+#endif

239248

240249-- $strict

241250--

@@ -324,9 +333,21 @@ instance Ord Text where

324333instance Read Text where

325334 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

326335

336+#if MIN_VERSION_base(4,9,0)

337+-- Semigroup orphan instances for older GHCs are provided by

338+-- 'semigroups` package

339+

340+instance Semigroup Text where

341+ (<>) = append

342+#endif

343+

327344instance Monoid Text where

328345 mempty = empty

346+#if MIN_VERSION_base(4,9,0)

347+ mappend = (<>) -- future-proof definition

348+#else

329349 mappend = append

350+#endif

330351 mconcat = concat

331352

332353instance IsString Text where

@@ -345,7 +366,11 @@ instance NFData Text where rnf !_ = ()

345366

346367instance Binary Text where

347368 put t = put (encodeUtf8 t)

348− get = P.fmap decodeUtf8 get

369+ get = do

370+ bs <- get

371+ case decodeUtf8' bs of

372+ P.Left exn -> P.fail (P.show exn)

373+ P.Right a -> P.return a

349374

350375-- | This instance preserves data abstraction at the cost of inefficiency.

351376-- We omit reflection services for the sake of data abstraction.

@@ -370,6 +395,12 @@ instance Data Text where

370395 _ -> P.error "gunfold"

371396 dataTypeOf _ = textDataType

372397

398+#if MIN_VERSION_base(4,7,0)

399+-- | Only defined for @base-4.7.0.0@ and later

400+instance PrintfArg Text where

401+ formatArg txt = formatString $ unpack txt

402+#endif

403+

373404packConstr :: Constr

374405packConstr = mkConstr textDataType "pack" [] Prefix

375406

@@ -455,8 +486,8 @@ head t = S.head (stream t)

455486uncons :: Text -> Maybe (Char, Text)

456487uncons t@(Text arr off len)

457488 | len <= 0 = Nothing

458− | otherwise = Just (c, text arr (off+d) (len-d))

459− where Iter c d = iter t 0

489+ | otherwise = Just $ let !(Iter c d) = iter t 0

490+ in (c, text arr (off+d) (len-d))

460491{-# INLINE [1] uncons #-}

461492

462493-- | Lifted from Control.Arrow and specialized.

@@ -541,7 +572,9 @@ isSingleton = S.isSingleton . stream

541572-- Subject to fusion.

542573length :: Text -> Int

543574length t = S.length (stream t)

544−{-# INLINE length #-}

575+{-# INLINE [0] length #-}

576+-- length needs to be phased after the compareN/length rules otherwise

577+-- it may inline before the rules have an opportunity to fire.

545578

546579-- | /O(n)/ Compare the count of characters in a 'Text' to a number.

547580-- Subject to fusion.

@@ -560,32 +593,32 @@ compareLength t n = S.compareLengthI (stream t) n

560593

561594{-# RULES

562595"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.

563− (==) (length t) n = compareLength t n == EQ

596+ eqInt (length t) n = compareLength t n == EQ

564597 #-}

565598

566599{-# RULES

567600"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.

568− (/=) (length t) n = compareLength t n /= EQ

601+ neInt (length t) n = compareLength t n /= EQ

569602 #-}

570603

571604{-# RULES

572605"TEXT <N/length -> compareLength/==LT" [~1] forall t n.

573− (<) (length t) n = compareLength t n == LT

606+ ltInt (length t) n = compareLength t n == LT

574607 #-}

575608

576609{-# RULES

577610"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.

578− (<=) (length t) n = compareLength t n /= GT

611+ leInt (length t) n = compareLength t n /= GT

579612 #-}

580613

581614{-# RULES

582615"TEXT >N/length -> compareLength/==GT" [~1] forall t n.

583− (>) (length t) n = compareLength t n == GT

616+ gtInt (length t) n = compareLength t n == GT

584617 #-}

585618

586619{-# RULES

587620"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.

588− (>=) (length t) n = compareLength t n /= LT

621+ geInt (length t) n = compareLength t n /= LT

589622 #-}

590623

591624-- -----------------------------------------------------------------------------

@@ -882,13 +915,13 @@ concatMap f = concat . foldr ((:) . f) []

882915{-# INLINE concatMap #-}

883916

884917-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the

885−-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion.

918+-- 'Text' @t@ satisfies the predicate @p@. Subject to fusion.

886919any :: (Char -> Bool) -> Text -> Bool

887920any p t = S.any p (stream t)

888921{-# INLINE any #-}

889922

890923-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the

891−-- 'Text' @t@ satisify the predicate @p@. Subject to fusion.

924+-- 'Text' @t@ satisfy the predicate @p@. Subject to fusion.

892925all :: (Char -> Bool) -> Text -> Bool

893926all p t = S.all p (stream t)

894927{-# INLINE all #-}

@@ -1119,6 +1152,27 @@ takeWhile p t@(Text arr off len) = loop 0

11191152 unstream (S.takeWhile p (stream t)) = takeWhile p t

11201153 #-}

11211154

1155+-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',

1156+-- returns the longest suffix (possibly empty) of elements that

1157+-- satisfy @p@. Subject to fusion.

1158+-- Examples:

1159+--

1160+-- > takeWhileEnd (=='o') "foo" == "oo"

1161+takeWhileEnd :: (Char -> Bool) -> Text -> Text

1162+takeWhileEnd p t@(Text arr off len) = loop (len-1) len

1163+ where loop !i !l | l <= 0 = t

1164+ | p c = loop (i+d) (l+d)

1165+ | otherwise = text arr (off+l) (len-l)

1166+ where (c,d) = reverseIter t i

1167+{-# INLINE [1] takeWhileEnd #-}

1168+

1169+{-# RULES

1170+"TEXT takeWhileEnd -> fused" [~1] forall p t.

1171+ takeWhileEnd p t = S.reverse (S.takeWhile p (S.reverseStream t))

1172+"TEXT takeWhileEnd -> unfused" [1] forall p t.

1173+ S.reverse (S.takeWhile p (S.reverseStream t)) = takeWhileEnd p t

1174+ #-}

1175+

11221176-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after

11231177-- 'takeWhile' @p@ @t@. Subject to fusion.

11241178dropWhile :: (Char -> Bool) -> Text -> Text

@@ -1137,8 +1191,9 @@ dropWhile p t@(Text arr off len) = loop 0 0

11371191 #-}

11381192

11391193-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after

1140−-- dropping characters that fail the predicate @p@ from the end of

1194+-- dropping characters that satisfy the predicate @p@ from the end of

11411195-- @t@. Subject to fusion.

1196+--

11421197-- Examples:

11431198--

11441199-- > dropWhileEnd (=='.') "foo..." == "foo"

@@ -1158,7 +1213,7 @@ dropWhileEnd p t@(Text arr off len) = loop (len-1) len

11581213 #-}

11591214

11601215-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after

1161−-- dropping characters that fail the predicate @p@ from both the

1216+-- dropping characters that satisfy the predicate @p@ from both the

11621217-- beginning and end of @t@. Subject to fusion.

11631218dropAround :: (Char -> Bool) -> Text -> Text

11641219dropAround p = dropWhile p . dropWhileEnd p

Data/Text/Array.hs+5−4

@@ -20,7 +20,7 @@

2020-- > import qualified Data.Text.Array as A

2121--

2222-- The names in this module resemble those in the 'Data.Array' family

23−-- of modules, but are shorter due to the assumption of qualifid

23+-- of modules, but are shorter due to the assumption of qualified

2424-- naming.

2525module Data.Text.Array

2626 (

@@ -74,7 +74,7 @@ import Foreign.C.Types (CInt, CSize)

7474#endif

7575import GHC.Base (ByteArray#, MutableByteArray#, Int(..),

7676 indexWord16Array#, newByteArray#,

77− unsafeCoerce#, writeWord16Array#)

77+ unsafeFreezeByteArray#, writeWord16Array#)

7878import GHC.ST (ST(..), runST)

7979import GHC.Word (Word16(..))

8080import Prelude hiding (length, read)

@@ -130,8 +130,9 @@ array_size_error = error "Data.Text.Array.new: size overflow"

130130

131131-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards!

132132unsafeFreeze :: MArray s -> ST s Array

133−unsafeFreeze MArray{..} = ST $ \s# ->

134− (# s#, Array (unsafeCoerce# maBA)

133+unsafeFreeze MArray{..} = ST $ \s1# ->

134+ case unsafeFreezeByteArray# maBA s1# of

135+ (# s2#, ba# #) -> (# s2#, Array ba#

135136#if defined(ASSERTS)

136137 maLen

137138#endif

Data/Text/Encoding.hs+7−18

@@ -55,13 +55,9 @@ module Data.Text.Encoding

5555 , encodeUtf32LE

5656 , encodeUtf32BE

5757

58−#if MIN_VERSION_bytestring(0,10,4)

5958 -- * Encoding Text using ByteString Builders

60− -- | /Note/ that these functions are only available if built against

61− -- @bytestring >= 0.10.4.0@.

6259 , encodeUtf8Builder

6360 , encodeUtf8BuilderEscaped

64−#endif

6561 ) where

6662

6763#if __GLASGOW_HASKELL__ >= 702

@@ -70,24 +66,15 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

7066import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

7167#endif

7268

73−#if MIN_VERSION_bytestring(0,10,4)

74−import Data.Bits ((.&.))

75−import Data.Text.Internal.Unsafe.Char (ord)

76−import qualified Data.ByteString.Builder as B

77−import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)

78−import qualified Data.ByteString.Builder.Prim as BP

79−import qualified Data.ByteString.Builder.Prim.Internal as BP

80−import qualified Data.Text.Internal.Encoding.Utf16 as U16

81−#endif

82−

8369import Control.Exception (evaluate, try)

8470import Control.Monad.ST (runST)

71+import Data.Bits ((.&.))

8572import Data.ByteString as B

8673import Data.ByteString.Internal as B hiding (c2w)

8774import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

8875import Data.Text.Internal (Text(..), safe, text)

8976import Data.Text.Internal.Private (runText)

90−import Data.Text.Internal.Unsafe.Char (unsafeWrite)

77+import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)

9178import Data.Text.Internal.Unsafe.Shift (shiftR)

9279import Data.Text.Show ()

9380import Data.Text.Unsafe (unsafeDupablePerformIO)

@@ -98,8 +85,13 @@ import Foreign.Marshal.Utils (with)

9885import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

9986import Foreign.Storable (Storable, peek, poke)

10087import GHC.Base (ByteArray#, MutableByteArray#)

88+import qualified Data.ByteString.Builder as B

89+import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)

90+import qualified Data.ByteString.Builder.Prim as BP

91+import qualified Data.ByteString.Builder.Prim.Internal as BP

10192import qualified Data.Text.Array as A

10293import qualified Data.Text.Internal.Encoding.Fusion as E

94+import qualified Data.Text.Internal.Encoding.Utf16 as U16

10395import qualified Data.Text.Internal.Fusion as F

10496

10597#include "text_cbits.h"

@@ -315,8 +307,6 @@ decodeUtf8' :: ByteString -> Either UnicodeException Text

315307decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode

316308{-# INLINE decodeUtf8' #-}

317309

318−#if MIN_VERSION_bytestring(0,10,4)

319−

320310-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

321311encodeUtf8Builder :: Text -> B.Builder

322312encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

@@ -377,7 +367,6 @@ encodeUtf8BuilderEscaped be =

377367 outerLoop i (B.BufferRange op ope)

378368 where

379369 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)

380−#endif

381370

382371-- | Encode text using UTF-8 encoding.

383372encodeUtf8 :: Text -> ByteString

Data/Text/Encoding/Error.hs+3−1

@@ -1,5 +1,7 @@

11{-# LANGUAGE CPP, DeriveDataTypeable #-}

2−#if __GLASGOW_HASKELL__ >= 702

2+#if __GLASGOW_HASKELL__ >= 704

3+{-# LANGUAGE Safe #-}

4+#elif __GLASGOW_HASKELL__ >= 702

35{-# LANGUAGE Trustworthy #-}

46#endif

57-- |

Data/Text/Internal/Builder.hs+17−2

@@ -6,7 +6,7 @@

66-- Module : Data.Text.Internal.Builder

9−-- License : BSD3-style (see LICENSE)

9+-- License : BSD-style (see LICENSE)

1010--

1111-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

1212-- Stability : experimental

@@ -59,6 +59,9 @@ module Data.Text.Internal.Builder

5959

6060import Control.Monad.ST (ST, runST)

6161import Data.Monoid (Monoid(..))

62+#if MIN_VERSION_base(4,9,0)

63+import Data.Semigroup (Semigroup(..))

64+#endif

6265import Data.Text.Internal (Text(..))

6366import Data.Text.Internal.Lazy (smallChunkSize)

6467import Data.Text.Unsafe (inlineInterleaveST)

@@ -89,12 +92,22 @@ newtype Builder = Builder {

8992 -> ST s [S.Text]

9093 }

9194

95+#if MIN_VERSION_base(4,9,0)

96+instance Semigroup Builder where

97+ (<>) = append

98+ {-# INLINE (<>) #-}

99+#endif

100+

92101instance Monoid Builder where

93102 mempty = empty

94103 {-# INLINE mempty #-}

104+#if MIN_VERSION_base(4,9,0)

105+ mappend = (<>) -- future-proof definition

106+#else

95107 mappend = append

108+#endif

96109 {-# INLINE mappend #-}

97− mconcat = foldr mappend mempty

110+ mconcat = foldr mappend Data.Monoid.mempty

98111 {-# INLINE mconcat #-}

99112

100113instance String.IsString Builder where

@@ -229,6 +242,8 @@ flush = Builder $ \ k buf@(Buffer p o u l) ->

229242 !t = Text arr o u

230243 ts <- inlineInterleaveST (k b)

231244 return $! t : ts

245+{-# INLINE [1] flush #-}

246+-- defer inlining so that flush/flush rule may fire.

232247

233248------------------------------------------------------------------------

234249

Data/Text/Internal/Builder/Functions.hs+2−1

@@ -23,7 +23,8 @@ module Data.Text.Internal.Builder.Functions

2323

2424import Data.Monoid (mappend)

2525import Data.Text.Lazy.Builder (Builder)

26−import GHC.Base

26+import GHC.Base (chr#,ord#,(+#),Int(I#),Char(C#))

27+import Prelude ()

2728

2829-- | Unsafe conversion for decimal digits.

2930{-# INLINE i2d #-}

Data/Text/Internal/Builder/Int/Digits.hs+1−1

@@ -2,7 +2,7 @@

33-- Module: Data.Text.Internal.Builder.Int.Digits

5−-- License: BSD3

5+-- License: BSD-style

66-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

77-- Stability: experimental

88-- Portability: portable

Data/Text/Internal/Fusion.hs+27−17

@@ -98,25 +98,35 @@ reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `sh

9898-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.

9999unstream :: Stream Char -> Text

100100unstream (Stream next0 s0 len) = runText $ \done -> do

101− let mlen = upperBound 4 len

101+ -- Before encoding each char we perform a buffer realloc check assuming

102+ -- worst case encoding size of two 16-bit units for the char. Just add an

103+ -- extra space to the buffer so that we do not end up reallocating even when

104+ -- all the chars are encoded as single unit.

105+ let mlen = upperBound 4 len + 1

102106 arr0 <- A.new mlen

103− let outer arr top = loop

107+ let outer !arr !maxi = encode

104108 where

105− loop !s !i =

106− case next0 s of

107− Done -> done arr i

108− Skip s' -> loop s' i

109− Yield x s'

110− | j >= top -> {-# SCC "unstream/resize" #-} do

111− let top' = (top + 1) `shiftL` 1

112− arr' <- A.new top'

113− A.copyM arr' 0 arr 0 top

114− outer arr' top' s i

115− | otherwise -> do d <- unsafeWrite arr i x

116− loop s' (i+d)

117− where j | ord x < 0x10000 = i

118− | otherwise = i + 1

119− outer arr0 mlen s0 0

109+ -- keep the common case loop as small as possible

110+ encode !si !di =

111+ case next0 si of

112+ Done -> done arr di

113+ Skip si' -> encode si' di

114+ Yield c si'

115+ -- simply check for the worst case

116+ | maxi < di + 1 -> realloc si di

117+ | otherwise -> do

118+ n <- unsafeWrite arr di c

119+ encode si' (di + n)

120+

121+ -- keep uncommon case separate from the common case code

122+ {-# NOINLINE realloc #-}

123+ realloc !si !di = do

124+ let newlen = (maxi + 1) * 2

125+ arr' <- A.new newlen

126+ A.copyM arr' 0 arr 0 di

127+ outer arr' (newlen - 1) si di

128+

129+ outer arr0 (mlen - 1) s0 0

120130{-# INLINE [0] unstream #-}

121131{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}

122132

Data/Text/Internal/Fusion/CaseMapping.hs+286−112

@@ -1,10 +1,10 @@

11{-# LANGUAGE Rank2Types #-}

22-- AUTOMATICALLY GENERATED - DO NOT EDIT

33-- Generated by scripts/CaseMapping.hs

4−-- CaseFolding-7.0.0.txt

5−-- Date: 2014-04-09, 20:00:56 GMT [MD]

6−-- SpecialCasing-7.0.0.txt

7−-- Date: 2014-03-18, 07:18:02 GMT [MD]

4+-- CaseFolding-8.0.0.txt

5+-- Date: 2015-01-13, 18:16:36 GMT [MD]

6+-- SpecialCasing-8.0.0.txt

7+-- Date: 2014-12-16, 23:08:04 GMT [MD]

99module Data.Text.Internal.Fusion.CaseMapping where

1010import Data.Char

@@ -337,8 +337,6 @@ foldMapping '\x017f' s = Yield '\x0073' (CC s '\x0000' '\x0000')

337337foldMapping '\x01f0' s = Yield '\x006a' (CC s '\x030c' '\x0000')

338338-- COMBINING GREEK YPOGEGRAMMENI

339339foldMapping '\x0345' s = Yield '\x03b9' (CC s '\x0000' '\x0000')

340−-- GREEK CAPITAL LETTER YOT

341−foldMapping '\x037f' s = Yield '\x03f3' (CC s '\x0000' '\x0000')

342340-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS

343341foldMapping '\x0390' s = Yield '\x03b9' (CC s '\x0308' '\x0301')

344342-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS

@@ -359,20 +357,20 @@ foldMapping '\x03f0' s = Yield '\x03ba' (CC s '\x0000' '\x0000')

359357foldMapping '\x03f1' s = Yield '\x03c1' (CC s '\x0000' '\x0000')

360358-- GREEK LUNATE EPSILON SYMBOL

361359foldMapping '\x03f5' s = Yield '\x03b5' (CC s '\x0000' '\x0000')

362−-- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK

363−foldMapping '\x0528' s = Yield '\x0529' (CC s '\x0000' '\x0000')

364−-- CYRILLIC CAPITAL LETTER DZZHE

365−foldMapping '\x052a' s = Yield '\x052b' (CC s '\x0000' '\x0000')

366−-- CYRILLIC CAPITAL LETTER DCHE

367−foldMapping '\x052c' s = Yield '\x052d' (CC s '\x0000' '\x0000')

368−-- CYRILLIC CAPITAL LETTER EL WITH DESCENDER

369−foldMapping '\x052e' s = Yield '\x052f' (CC s '\x0000' '\x0000')

370360-- ARMENIAN SMALL LIGATURE ECH YIWN

371361foldMapping '\x0587' s = Yield '\x0565' (CC s '\x0582' '\x0000')

372−-- GEORGIAN CAPITAL LETTER YN

373−foldMapping '\x10c7' s = Yield '\x2d27' (CC s '\x0000' '\x0000')

374−-- GEORGIAN CAPITAL LETTER AEN

375−foldMapping '\x10cd' s = Yield '\x2d2d' (CC s '\x0000' '\x0000')

362+-- CHEROKEE SMALL LETTER YE

363+foldMapping '\x13f8' s = Yield '\x13f0' (CC s '\x0000' '\x0000')

364+-- CHEROKEE SMALL LETTER YI

365+foldMapping '\x13f9' s = Yield '\x13f1' (CC s '\x0000' '\x0000')

366+-- CHEROKEE SMALL LETTER YO

367+foldMapping '\x13fa' s = Yield '\x13f2' (CC s '\x0000' '\x0000')

368+-- CHEROKEE SMALL LETTER YU

369+foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')

370+-- CHEROKEE SMALL LETTER YV

371+foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')

372+-- CHEROKEE SMALL LETTER MV

373+foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')

376374-- LATIN SMALL LETTER H WITH LINE BELOW

377375foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')

378376-- LATIN SMALL LETTER T WITH DIAERESIS

@@ -547,36 +545,174 @@ foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')

547545foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')

548546-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI

549547foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')

550−-- COPTIC CAPITAL LETTER BOHAIRIC KHEI

551−foldMapping '\x2cf2' s = Yield '\x2cf3' (CC s '\x0000' '\x0000')

552−-- CYRILLIC CAPITAL LETTER DOUBLE O

553−foldMapping '\xa698' s = Yield '\xa699' (CC s '\x0000' '\x0000')

554−-- CYRILLIC CAPITAL LETTER CROSSED O

555−foldMapping '\xa69a' s = Yield '\xa69b' (CC s '\x0000' '\x0000')

556−-- LATIN CAPITAL LETTER C WITH BAR

557−foldMapping '\xa792' s = Yield '\xa793' (CC s '\x0000' '\x0000')

558−-- LATIN CAPITAL LETTER B WITH FLOURISH

559−foldMapping '\xa796' s = Yield '\xa797' (CC s '\x0000' '\x0000')

560−-- LATIN CAPITAL LETTER F WITH STROKE

561−foldMapping '\xa798' s = Yield '\xa799' (CC s '\x0000' '\x0000')

562−-- LATIN CAPITAL LETTER VOLAPUK AE

563−foldMapping '\xa79a' s = Yield '\xa79b' (CC s '\x0000' '\x0000')

564−-- LATIN CAPITAL LETTER VOLAPUK OE

565−foldMapping '\xa79c' s = Yield '\xa79d' (CC s '\x0000' '\x0000')

566−-- LATIN CAPITAL LETTER VOLAPUK UE

567−foldMapping '\xa79e' s = Yield '\xa79f' (CC s '\x0000' '\x0000')

568−-- LATIN CAPITAL LETTER H WITH HOOK

569−foldMapping '\xa7aa' s = Yield '\x0266' (CC s '\x0000' '\x0000')

570−-- LATIN CAPITAL LETTER REVERSED OPEN E

571−foldMapping '\xa7ab' s = Yield '\x025c' (CC s '\x0000' '\x0000')

572−-- LATIN CAPITAL LETTER SCRIPT G

573−foldMapping '\xa7ac' s = Yield '\x0261' (CC s '\x0000' '\x0000')

574−-- LATIN CAPITAL LETTER L WITH BELT

575−foldMapping '\xa7ad' s = Yield '\x026c' (CC s '\x0000' '\x0000')

576−-- LATIN CAPITAL LETTER TURNED K

577−foldMapping '\xa7b0' s = Yield '\x029e' (CC s '\x0000' '\x0000')

578−-- LATIN CAPITAL LETTER TURNED T

579−foldMapping '\xa7b1' s = Yield '\x0287' (CC s '\x0000' '\x0000')

548+-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL

549+foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')

550+-- LATIN CAPITAL LETTER CHI

551+foldMapping '\xa7b3' s = Yield '\xab53' (CC s '\x0000' '\x0000')

552+-- LATIN CAPITAL LETTER BETA

553+foldMapping '\xa7b4' s = Yield '\xa7b5' (CC s '\x0000' '\x0000')

554+-- LATIN CAPITAL LETTER OMEGA

555+foldMapping '\xa7b6' s = Yield '\xa7b7' (CC s '\x0000' '\x0000')

556+-- CHEROKEE SMALL LETTER A

557+foldMapping '\xab70' s = Yield '\x13a0' (CC s '\x0000' '\x0000')

558+-- CHEROKEE SMALL LETTER E

559+foldMapping '\xab71' s = Yield '\x13a1' (CC s '\x0000' '\x0000')

560+-- CHEROKEE SMALL LETTER I

561+foldMapping '\xab72' s = Yield '\x13a2' (CC s '\x0000' '\x0000')

562+-- CHEROKEE SMALL LETTER O

563+foldMapping '\xab73' s = Yield '\x13a3' (CC s '\x0000' '\x0000')

564+-- CHEROKEE SMALL LETTER U

565+foldMapping '\xab74' s = Yield '\x13a4' (CC s '\x0000' '\x0000')

566+-- CHEROKEE SMALL LETTER V

567+foldMapping '\xab75' s = Yield '\x13a5' (CC s '\x0000' '\x0000')

568+-- CHEROKEE SMALL LETTER GA

569+foldMapping '\xab76' s = Yield '\x13a6' (CC s '\x0000' '\x0000')

570+-- CHEROKEE SMALL LETTER KA

571+foldMapping '\xab77' s = Yield '\x13a7' (CC s '\x0000' '\x0000')

572+-- CHEROKEE SMALL LETTER GE

573+foldMapping '\xab78' s = Yield '\x13a8' (CC s '\x0000' '\x0000')

574+-- CHEROKEE SMALL LETTER GI

575+foldMapping '\xab79' s = Yield '\x13a9' (CC s '\x0000' '\x0000')

576+-- CHEROKEE SMALL LETTER GO

577+foldMapping '\xab7a' s = Yield '\x13aa' (CC s '\x0000' '\x0000')

578+-- CHEROKEE SMALL LETTER GU

579+foldMapping '\xab7b' s = Yield '\x13ab' (CC s '\x0000' '\x0000')

580+-- CHEROKEE SMALL LETTER GV

581+foldMapping '\xab7c' s = Yield '\x13ac' (CC s '\x0000' '\x0000')

582+-- CHEROKEE SMALL LETTER HA

583+foldMapping '\xab7d' s = Yield '\x13ad' (CC s '\x0000' '\x0000')

584+-- CHEROKEE SMALL LETTER HE

585+foldMapping '\xab7e' s = Yield '\x13ae' (CC s '\x0000' '\x0000')

586+-- CHEROKEE SMALL LETTER HI

587+foldMapping '\xab7f' s = Yield '\x13af' (CC s '\x0000' '\x0000')

588+-- CHEROKEE SMALL LETTER HO

589+foldMapping '\xab80' s = Yield '\x13b0' (CC s '\x0000' '\x0000')

590+-- CHEROKEE SMALL LETTER HU

591+foldMapping '\xab81' s = Yield '\x13b1' (CC s '\x0000' '\x0000')

592+-- CHEROKEE SMALL LETTER HV

593+foldMapping '\xab82' s = Yield '\x13b2' (CC s '\x0000' '\x0000')

594+-- CHEROKEE SMALL LETTER LA

595+foldMapping '\xab83' s = Yield '\x13b3' (CC s '\x0000' '\x0000')

596+-- CHEROKEE SMALL LETTER LE

597+foldMapping '\xab84' s = Yield '\x13b4' (CC s '\x0000' '\x0000')

598+-- CHEROKEE SMALL LETTER LI

599+foldMapping '\xab85' s = Yield '\x13b5' (CC s '\x0000' '\x0000')

600+-- CHEROKEE SMALL LETTER LO

601+foldMapping '\xab86' s = Yield '\x13b6' (CC s '\x0000' '\x0000')

602+-- CHEROKEE SMALL LETTER LU

603+foldMapping '\xab87' s = Yield '\x13b7' (CC s '\x0000' '\x0000')

604+-- CHEROKEE SMALL LETTER LV

605+foldMapping '\xab88' s = Yield '\x13b8' (CC s '\x0000' '\x0000')

606+-- CHEROKEE SMALL LETTER MA

607+foldMapping '\xab89' s = Yield '\x13b9' (CC s '\x0000' '\x0000')

608+-- CHEROKEE SMALL LETTER ME

609+foldMapping '\xab8a' s = Yield '\x13ba' (CC s '\x0000' '\x0000')

610+-- CHEROKEE SMALL LETTER MI

611+foldMapping '\xab8b' s = Yield '\x13bb' (CC s '\x0000' '\x0000')

612+-- CHEROKEE SMALL LETTER MO

613+foldMapping '\xab8c' s = Yield '\x13bc' (CC s '\x0000' '\x0000')

614+-- CHEROKEE SMALL LETTER MU

615+foldMapping '\xab8d' s = Yield '\x13bd' (CC s '\x0000' '\x0000')

616+-- CHEROKEE SMALL LETTER NA

617+foldMapping '\xab8e' s = Yield '\x13be' (CC s '\x0000' '\x0000')

618+-- CHEROKEE SMALL LETTER HNA

619+foldMapping '\xab8f' s = Yield '\x13bf' (CC s '\x0000' '\x0000')

620+-- CHEROKEE SMALL LETTER NAH

621+foldMapping '\xab90' s = Yield '\x13c0' (CC s '\x0000' '\x0000')

622+-- CHEROKEE SMALL LETTER NE

623+foldMapping '\xab91' s = Yield '\x13c1' (CC s '\x0000' '\x0000')

624+-- CHEROKEE SMALL LETTER NI

625+foldMapping '\xab92' s = Yield '\x13c2' (CC s '\x0000' '\x0000')

626+-- CHEROKEE SMALL LETTER NO

627+foldMapping '\xab93' s = Yield '\x13c3' (CC s '\x0000' '\x0000')

628+-- CHEROKEE SMALL LETTER NU

629+foldMapping '\xab94' s = Yield '\x13c4' (CC s '\x0000' '\x0000')

630+-- CHEROKEE SMALL LETTER NV

631+foldMapping '\xab95' s = Yield '\x13c5' (CC s '\x0000' '\x0000')

632+-- CHEROKEE SMALL LETTER QUA

633+foldMapping '\xab96' s = Yield '\x13c6' (CC s '\x0000' '\x0000')

634+-- CHEROKEE SMALL LETTER QUE

635+foldMapping '\xab97' s = Yield '\x13c7' (CC s '\x0000' '\x0000')

636+-- CHEROKEE SMALL LETTER QUI

637+foldMapping '\xab98' s = Yield '\x13c8' (CC s '\x0000' '\x0000')

638+-- CHEROKEE SMALL LETTER QUO

639+foldMapping '\xab99' s = Yield '\x13c9' (CC s '\x0000' '\x0000')

640+-- CHEROKEE SMALL LETTER QUU

641+foldMapping '\xab9a' s = Yield '\x13ca' (CC s '\x0000' '\x0000')

642+-- CHEROKEE SMALL LETTER QUV

643+foldMapping '\xab9b' s = Yield '\x13cb' (CC s '\x0000' '\x0000')

644+-- CHEROKEE SMALL LETTER SA

645+foldMapping '\xab9c' s = Yield '\x13cc' (CC s '\x0000' '\x0000')

646+-- CHEROKEE SMALL LETTER S

647+foldMapping '\xab9d' s = Yield '\x13cd' (CC s '\x0000' '\x0000')

648+-- CHEROKEE SMALL LETTER SE

649+foldMapping '\xab9e' s = Yield '\x13ce' (CC s '\x0000' '\x0000')

650+-- CHEROKEE SMALL LETTER SI

651+foldMapping '\xab9f' s = Yield '\x13cf' (CC s '\x0000' '\x0000')

652+-- CHEROKEE SMALL LETTER SO

653+foldMapping '\xaba0' s = Yield '\x13d0' (CC s '\x0000' '\x0000')

654+-- CHEROKEE SMALL LETTER SU

655+foldMapping '\xaba1' s = Yield '\x13d1' (CC s '\x0000' '\x0000')

656+-- CHEROKEE SMALL LETTER SV

657+foldMapping '\xaba2' s = Yield '\x13d2' (CC s '\x0000' '\x0000')

658+-- CHEROKEE SMALL LETTER DA

659+foldMapping '\xaba3' s = Yield '\x13d3' (CC s '\x0000' '\x0000')

660+-- CHEROKEE SMALL LETTER TA

661+foldMapping '\xaba4' s = Yield '\x13d4' (CC s '\x0000' '\x0000')

662+-- CHEROKEE SMALL LETTER DE

663+foldMapping '\xaba5' s = Yield '\x13d5' (CC s '\x0000' '\x0000')

664+-- CHEROKEE SMALL LETTER TE

665+foldMapping '\xaba6' s = Yield '\x13d6' (CC s '\x0000' '\x0000')

666+-- CHEROKEE SMALL LETTER DI

667+foldMapping '\xaba7' s = Yield '\x13d7' (CC s '\x0000' '\x0000')

668+-- CHEROKEE SMALL LETTER TI

669+foldMapping '\xaba8' s = Yield '\x13d8' (CC s '\x0000' '\x0000')

670+-- CHEROKEE SMALL LETTER DO

671+foldMapping '\xaba9' s = Yield '\x13d9' (CC s '\x0000' '\x0000')

672+-- CHEROKEE SMALL LETTER DU

673+foldMapping '\xabaa' s = Yield '\x13da' (CC s '\x0000' '\x0000')

674+-- CHEROKEE SMALL LETTER DV

675+foldMapping '\xabab' s = Yield '\x13db' (CC s '\x0000' '\x0000')

676+-- CHEROKEE SMALL LETTER DLA

677+foldMapping '\xabac' s = Yield '\x13dc' (CC s '\x0000' '\x0000')

678+-- CHEROKEE SMALL LETTER TLA

679+foldMapping '\xabad' s = Yield '\x13dd' (CC s '\x0000' '\x0000')

680+-- CHEROKEE SMALL LETTER TLE

681+foldMapping '\xabae' s = Yield '\x13de' (CC s '\x0000' '\x0000')

682+-- CHEROKEE SMALL LETTER TLI

683+foldMapping '\xabaf' s = Yield '\x13df' (CC s '\x0000' '\x0000')

684+-- CHEROKEE SMALL LETTER TLO

685+foldMapping '\xabb0' s = Yield '\x13e0' (CC s '\x0000' '\x0000')

686+-- CHEROKEE SMALL LETTER TLU

687+foldMapping '\xabb1' s = Yield '\x13e1' (CC s '\x0000' '\x0000')

688+-- CHEROKEE SMALL LETTER TLV

689+foldMapping '\xabb2' s = Yield '\x13e2' (CC s '\x0000' '\x0000')

690+-- CHEROKEE SMALL LETTER TSA

691+foldMapping '\xabb3' s = Yield '\x13e3' (CC s '\x0000' '\x0000')

692+-- CHEROKEE SMALL LETTER TSE

693+foldMapping '\xabb4' s = Yield '\x13e4' (CC s '\x0000' '\x0000')

694+-- CHEROKEE SMALL LETTER TSI

695+foldMapping '\xabb5' s = Yield '\x13e5' (CC s '\x0000' '\x0000')

696+-- CHEROKEE SMALL LETTER TSO

697+foldMapping '\xabb6' s = Yield '\x13e6' (CC s '\x0000' '\x0000')

698+-- CHEROKEE SMALL LETTER TSU

699+foldMapping '\xabb7' s = Yield '\x13e7' (CC s '\x0000' '\x0000')

700+-- CHEROKEE SMALL LETTER TSV

701+foldMapping '\xabb8' s = Yield '\x13e8' (CC s '\x0000' '\x0000')

702+-- CHEROKEE SMALL LETTER WA

703+foldMapping '\xabb9' s = Yield '\x13e9' (CC s '\x0000' '\x0000')

704+-- CHEROKEE SMALL LETTER WE

705+foldMapping '\xabba' s = Yield '\x13ea' (CC s '\x0000' '\x0000')

706+-- CHEROKEE SMALL LETTER WI

707+foldMapping '\xabbb' s = Yield '\x13eb' (CC s '\x0000' '\x0000')

708+-- CHEROKEE SMALL LETTER WO

709+foldMapping '\xabbc' s = Yield '\x13ec' (CC s '\x0000' '\x0000')

710+-- CHEROKEE SMALL LETTER WU

711+foldMapping '\xabbd' s = Yield '\x13ed' (CC s '\x0000' '\x0000')

712+-- CHEROKEE SMALL LETTER WV

713+foldMapping '\xabbe' s = Yield '\x13ee' (CC s '\x0000' '\x0000')

714+-- CHEROKEE SMALL LETTER YA

715+foldMapping '\xabbf' s = Yield '\x13ef' (CC s '\x0000' '\x0000')

580716-- LATIN SMALL LIGATURE FF

581717foldMapping '\xfb00' s = Yield '\x0066' (CC s '\x0066' '\x0000')

582718-- LATIN SMALL LIGATURE FI

@@ -601,68 +737,106 @@ foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')

601737foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')

602738-- ARMENIAN SMALL LIGATURE MEN XEH

603739foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')

604−-- WARANG CITI CAPITAL LETTER NGAA

605−foldMapping '\x118a0' s = Yield '\x118c0' (CC s '\x0000' '\x0000')

606−-- WARANG CITI CAPITAL LETTER A

607−foldMapping '\x118a1' s = Yield '\x118c1' (CC s '\x0000' '\x0000')

608−-- WARANG CITI CAPITAL LETTER WI

609−foldMapping '\x118a2' s = Yield '\x118c2' (CC s '\x0000' '\x0000')

610−-- WARANG CITI CAPITAL LETTER YU

611−foldMapping '\x118a3' s = Yield '\x118c3' (CC s '\x0000' '\x0000')

612−-- WARANG CITI CAPITAL LETTER YA

613−foldMapping '\x118a4' s = Yield '\x118c4' (CC s '\x0000' '\x0000')

614−-- WARANG CITI CAPITAL LETTER YO

615−foldMapping '\x118a5' s = Yield '\x118c5' (CC s '\x0000' '\x0000')

616−-- WARANG CITI CAPITAL LETTER II

617−foldMapping '\x118a6' s = Yield '\x118c6' (CC s '\x0000' '\x0000')

618−-- WARANG CITI CAPITAL LETTER UU

619−foldMapping '\x118a7' s = Yield '\x118c7' (CC s '\x0000' '\x0000')

620−-- WARANG CITI CAPITAL LETTER E

621−foldMapping '\x118a8' s = Yield '\x118c8' (CC s '\x0000' '\x0000')

622−-- WARANG CITI CAPITAL LETTER O

623−foldMapping '\x118a9' s = Yield '\x118c9' (CC s '\x0000' '\x0000')

624−-- WARANG CITI CAPITAL LETTER ANG

625−foldMapping '\x118aa' s = Yield '\x118ca' (CC s '\x0000' '\x0000')

626−-- WARANG CITI CAPITAL LETTER GA

627−foldMapping '\x118ab' s = Yield '\x118cb' (CC s '\x0000' '\x0000')

628−-- WARANG CITI CAPITAL LETTER KO

629−foldMapping '\x118ac' s = Yield '\x118cc' (CC s '\x0000' '\x0000')

630−-- WARANG CITI CAPITAL LETTER ENY

631−foldMapping '\x118ad' s = Yield '\x118cd' (CC s '\x0000' '\x0000')

632−-- WARANG CITI CAPITAL LETTER YUJ

633−foldMapping '\x118ae' s = Yield '\x118ce' (CC s '\x0000' '\x0000')

634−-- WARANG CITI CAPITAL LETTER UC

635−foldMapping '\x118af' s = Yield '\x118cf' (CC s '\x0000' '\x0000')

636−-- WARANG CITI CAPITAL LETTER ENN

637−foldMapping '\x118b0' s = Yield '\x118d0' (CC s '\x0000' '\x0000')

638−-- WARANG CITI CAPITAL LETTER ODD

639−foldMapping '\x118b1' s = Yield '\x118d1' (CC s '\x0000' '\x0000')

640−-- WARANG CITI CAPITAL LETTER TTE

641−foldMapping '\x118b2' s = Yield '\x118d2' (CC s '\x0000' '\x0000')

642−-- WARANG CITI CAPITAL LETTER NUNG

643−foldMapping '\x118b3' s = Yield '\x118d3' (CC s '\x0000' '\x0000')

644−-- WARANG CITI CAPITAL LETTER DA

645−foldMapping '\x118b4' s = Yield '\x118d4' (CC s '\x0000' '\x0000')

646−-- WARANG CITI CAPITAL LETTER AT

647−foldMapping '\x118b5' s = Yield '\x118d5' (CC s '\x0000' '\x0000')

648−-- WARANG CITI CAPITAL LETTER AM

649−foldMapping '\x118b6' s = Yield '\x118d6' (CC s '\x0000' '\x0000')

650−-- WARANG CITI CAPITAL LETTER BU

651−foldMapping '\x118b7' s = Yield '\x118d7' (CC s '\x0000' '\x0000')

652−-- WARANG CITI CAPITAL LETTER PU

653−foldMapping '\x118b8' s = Yield '\x118d8' (CC s '\x0000' '\x0000')

654−-- WARANG CITI CAPITAL LETTER HIYO

655−foldMapping '\x118b9' s = Yield '\x118d9' (CC s '\x0000' '\x0000')

656−-- WARANG CITI CAPITAL LETTER HOLO

657−foldMapping '\x118ba' s = Yield '\x118da' (CC s '\x0000' '\x0000')

658−-- WARANG CITI CAPITAL LETTER HORR

659−foldMapping '\x118bb' s = Yield '\x118db' (CC s '\x0000' '\x0000')

660−-- WARANG CITI CAPITAL LETTER HAR

661−foldMapping '\x118bc' s = Yield '\x118dc' (CC s '\x0000' '\x0000')

662−-- WARANG CITI CAPITAL LETTER SSUU

663−foldMapping '\x118bd' s = Yield '\x118dd' (CC s '\x0000' '\x0000')

664−-- WARANG CITI CAPITAL LETTER SII

665−foldMapping '\x118be' s = Yield '\x118de' (CC s '\x0000' '\x0000')

666−-- WARANG CITI CAPITAL LETTER VIYO

667−foldMapping '\x118bf' s = Yield '\x118df' (CC s '\x0000' '\x0000')

740+-- OLD HUNGARIAN CAPITAL LETTER A

741+foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')

742+-- OLD HUNGARIAN CAPITAL LETTER AA

743+foldMapping '\x10c81' s = Yield '\x10cc1' (CC s '\x0000' '\x0000')

744+-- OLD HUNGARIAN CAPITAL LETTER EB

745+foldMapping '\x10c82' s = Yield '\x10cc2' (CC s '\x0000' '\x0000')

746+-- OLD HUNGARIAN CAPITAL LETTER AMB

747+foldMapping '\x10c83' s = Yield '\x10cc3' (CC s '\x0000' '\x0000')

748+-- OLD HUNGARIAN CAPITAL LETTER EC

749+foldMapping '\x10c84' s = Yield '\x10cc4' (CC s '\x0000' '\x0000')

750+-- OLD HUNGARIAN CAPITAL LETTER ENC

751+foldMapping '\x10c85' s = Yield '\x10cc5' (CC s '\x0000' '\x0000')

752+-- OLD HUNGARIAN CAPITAL LETTER ECS

753+foldMapping '\x10c86' s = Yield '\x10cc6' (CC s '\x0000' '\x0000')

754+-- OLD HUNGARIAN CAPITAL LETTER ED

755+foldMapping '\x10c87' s = Yield '\x10cc7' (CC s '\x0000' '\x0000')

756+-- OLD HUNGARIAN CAPITAL LETTER AND

757+foldMapping '\x10c88' s = Yield '\x10cc8' (CC s '\x0000' '\x0000')

758+-- OLD HUNGARIAN CAPITAL LETTER E

759+foldMapping '\x10c89' s = Yield '\x10cc9' (CC s '\x0000' '\x0000')

760+-- OLD HUNGARIAN CAPITAL LETTER CLOSE E

761+foldMapping '\x10c8a' s = Yield '\x10cca' (CC s '\x0000' '\x0000')

762+-- OLD HUNGARIAN CAPITAL LETTER EE

763+foldMapping '\x10c8b' s = Yield '\x10ccb' (CC s '\x0000' '\x0000')

764+-- OLD HUNGARIAN CAPITAL LETTER EF

765+foldMapping '\x10c8c' s = Yield '\x10ccc' (CC s '\x0000' '\x0000')

766+-- OLD HUNGARIAN CAPITAL LETTER EG

767+foldMapping '\x10c8d' s = Yield '\x10ccd' (CC s '\x0000' '\x0000')

768+-- OLD HUNGARIAN CAPITAL LETTER EGY

769+foldMapping '\x10c8e' s = Yield '\x10cce' (CC s '\x0000' '\x0000')

770+-- OLD HUNGARIAN CAPITAL LETTER EH

771+foldMapping '\x10c8f' s = Yield '\x10ccf' (CC s '\x0000' '\x0000')

772+-- OLD HUNGARIAN CAPITAL LETTER I

773+foldMapping '\x10c90' s = Yield '\x10cd0' (CC s '\x0000' '\x0000')

774+-- OLD HUNGARIAN CAPITAL LETTER II

775+foldMapping '\x10c91' s = Yield '\x10cd1' (CC s '\x0000' '\x0000')

776+-- OLD HUNGARIAN CAPITAL LETTER EJ

777+foldMapping '\x10c92' s = Yield '\x10cd2' (CC s '\x0000' '\x0000')

778+-- OLD HUNGARIAN CAPITAL LETTER EK

779+foldMapping '\x10c93' s = Yield '\x10cd3' (CC s '\x0000' '\x0000')

780+-- OLD HUNGARIAN CAPITAL LETTER AK

781+foldMapping '\x10c94' s = Yield '\x10cd4' (CC s '\x0000' '\x0000')

782+-- OLD HUNGARIAN CAPITAL LETTER UNK

783+foldMapping '\x10c95' s = Yield '\x10cd5' (CC s '\x0000' '\x0000')

784+-- OLD HUNGARIAN CAPITAL LETTER EL

785+foldMapping '\x10c96' s = Yield '\x10cd6' (CC s '\x0000' '\x0000')

786+-- OLD HUNGARIAN CAPITAL LETTER ELY

787+foldMapping '\x10c97' s = Yield '\x10cd7' (CC s '\x0000' '\x0000')

788+-- OLD HUNGARIAN CAPITAL LETTER EM

789+foldMapping '\x10c98' s = Yield '\x10cd8' (CC s '\x0000' '\x0000')

790+-- OLD HUNGARIAN CAPITAL LETTER EN

791+foldMapping '\x10c99' s = Yield '\x10cd9' (CC s '\x0000' '\x0000')

792+-- OLD HUNGARIAN CAPITAL LETTER ENY

793+foldMapping '\x10c9a' s = Yield '\x10cda' (CC s '\x0000' '\x0000')

794+-- OLD HUNGARIAN CAPITAL LETTER O

795+foldMapping '\x10c9b' s = Yield '\x10cdb' (CC s '\x0000' '\x0000')

796+-- OLD HUNGARIAN CAPITAL LETTER OO

797+foldMapping '\x10c9c' s = Yield '\x10cdc' (CC s '\x0000' '\x0000')

798+-- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE

799+foldMapping '\x10c9d' s = Yield '\x10cdd' (CC s '\x0000' '\x0000')

800+-- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE

801+foldMapping '\x10c9e' s = Yield '\x10cde' (CC s '\x0000' '\x0000')

802+-- OLD HUNGARIAN CAPITAL LETTER OEE

803+foldMapping '\x10c9f' s = Yield '\x10cdf' (CC s '\x0000' '\x0000')

804+-- OLD HUNGARIAN CAPITAL LETTER EP

805+foldMapping '\x10ca0' s = Yield '\x10ce0' (CC s '\x0000' '\x0000')

806+-- OLD HUNGARIAN CAPITAL LETTER EMP

807+foldMapping '\x10ca1' s = Yield '\x10ce1' (CC s '\x0000' '\x0000')

808+-- OLD HUNGARIAN CAPITAL LETTER ER

809+foldMapping '\x10ca2' s = Yield '\x10ce2' (CC s '\x0000' '\x0000')

810+-- OLD HUNGARIAN CAPITAL LETTER SHORT ER

811+foldMapping '\x10ca3' s = Yield '\x10ce3' (CC s '\x0000' '\x0000')

812+-- OLD HUNGARIAN CAPITAL LETTER ES

813+foldMapping '\x10ca4' s = Yield '\x10ce4' (CC s '\x0000' '\x0000')

814+-- OLD HUNGARIAN CAPITAL LETTER ESZ

815+foldMapping '\x10ca5' s = Yield '\x10ce5' (CC s '\x0000' '\x0000')

816+-- OLD HUNGARIAN CAPITAL LETTER ET

817+foldMapping '\x10ca6' s = Yield '\x10ce6' (CC s '\x0000' '\x0000')

818+-- OLD HUNGARIAN CAPITAL LETTER ENT

819+foldMapping '\x10ca7' s = Yield '\x10ce7' (CC s '\x0000' '\x0000')

820+-- OLD HUNGARIAN CAPITAL LETTER ETY

821+foldMapping '\x10ca8' s = Yield '\x10ce8' (CC s '\x0000' '\x0000')

822+-- OLD HUNGARIAN CAPITAL LETTER ECH

823+foldMapping '\x10ca9' s = Yield '\x10ce9' (CC s '\x0000' '\x0000')

824+-- OLD HUNGARIAN CAPITAL LETTER U

825+foldMapping '\x10caa' s = Yield '\x10cea' (CC s '\x0000' '\x0000')

826+-- OLD HUNGARIAN CAPITAL LETTER UU

827+foldMapping '\x10cab' s = Yield '\x10ceb' (CC s '\x0000' '\x0000')

828+-- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE

829+foldMapping '\x10cac' s = Yield '\x10cec' (CC s '\x0000' '\x0000')

830+-- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE

831+foldMapping '\x10cad' s = Yield '\x10ced' (CC s '\x0000' '\x0000')

832+-- OLD HUNGARIAN CAPITAL LETTER EV

833+foldMapping '\x10cae' s = Yield '\x10cee' (CC s '\x0000' '\x0000')

834+-- OLD HUNGARIAN CAPITAL LETTER EZ

835+foldMapping '\x10caf' s = Yield '\x10cef' (CC s '\x0000' '\x0000')

836+-- OLD HUNGARIAN CAPITAL LETTER EZS

837+foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')

838+-- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN

839+foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')

840+-- OLD HUNGARIAN CAPITAL LETTER US

841+foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')

668842foldMapping c s = Yield (toLower c) (CC s '\0' '\0')

Data/Text/Internal/Fusion/Common.hs+5−5

@@ -593,7 +593,7 @@ concatMap f = foldr (append . f) empty

593593{-# INLINE [0] concatMap #-}

594594

595595-- | /O(n)/ any @p @xs determines if any character in the stream

596−-- @xs@ satisifes the predicate @p@.

596+-- @xs@ satisfies the predicate @p@.

597597any :: (Char -> Bool) -> Stream Char -> Bool

598598any p (Stream next0 s0 _len) = loop_any s0

599599 where

@@ -605,7 +605,7 @@ any p (Stream next0 s0 _len) = loop_any s0

605605{-# INLINE [0] any #-}

606606

607607-- | /O(n)/ all @p @xs determines if all characters in the 'Text'

608−-- @xs@ satisify the predicate @p@.

608+-- @xs@ satisfy the predicate @p@.

609609all :: (Char -> Bool) -> Stream Char -> Bool

610610all p (Stream next0 s0 _len) = loop_all s0

611611 where

@@ -669,12 +669,12 @@ scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT may

669669-- ** Generating and unfolding streams

670670

671671replicateCharI :: Integral a => a -> Char -> Stream Char

672−replicateCharI n c

672+replicateCharI !n !c

673673 | n < 0 = empty

674674 | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low

675675 where

676− next i | i >= n = Done

677− | otherwise = Yield c (i + 1)

676+ next !i | i >= n = Done

677+ | otherwise = Yield c (i + 1)

678678{-# INLINE [0] replicateCharI #-}

679679

680680data RI s = RI !s {-# UNPACK #-} !Int64

Data/Text/Internal/Read.hs+3−3

@@ -7,7 +7,7 @@

77-- Stability : experimental

88-- Portability : GHC

99--

10−-- Common internal functiopns for reading textual data.

10+-- Common internal functions for reading textual data.

1111module Data.Text.Internal.Read

1212 (

1313 IReader

@@ -18,7 +18,7 @@ module Data.Text.Internal.Read

1818 , perhaps

1919 ) where

2020

21−import Control.Applicative (Applicative(..))

21+import Control.Applicative as App (Applicative(..))

2222import Control.Arrow (first)

2323import Control.Monad (ap)

2424import Data.Char (ord)

@@ -38,7 +38,7 @@ instance Applicative (IParser t) where

3838 (<*>) = ap

3939

4040instance Monad (IParser t) where

41− return = pure

41+ return = App.pure

4242 m >>= k = P $ \t -> case runP m t of

4343 Left err -> Left err

4444 Right (a,t') -> runP (k a) t'

Data/Text/Lazy.hs+53−8

@@ -140,6 +140,7 @@ module Data.Text.Lazy

140140 , drop

141141 , dropEnd

142142 , takeWhile

143+ , takeWhileEnd

143144 , dropWhile

144145 , dropWhileEnd

145146 , dropAround

@@ -214,6 +215,9 @@ import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,

214215 Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))

215216import Data.Binary (Binary(get, put))

216217import Data.Monoid (Monoid(..))

218+#if MIN_VERSION_base(4,9,0)

219+import Data.Semigroup (Semigroup(..))

220+#endif

217221import Data.String (IsString(..))

218222import qualified Data.Text as T

219223import qualified Data.Text.Internal as T

@@ -225,7 +229,7 @@ import Data.Text.Internal.Lazy.Fusion (stream, unstream)

225229import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks,

226230 foldrChunks, smallChunkSize)

227231import Data.Text.Internal (firstf, safe, text)

228−import Data.Text.Lazy.Encoding (decodeUtf8, encodeUtf8)

232+import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8)

229233import qualified Data.Text.Internal.Functions as F

230234import Data.Text.Internal.Lazy.Search (indices)

231235#if __GLASGOW_HASKELL__ >= 702

@@ -237,6 +241,9 @@ import qualified GHC.Base as GHC

237241import qualified GHC.Exts as Exts

238242#endif

239243import GHC.Prim (Addr#)

244+#if MIN_VERSION_base(4,7,0)

245+import Text.Printf (PrintfArg, formatArg, formatString)

246+#endif

240247

241248-- $fusion

242249--

@@ -334,9 +341,21 @@ instance Show Text where

334341instance Read Text where

335342 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]

336343

344+#if MIN_VERSION_base(4,9,0)

345+-- Semigroup orphan instances for older GHCs are provided by

346+-- 'semigroups` package

347+

348+instance Semigroup Text where

349+ (<>) = append

350+#endif

351+

337352instance Monoid Text where

338353 mempty = empty

354+#if MIN_VERSION_base(4,9,0)

355+ mappend = (<>) -- future-proof definition

356+#else

339357 mappend = append

358+#endif

340359 mconcat = concat

341360

342361instance IsString Text where

@@ -357,7 +376,11 @@ instance NFData Text where

357376

358377instance Binary Text where

359378 put t = put (encodeUtf8 t)

360− get = P.fmap decodeUtf8 get

379+ get = do

380+ bs <- get

381+ case decodeUtf8' bs of

382+ P.Left exn -> P.fail (P.show exn)

383+ P.Right a -> P.return a

361384

362385-- | This instance preserves data abstraction at the cost of inefficiency.

363386-- We omit reflection services for the sake of data abstraction.

@@ -372,6 +395,12 @@ instance Data Text where

372395 _ -> error "Data.Text.Lazy.Text.gunfold"

373396 dataTypeOf _ = textDataType

374397

398+#if MIN_VERSION_base(4,7,0)

399+-- | Only defined for @base-4.7.0.0@ and later

400+instance PrintfArg Text where

401+ formatArg txt = formatString $ unpack txt

402+#endif

403+

375404packConstr :: Constr

376405packConstr = mkConstr textDataType "pack" [] Prefix

377406

@@ -516,7 +545,7 @@ tail Empty = emptyError "tail"

516545 unstream (S.tail (stream t)) = tail t

517546 #-}

518547

519−-- | /O(1)/ Returns all but the last character of a 'Text', which must

548+-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must

520549-- be non-empty. Subject to fusion.

521550init :: Text -> Text

522551init (Chunk t0 ts0) = go t0 ts0

@@ -552,7 +581,7 @@ isSingleton :: Text -> Bool

552581isSingleton = S.isSingleton . stream

553582{-# INLINE isSingleton #-}

554583

555−-- | /O(1)/ Returns the last character of a 'Text', which must be

584+-- | /O(n\/c)/ Returns the last character of a 'Text', which must be

556585-- non-empty. Subject to fusion.

557586last :: Text -> Char

558587last Empty = emptyError "last"

@@ -852,13 +881,13 @@ concatMap f = concat . foldr ((:) . f) []

852881{-# INLINE concatMap #-}

853882

854883-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the

855−-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion.

884+-- 'Text' @t@ satisfies the predicate @p@. Subject to fusion.

856885any :: (Char -> Bool) -> Text -> Bool

857886any p t = S.any p (stream t)

858887{-# INLINE any #-}

859888

860889-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the

861−-- 'Text' @t@ satisify the predicate @p@. Subject to fusion.

890+-- 'Text' @t@ satisfy the predicate @p@. Subject to fusion.

862891all :: (Char -> Bool) -> Text -> Bool

863892all p t = S.all p (stream t)

864893{-# INLINE all #-}

@@ -1115,6 +1144,20 @@ takeWhile p t0 = takeWhile' t0

11151144"LAZY TEXT takeWhile -> unfused" [1] forall p t.

11161145 unstream (S.takeWhile p (stream t)) = takeWhile p t

11171146 #-}

1147+-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',

1148+-- returns the longest suffix (possibly empty) of elements that

1149+-- satisfy @p@.

1150+-- Examples:

1151+--

1152+-- > takeWhileEnd (=='o') "foo" == "oo"

1153+takeWhileEnd :: (Char -> Bool) -> Text -> Text

1154+takeWhileEnd p = takeChunk empty . L.reverse . toChunks

1155+ where takeChunk acc [] = acc

1156+ takeChunk acc (t:ts) = if T.length t' < T.length t

1157+ then (Chunk t' acc)

1158+ else takeChunk (Chunk t' acc) ts

1159+ where t' = T.takeWhileEnd p t

1160+{-# INLINE takeWhileEnd #-}

11181161

11191162-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after

11201163-- 'takeWhile' @p@ @t@. Subject to fusion.

@@ -1133,9 +1176,11 @@ dropWhile p t0 = dropWhile' t0

11331176"LAZY TEXT dropWhile -> unfused" [1] forall p t.

11341177 unstream (S.dropWhile p (stream t)) = dropWhile p t

11351178 #-}

1179+

11361180-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after

1137−-- dropping characters that fail the predicate @p@ from the end of

1181+-- dropping characters that satisfy the predicate @p@ from the end of

11381182-- @t@.

1183+--

11391184-- Examples:

11401185--

11411186-- > dropWhileEnd (=='.') "foo..." == "foo"

@@ -1152,7 +1197,7 @@ dropWhileEnd p = go

11521197{-# INLINE dropWhileEnd #-}

11531198

11541199-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after

1155−-- dropping characters that fail the predicate @p@ from both the

1200+-- dropping characters that satisfy the predicate @p@ from both the

11561201-- beginning and end of @t@. Subject to fusion.

11571202dropAround :: (Char -> Bool) -> Text -> Text

11581203dropAround p = dropWhile p . dropWhileEnd p

Data/Text/Lazy/Builder.hs+1−1

@@ -8,7 +8,7 @@

88-- Module : Data.Text.Lazy.Builder

11−-- License : BSD3-style (see LICENSE)

11+-- License : BSD-style (see LICENSE)

1212--

1313-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

1414-- Stability : experimental

Data/Text/Lazy/Builder/Int.hs+4−4

@@ -7,7 +7,7 @@

77-- Module: Data.Text.Lazy.Builder.Int

10−-- License: BSD3

10+-- License: BSD-style

1111-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

1212-- Stability: experimental

1313-- Portability: portable

@@ -35,7 +35,7 @@ import Control.Monad.ST

3535

3636#ifdef __GLASGOW_HASKELL__

3737# if defined(INTEGER_GMP)

38−import GHC.Integer.GMP.Internals

38+import GHC.Integer.GMP.Internals (Integer(S#))

3939# elif defined(INTEGER_SIMPLE)

4040import GHC.Integer

4141# else

@@ -55,7 +55,7 @@ decimal :: Integral a => a -> Builder

5555{-# RULES "decimal/Int16" decimal = boundedDecimal :: Int16 -> Builder #-}

5656{-# RULES "decimal/Int32" decimal = boundedDecimal :: Int32 -> Builder #-}

5757{-# RULES "decimal/Int64" decimal = boundedDecimal :: Int64 -> Builder #-}

58−{-# RULES "decimal/Word" decimal = positive :: Word -> Builder #-}

58+{-# RULES "decimal/Word" decimal = positive :: Data.Word.Word -> Builder #-}

5959{-# RULES "decimal/Word8" decimal = positive :: Word8 -> Builder #-}

6060{-# RULES "decimal/Word16" decimal = positive :: Word16 -> Builder #-}

6161{-# RULES "decimal/Word32" decimal = positive :: Word32 -> Builder #-}

@@ -247,7 +247,7 @@ integer base i

247247 PAIR(x,y) -> pblock q <> pblock r <> putB ns

248248 where q = fromInteger x

249249 r = fromInteger y

250− putB _ = mempty

250+ putB _ = Data.Monoid.mempty

251251

252252 int :: Int -> Builder

253253 int x | base == 10 = decimal x

Data/Text/Lazy/Encoding.hs+6−16

@@ -46,27 +46,23 @@ module Data.Text.Lazy.Encoding

4646 , encodeUtf32LE

4747 , encodeUtf32BE

4848

49−#if MIN_VERSION_bytestring(0,10,4)

5049 -- * Encoding Text using ByteString Builders

5150 , encodeUtf8Builder

5251 , encodeUtf8BuilderEscaped

53−#endif

5452 ) where

5553

5654import Control.Exception (evaluate, try)

55+import Data.Monoid (Monoid(..))

5756import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

5857import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)

59−import qualified Data.ByteString as S

60−import qualified Data.ByteString.Lazy as B

61−import qualified Data.ByteString.Lazy.Internal as B

62−import qualified Data.ByteString.Unsafe as B

63−#if MIN_VERSION_bytestring(0,10,4)

6458import Data.Word (Word8)

65−import Data.Monoid (mappend, mempty)

59+import qualified Data.ByteString as S

6660import qualified Data.ByteString.Builder as B

6761import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)

6862import qualified Data.ByteString.Builder.Prim as BP

69−#endif

63+import qualified Data.ByteString.Lazy as B

64+import qualified Data.ByteString.Lazy.Internal as B

65+import qualified Data.ByteString.Unsafe as B

7066import qualified Data.Text as T

7167import qualified Data.Text.Encoding as TE

7268import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E

@@ -146,7 +142,6 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

146142{-# INLINE decodeUtf8' #-}

147143

148144encodeUtf8 :: Text -> B.ByteString

149−#if MIN_VERSION_bytestring(0,10,4)

150145encodeUtf8 Empty = B.empty

151146encodeUtf8 lt@(Chunk t _) =

152147 B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt

@@ -161,18 +156,13 @@ encodeUtf8 lt@(Chunk t _) =

161156

162157encodeUtf8Builder :: Text -> B.Builder

163158encodeUtf8Builder =

164− foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) mempty

159+ foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty

165160

166161{-# INLINE encodeUtf8BuilderEscaped #-}

167162encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

168163encodeUtf8BuilderEscaped prim =

169164 foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty

170165

171−#else

172−encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)

173−encodeUtf8 Empty = B.Empty

174−#endif

175−

176166-- | Decode text from little endian UTF-16 encoding.

177167decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text

178168decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

Data/Text/Lazy/Read.hs+4−2

@@ -1,5 +1,7 @@

11{-# LANGUAGE OverloadedStrings, CPP #-}

2−#if __GLASGOW_HASKELL__ >= 702

2+#if __GLASGOW_HASKELL__ >= 704

3+{-# LANGUAGE Safe #-}

4+#elif __GLASGOW_HASKELL__ >= 702

35{-# LANGUAGE Trustworthy #-}

46#endif

@@ -54,7 +56,7 @@ decimal :: Integral a => Reader a

5456{-# SPECIALIZE decimal :: Reader Int32 #-}

5557{-# SPECIALIZE decimal :: Reader Int64 #-}

5658{-# SPECIALIZE decimal :: Reader Integer #-}

57−{-# SPECIALIZE decimal :: Reader Word #-}

59+{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}

5860{-# SPECIALIZE decimal :: Reader Word8 #-}

5961{-# SPECIALIZE decimal :: Reader Word16 #-}

6062{-# SPECIALIZE decimal :: Reader Word32 #-}

Data/Text/Read.hs+1−1

@@ -55,7 +55,7 @@ decimal :: Integral a => Reader a

5555{-# SPECIALIZE decimal :: Reader Int32 #-}

5656{-# SPECIALIZE decimal :: Reader Int64 #-}

5757{-# SPECIALIZE decimal :: Reader Integer #-}

58−{-# SPECIALIZE decimal :: Reader Word #-}

58+{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}

5959{-# SPECIALIZE decimal :: Reader Word8 #-}

6060{-# SPECIALIZE decimal :: Reader Word16 #-}

6161{-# SPECIALIZE decimal :: Reader Word32 #-}

Data/Text/Show.hs+9−5

@@ -17,6 +17,7 @@ module Data.Text.Show

1717 (

1818 singleton

1919 , unpack

20+ , unpackCString#

2021 ) where

2122

2223import Control.Monad.ST (ST)

@@ -41,24 +42,27 @@ unpack :: Text -> String

4142unpack = S.unstreamList . stream

4243{-# INLINE [1] unpack #-}

4344

44−-- | /O(n)/ Convert a literal string into a Text. Subject to fusion.

45+-- | /O(n)/ Convert a literal string into a 'Text'. Subject to

46+-- fusion.

47+--

48+-- This is exposed solely for people writing GHC rewrite rules.

4549unpackCString# :: Addr# -> Text

4650unpackCString# addr# = unstream (S.streamCString# addr#)

4751{-# NOINLINE unpackCString# #-}

4852

49−{-# RULES "TEXT literal" forall a.

53+{-# RULES "TEXT literal" [1] forall a.

5054 unstream (S.map safe (S.streamList (GHC.unpackCString# a)))

5155 = unpackCString# a #-}

5256

53−{-# RULES "TEXT literal UTF8" forall a.

57+{-# RULES "TEXT literal UTF8" [1] forall a.

5458 unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a)))

5559 = unpackCString# a #-}

5660

57−{-# RULES "TEXT empty literal"

61+{-# RULES "TEXT empty literal" [1]

5862 unstream (S.map safe (S.streamList []))

5963 = empty_ #-}

6064

61−{-# RULES "TEXT singleton literal" forall a.

65+{-# RULES "TEXT singleton literal" [1] forall a.

6266 unstream (S.map safe (S.streamList [a]))

6367 = singleton_ a #-}

6468

benchmarks/haskell/Benchmarks.hs+1−1

@@ -53,7 +53,7 @@ benchmarks = do

5353 , Mul.benchmark

5454 , Pure.benchmark "tiny" (tf "tiny.txt")

5555 , Pure.benchmark "ascii" (tf "ascii-small.txt")

56− , Pure.benchmark "france" (tf "france.html")

56+ -- , Pure.benchmark "france" (tf "france.html")

5757 , Pure.benchmark "russian" (tf "russian-small.txt")

5858 , Pure.benchmark "japanese" (tf "japanese.txt")

5959 , ReadNumbers.benchmark (tf "numbers.txt")

benchmarks/haskell/Benchmarks/Pure.hs+7

@@ -262,6 +262,13 @@ benchmark kind fp = do

262262 , benchBSL $ nf (BL.map toUpper) bla

263263 , benchS $ nf (L.map toUpper) sa

264264 ]

265+ , bgroup "uncons"

266+ [ benchT $ nf T.uncons ta

267+ , benchTL $ nf TL.uncons tla

268+ , benchBS $ nf BS.uncons bsa

269+ , benchBSL $ nf BL.uncons bla

270+ , benchS $ nf L.uncons sa

271+ ]

265272 , bgroup "words"

266273 [ benchT $ nf T.words ta

267274 , benchTL $ nf TL.words tla

benchmarks/haskell/Benchmarks/ReadNumbers.hs+2−5

@@ -25,8 +25,7 @@ import Data.List (foldl')

2525import Numeric (readDec, readFloat, readHex)

2626import qualified Data.ByteString.Char8 as B

2727import qualified Data.ByteString.Lazy.Char8 as BL

28−import qualified Data.ByteString.Lex.Double as B

29−import qualified Data.ByteString.Lex.Lazy.Double as BL

28+import qualified Data.ByteString.Lex.Fractional as B

3029import qualified Data.Text as T

3130import qualified Data.Text.IO as T

3231import qualified Data.Text.Lazy as TL

@@ -63,12 +62,10 @@ benchmark fp = do

6362 whnf (double . text TL.rational) tl

6463

6564 , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b

66− , bench "DoubleByteString" $ whnf (double . byteString B.readDouble) b

65+ , bench "DoubleByteString" $ whnf (double . byteString B.readDecimal) b

6766

6867 , bench "DecimalLazyByteString" $

6968 whnf (int . byteString BL.readInt) bl

70− , bench "DoubleLazyByteString" $

71− whnf (double . byteString BL.readDouble) bl

7269 ]

7370 where

7471 -- Used for fixing types

benchmarks/haskell/Benchmarks/Stream.hs+16−5

@@ -18,11 +18,11 @@ import qualified Data.Text.Encoding as T

1818import qualified Data.Text.Encoding.Error as E

1919import qualified Data.Text.Internal.Encoding.Fusion as T

2020import qualified Data.Text.Internal.Encoding.Fusion.Common as F

21−import qualified Data.Text.Internal.Fusion as T

21+import qualified Data.Text.Internal.Fusion as F

2222import qualified Data.Text.IO as T

2323import qualified Data.Text.Lazy.Encoding as TL

2424import qualified Data.Text.Internal.Lazy.Encoding.Fusion as TL

25−import qualified Data.Text.Internal.Lazy.Fusion as TL

25+import qualified Data.Text.Internal.Lazy.Fusion as FL

2626import qualified Data.Text.Lazy.IO as TL

2727

2828instance NFData a => NFData (Stream a) where

@@ -53,14 +53,25 @@ benchmark fp = do

5353 !utf32beL = TL.encodeUtf32BE tl

5454

5555 -- For the functions which operate on streams

56− let !s = T.stream t

56+ let !s = F.stream t

5757

5858 return $ bgroup "Stream"

5959

6060 -- Fusion

6161 [ bgroup "stream" $

62− [ bench "Text" $ nf T.stream t

63− , bench "LazyText" $ nf TL.stream tl

62+ [ bench "Text" $ nf F.stream t

63+ , bench "LazyText" $ nf FL.stream tl

64+ ]

65+ -- must perform exactly the same as stream above due to

66+ -- stream/unstream (i.e. stream after unstream) fusion

67+ , bgroup "stream-fusion" $

68+ [ bench "Text" $ nf (F.stream . F.unstream . F.stream) t

69+ , bench "LazyText" $ nf (FL.stream . FL.unstream . FL.stream) tl

70+ ]

71+ -- measure the overhead of unstream after stream

72+ , bgroup "stream-unstream" $

73+ [ bench "Text" $ nf (F.unstream . F.stream) t

74+ , bench "LazyText" $ nf (FL.unstream . FL.stream) tl

6475 ]

6576

6677 -- Encoding.Fusion

benchmarks/text-benchmarks.cabal+13−3

@@ -3,7 +3,7 @@ version: 0.0.0.0

33synopsis: Benchmarks for the text package

44description: Benchmarks for the text package

55homepage: https://bitbucket.org/bos/text

6−license: BSD3

6+license: BSD2

77license-file: ../LICENSE

88author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

99 Bryan O'Sullivan <bos@serpentine.com>,

@@ -15,6 +15,11 @@ build-type: Simple

1515

1616cabal-version: >=1.2

1717

18+flag bytestring-builder

19+ description: Depend on the bytestring-builder package for backwards compatibility.

20+ default: False

21+ manual: False

22+

1823flag llvm

1924 description: use LLVM

2025 default: False

@@ -33,8 +38,7 @@ executable text-benchmarks

3338 build-depends: base == 4.*,

3439 binary,

3540 blaze-builder,

36− bytestring,

37− bytestring-lexing,

41+ bytestring-lexing >= 0.5.0,

3842 containers,

3943 criterion >= 0.10.0.0,

4044 deepseq,

@@ -46,6 +50,12 @@ executable text-benchmarks

4650 utf8-string,

4751 vector

4852

53+ if flag(bytestring-builder)

54+ build-depends: bytestring >= 0.9 && < 0.10.4,

55+ bytestring-builder >= 0.10.4

56+ else

57+ build-depends: bytestring >= 0.10.4

58+

4959executable text-multilang

5060 hs-source-dirs: haskell

5161 main-is: Multilang.hs

changelog.md+53

@@ -1,3 +1,56 @@

1+1.2.2.1

3+* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

4+ The build flag has been renamed accordingly. Your army of diligent

5+ maintainers apologizes for the churn.

7+* Spec compliance: toCaseFold now follows the Unicode 8.0 spec

8+ (updated from 7.0)

10+* An STG lint error has been fixed

11+

12+1.2.2.0

13+

14+* The `integer-simple` package, upon which this package optionally

15+ depended, has been replaced with `integer-pure`. The build flag has

16+ been renamed accordingly.

17+

18+* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a

19+ `get`, the error is propagated via `fail` instead of an uncatchable

20+ crash.

21+

22+* New function: `takeWhileEnd`

23+

24+* New instances for the `Text` types:

25+ * if `base` >= 4.7: `PrintfArg`

26+ * if `base` >= 4.9: `Semigroup`

27+

28+1.2.1.3

29+

30+* Bug fix: As it turns out, moving the literal rewrite rules to simplifier

31+ phase 2 does not prevent competition with the `unpack` rule, which is

32+ also active in this phase. Unfortunately this was hidden due to a silly

33+ test environment mistake. Moving literal rules back to phase 1 finally

34+ fixes GHC Trac #10528 correctly.

35+

36+1.2.1.2

37+

38+* Bug fix: Run literal rewrite rules in simplifier phase 2.

39+ The behavior of the simplifier changed in GHC 7.10.2,

40+ causing these rules to fail to fire, leading to poor code generation

41+ and long compilation times. See

42+ [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).

43+

44+1.2.1.1

45+

46+* Expose unpackCString#, which you should never use.

47+

48+1.2.1.0

49+

50+* Added Binary instances for both Text types. (If you have previously

51+ been using the text-binary package to get a Binary instance, it is

52+ now obsolete.)

53+

1541.2.0.6

255

356* Fixed a space leak in UTF-8 decoding

tests/LiteralRuleTest.hsnew file+21

@@ -0,0 +1,21 @@

1+{-# LANGUAGE OverloadedStrings #-}

3+module LiteralRuleTest where

5+import Data.Text (Text)

7+-- This should produce 8 firings of the "TEXT literal" rule

8+strings :: [Text]

9+strings = [ "abstime", "aclitem", "bit", "bool", "box", "bpchar", "bytea", "char" ]

10+

11+-- This should produce 7 firings of the "TEXT literal UTF8" rule

12+utf8Strings :: [Text]

13+utf8Strings = [ "\0abstime", "\0aclitem", "\xfefe bit", "\0bool", "\0box", "\0bpchar", "\0bytea" ]

14+

15+-- This should produce 4 firings of the "TEXT empty literal" rule

16+empties :: [Text]

17+empties = [ "", "", "", "" ]

18+

19+-- This should produce 5 firings of the "TEXT empty literal" rule

20+--singletons :: [Text]

21+--singletons = [ "a", "b", "c", "d", "e" ]

tests/Makefile+6−1

@@ -1,5 +1,10 @@

11count = 1000

3+all: coverage literal-rule-test

5+literal-rule-test:

6+ ./literal-rule-test.sh

38coverage: build coverage/hpc_index.html

510build: text-test-data

@@ -32,4 +37,4 @@ coverage/hpc_index.html: coverage/coverage.tix

3237clean:

3338 rm -rf dist coverage .hpc

3439

35−.PHONY: build coverage

40+.PHONY: build coverage all literal-rule-test

tests/Tests/Properties.hs+6

@@ -516,6 +516,10 @@ sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`

516516 (unpackS . S.takeWhile p . S.filter q)

517517t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)

518518tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)

519+t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

520+ (unpackS . T.takeWhileEnd p)

521+tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`

522+ (unpackS . TL.takeWhileEnd p)

519523s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)

520524s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)

521525sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`

@@ -1142,6 +1146,8 @@ tests =

11421146 testProperty "sf_takeWhile" sf_takeWhile,

11431147 testProperty "t_takeWhile" t_takeWhile,

11441148 testProperty "tl_takeWhile" tl_takeWhile,

1149+ testProperty "t_takeWhileEnd" t_takeWhileEnd,

1150+ testProperty "tl_takeWhileEnd" tl_takeWhileEnd,

11451151 testProperty "sf_dropWhile" sf_dropWhile,

11461152 testProperty "s_dropWhile" s_dropWhile,

11471153 testProperty "s_dropWhile_s" s_dropWhile_s,

tests/literal-rule-test.shnew file+29

@@ -0,0 +1,29 @@

1+#!/bin/bash -e

3+failed=0

5+function check_firings() {

6+ rule=$1

7+ expected=$2

8+ build="ghc -O -ddump-rule-firings LiteralRuleTest.hs"

9+ build="$build -i.. -I../include"

10+ touch LiteralRuleTest.hs

11+ echo -n "Want to see $expected firings of rule $rule... " >&2

12+ firings=$($build 2>&1 | grep "Rule fired: $rule\$" | wc -l)

13+ rm -f LiteralRuleTest.{o.hi}

14+

15+ if [ $firings != $expected ]; then

16+ echo "failed, saw $firings" >&2

17+ failed=1

18+ else

19+ echo "pass" >&2

20+ fi

21+}

22+

23+check_firings "TEXT literal" 8

24+check_firings "TEXT literal UTF8" 7

25+check_firings "TEXT empty literal" 4

26+# This is broken at the moment. "TEXT literal" rule fires instead.

27+#check_firings "TEXT singleton literal" 5

28+

29+exit $failed

tests/text-tests.cabal+1−1

@@ -3,7 +3,7 @@ version: 0.0.0.0

33synopsis: Functional tests for the text package

44description: Functional tests for the text package

55homepage: https://github.com/bos/text

6−license: BSD3

6+license: BSD2

77license-file: ../LICENSE

88author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

99 Bryan O'Sullivan <bos@serpentine.com>,

text.cabal+19−6

@@ -1,5 +1,5 @@

11name: text

2−version: 1.2.0.6

2+version: 1.2.2.1

33homepage: https://github.com/bos/text

44bug-reports: https://github.com/bos/text/issues

55synopsis: An efficient packed Unicode text type.

@@ -31,7 +31,7 @@ description:

3131 the @text-icu@ package:

3232 <http://hackage.haskell.org/package/text-icu>

3333

34−license: BSD3

34+license: BSD2

3535license-file: LICENSE

3636author: Bryan O'Sullivan <bos@serpentine.com>

3737maintainer: Bryan O'Sullivan <bos@serpentine.com>

@@ -64,6 +64,11 @@ extra-source-files:

6464 tests/scripts/*.sh

6565 tests/text-tests.cabal

6666

67+flag bytestring-builder

68+ description: Depend on the bytestring-builder package for backwards compatibility.

69+ default: False

70+ manual: False

71+

6772flag developer

6873 description: operate in developer mode

6974 default: False

@@ -71,6 +76,8 @@ flag developer

7176

7277flag integer-simple

7378 description: Use the simple integer library instead of GMP

79+ default: False

80+ manual: False

7481

7582library

7683 c-sources: cbits/cbits.c

@@ -131,10 +138,11 @@ library

131138 deepseq >= 1.1.0.0,

132139 ghc-prim >= 0.2

133140

134− if impl(ghc >= 7.7)

135− build-depends: bytestring >= 0.10.4.0

141+ if flag(bytestring-builder)

142+ build-depends: bytestring >= 0.9 && < 0.10.4,

143+ bytestring-builder >= 0.10.4

136144 else

137− build-depends: bytestring >= 0.9

145+ build-depends: bytestring >= 0.10.4

138146

139147 cpp-options: -DHAVE_DEEPSEQ

140148 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2

@@ -169,7 +177,6 @@ test-suite tests

169177 array,

170178 base,

171179 binary,

172− bytestring,

173180 deepseq,

174181 directory,

175182 ghc-prim,

@@ -179,6 +186,12 @@ test-suite tests

179186 test-framework-hunit >= 0.2,

180187 test-framework-quickcheck2 >= 0.2

181188

189+ if flag(bytestring-builder)

190+ build-depends: bytestring >= 0.9 && < 0.10.4,

191+ bytestring-builder >= 0.10.4

192+ else

193+ build-depends: bytestring >= 0.10.4

194+

182195 if flag(integer-simple)

183196 cpp-options: -DINTEGER_SIMPLE

184197 build-depends: integer-simple >= 0.1 && < 0.5