c59fff3..master · text

git.iko.soy/text

10 files changed+42−86

unifiedsplit

Data/Text.hs+1−1 Data/Text/Foreign.hs+1−1 Data/Text/Internal/Fusion/Common.hs+9−10 Data/Text/Lazy/Read.hs+3−3 Data/Text/Read.hs+3−3 changelog.md−11 tests/Tests/Properties.hs+20−37 tests/Tests/QuickCheckUtils.hs+1−1 tests/text-tests.cabal+3−18 text.cabal+1−1

Data/Text.hs+1−1

@@ -1099,8 +1099,8 @@ takeEnd n t@(Text arr off len)

10991099iterNEnd :: Int -> Text -> Int

11001100iterNEnd n t@(Text _arr _off len) = loop (len-1) n

11011101 where loop i !m

1102− | m <= 0 = i+1

11031102 | i <= 0 = 0

1103+ | m <= 1 = i

11041104 | otherwise = loop (i+d) (m-1)

11051105 where d = reverseIter_ t i

11061106

Data/Text/Foreign.hs+1−1

@@ -108,7 +108,7 @@ takeWord16 (I16 n) t@(Text arr off len)

108108 | n >= len || m >= len = t

109109 | otherwise = Text arr off m

110110 where

111− m | w < 0xD800 || w > 0xDBFF = n

111+ m | w < 0xDB00 || w > 0xD8FF = n

112112 | otherwise = n+1

113113 w = A.unsafeIndex arr (off+n-1)

114114

Data/Text/Internal/Fusion/Common.hs+9−10

@@ -107,7 +107,7 @@ import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..),

107107import qualified Data.List as L

108108import qualified Prelude as P

109109import Data.Bits (shiftL)

110−import Data.Char (isLetter, isSpace)

110+import Data.Char (isLetter)

111111import Data.Int (Int64)

112112import Data.Text.Internal.Fusion.Types

113113import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,

@@ -462,16 +462,15 @@ toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') len

462462 where

463463 next (CC (letter :*: s) '\0' _) =

464464 case next0 s of

465− Done -> Done

466− Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

465+ Done -> Done

466+ Skip s' -> Skip (CC (letter :*: s') '\0' '\0')

467467 Yield c s'

468− | nonSpace -> if letter

469− then lowerMapping c (nonSpace :*: s')

470− else titleMapping c (letter' :*: s')

471− | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

472− where nonSpace = P.not (isSpace c)

473− letter' = isLetter c

474− next (CC s a b) = Yield a (CC s b '\0')

468+ | letter' -> if letter

469+ then lowerMapping c (letter' :*: s')

470+ else titleMapping c (letter' :*: s')

471+ | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')

472+ where letter' = isLetter c

473+ next (CC s a b) = Yield a (CC s b '\0')

475474{-# INLINE [0] toTitle #-}

476475

477476data Justify i s = Just1 !i !s

Data/Text/Lazy/Read.hs+3−3

@@ -68,9 +68,9 @@ decimal txt

6868 go n d = (n * 10 + fromIntegral (digitToInt d))

6969

7070-- | Read a hexadecimal integer, consisting of an optional leading

71−-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

72−-- consumed until a non-hex-digit or end of string is reached.

73−-- This function is case insensitive.

71+-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

72+-- until a non-hex-digit or end of string is reached. This function

73+-- is case insensitive.

7474--

7575-- This function does not handle leading sign characters. If you need

7676-- to handle signed input, use @'signed' 'hexadecimal'@.

Data/Text/Read.hs+3−3

@@ -67,9 +67,9 @@ decimal txt

6767 go n d = (n * 10 + fromIntegral (digitToInt d))

6868

6969-- | Read a hexadecimal integer, consisting of an optional leading

70−-- @\"0x\"@ followed by at least one hexadecimal digit. Input is

71−-- consumed until a non-hex-digit or end of string is reached.

72−-- This function is case insensitive.

70+-- @\"0x\"@ followed by at least one decimal digit. Input is consumed

71+-- until a non-hex-digit or end of string is reached. This function

72+-- is case insensitive.

7373--

7474-- This function does not handle leading sign characters. If you need

7575-- to handle signed input, use @'signed' 'hexadecimal'@.

changelog.md−11

@@ -1,14 +1,3 @@

1−* The `toTitle` function now correctly handles letters that

2− immediately follow puncutation. Before, `"there's"` would turn into

3− `"There'S"`. Now, it becomes `"There's"`.

4−

5−* The implementation of unstreaming is faster, resulting in operations

6− such as `map` and `intersperse` speeding up by up to 30%, with

7− smaller code generated.

8−

9−* The optimised length comparison function is now more likely to be

10− used after some rewrite rule tweaking.

11−

1211.2.2.1

132

143* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.

tests/Tests/Properties.hs+20−37

@@ -11,7 +11,7 @@ module Tests.Properties

1111import Control.Applicative ((<$>), (<*>))

1212import Control.Arrow ((***), second)

1313import Data.Bits ((.&.))

14−import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord)

14+import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)

1515import Data.Int (Int8, Int16, Int32, Int64)

1616import Data.Monoid (Monoid(..))

1717import Data.String (IsString(fromString))

@@ -23,7 +23,6 @@ import Data.Text.Internal.Search (indices)

2323import Data.Text.Lazy.Read as TL

2424import Data.Text.Read as T

2525import Data.Word (Word, Word8, Word16, Word32, Word64)

26−import Data.Maybe (mapMaybe)

2726import Numeric (showEFloat, showFFloat, showGFloat, showHex)

2827import Prelude hiding (replicate)

2928import Test.Framework (Test, testGroup)

@@ -38,7 +37,6 @@ import qualified Control.Exception as Exception

3837import qualified Data.Bits as Bits (shiftL, shiftR)

3938import qualified Data.ByteString as B

4039import qualified Data.ByteString.Lazy as BL

41−import qualified Data.Char as C

4240import qualified Data.List as L

4341import qualified Data.Text as T

4442import qualified Data.Text.Encoding as E

@@ -62,10 +60,10 @@ t_pack_unpack = (T.unpack . T.pack) `eq` id

6260tl_pack_unpack = (TL.unpack . TL.pack) `eq` id

6361t_stream_unstream = (S.unstream . S.stream) `eq` id

6462tl_stream_unstream = (SL.unstream . SL.stream) `eq` id

65−t_reverse_stream t = (S.reverse . S.reverseStream) t === t

66−t_singleton c = [c] === (T.unpack . T.singleton) c

67−tl_singleton c = [c] === (TL.unpack . TL.singleton) c

68−tl_unstreamChunks x = f 11 x === f 1000 x

63+t_reverse_stream t = (S.reverse . S.reverseStream) t == t

64+t_singleton c = [c] == (T.unpack . T.singleton) c

65+tl_singleton c = [c] == (TL.unpack . TL.singleton) c

66+tl_unstreamChunks x = f 11 x == f 1000 x

6967 where f n = SL.unstreamChunks n . S.streamList

7068tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id

7169tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id

@@ -76,13 +74,13 @@ encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack

7674encodeLazyL1 :: TL.Text -> BL.ByteString

7775encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks

7876

79−t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a

77+t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

8078 where a = T.map (\c -> chr (ord c `mod` 128)) t

81−tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a

79+tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a

8280 where a = TL.map (\c -> chr (ord c `mod` 128)) t

83−t_latin1 t = E.decodeLatin1 (encodeL1 a) === a

81+t_latin1 t = E.decodeLatin1 (encodeL1 a) == a

8482 where a = T.map (\c -> chr (ord c `mod` 256)) t

85−tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a

83+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a

8684 where a = TL.map (\c -> chr (ord c `mod` 256)) t

8785t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id

8886t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)

@@ -113,7 +111,7 @@ t_utf8_undecoded = forAll genUnicode $ \t ->

113111 let b = E.encodeUtf8 t

114112 ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t

115113 leftover = (++ [B.empty]) . init . tail . B.inits

116− in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls

114+ in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b == ls

117115

118116data Badness = Solo | Leading | Trailing

119117 deriving (Eq, Show)

@@ -203,8 +201,8 @@ t_mconcat = unsquare $

203201 mconcat `eq` (unpackS . mconcat . L.map T.pack)

204202tl_mconcat = unsquare $

205203 mconcat `eq` (unpackS . mconcat . L.map TL.pack)

206−t_mempty = mempty === (unpackS (mempty :: T.Text))

207−tl_mempty = mempty === (unpackS (mempty :: TL.Text))

204+t_mempty = mempty == (unpackS (mempty :: T.Text))

205+tl_mempty = mempty == (unpackS (mempty :: TL.Text))

208206t_IsString = fromString `eqP` (T.unpack . fromString)

209207tl_IsString = fromString `eqP` (TL.unpack . fromString)

210208

@@ -328,19 +326,6 @@ t_toUpper_upper t = p (T.toUpper t) >= p t

328326 where p = T.length . T.filter isUpper

329327tl_toUpper_upper t = p (TL.toUpper t) >= p t

330328 where p = TL.length . TL.filter isUpper

331−t_toTitle_title t = all (<= 1) (caps w)

332− where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle

333− -- TIL: there exist uppercase-only letters

334− w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t

335−t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable

336− where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words

337− -- Surprise! The Spanish/Portuguese ordinal indicators changed

338− -- from category Ll (letter, lowercase) to Lo (letter, other)

339− -- in Unicode 7.0

340− -- Oh, and there exist lowercase-only letters (see previous test)

341− stable c = if isLower c

342− then C.toUpper c /= c

343− else c /= '\170' && c /= '\186'

344329

345330justifyLeft k c xs = xs ++ L.replicate (k - length xs) c

346331justifyRight m n xs = L.replicate (m - length xs) n ++ xs

@@ -735,7 +720,7 @@ tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s

735720 conc = T.concat . TL.toChunks

736721t_indices_occurs = unsquare $ \(NotEmpty t) ts ->

737722 let s = T.intercalate t ts

738− in Slow.indices t s === indices t s

723+ in Slow.indices t s == indices t s

739724

740725-- Bit shifts.

741726shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

@@ -827,18 +812,18 @@ tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a

827812-- Reading.

828813

829814t_decimal (n::Int) s =

830− T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)

815+ T.signed T.decimal (T.pack (show n) `T.append` t) == Right (n,t)

831816 where t = T.dropWhile isDigit s

832817tl_decimal (n::Int) s =

833− TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)

818+ TL.signed TL.decimal (TL.pack (show n) `TL.append` t) == Right (n,t)

834819 where t = TL.dropWhile isDigit s

835820t_hexadecimal m s ox =

836− T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)

821+ T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) == Right (n,t)

837822 where t = T.dropWhile isHexDigit s

838823 p = if ox then "0x" else ""

839824 n = getPositive m :: Int

840825tl_hexadecimal m s ox =

841− TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)

826+ TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) == Right (n,t)

842827 where t = TL.dropWhile isHexDigit s

843828 p = if ox then "0x" else ""

844829 n = getPositive m :: Int

@@ -882,11 +867,11 @@ tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn

882867

883868t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t

884869t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t

885−t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t

870+t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) == t

886871 where n = small m

887872t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)

888873

889−t_copy t = T.copy t === t

874+t_copy t = T.copy t == t

890875

891876-- Regression tests.

892877s_filter_eq s = S.filter p t == S.streamList (filter p s)

@@ -1045,9 +1030,7 @@ tests =

10451030 testProperty "tl_toLower_lower" tl_toLower_lower,

10461031 testProperty "t_toUpper_length" t_toUpper_length,

10471032 testProperty "t_toUpper_upper" t_toUpper_upper,

1048− testProperty "tl_toUpper_upper" tl_toUpper_upper,

1049− testProperty "t_toTitle_title" t_toTitle_title,

1050− testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower

1033+ testProperty "tl_toUpper_upper" tl_toUpper_upper

10511034 ],

10521035

10531036 testGroup "justification" [

tests/Tests/QuickCheckUtils.hs+1−1

@@ -106,7 +106,7 @@ smallArbitrary = sized $ \n -> resize (smallish n) arbitrary

106106 where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs

107107

108108instance Arbitrary T.Text where

109− arbitrary = T.pack `fmap` string

109+ arbitrary = T.pack `fmap` arbitrary

110110 shrink = map T.pack . shrink . T.unpack

111111

112112instance Arbitrary TL.Text where

tests/text-tests.cabal+3−18

@@ -20,11 +20,6 @@ flag hpc

2020 default: False

2121 manual: True

2222

23−flag bytestring-builder

24− description: Depend on the bytestring-builder package for backwards compatibility.

25− default: False

26− manual: False

27−

2823executable text-tests

2924 main-is: Tests.hs

3025

@@ -53,21 +48,16 @@ executable text-tests

5348 HUnit >= 1.2,

5449 QuickCheck >= 2.7,

5550 base == 4.*,

51+ bytestring,

5652 deepseq,

5753 directory,

58− quickcheck-unicode >= 1.0.1.0,

54+ quickcheck-unicode,

5955 random,

6056 test-framework >= 0.4,

6157 test-framework-hunit >= 0.2,

6258 test-framework-quickcheck2 >= 0.2,

6359 text-tests

6460

65− if flag(bytestring-builder)

66− build-depends: bytestring >= 0.9 && < 0.10.4,

67− bytestring-builder >= 0.10.4

68− else

69− build-depends: bytestring >= 0.10.4

70−

7161executable text-tests-stdio

7262 main-is: Tests/IO.hs

7363

@@ -147,12 +137,7 @@ library

147137 array,

148138 base == 4.*,

149139 binary,

140+ bytestring,

150141 deepseq,

151142 ghc-prim,

152143 integer-gmp

153−

154− if flag(bytestring-builder)

155− build-depends: bytestring >= 0.9 && < 0.10.4,

156− bytestring-builder >= 0.10.4

157− else

158− build-depends: bytestring >= 0.10.4

text.cabal+1−1

@@ -180,7 +180,7 @@ test-suite tests

180180 deepseq,

181181 directory,

182182 ghc-prim,

183− quickcheck-unicode >= 1.0.1.0,

183+ quickcheck-unicode,

184184 random,

185185 test-framework >= 0.4,

186186 test-framework-hunit >= 0.2,