18f9a7b...master · text

git.iko.soy/text

17 files changed+108−82

since splitsnapshots

unifiedsplit

Data/Text.hs+10−7 Data/Text/Array.hs+1−1 Data/Text/Encoding.hs+7−18 Data/Text/Internal/Builder.hs+3−1 Data/Text/Internal/Builder/Int/Digits.hs+1−1 Data/Text/Internal/Fusion.hs+27−17 Data/Text/Internal/Fusion/Common.hs+3−3 Data/Text/Lazy.hs+2−2 Data/Text/Lazy/Builder.hs+1−1 Data/Text/Lazy/Builder/Int.hs+1−1 Data/Text/Lazy/Encoding.hs+5−15 benchmarks/haskell/Benchmarks.hs+1−1 benchmarks/haskell/Benchmarks/Stream.hs+16−5 benchmarks/text-benchmarks.cabal+12−2 tests/literal-rule-test.sh+1−1 tests/text-tests.cabal+1−1 text.cabal+16−5

Data/Text.hs+10−7

@@ -238,6 +238,7 @@ import Data.ByteString (ByteString)

238238import qualified Data.Text.Lazy as L

239239import Data.Int (Int64)

240240#endif

241+import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt)

241242#if __GLASGOW_HASKELL__ >= 708

242243import qualified GHC.Exts as Exts

243244#endif

@@ -571,7 +572,9 @@ isSingleton = S.isSingleton . stream

571572-- Subject to fusion.

572573length :: Text -> Int

573574length t = S.length (stream t)

574−{-# INLINE length #-}

575+{-# INLINE [0] length #-}

576+-- length needs to be phased after the compareN/length rules otherwise

577+-- it may inline before the rules have an opportunity to fire.

575578

576579-- | /O(n)/ Compare the count of characters in a 'Text' to a number.

577580-- Subject to fusion.

@@ -590,32 +593,32 @@ compareLength t n = S.compareLengthI (stream t) n

590593

591594{-# RULES

592595"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.

593− (==) (length t) n = compareLength t n == EQ

596+ eqInt (length t) n = compareLength t n == EQ

594597 #-}

595598

596599{-# RULES

597600"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.

598− (/=) (length t) n = compareLength t n /= EQ

601+ neInt (length t) n = compareLength t n /= EQ

599602 #-}

600603

601604{-# RULES

602605"TEXT <N/length -> compareLength/==LT" [~1] forall t n.

603− (<) (length t) n = compareLength t n == LT

606+ ltInt (length t) n = compareLength t n == LT

604607 #-}

605608

606609{-# RULES

607610"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.

608− (<=) (length t) n = compareLength t n /= GT

611+ leInt (length t) n = compareLength t n /= GT

609612 #-}

610613

611614{-# RULES

612615"TEXT >N/length -> compareLength/==GT" [~1] forall t n.

613− (>) (length t) n = compareLength t n == GT

616+ gtInt (length t) n = compareLength t n == GT

614617 #-}

615618

616619{-# RULES

617620"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.

618− (>=) (length t) n = compareLength t n /= LT

621+ geInt (length t) n = compareLength t n /= LT

619622 #-}

620623

621624-- -----------------------------------------------------------------------------

Data/Text/Array.hs+1−1

@@ -20,7 +20,7 @@

2020-- > import qualified Data.Text.Array as A

2121--

2222-- The names in this module resemble those in the 'Data.Array' family

23−-- of modules, but are shorter due to the assumption of qualifid

23+-- of modules, but are shorter due to the assumption of qualified

2424-- naming.

2525module Data.Text.Array

2626 (

Data/Text/Encoding.hs+7−18

@@ -55,13 +55,9 @@ module Data.Text.Encoding

5555 , encodeUtf32LE

5656 , encodeUtf32BE

5757

58−#if MIN_VERSION_bytestring(0,10,4)

5958 -- * Encoding Text using ByteString Builders

60− -- | /Note/ that these functions are only available if built against

61− -- @bytestring >= 0.10.4.0@.

6259 , encodeUtf8Builder

6360 , encodeUtf8BuilderEscaped

64−#endif

6561 ) where

6662

6763#if __GLASGOW_HASKELL__ >= 702

@@ -70,24 +66,15 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)

7066import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)

7167#endif

7268

73−#if MIN_VERSION_bytestring(0,10,4)

74−import Data.Bits ((.&.))

75−import Data.Text.Internal.Unsafe.Char (ord)

76−import qualified Data.ByteString.Builder as B

77−import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)

78−import qualified Data.ByteString.Builder.Prim as BP

79−import qualified Data.ByteString.Builder.Prim.Internal as BP

80−import qualified Data.Text.Internal.Encoding.Utf16 as U16

81−#endif

82−

8369import Control.Exception (evaluate, try)

8470import Control.Monad.ST (runST)

71+import Data.Bits ((.&.))

8572import Data.ByteString as B

8673import Data.ByteString.Internal as B hiding (c2w)

8774import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

8875import Data.Text.Internal (Text(..), safe, text)

8976import Data.Text.Internal.Private (runText)

90−import Data.Text.Internal.Unsafe.Char (unsafeWrite)

77+import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)

9178import Data.Text.Internal.Unsafe.Shift (shiftR)

9279import Data.Text.Show ()

9380import Data.Text.Unsafe (unsafeDupablePerformIO)

@@ -98,8 +85,13 @@ import Foreign.Marshal.Utils (with)

9885import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)

9986import Foreign.Storable (Storable, peek, poke)

10087import GHC.Base (ByteArray#, MutableByteArray#)

88+import qualified Data.ByteString.Builder as B

89+import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)

90+import qualified Data.ByteString.Builder.Prim as BP

91+import qualified Data.ByteString.Builder.Prim.Internal as BP

10192import qualified Data.Text.Array as A

10293import qualified Data.Text.Internal.Encoding.Fusion as E

94+import qualified Data.Text.Internal.Encoding.Utf16 as U16

10395import qualified Data.Text.Internal.Fusion as F

10496

10597#include "text_cbits.h"

@@ -315,8 +307,6 @@ decodeUtf8' :: ByteString -> Either UnicodeException Text

315307decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode

316308{-# INLINE decodeUtf8' #-}

317309

318−#if MIN_VERSION_bytestring(0,10,4)

319−

320310-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.

321311encodeUtf8Builder :: Text -> B.Builder

322312encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)

@@ -377,7 +367,6 @@ encodeUtf8BuilderEscaped be =

377367 outerLoop i (B.BufferRange op ope)

378368 where

379369 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)

380−#endif

381370

382371-- | Encode text using UTF-8 encoding.

383372encodeUtf8 :: Text -> ByteString

Data/Text/Internal/Builder.hs+3−1

@@ -6,7 +6,7 @@

66-- Module : Data.Text.Internal.Builder

9−-- License : BSD3-style (see LICENSE)

9+-- License : BSD-style (see LICENSE)

1010--

1111-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

1212-- Stability : experimental

@@ -242,6 +242,8 @@ flush = Builder $ \ k buf@(Buffer p o u l) ->

242242 !t = Text arr o u

243243 ts <- inlineInterleaveST (k b)

244244 return $! t : ts

245+{-# INLINE [1] flush #-}

246+-- defer inlining so that flush/flush rule may fire.

245247

246248------------------------------------------------------------------------

247249

Data/Text/Internal/Builder/Int/Digits.hs+1−1

@@ -2,7 +2,7 @@

33-- Module: Data.Text.Internal.Builder.Int.Digits

5−-- License: BSD3

5+-- License: BSD-style

66-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

77-- Stability: experimental

88-- Portability: portable

Data/Text/Internal/Fusion.hs+27−17

@@ -98,25 +98,35 @@ reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `sh

9898-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.

9999unstream :: Stream Char -> Text

100100unstream (Stream next0 s0 len) = runText $ \done -> do

101− let mlen = upperBound 4 len

101+ -- Before encoding each char we perform a buffer realloc check assuming

102+ -- worst case encoding size of two 16-bit units for the char. Just add an

103+ -- extra space to the buffer so that we do not end up reallocating even when

104+ -- all the chars are encoded as single unit.

105+ let mlen = upperBound 4 len + 1

102106 arr0 <- A.new mlen

103− let outer arr top = loop

107+ let outer !arr !maxi = encode

104108 where

105− loop !s !i =

106− case next0 s of

107− Done -> done arr i

108− Skip s' -> loop s' i

109− Yield x s'

110− | j >= top -> {-# SCC "unstream/resize" #-} do

111− let top' = (top + 1) `shiftL` 1

112− arr' <- A.new top'

113− A.copyM arr' 0 arr 0 top

114− outer arr' top' s i

115− | otherwise -> do d <- unsafeWrite arr i x

116− loop s' (i+d)

117− where j | ord x < 0x10000 = i

118− | otherwise = i + 1

119− outer arr0 mlen s0 0

109+ -- keep the common case loop as small as possible

110+ encode !si !di =

111+ case next0 si of

112+ Done -> done arr di

113+ Skip si' -> encode si' di

114+ Yield c si'

115+ -- simply check for the worst case

116+ | maxi < di + 1 -> realloc si di

117+ | otherwise -> do

118+ n <- unsafeWrite arr di c

119+ encode si' (di + n)

120+

121+ -- keep uncommon case separate from the common case code

122+ {-# NOINLINE realloc #-}

123+ realloc !si !di = do

124+ let newlen = (maxi + 1) * 2

125+ arr' <- A.new newlen

126+ A.copyM arr' 0 arr 0 di

127+ outer arr' (newlen - 1) si di

128+

129+ outer arr0 (mlen - 1) s0 0

120130{-# INLINE [0] unstream #-}

121131{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}

122132

Data/Text/Internal/Fusion/Common.hs+3−3

@@ -669,12 +669,12 @@ scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT may

669669-- ** Generating and unfolding streams

670670

671671replicateCharI :: Integral a => a -> Char -> Stream Char

672−replicateCharI n c

672+replicateCharI !n !c

673673 | n < 0 = empty

674674 | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low

675675 where

676− next i | i >= n = Done

677− | otherwise = Yield c (i + 1)

676+ next !i | i >= n = Done

677+ | otherwise = Yield c (i + 1)

678678{-# INLINE [0] replicateCharI #-}

679679

680680data RI s = RI !s {-# UNPACK #-} !Int64

Data/Text/Lazy.hs+2−2

@@ -545,7 +545,7 @@ tail Empty = emptyError "tail"

545545 unstream (S.tail (stream t)) = tail t

546546 #-}

547547

548−-- | /O(1)/ Returns all but the last character of a 'Text', which must

548+-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must

549549-- be non-empty. Subject to fusion.

550550init :: Text -> Text

551551init (Chunk t0 ts0) = go t0 ts0

@@ -581,7 +581,7 @@ isSingleton :: Text -> Bool

581581isSingleton = S.isSingleton . stream

582582{-# INLINE isSingleton #-}

583583

584−-- | /O(1)/ Returns the last character of a 'Text', which must be

584+-- | /O(n\/c)/ Returns the last character of a 'Text', which must be

585585-- non-empty. Subject to fusion.

586586last :: Text -> Char

587587last Empty = emptyError "last"

Data/Text/Lazy/Builder.hs+1−1

@@ -8,7 +8,7 @@

88-- Module : Data.Text.Lazy.Builder

11−-- License : BSD3-style (see LICENSE)

11+-- License : BSD-style (see LICENSE)

1212--

1313-- Maintainer : Johan Tibell <johan.tibell@gmail.com>

1414-- Stability : experimental

Data/Text/Lazy/Builder/Int.hs+1−1

@@ -7,7 +7,7 @@

77-- Module: Data.Text.Lazy.Builder.Int

10−-- License: BSD3

10+-- License: BSD-style

1111-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>

1212-- Stability: experimental

1313-- Portability: portable

Data/Text/Lazy/Encoding.hs+5−15

@@ -46,27 +46,23 @@ module Data.Text.Lazy.Encoding

4646 , encodeUtf32LE

4747 , encodeUtf32BE

4848

49−#if MIN_VERSION_bytestring(0,10,4)

5049 -- * Encoding Text using ByteString Builders

5150 , encodeUtf8Builder

5251 , encodeUtf8BuilderEscaped

53−#endif

5452 ) where

5553

5654import Control.Exception (evaluate, try)

55+import Data.Monoid (Monoid(..))

5756import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)

5857import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)

59−import qualified Data.ByteString as S

60−import qualified Data.ByteString.Lazy as B

61−import qualified Data.ByteString.Lazy.Internal as B

62−import qualified Data.ByteString.Unsafe as B

63−#if MIN_VERSION_bytestring(0,10,4)

6458import Data.Word (Word8)

65−import Data.Monoid (Monoid(..))

59+import qualified Data.ByteString as S

6660import qualified Data.ByteString.Builder as B

6761import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)

6862import qualified Data.ByteString.Builder.Prim as BP

69−#endif

63+import qualified Data.ByteString.Lazy as B

64+import qualified Data.ByteString.Lazy.Internal as B

65+import qualified Data.ByteString.Unsafe as B

7066import qualified Data.Text as T

7167import qualified Data.Text.Encoding as TE

7268import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E

@@ -146,7 +142,6 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do

146142{-# INLINE decodeUtf8' #-}

147143

148144encodeUtf8 :: Text -> B.ByteString

149−#if MIN_VERSION_bytestring(0,10,4)

150145encodeUtf8 Empty = B.empty

151146encodeUtf8 lt@(Chunk t _) =

152147 B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt

@@ -168,11 +163,6 @@ encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder

168163encodeUtf8BuilderEscaped prim =

169164 foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty

170165

171−#else

172−encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)

173−encodeUtf8 Empty = B.Empty

174−#endif

175−

176166-- | Decode text from little endian UTF-16 encoding.

177167decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text

178168decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

benchmarks/haskell/Benchmarks.hs+1−1

@@ -53,7 +53,7 @@ benchmarks = do

5353 , Mul.benchmark

5454 , Pure.benchmark "tiny" (tf "tiny.txt")

5555 , Pure.benchmark "ascii" (tf "ascii-small.txt")

56− , Pure.benchmark "france" (tf "france.html")

56+ -- , Pure.benchmark "france" (tf "france.html")

5757 , Pure.benchmark "russian" (tf "russian-small.txt")

5858 , Pure.benchmark "japanese" (tf "japanese.txt")

5959 , ReadNumbers.benchmark (tf "numbers.txt")

benchmarks/haskell/Benchmarks/Stream.hs+16−5

@@ -18,11 +18,11 @@ import qualified Data.Text.Encoding as T

1818import qualified Data.Text.Encoding.Error as E

1919import qualified Data.Text.Internal.Encoding.Fusion as T

2020import qualified Data.Text.Internal.Encoding.Fusion.Common as F

21−import qualified Data.Text.Internal.Fusion as T

21+import qualified Data.Text.Internal.Fusion as F

2222import qualified Data.Text.IO as T

2323import qualified Data.Text.Lazy.Encoding as TL

2424import qualified Data.Text.Internal.Lazy.Encoding.Fusion as TL

25−import qualified Data.Text.Internal.Lazy.Fusion as TL

25+import qualified Data.Text.Internal.Lazy.Fusion as FL

2626import qualified Data.Text.Lazy.IO as TL

2727

2828instance NFData a => NFData (Stream a) where

@@ -53,14 +53,25 @@ benchmark fp = do

5353 !utf32beL = TL.encodeUtf32BE tl

5454

5555 -- For the functions which operate on streams

56− let !s = T.stream t

56+ let !s = F.stream t

5757

5858 return $ bgroup "Stream"

5959

6060 -- Fusion

6161 [ bgroup "stream" $

62− [ bench "Text" $ nf T.stream t

63− , bench "LazyText" $ nf TL.stream tl

62+ [ bench "Text" $ nf F.stream t

63+ , bench "LazyText" $ nf FL.stream tl

64+ ]

65+ -- must perform exactly the same as stream above due to

66+ -- stream/unstream (i.e. stream after unstream) fusion

67+ , bgroup "stream-fusion" $

68+ [ bench "Text" $ nf (F.stream . F.unstream . F.stream) t

69+ , bench "LazyText" $ nf (FL.stream . FL.unstream . FL.stream) tl

70+ ]

71+ -- measure the overhead of unstream after stream

72+ , bgroup "stream-unstream" $

73+ [ bench "Text" $ nf (F.unstream . F.stream) t

74+ , bench "LazyText" $ nf (FL.unstream . FL.stream) tl

6475 ]

6576

6677 -- Encoding.Fusion

benchmarks/text-benchmarks.cabal+12−2

@@ -3,7 +3,7 @@ version: 0.0.0.0

33synopsis: Benchmarks for the text package

44description: Benchmarks for the text package

55homepage: https://bitbucket.org/bos/text

6−license: BSD3

6+license: BSD2

77license-file: ../LICENSE

88author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

99 Bryan O'Sullivan <bos@serpentine.com>,

@@ -15,6 +15,11 @@ build-type: Simple

1515

1616cabal-version: >=1.2

1717

18+flag bytestring-builder

19+ description: Depend on the bytestring-builder package for backwards compatibility.

20+ default: False

21+ manual: False

22+

1823flag llvm

1924 description: use LLVM

2025 default: False

@@ -33,7 +38,6 @@ executable text-benchmarks

3338 build-depends: base == 4.*,

3439 binary,

3540 blaze-builder,

36− bytestring,

3741 bytestring-lexing >= 0.5.0,

3842 containers,

3943 criterion >= 0.10.0.0,

@@ -46,6 +50,12 @@ executable text-benchmarks

4650 utf8-string,

4751 vector

4852

53+ if flag(bytestring-builder)

54+ build-depends: bytestring >= 0.9 && < 0.10.4,

55+ bytestring-builder >= 0.10.4

56+ else

57+ build-depends: bytestring >= 0.10.4

58+

4959executable text-multilang

5060 hs-source-dirs: haskell

5161 main-is: Multilang.hs

tests/literal-rule-test.sh+1−1

@@ -6,7 +6,7 @@ function check_firings() {

66 rule=$1

77 expected=$2

88 build="ghc -O -ddump-rule-firings LiteralRuleTest.hs"

9− build="$build -i.. -I../include -DMIN_VERSION_bytestring(a,b,c)=1"

9+ build="$build -i.. -I../include"

1010 touch LiteralRuleTest.hs

1111 echo -n "Want to see $expected firings of rule $rule... " >&2

1212 firings=$($build 2>&1 | grep "Rule fired: $rule\$" | wc -l)

tests/text-tests.cabal+1−1

@@ -3,7 +3,7 @@ version: 0.0.0.0

33synopsis: Functional tests for the text package

44description: Functional tests for the text package

55homepage: https://github.com/bos/text

6−license: BSD3

6+license: BSD2

77license-file: ../LICENSE

88author: Jasper Van der Jeugt <jaspervdj@gmail.com>,

99 Bryan O'Sullivan <bos@serpentine.com>,

text.cabal+16−5

@@ -31,7 +31,7 @@ description:

3131 the @text-icu@ package:

3232 <http://hackage.haskell.org/package/text-icu>

3333

34−license: BSD3

34+license: BSD2

3535license-file: LICENSE

3636author: Bryan O'Sullivan <bos@serpentine.com>

3737maintainer: Bryan O'Sullivan <bos@serpentine.com>

@@ -64,6 +64,11 @@ extra-source-files:

6464 tests/scripts/*.sh

6565 tests/text-tests.cabal

6666

67+flag bytestring-builder

68+ description: Depend on the bytestring-builder package for backwards compatibility.

69+ default: False

70+ manual: False

71+

6772flag developer

6873 description: operate in developer mode

6974 default: False

@@ -133,10 +138,11 @@ library

133138 deepseq >= 1.1.0.0,

134139 ghc-prim >= 0.2

135140

136− if impl(ghc >= 7.7)

137− build-depends: bytestring >= 0.10.4.0

141+ if flag(bytestring-builder)

142+ build-depends: bytestring >= 0.9 && < 0.10.4,

143+ bytestring-builder >= 0.10.4

138144 else

139− build-depends: bytestring >= 0.9

145+ build-depends: bytestring >= 0.10.4

140146

141147 cpp-options: -DHAVE_DEEPSEQ

142148 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2

@@ -171,7 +177,6 @@ test-suite tests

171177 array,

172178 base,

173179 binary,

174− bytestring,

175180 deepseq,

176181 directory,

177182 ghc-prim,

@@ -181,6 +186,12 @@ test-suite tests

181186 test-framework-hunit >= 0.2,

182187 test-framework-quickcheck2 >= 0.2

183188

189+ if flag(bytestring-builder)

190+ build-depends: bytestring >= 0.9 && < 0.10.4,

191+ bytestring-builder >= 0.10.4

192+ else

193+ build-depends: bytestring >= 0.10.4

194+

184195 if flag(integer-simple)

185196 cpp-options: -DINTEGER_SIMPLE

186197 build-depends: integer-simple >= 0.1 && < 0.5