f23938f · text · git.iko.soy

tests/benchmarks/python/case_map.pydeleted−7

compare with master

unifiedsplit

@@ -1,7 +0,0 @@

1−#!/usr/bin/env python

2−

3−import utils, sys

4−

5−for f in sys.argv[1:]:

6− t = utils.benchmark(lambda: utils.with_utf8_file(f, lambda c: c.upper()))

7− sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/cut.pynew file+12

@@ -0,0 +1,12 @@

1+#!/usr/bin/env python

2+

3+import utils, sys, codecs

4+

5+def cut(filename, l, r):

6+ content = open(filename, encoding='utf-8')

7+ for line in content:

8+ print(line[l:r])

9+

10+for f in sys.argv[1:]:

11+ t = utils.benchmark(lambda: cut(f, 20, 40))

12+ sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/file_read.pydeleted−7

@@ -1,7 +0,0 @@

1−#!/usr/bin/env python

2−

3−import utils, sys

4−

5−for f in sys.argv[1:]:

6− t = utils.benchmark(lambda: utils.with_utf8_file(f, lambda c: len(c)))

7− sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/sort.py+6−8

@@ -1,15 +1,13 @@

1#!/usr/bin/env python

2

3−import utils, sys

3+import utils, sys, codecs

4

5−def sort(string):

5+def sort(filename):

6+ content = open(filename, encoding='utf-8').read()

6− lines = string.splitlines()

7+ lines = content.splitlines()

7 lines.sort()

8 lines.sort()

8− return '\n'.join(lines)

9+ print('\n'.join(lines))

9

10

10for f in sys.argv[1:]:

11for f in sys.argv[1:]:

11− t = utils.benchmark(lambda: sys.stdout.write(

12+ t = utils.benchmark(lambda: sort(f))

12− utils.with_utf8_file(f,sort).encode('utf-8'))

13− )

14 sys.stderr.write('{0}: {1}\n'.format(f, t))

13 sys.stderr.write('{0}: {1}\n'.format(f, t))

15−

tests/benchmarks/python/strip_brackets.py → tests/benchmarks/python/strip_tags.pyrenamed+9−6

@@ -2,21 +2,24 @@

2

3import utils, sys

4

5−def strip_brackets(string):

5+def strip_tags(filename):

6+ string = open(filename, encoding='utf-8').read()

7+

6 d = 0

8 d = 0

7− out = ''

9+ out = []

10+

8 for c in string:

11 for c in string:

9− if c == '{' or c == '[': d += 1

12+ if c == '<': d += 1

10

13

11 if d > 0:

14 if d > 0:

12 out += ' '

15 out += ' '

13 else:

16 else:

14 out += c

17 out += c

15

18

16− if c == '}' or c == ']': d -= 1

19+ if c == '>': d -= 1

17

20

18− return out

21+ print(''.join(out))

19

22

20for f in sys.argv[1:]:

23for f in sys.argv[1:]:

21− t = utils.benchmark(lambda: utils.with_utf8_file(f, strip_brackets))

24+ t = utils.benchmark(lambda: strip_tags(f))

22 sys.stderr.write('{0}: {1}\n'.format(f, t))

25 sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/utils.py−4

@@ -16,7 +16,3 @@ def benchmark(f):

16 sys.stderr.write('Run {0}: {1}\n'.format(i, result))

17 total += result

18 return total / runs

19−

20−def with_utf8_file(filename, f):

21− contents = open(filename).read().decode('utf-8')

22− return f(contents)

tests/benchmarks/python/word_count.pydeleted−17

@@ -1,17 +0,0 @@

1−#!/usr/bin/env python

2−

3−import utils, sys

4−

5−def word_count(string):

6− freqs = {}

7− for w in string.split():

8− w = w.lower()

9− if freqs.get(w):

10− freqs[w] += 1

11− else:

12− freqs[w] = 1

13− return freqs

14−

15−for f in sys.argv[1:]:

16− t = utils.benchmark(lambda: utils.with_utf8_file(f, word_count))

17− sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/ruby/case_map.rbdeleted−8

@@ -1,8 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−ARGV.each do |f|

6− t = benchmark { with_utf8_file(f) { |c| c.upcase } }

7− STDERR.puts "#{f}: #{t}"

8−end

tests/benchmarks/ruby/cut.rbnew file+16

@@ -0,0 +1,16 @@

1+#!/usr/bin/env ruby

2+

3+require './utils.rb'

4+

5+def cut(filename, l, r)

6+ File.open(filename, 'r:utf-8') do |file|

7+ file.each_line do |line|

8+ puts line[l, r - l]

9+ end

10+ end

11+end

12+

13+ARGV.each do |f|

14+ t = benchmark { cut(f, 20, 40) }

15+ STDERR.puts "#{f}: #{t}"

16+end

tests/benchmarks/ruby/file_read.rbdeleted−8

@@ -1,8 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−ARGV.each do |f|

6− t = benchmark { with_utf8_file(f) { |c| c.size } }

7− STDERR.puts "#{f}: #{t}"

8−end

tests/benchmarks/ruby/fold.rbnew file+50

@@ -0,0 +1,50 @@

1+#!/usr/bin/env ruby

2+

3+require './utils.rb'

4+

5+def fold(filename, max_width)

6+ File.open(filename, 'r:utf-8') do |file|

7+ # Words in this paragraph

8+ paragraph = []

9+

10+ file.each_line do |line|

11+ # If we encounter an empty line, we reformat and dump the current

12+ # paragraph

13+ if line.strip.empty?

14+ puts fold_paragraph(paragraph, max_width)

15+ puts

16+ paragraph = []

17+ # Otherwise, we append the words found in the line to the paragraph

18+ else

19+ paragraph.concat line.split

20+ end

21+ end

22+

23+ # Last paragraph

24+ puts fold_paragraph(paragraph, max_width) unless paragraph.empty?

25+ end

26+end

27+

28+# Fold a single paragraph to the desired width

29+def fold_paragraph(paragraph, max_width)

30+ # Gradually build our output

31+ str, *rest = paragraph

32+ width = str.length

33+

34+ rest.each do |word|

35+ if width + word.length + 1 <= max_width

36+ str << ' ' << word

37+ width += word.length + 1

38+ else

39+ str << "\n" << word

40+ width = word.length

41+ end

42+ end

43+

44+ str

45+end

46+

47+ARGV.each do |f|

48+ t = benchmark { fold(f, 80) }

49+ STDERR.puts "#{f}: #{t}"

50+end

tests/benchmarks/ruby/sort.rb+6−5

@@ -2,13 +2,14 @@

2

3require './utils.rb'

4

5−def sort(str)

5+def sort(filename)

6+ File.open(filename, 'r:utf-8') do |file|

7+ content = file.read

6− str.lines.sort.join

8+ puts content.lines.sort.join

9+ end

7end

10end

8

11

9ARGV.each do |f|

12ARGV.each do |f|

10− t = benchmark do

13+ t = benchmark { sort(f) }

11− with_utf8_file(f) { |c| puts sort(c) }

12− end

13 STDERR.puts "#{f}: #{t}"

14 STDERR.puts "#{f}: #{t}"

14end

15end

tests/benchmarks/ruby/strip_brackets.rbdeleted−21

@@ -1,21 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−def strip_brackets(str)

6− d = 0

7− out = ''

8−

9− str.each_char do |c|

10− d += 1 if c == '{' || c == '['

11− out << if d > 0 then ' ' else c end

12− d -= 1 if c == '}' || c == ']'

13− end

14−

15− out

16−end

17−

18−ARGV.each do |f|

19− t = benchmark { with_utf8_file(f) { |c| strip_brackets(c) } }

20− STDERR.puts "#{f}: #{t}"

21−end

tests/benchmarks/ruby/strip_tags.rbnew file+22

@@ -0,0 +1,22 @@

1+#!/usr/bin/env ruby

2+

3+require './utils.rb'

4+

5+def strip_tags(filename)

6+ File.open(filename, 'r:utf-8') do |file|

7+ str = file.read

8+

9+ d = 0

10+

11+ str.each_char do |c|

12+ d += 1 if c == '<'

13+ putc(if d > 0 then ' ' else c end)

14+ d -= 1 if c == '>'

15+ end

16+ end

17+end

18+

19+ARGV.each do |f|

20+ t = benchmark { strip_tags(f) }

21+ STDERR.puts "#{f}: #{t}"

22+end

tests/benchmarks/ruby/utils.rb−6

@@ -12,9 +12,3 @@ def benchmark(&block)

12

13 total / runs

14end

15−

16−def with_utf8_file(filename)

17− File.open(filename, 'r:utf-8') do |file|

18− yield file.read

19− end

20−end

tests/benchmarks/ruby/word_count.rbdeleted−16

@@ -1,16 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−def word_count(str)

6− freqs = Hash.new 0

7− str.split.each do |w|

8− freqs[w.downcase] += 1

9− end

10− freqs

11−end

12−

13−ARGV.each do |f|

14− t = benchmark { with_utf8_file(f) { |c| word_count(c) } }

15− STDERR.puts "#{f}: #{t}"

16−end

tests/benchmarks/src/Data/Text/Benchmarks.hs+29−20

@@ -5,27 +5,28 @@ module Main

5 ( main

6 ) where

7

8−import Criterion.Main (Benchmark, defaultMain)

8+import Criterion.Main (Benchmark, defaultMain, bgroup)

9import System.FilePath ((</>))

10import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8)

11

12import qualified Data.Text.Benchmarks.Builder as Builder

13−import qualified Data.Text.Benchmarks.CaseMap as CaseMap

14−import qualified Data.Text.Benchmarks.Cut as Cut

15import qualified Data.Text.Benchmarks.DecodeUtf8 as DecodeUtf8

13import qualified Data.Text.Benchmarks.DecodeUtf8 as DecodeUtf8

16import qualified Data.Text.Benchmarks.EncodeUtf8 as EncodeUtf8

14import qualified Data.Text.Benchmarks.EncodeUtf8 as EncodeUtf8

17import qualified Data.Text.Benchmarks.Equality as Equality

15import qualified Data.Text.Benchmarks.Equality as Equality

18−import qualified Data.Text.Benchmarks.FileIndices as FileIndices

19import qualified Data.Text.Benchmarks.FileRead as FileRead

16import qualified Data.Text.Benchmarks.FileRead as FileRead

20import qualified Data.Text.Benchmarks.FoldLines as FoldLines

17import qualified Data.Text.Benchmarks.FoldLines as FoldLines

21−import qualified Data.Text.Benchmarks.HtmlCombinator as HtmlCombinator

22−import qualified Data.Text.Benchmarks.Ordering as Ordering

23import qualified Data.Text.Benchmarks.Pure as Pure

18import qualified Data.Text.Benchmarks.Pure as Pure

24import qualified Data.Text.Benchmarks.ReadNumbers as ReadNumbers

19import qualified Data.Text.Benchmarks.ReadNumbers as ReadNumbers

25import qualified Data.Text.Benchmarks.Replace as Replace

20import qualified Data.Text.Benchmarks.Replace as Replace

26−import qualified Data.Text.Benchmarks.Sort as Sort

21+import qualified Data.Text.Benchmarks.Search as Search

22+import qualified Data.Text.Benchmarks.WordFrequencies as WordFrequencies

23+

24+import qualified Data.Text.Benchmarks.Programs.BigTable as Programs.BigTable

25+import qualified Data.Text.Benchmarks.Programs.Cut as Programs.Cut

26+import qualified Data.Text.Benchmarks.Programs.Fold as Programs.Fold

27+import qualified Data.Text.Benchmarks.Programs.Sort as Programs.Sort

27−import qualified Data.Text.Benchmarks.StripBrackets as StripBrackets

28+import qualified Data.Text.Benchmarks.Programs.StripTags as Programs.StripTags

28−import qualified Data.Text.Benchmarks.WordCount as WordCount

29+import qualified Data.Text.Benchmarks.Programs.Throughput as Programs.Throughput

29

30

30main :: IO ()

31main :: IO ()

31main = benchmarks >>= defaultMain

32main = benchmarks >>= defaultMain

@@ -34,25 +35,33 @@ benchmarks :: IO [Benchmark]

34benchmarks = do

35benchmarks = do

35 sink <- openFile "/dev/null" WriteMode

36 sink <- openFile "/dev/null" WriteMode

36 hSetEncoding sink utf8

37 hSetEncoding sink utf8

38+

39+ -- Traditional benchmarks

37− sequence

40+ bs <- sequence

38 [ Builder.benchmark

41 [ Builder.benchmark

39− , CaseMap.benchmark (tf "russian.txt") sink

40− , Cut.benchmark (tf "russian.txt") sink 30 60

41 , DecodeUtf8.benchmark (tf "russian.txt")

42 , DecodeUtf8.benchmark (tf "russian.txt")

42− , EncodeUtf8.benchmark sink "επανάληψη 竺法蘭共譯"

43+ , EncodeUtf8.benchmark "επανάληψη 竺法蘭共譯"

43 , Equality.benchmark (tf "japanese.txt")

44 , Equality.benchmark (tf "japanese.txt")

44− , FileIndices.benchmark (tf "russian.txt") "принимая"

45 , FileRead.benchmark (tf "russian.txt")

46 , FoldLines.benchmark (tf "russian.txt")

47− , HtmlCombinator.benchmark sink

48− , Ordering.benchmark (tf "russian.txt")

49 , Pure.benchmark (tf "japanese.txt")

47 , Pure.benchmark (tf "japanese.txt")

50 , ReadNumbers.benchmark (tf "numbers.txt")

48 , ReadNumbers.benchmark (tf "numbers.txt")

51− , Replace.benchmark (tf "russian.txt") sink "принимая" "своем"

49+ , Replace.benchmark (tf "russian.txt") "принимая" "своем"

52− , Sort.benchmark (tf "russian.txt") sink

50+ , Search.benchmark (tf "russian.txt") "принимая"

53− , StripBrackets.benchmark (tf "russian.txt") sink

54− , WordCount.benchmark (tf "russian.txt")

51+ , WordFrequencies.benchmark (tf "russian.txt")

55 ]

52 ]

53+

54+ -- Program-like benchmarks

55+ ps <- bgroup "Programs" `fmap` sequence

56+ [ Programs.BigTable.benchmark sink

57+ , Programs.Cut.benchmark (tf "russian.txt") sink 20 40

58+ , Programs.Fold.benchmark (tf "russian.txt") sink

59+ , Programs.Sort.benchmark (tf "russian.txt") sink

60+ , Programs.StripTags.benchmark (tf "yiwiki.xml") sink

61+ , Programs.Throughput.benchmark (tf "russian.txt") sink

62+ ]

63+

64+ return $ bs ++ [ps]

56 where

65 where

57 -- Location of a test file

66 -- Location of a test file

58 tf = ("../text/test" </>)

67 tf = ("../text/test" </>)

tests/benchmarks/src/Data/Text/Benchmarks/Builder.hs+4

@@ -1,5 +1,9 @@

1-- | Testing the internal builder monoid

2--

3+-- Tested in this benchmark:

4+--

5+-- * Concatenating many small strings using a builder

6+--

3{-# LANGUAGE OverloadedStrings #-}

7{-# LANGUAGE OverloadedStrings #-}

4module Data.Text.Benchmarks.Builder

8module Data.Text.Benchmarks.Builder

5 ( benchmark

9 ( benchmark

tests/benchmarks/src/Data/Text/Benchmarks/CaseMap.hsdeleted−15

@@ -1,15 +0,0 @@

1−-- | This benchmark converts a number of UTF-8 encoded files to uppercase

2−--

3−module Data.Text.Benchmarks.CaseMap

4− ( benchmark

5− ) where

6−

7−import Criterion (Benchmark, bench)

8−import System.IO (Handle)

9−import qualified Data.ByteString as B

10−import qualified Data.Text as T

11−import qualified Data.Text.Encoding as T

12−

13−benchmark :: FilePath -> Handle -> IO Benchmark

14−benchmark fp sink = return $ bench "CaseMap" $

15− B.readFile fp >>= B.hPutStr sink . T.encodeUtf8 . T.toUpper . T.decodeUtf8

tests/benchmarks/src/Data/Text/Benchmarks/DecodeUtf8.hs+30−82

@@ -1,95 +1,43 @@

1+-- | Test decoding of UTF-8

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Decoding bytes using UTF-8

6+--

7+-- In some tests:

8+--

9+-- * Taking the length of the result

10+--

11+-- * Taking the init of the result

12+--

13+-- The latter are used for testing stream fusion.

14+--

1module Data.Text.Benchmarks.DecodeUtf8

15module Data.Text.Benchmarks.DecodeUtf8

2 ( benchmark

16 ( benchmark

3 ) where

17 ) where

4

18

5−import Control.DeepSeq (rnf)

6−import Criterion (Benchmark, bgroup, bench)

19+import Criterion (Benchmark, bgroup, bench, nf)

7−import System.IO (IOMode (ReadMode), openFile, hGetContents, hSetEncoding, utf8)

8import qualified Codec.Binary.UTF8.Generic as U8

20import qualified Codec.Binary.UTF8.Generic as U8

9import qualified Data.ByteString as B

21import qualified Data.ByteString as B

10import qualified Data.ByteString.Lazy as BL

22import qualified Data.ByteString.Lazy as BL

11import qualified Data.Text as T

23import qualified Data.Text as T

12import qualified Data.Text.Encoding as T

24import qualified Data.Text.Encoding as T

13−import qualified Data.Text.IO as T

14import qualified Data.Text.Lazy as TL

25import qualified Data.Text.Lazy as TL

15import qualified Data.Text.Lazy.Encoding as TL

26import qualified Data.Text.Lazy.Encoding as TL

16−import qualified Data.Text.Lazy.IO as TL

17

27

18benchmark :: FilePath -> IO Benchmark

28benchmark :: FilePath -> IO Benchmark

29+benchmark fp = do

30+ bs <- B.readFile fp

31+ lbs <- BL.readFile fp

19−benchmark fp = return $ bgroup "DecodeUtf8"

32+ return $ bgroup "DecodeUtf8"

20− [ bench "Strict" $ do

33+ [ bench "Strict" $ nf T.decodeUtf8 bs

21− bs <- B.readFile fp

22− rnf (T.decodeUtf8 bs) `seq` return ()

23−

24− , bench "StrictLength" $ do

34+ , bench "StrictLength" $ nf (T.length . T.decodeUtf8) bs

25− bs <- B.readFile fp

26− rnf (T.length $ T.decodeUtf8 bs) `seq` return ()

27−

28− , bench "StrictInitLength" $ do

35+ , bench "StrictInitLength" $ nf (T.length . T.init . T.decodeUtf8) bs

29− bs <- B.readFile fp

30− rnf (T.length $ T.init $ T.decodeUtf8 bs) `seq` return ()

31−

32− , bench "StrictIO" $ do

33− h <- openFile fp ReadMode

34− hSetEncoding h utf8

35− t <- T.hGetContents h

36− rnf t `seq` return ()

37−

38− , bench "StrictLengthIO" $ do

39− h <- openFile fp ReadMode

40− hSetEncoding h utf8

41− t <- T.hGetContents h

42− rnf (T.length t) `seq` return ()

43−

44− , bench "Lazy" $ do

36+ , bench "Lazy" $ nf TL.decodeUtf8 lbs

45− bs <- BL.readFile fp

46− rnf (TL.decodeUtf8 bs) `seq` return ()

47−

48− , bench "LazyLength" $ do

49− bs <- BL.readFile fp

50− rnf (TL.length $ TL.decodeUtf8 bs) `seq` return ()

37+ , bench "LazyLength" $ nf (TL.length . TL.decodeUtf8) lbs

51−

52− , bench "LazyInitLength" $ do

53− bs <- BL.readFile fp

54− rnf (TL.length $ TL.init $ TL.decodeUtf8 bs) `seq` return ()

38+ , bench "LazyInitLength" $ nf (TL.length . TL.init . TL.decodeUtf8) lbs

55−

56− , bench "LazyIO" $ do

57− h <- openFile fp ReadMode

58− hSetEncoding h utf8

59− t <- TL.hGetContents h

60− rnf t `seq` return ()

61−

62− , bench "LazyLengthIO" $ do

63− h <- openFile fp ReadMode

64− hSetEncoding h utf8

65− t <- TL.hGetContents h

66− rnf (TL.length t) `seq` return ()

67−

68− , bench "String" $ do

69− h <- openFile fp ReadMode

70− hSetEncoding h utf8

71− t <- hGetContents h

72− rnf t `seq` return ()

73−

74− , bench "StringLength" $ do

75− h <- openFile fp ReadMode

76− hSetEncoding h utf8

77− t <- hGetContents h

78− rnf (length t) `seq` return ()

79−

80− , bench "LazyStringUtf8" $ do

81− s <- U8.toString `fmap` BL.readFile fp

82− rnf s `seq` return ()

83−

84− , bench "LazyStringUtf8Length" $ do

85− s <- U8.toString `fmap` BL.readFile fp

86− rnf (length s) `seq` return ()

87−

88− , bench "StrictStringUtf8" $ do

39+ , bench "StrictStringUtf8" $ nf U8.toString bs

89− s <- U8.toString `fmap` B.readFile fp

90− rnf s `seq` return ()

91−

92− , bench "StrictStringUtf8Length" $ do

40+ , bench "StrictStringUtf8Length" $ nf (length . U8.toString) bs

93− s <- U8.toString `fmap` B.readFile fp

41+ , bench "LazyStringUtf8" $ nf U8.toString lbs

94− rnf (length s) `seq` return ()

42+ , bench "LazyStringUtf8Length" $ nf (length . U8.toString) lbs

95− ]

43+ ]

tests/benchmarks/src/Data/Text/Benchmarks/EncodeUtf8.hs+17−17

@@ -1,33 +1,33 @@

1+-- | UTF-8 encode a text

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Replicating a string a number of times

6+--

7+-- * UTF-8 encoding it

8+--

1module Data.Text.Benchmarks.EncodeUtf8

9module Data.Text.Benchmarks.EncodeUtf8

2 ( benchmark

10 ( benchmark

3 ) where

11 ) where

4

12

5−import Criterion (Benchmark, bgroup, bench)

13+import Criterion (Benchmark, bgroup, bench, whnf)

6−import System.IO (Handle, hPutStr)

7import qualified Data.ByteString as B

14import qualified Data.ByteString as B

8import qualified Data.ByteString.Lazy as BL

15import qualified Data.ByteString.Lazy as BL

9import qualified Data.Text as T

16import qualified Data.Text as T

10import qualified Data.Text.Encoding as T

17import qualified Data.Text.Encoding as T

11−import qualified Data.Text.IO as T

12import qualified Data.Text.Lazy as TL

18import qualified Data.Text.Lazy as TL

13import qualified Data.Text.Lazy.Encoding as TL

19import qualified Data.Text.Lazy.Encoding as TL

14−import qualified Data.Text.Lazy.IO as TL

15

20

16−benchmark :: Handle -> String -> IO Benchmark

21+benchmark :: String -> IO Benchmark

22+benchmark string = do

17−benchmark sink string = return $ bgroup "EncodeUtf8"

23+ return $ bgroup "EncodeUtf8"

18− [ bench "String" $ hPutStr sink $ concat $ replicate k string

24+ [ bench "Text" $ whnf (B.length . T.encodeUtf8) text

19− , bench "Text" $ T.hPutStr sink $ T.replicate k text

20− , bench "TextLazy" $ TL.hPutStr sink $

21− TL.replicate (fromIntegral k) lazyText

22− , bench "TextByteString" $ B.hPutStr sink $

23− T.encodeUtf8 $ T.replicate k text

24− , bench "TextByteStringLazy" $ BL.hPutStr sink $

25− TL.encodeUtf8 $ TL.replicate (fromIntegral k) lazyText

25+ , bench "LazyText" $ whnf (BL.length . TL.encodeUtf8) lazyText

26− ]

26+ ]

27 where

28 -- The string in different formats

29− text = T.pack string

29+ text = T.replicate k $ T.pack string

30− lazyText = TL.pack string

30+ lazyText = TL.replicate (fromIntegral k) $ TL.pack string

31

32 -- Amount

33 k = 100000

tests/benchmarks/src/Data/Text/Benchmarks/Equality.hs+5−2

@@ -1,9 +1,12 @@

1-- | Compare a string with a copy of itself that is identical except

2-- for the last character.

3--

4+-- Tested in this benchmark:

5+--

6+-- * Comparison of strings (Eq instance)

7+--

4module Data.Text.Benchmarks.Equality

8module Data.Text.Benchmarks.Equality

5− (

6− benchmark

9+ ( benchmark

7 ) where

10 ) where

8

11

9import Criterion (Benchmark, bgroup, bench, whnf)

12import Criterion (Benchmark, bgroup, bench, whnf)

tests/benchmarks/src/Data/Text/Benchmarks/FileIndices.hsdeleted−28

@@ -1,28 +0,0 @@

1−-- | Search for a pattern in a file, find the number of occurences

2−--

3−module Data.Text.Benchmarks.FileIndices

4− ( benchmark

5− ) where

6−

7−import Control.Exception (evaluate)

8−import Criterion (Benchmark, bench, bgroup)

9−import qualified Data.ByteString as B

10−import qualified Data.ByteString.Lazy as BL

11−import qualified Data.ByteString.Lazy.Search as BL

12−import qualified Data.Text.Lazy as TL

13−import qualified Data.Text.Lazy.Encoding as TL

14−import qualified Data.Text.Lazy.IO as TL

15−

16−benchmark :: FilePath -> TL.Text -> IO Benchmark

17−benchmark fp t = return $ bgroup "FileIndices"

18− [ bench "LazyText" $ TL.readFile fp >>= evaluate . text t

19− , bench "LazyByteString" $ BL.readFile fp >>= evaluate . byteString b

20− ]

21− where

22− b = B.concat $ BL.toChunks $ TL.encodeUtf8 t

23−

24−text :: TL.Text -> TL.Text -> Int

25−text needle = fromIntegral . TL.count needle

26−

27−byteString :: B.ByteString -> BL.ByteString -> Int

28−byteString needle = length . BL.indices needle

tests/benchmarks/src/Data/Text/Benchmarks/FileRead.hs+4

@@ -1,5 +1,9 @@

1-- | Benchmarks simple file reading

2--

3+-- Tested in this benchmark:

4+--

5+-- * Reading a file from the disk

6+--

3module Data.Text.Benchmarks.FileRead

7module Data.Text.Benchmarks.FileRead

4 ( benchmark

8 ( benchmark

5 ) where

9 ) where

tests/benchmarks/src/Data/Text/Benchmarks/FoldLines.hs+9−4

@@ -1,4 +1,9 @@

1−-- | Fold over the lines in a file

1+-- | Read a file line-by-line using handles, and perform a fold over the lines.

2+-- The fold is used here to calculate the number of lines in the file.

3+--

4+-- Tested in this benchmark:

5+--

6+-- * Buffered, line-based IO

2--

7--

3{-# LANGUAGE BangPatterns #-}

8{-# LANGUAGE BangPatterns #-}

4module Data.Text.Benchmarks.FoldLines

9module Data.Text.Benchmarks.FoldLines

@@ -12,8 +17,8 @@ import qualified Data.Text as T

12import qualified Data.Text.IO as T

17import qualified Data.Text.IO as T

13

18

14benchmark :: FilePath -> IO Benchmark

19benchmark :: FilePath -> IO Benchmark

15−benchmark fp = return $ bgroup "FoldLines"

20+benchmark fp = return $ bgroup "ReadLines"

16− [ bench "Text" $ withHandle $ foldLinesT (\n _ -> n + 1) (0 :: Int)

21+ [ bench "Text" $ withHandle $ foldLinesT (\n _ -> n + 1) (0 :: Int)

17 , bench "ByteString" $ withHandle $ foldLinesB (\n _ -> n + 1) (0 :: Int)

22 , bench "ByteString" $ withHandle $ foldLinesB (\n _ -> n + 1) (0 :: Int)

18 ]

23 ]

19 where

24 where

@@ -24,7 +29,7 @@ benchmark fp = return $ bgroup "FoldLines"

24 hClose h

29 hClose h

25 return x

30 return x

26

31

27−-- Text line fold

32+-- | Text line fold

28--

33--

29foldLinesT :: (a -> T.Text -> a) -> a -> Handle -> IO a

34foldLinesT :: (a -> T.Text -> a) -> a -> Handle -> IO a

30foldLinesT f z0 h = go z0

35foldLinesT f z0 h = go z0

tests/benchmarks/src/Data/Text/Benchmarks/Ordering.hsdeleted−58

@@ -1,58 +0,0 @@

1−-- | For every 1000th line of a file, check how many lines in the file are

2−-- lexicographically smaller

3−--

4−module Data.Text.Benchmarks.Ordering

5− ( benchmark

6− ) where

7−

8−import Control.Exception (evaluate)

9−import Criterion (Benchmark, bgroup, bench)

10−import qualified Data.ByteString.Char8 as B

11−import qualified Data.ByteString.Lazy.Char8 as BL

12−import qualified Data.Text as T

13−import qualified Data.Text.Encoding as T

14−import qualified Data.Text.Lazy as TL

15−import qualified Data.Text.Lazy.Encoding as TL

16−

17−benchmark :: FilePath -> IO Benchmark

18−benchmark fp = return $ bgroup "Ordering"

19− [ bench "ByteString" $ B.readFile fp >>=

20− evaluate . numSmaller . B.lines

21− , bench "LazyByteString" $ BL.readFile fp >>=

22− evaluate . numSmaller . BL.lines

23−

24− , bench "Text" $ B.readFile fp >>=

25− evaluate . numSmaller . T.lines . T.decodeUtf8

26− , bench "TextFusion" $ B.readFile fp >>=

27− evaluate . numSmallerFusion . T.lines . T.decodeUtf8

28− , bench "LazyText" $ BL.readFile fp >>=

29− evaluate . numSmaller . TL.lines . TL.decodeUtf8

30−

31− , bench "String" $ readFile fp >>=

32− evaluate . numSmaller . lines

33− ]

34−

35−-- | Take every Nth item from a list

36−--

37−every :: Int -> [a] -> [a]

38−every k = go k

39− where

40− go n (x : xs)

41− | n < k = go (n+1) xs

42− | otherwise = x : go 1 xs

43− go _ _ = []

44−

45−-- | Benchmark logic

46−--

47−numSmaller :: (Ord a) => [a] -> Int

48−numSmaller ls = sum . map f $ every 1000 ls

49− where

50− f x = length . filter ((== GT) . compare x) $ ls

51−

52−-- | Test a comparison that could be fused: compare (toLower a) (toLower b)

53−-- Currently, this funcion performs very badly!

54−--

55−numSmallerFusion :: [T.Text] -> Int

56−numSmallerFusion ls = sum . map f $ every 1000 ls

57− where

58− f x = length . filter ((== GT) . compare (T.toLower x) . T.toLower) $ ls

tests/benchmarks/src/Data/Text/Benchmarks/HtmlCombinator.hs → tests/benchmarks/src/Data/Text/Benchmarks/Programs/BigTable.hsrenamed+8−2

@@ -1,7 +1,13 @@

1-- | Create a large HTML table and dump it to a handle

2--

3+-- Tested in this benchmark:

4+--

5+-- * Creating a large HTML document using a builder

6+--

7+-- * Writing to a handle

8+--

3{-# LANGUAGE OverloadedStrings #-}

9{-# LANGUAGE OverloadedStrings #-}

4−module Data.Text.Benchmarks.HtmlCombinator

10+module Data.Text.Benchmarks.Programs.BigTable

5 ( benchmark

11 ( benchmark

6 ) where

12 ) where

7

13

@@ -13,7 +19,7 @@ import System.IO (Handle)

13import qualified Data.Text as T

19import qualified Data.Text as T

14

20

15benchmark :: Handle -> IO Benchmark

21benchmark :: Handle -> IO Benchmark

16−benchmark sink = return $ bench "HtmlCombinator" $ do

22+benchmark sink = return $ bench "BigTable" $ do

17 hPutStr sink "Content-Type: text/html\n\n<table>"

23 hPutStr sink "Content-Type: text/html\n\n<table>"

18 hPutStr sink . toLazyText . makeTable =<< rows

24 hPutStr sink . toLazyText . makeTable =<< rows

19 hPutStr sink "</table>"

25 hPutStr sink "</table>"

tests/benchmarks/src/Data/Text/Benchmarks/Cut.hs → tests/benchmarks/src/Data/Text/Benchmarks/Programs/Cut.hsrenamed+14−2

@@ -1,6 +1,18 @@

1−-- | Cut into a file, selecting certain columns (e.g. lines 10 to 40)

1+-- | Cut into a file, selecting certain columns (e.g. columns 10 to 40)

2--

3+-- Tested in this benchmark:

4+--

5+-- * Reading the file

6+--

7+-- * Splitting into lines

8+--

9+-- * Taking a number of characters from the lines

10+--

11+-- * Joining the lines

12+--

13+-- * Writing back to a handle

14+--

3−module Data.Text.Benchmarks.Cut

15+module Data.Text.Benchmarks.Programs.Cut

4 ( benchmark

16 ( benchmark

5 ) where

17 ) where

6

18

tests/benchmarks/src/Data/Text/Benchmarks/Programs/Fold.hsnew file+68

@@ -0,0 +1,68 @@

1+-- | Benchmark which formats paragraph, like the @sort@ unix utility.

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Reading the file

6+--

7+-- * Splitting into paragraphs

8+--

9+-- * Reformatting the paragraphs to a certain line width

10+--

11+-- * Concatenating the results using the text builder

12+--

13+-- * Writing back to a handle

14+--

15+{-# LANGUAGE OverloadedStrings #-}

16+module Data.Text.Benchmarks.Programs.Fold

17+ ( benchmark

18+ ) where

19+

20+import Data.List (foldl')

21+import Data.List (intersperse)

22+import Data.Monoid (mempty, mappend, mconcat)

23+import System.IO (Handle)

24+import Criterion (Benchmark, bench)

25+import qualified Data.Text as T

26+import qualified Data.Text.IO as T

27+import qualified Data.Text.Lazy.Builder as TLB

28+import qualified Data.Text.Lazy as TL

29+import qualified Data.Text.Lazy.IO as TL

30+

31+benchmark :: FilePath -> Handle -> IO Benchmark

32+benchmark i o = return $

33+ bench "Fold" $ T.readFile i >>= TL.hPutStr o . fold 80

34+

35+-- | We represent a paragraph by a word list

36+--

37+type Paragraph = [T.Text]

38+

39+-- | Fold a text

40+--

41+fold :: Int -> T.Text -> TL.Text

42+fold maxWidth = TLB.toLazyText . mconcat .

43+ intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs

44+

45+-- | Fold a paragraph

46+--

47+foldParagraph :: Int -> Paragraph -> TLB.Builder

48+foldParagraph _ [] = mempty

49+foldParagraph max' (w : ws) = fst $ foldl' go (TLB.fromText w, T.length w) ws

50+ where

51+ go (builder, width) word

52+ | width + len + 1 <= max' =

53+ (builder `mappend` " " `mappend` word', width + len + 1)

54+ | otherwise =

55+ (builder `mappend` "\n" `mappend` word', len)

56+ where

57+ word' = TLB.fromText word

58+ len = T.length word

59+

60+-- | Divide a text into paragraphs

61+--

62+paragraphs :: T.Text -> [Paragraph]

63+paragraphs = splitParagraphs . map T.words . T.lines

64+ where

65+ splitParagraphs ls = case break null ls of

66+ ([], []) -> []

67+ (p, []) -> [concat p]

68+ (p, lr) -> concat p : splitParagraphs (dropWhile null lr)

tests/benchmarks/src/Data/Text/Benchmarks/Programs/Sort.hsnew file+63

@@ -0,0 +1,63 @@

1+-- | This benchmark sorts the lines of a file, like the @sort@ unix utility.

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Reading the file

6+--

7+-- * Splitting into lines

8+--

9+-- * Sorting the lines

10+--

11+-- * Joining the lines

12+--

13+-- * Writing back to a handle

14+--

15+{-# LANGUAGE OverloadedStrings #-}

16+module Data.Text.Benchmarks.Programs.Sort

17+ ( benchmark

18+ ) where

19+

20+import Criterion (Benchmark, bgroup, bench)

21+import Data.Monoid (mconcat)

22+import System.IO (Handle, hPutStr)

23+import qualified Data.ByteString as B

24+import qualified Data.ByteString.Lazy as BL

25+import qualified Data.ByteString.Lazy.Char8 as BLC

26+import qualified Data.ByteString.Char8 as BC

27+import qualified Data.List as L

28+import qualified Data.Text as T

29+import qualified Data.Text.IO as T

30+import qualified Data.Text.Lazy as TL

31+import qualified Data.Text.Lazy.IO as TL

32+import qualified Data.Text.Lazy.Builder as TLB

33+

34+benchmark :: FilePath -> Handle -> IO Benchmark

35+benchmark i o = return $ bgroup "Sort"

36+ [ bench "String" $ readFile i >>= hPutStr o . string

37+ , bench "ByteString" $ B.readFile i >>= B.hPutStr o . byteString

38+ , bench "LazyByteString" $ BL.readFile i >>= BL.hPutStr o . lazyByteString

39+ , bench "Text" $ T.readFile i >>= T.hPutStr o . text

40+ , bench "LazyText" $ TL.readFile i >>= TL.hPutStr o . lazyText

41+ , bench "TextBuilder" $ T.readFile i >>= TL.hPutStr o . textBuilder

42+ ]

43+

44+string :: String -> String

45+string = unlines . L.sort . lines

46+

47+byteString :: B.ByteString -> B.ByteString

48+byteString = BC.unlines . L.sort . BC.lines

49+

50+lazyByteString :: BL.ByteString -> BL.ByteString

51+lazyByteString = BLC.unlines . L.sort . BLC.lines

52+

53+text :: T.Text -> T.Text

54+text = T.unlines . L.sort . T.lines

55+

56+lazyText :: TL.Text -> TL.Text

57+lazyText = TL.unlines . L.sort . TL.lines

58+

59+-- | Text variant using a builder monoid for the final concatenation

60+--

61+textBuilder :: T.Text -> TL.Text

62+textBuilder = TLB.toLazyText . mconcat . L.intersperse (TLB.singleton '\n') .

63+ map TLB.fromText . L.sort . T.lines

tests/benchmarks/src/Data/Text/Benchmarks/Programs/StripTags.hsnew file+51

@@ -0,0 +1,51 @@

1+-- | Program to replace HTML tags by whitespace

2+--

3+-- This program was originally contributed by Petr Prokhorenkov.

4+--

5+-- Tested in this benchmark:

6+--

7+-- * Reading the file

8+--

9+-- * Replacing text between HTML tags (<>) with whitespace

10+--

11+-- * Writing back to a handle

12+--

13+{-# OPTIONS_GHC -fspec-constr-count=5 #-}

14+module Data.Text.Benchmarks.Programs.StripTags

15+ ( benchmark

16+ ) where

17+

18+import Criterion (Benchmark, bgroup, bench)

19+import Data.List (mapAccumL)

20+import System.IO (Handle, hPutStr)

21+import qualified Data.ByteString as B

22+import qualified Data.ByteString.Char8 as BC

23+import qualified Data.Text as T

24+import qualified Data.Text.Encoding as T

25+

26+benchmark :: FilePath -> Handle -> IO Benchmark

27+benchmark i o = return $ bgroup "StripTags"

28+ [ bench "String" $ readFile i >>= hPutStr o . string

29+ , bench "ByteString" $ B.readFile i >>= B.hPutStr o . byteString

30+ , bench "Text" $

31+ B.readFile i >>= B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8

32+ ]

33+

34+string :: String -> String

35+string = snd . mapAccumL step 0

36+

37+text :: T.Text -> T.Text

38+text = snd . T.mapAccumL step 0

39+

40+byteString :: B.ByteString -> B.ByteString

41+byteString = snd . BC.mapAccumL step 0

42+

43+step :: Int -> Char -> (Int, Char)

44+step d c

45+ | d > 0 || d' > 0 = (d', ' ')

46+ | otherwise = (d', c)

47+ where

48+ d' = d + depth c

49+ depth '>' = 1

50+ depth '<' = -1

51+ depth _ = 0

tests/benchmarks/src/Data/Text/Benchmarks/Programs/Throughput.hsnew file+35

@@ -0,0 +1,35 @@

1+-- | This benchmark simply reads and writes a file using the various string

2+-- libraries. The point of it is that we can make better estimations on how

3+-- much time the other benchmarks spend doing IO.

4+--

5+-- Note that we expect ByteStrings to be a whole lot faster, since they do not

6+-- do any actual encoding/decoding here, while String and Text do have UTF-8

7+-- encoding/decoding.

8+--

9+-- Tested in this benchmark:

10+--

11+-- * Reading the file

12+--

13+-- * Replacing text between HTML tags (<>) with whitespace

14+--

15+-- * Writing back to a handle

16+--

17+module Data.Text.Benchmarks.Programs.Throughput

18+ ( benchmark

19+ ) where

20+

21+import Criterion (Benchmark, bgroup, bench)

22+import System.IO (Handle, hPutStr)

23+import qualified Data.ByteString as B

24+import qualified Data.ByteString.Lazy as BL

25+import qualified Data.Text.IO as T

26+import qualified Data.Text.Lazy.IO as TL

27+

28+benchmark :: FilePath -> Handle -> IO Benchmark

29+benchmark fp sink = return $ bgroup "Throughput"

30+ [ bench "String" $ readFile fp >>= hPutStr sink

31+ , bench "ByteString" $ B.readFile fp >>= B.hPutStr sink

32+ , bench "LazyByteString" $ BL.readFile fp >>= BL.hPutStr sink

33+ , bench "Text" $ T.readFile fp >>= T.hPutStr sink

34+ , bench "LazyText" $ TL.readFile fp >>= TL.hPutStr sink

35+ ]

tests/benchmarks/src/Data/Text/Benchmarks/Pure.hs+4

@@ -1,5 +1,9 @@

1-- | Benchmarks various pure functions from the Text library

2--

3+-- Tested in this benchmark:

4+--

5+-- * Most pure functions defined the string types

6+--

3{-# LANGUAGE BangPatterns, GADTs, MagicHash #-}

7{-# LANGUAGE BangPatterns, GADTs, MagicHash #-}

4{-# OPTIONS_GHC -fno-warn-orphans #-}

8{-# OPTIONS_GHC -fno-warn-orphans #-}

5module Data.Text.Benchmarks.Pure

9module Data.Text.Benchmarks.Pure

tests/benchmarks/src/Data/Text/Benchmarks/ReadNumbers.hs+50−37

@@ -1,12 +1,26 @@

1-- | Read numbers from a file with a just a number on each line, find the

2−-- minimum of those numbers.

2+-- minimum of those numbers. The file contains different kinds of numbers:

3+--

4+-- * Decimals

5+--

6+-- * Hexadecimals

7+--

8+-- * Floating point numbers

9+--

10+-- * Floating point numbers in scientific notation

11+--

12+-- The different benchmarks will only take into account the values they can

13+-- parse.

14+--

15+-- Tested in this benchmark:

16+--

17+-- * Lexing/parsing of different numerical types

3--

18--

4module Data.Text.Benchmarks.ReadNumbers

19module Data.Text.Benchmarks.ReadNumbers

5 ( benchmark

20 ( benchmark

6 ) where

21 ) where

7

22

8−import Control.Exception (evaluate)

9−import Criterion (Benchmark, bgroup, bench)

23+import Criterion (Benchmark, bgroup, bench, whnf)

10import Data.List (foldl')

24import Data.List (foldl')

11import Numeric (readDec, readFloat, readHex)

25import Numeric (readDec, readFloat, readHex)

12import qualified Data.ByteString.Char8 as B

26import qualified Data.ByteString.Char8 as B

@@ -14,49 +28,48 @@ import qualified Data.ByteString.Lazy.Char8 as BL

14import qualified Data.ByteString.Lex.Double as B

28import qualified Data.ByteString.Lex.Double as B

15import qualified Data.ByteString.Lex.Lazy.Double as BL

29import qualified Data.ByteString.Lex.Lazy.Double as BL

16import qualified Data.Text as T

30import qualified Data.Text as T

17−import qualified Data.Text.Encoding as T

31+import qualified Data.Text.IO as T

18import qualified Data.Text.Lazy as TL

32import qualified Data.Text.Lazy as TL

19−import qualified Data.Text.Lazy.Encoding as TL

33+import qualified Data.Text.Lazy.IO as TL

20import qualified Data.Text.Lazy.Read as TL

34import qualified Data.Text.Lazy.Read as TL

21import qualified Data.Text.Read as T

35import qualified Data.Text.Read as T

22

36

23benchmark :: FilePath -> IO Benchmark

37benchmark :: FilePath -> IO Benchmark

38+benchmark fp = do

39+ -- Read all files into lines: string, text, lazy text, bytestring, lazy

40+ -- bytestring

41+ s <- lines `fmap` readFile fp

42+ t <- T.lines `fmap` T.readFile fp

43+ tl <- TL.lines `fmap` TL.readFile fp

44+ b <- B.lines `fmap` B.readFile fp

45+ bl <- BL.lines `fmap` BL.readFile fp

24−benchmark fp = return $ bgroup "ReadNumbers"

46+ return $ bgroup "ReadNumbers"

25− [ bench "DecimalString" $ readFile fp >>= evaluate .

26− int . string readDec . lines

47+ [ bench "DecimalString" $ whnf (int . string readDec) s

27− , bench "HexadecimalString" $ readFile fp >>= evaluate .

48+ , bench "HexadecimalString" $ whnf (int . string readHex) s

28− int . string readHex . lines

29− , bench "DoubleString" $ readFile fp >>= evaluate .

30− double . string readFloat . lines

49+ , bench "DoubleString" $ whnf (double . string readFloat) s

31

50

32− , bench "DecimalText" $ B.readFile fp >>= evaluate .

33− int . text (T.signed T.decimal) . T.lines . T.decodeUtf8

51+ , bench "DecimalText" $ whnf (int . text (T.signed T.decimal)) t

34− , bench "HexadecimalText" $ B.readFile fp >>= evaluate .

35− int . text (T.signed T.hexadecimal) . T.lines . T.decodeUtf8

52+ , bench "HexadecimalText" $ whnf (int . text (T.signed T.hexadecimal)) t

36− , bench "DoubleText" $ B.readFile fp >>= evaluate .

37− double . text T.double . T.lines . T.decodeUtf8

53+ , bench "DoubleText" $ whnf (double . text T.double) t

38− , bench "RationalText" $ B.readFile fp >>= evaluate .

39− double . text T.rational . T.lines . T.decodeUtf8

54+ , bench "RationalText" $ whnf (double . text T.rational) t

40

55

41− , bench "DecimalLazyText" $ BL.readFile fp >>= evaluate .

56+ , bench "DecimalLazyText" $

42− int . text (TL.signed TL.decimal) . TL.lines . TL.decodeUtf8

57+ whnf (int . text (TL.signed TL.decimal)) tl

43− , bench "HexadecimalLazyText" $ BL.readFile fp >>= evaluate .

58+ , bench "HexadecimalLazyText" $

44− int . text (TL.signed TL.hexadecimal) . TL.lines . TL.decodeUtf8

59+ whnf (int . text (TL.signed TL.hexadecimal)) tl

45− , bench "DoubleLazyText" $ BL.readFile fp >>= evaluate .

60+ , bench "DoubleLazyText" $

46− double . text TL.double . TL.lines . TL.decodeUtf8

61+ whnf (double . text TL.double) tl

47− , bench "RationalLazyText" $ BL.readFile fp >>= evaluate .

62+ , bench "RationalLazyText" $

48− double . text TL.rational . TL.lines . TL.decodeUtf8

63+ whnf (double . text TL.rational) tl

49

64

50− , bench "DecimalByteString" $ B.readFile fp >>= evaluate .

51− int . byteString B.readInt . B.lines

65+ , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b

52− , bench "DoubleByteString" $ B.readFile fp >>= evaluate .

53− double . byteString B.readDouble . B.lines

66+ , bench "DoubleByteString" $ whnf (double . byteString B.readDouble) b

54

67

55− , bench "DecimalLazyByteString" $ BL.readFile fp >>= evaluate .

68+ , bench "DecimalLazyByteString" $

56− int . byteString BL.readInt . BL.lines

69+ whnf (int . byteString BL.readInt) bl

57− , bench "DoubleLazyByteString" $ BL.readFile fp >>= evaluate .

70+ , bench "DoubleLazyByteString" $

58− double . byteString BL.readDouble . BL.lines

71+ whnf (double . byteString BL.readDouble) bl

59− ]

72+ ]

60 where

73 where

61 -- Used for fixing types

74 -- Used for fixing types

62 int :: Int -> Int

75 int :: Int -> Int

tests/benchmarks/src/Data/Text/Benchmarks/Replace.hs+14−16

@@ -1,11 +1,14 @@

1−-- | Replace a string by another string in a file

1+-- | Replace a string by another string

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Search and replace of a pattern in a text

2--

6--

3module Data.Text.Benchmarks.Replace

7module Data.Text.Benchmarks.Replace

4 ( benchmark

8 ( benchmark

5 ) where

9 ) where

6

10

7−import Criterion (Benchmark, bgroup, bench)

11+import Criterion (Benchmark, bgroup, bench, nf)

8−import System.IO (Handle)

9import qualified Data.ByteString.Char8 as B

12import qualified Data.ByteString.Char8 as B

10import qualified Data.ByteString.Lazy as BL

13import qualified Data.ByteString.Lazy as BL

11import qualified Data.ByteString.Lazy.Search as BL

14import qualified Data.ByteString.Lazy.Search as BL

@@ -13,19 +16,14 @@ import qualified Data.Text.Lazy as TL

13import qualified Data.Text.Lazy.Encoding as TL

16import qualified Data.Text.Lazy.Encoding as TL

14import qualified Data.Text.Lazy.IO as TL

17import qualified Data.Text.Lazy.IO as TL

15

18

16−benchmark :: FilePath -> Handle -> String -> String -> IO Benchmark

19+benchmark :: FilePath -> String -> String -> IO Benchmark

20+benchmark fp pat sub = do

21+ tl <- TL.readFile fp

22+ bl <- BL.readFile fp

17−benchmark fp sink pat sub = return $ bgroup "Replace"

23+ return $ bgroup "Replace"

18− -- We have benchmarks for lazy text and lazy bytestrings. We also benchmark

19− -- without the acual replacement, so we can get an idea of what time is

20− -- spent on IO and computations.

21− [ bench "LazyText" $ TL.readFile fp >>=

22− TL.hPutStr sink . TL.replace tpat tsub

24+ [ bench "LazyText" $ nf (TL.length . TL.replace tpat tsub) tl

23− , bench "LazyTextNull" $ TL.readFile fp >>= TL.hPutStr sink

24−

25− , bench "LazyByteString" $ BL.readFile fp >>=

25+ , bench "LazyByteString" $ nf (BL.length . BL.replace bpat bsub) bl

26− BL.hPutStr sink . BL.replace bpat bsub

27− , bench "LazyByteStringNull" $ BL.readFile fp >>= BL.hPutStr sink

28− ]

26+ ]

29 where

27 where

30 tpat = TL.pack pat

28 tpat = TL.pack pat

31 tsub = TL.pack sub

29 tsub = TL.pack sub

tests/benchmarks/src/Data/Text/Benchmarks/Search.hsnew file+48

@@ -0,0 +1,48 @@

1+-- | Search for a pattern in a file, find the number of occurences

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Searching all occurences of a pattern using library routines

6+--

7+module Data.Text.Benchmarks.Search

8+ ( benchmark

9+ ) where

10+

11+import Criterion (Benchmark, bench, bgroup, whnf)

12+import qualified Data.ByteString as B

13+import qualified Data.ByteString.Lazy as BL

14+import qualified Data.ByteString.Lazy.Search as BL

15+import qualified Data.ByteString.Search as B

16+import qualified Data.Text as T

17+import qualified Data.Text.Encoding as T

18+import qualified Data.Text.IO as T

19+import qualified Data.Text.Lazy as TL

20+import qualified Data.Text.Lazy.IO as TL

21+

22+benchmark :: FilePath -> T.Text -> IO Benchmark

23+benchmark fp needleT = do

24+ b <- B.readFile fp

25+ bl <- BL.readFile fp

26+ t <- T.readFile fp

27+ tl <- TL.readFile fp

28+ return $ bgroup "FileIndices"

29+ [ bench "ByteString" $ whnf (byteString needleB) b

30+ , bench "LazyByteString" $ whnf (lazyByteString needleB) bl

31+ , bench "Text" $ whnf (text needleT) t

32+ , bench "LazyText" $ whnf (lazyText needleTL) tl

33+ ]

34+ where

35+ needleB = T.encodeUtf8 needleT

36+ needleTL = TL.fromChunks [needleT]

37+

38+byteString :: B.ByteString -> B.ByteString -> Int

39+byteString needle = length . B.indices needle

40+

41+lazyByteString :: B.ByteString -> BL.ByteString -> Int

42+lazyByteString needle = length . BL.indices needle

43+

44+text :: T.Text -> T.Text -> Int

45+text = T.count

46+

47+lazyText :: TL.Text -> TL.Text -> Int

48+lazyText needle = fromIntegral . TL.count needle

tests/benchmarks/src/Data/Text/Benchmarks/Sort.hsdeleted−27

@@ -1,27 +0,0 @@

1−-- | Implements the unix @sort@ program

2−--

3−{-# LANGUAGE OverloadedStrings #-}

4−module Data.Text.Benchmarks.Sort

5− ( benchmark

6− ) where

7−

8−import Criterion (Benchmark, bench)

9−import Data.Monoid (mconcat)

10−import System.IO (Handle)

11−import qualified Data.ByteString as B

12−import qualified Data.ByteString.Lazy as BL

13−import qualified Data.List as L

14−import qualified Data.Text as T

15−import qualified Data.Text.Encoding as T

16−import qualified Data.Text.Lazy as TL

17−import qualified Data.Text.Lazy.Builder as TLB

18−import qualified Data.Text.Lazy.Encoding as TL

19−

20−benchmark :: FilePath -> Handle -> IO Benchmark

21−benchmark fp sink = return $ bench "Sort" $ do

22− t <- T.decodeUtf8 `fmap` B.readFile fp

23− BL.hPutStr sink $ TL.encodeUtf8 $ sort t

24−

25−sort :: T.Text -> TL.Text

26−sort = TLB.toLazyText . mconcat . L.intersperse (TLB.fromText "\n") .

27− map TLB.fromText . L.sort . T.lines

tests/benchmarks/src/Data/Text/Benchmarks/StripBrackets.hsdeleted−33

@@ -1,33 +0,0 @@

1−-- | Program to replace everything between brackets by spaces

2−--

3−-- This program was originally contributed by Petr Prokhorenkov.

4−--

5−module Data.Text.Benchmarks.StripBrackets

6− ( benchmark

7− ) where

8−

9−import Criterion (Benchmark, bench)

10−import System.IO (Handle)

11−import qualified Data.ByteString as B

12−import qualified Data.Text as T

13−import qualified Data.Text.Encoding as T

14−

15−benchmark :: FilePath -> Handle -> IO Benchmark

16−benchmark fp sink = return $ bench "StripBrackets" $ do

17− t <- T.decodeUtf8 `fmap` B.readFile fp

18− B.hPutStr sink $ T.encodeUtf8 $ stripBrackets t

19−

20−stripBrackets :: T.Text -> T.Text

21−stripBrackets = snd . T.mapAccumL f (0 :: Int)

22− where

23− f depth c =

24− let depth' = depth + d' c

25− c' | depth > 0 || depth' > 0 = ' '

26− | otherwise = c

27− in (depth', c')

28−

29− d' '{' = 1

30− d' '[' = 1

31− d' '}' = -1

32− d' ']' = -1

33− d' _ = 0

tests/benchmarks/src/Data/Text/Benchmarks/WordCount.hsdeleted−22

@@ -1,22 +0,0 @@

1−-- | A word frequence count program

2−--

3−module Data.Text.Benchmarks.WordCount

4− ( benchmark

5− ) where

6−

7−import Control.Exception (evaluate)

8−import Criterion (Benchmark, bench)

9−import Data.List (foldl')

10−import Data.Map (Map)

11−import qualified Data.Map as M

12−import qualified Data.Text as T

13−import qualified Data.Text.IO as T

14−

15−benchmark :: FilePath -> IO Benchmark

16−benchmark fp = return $ bench "WordCount" $ do

17− t <- T.readFile fp

18− evaluate $ M.size $ wordCount t

19−

20−wordCount :: T.Text -> Map T.Text Int

21−wordCount =

22− foldl' (\m k -> M.insertWith (+) k 1 m) M.empty . map T.toLower . T.words

tests/benchmarks/src/Data/Text/Benchmarks/WordFrequencies.hsnew file+36

@@ -0,0 +1,36 @@

1+-- | A word frequency count using the different string types

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Splitting into words

6+--

7+-- * Converting to lowercase

8+--

9+-- * Comparing: Eq/Ord instances

10+--

11+module Data.Text.Benchmarks.WordFrequencies

12+ ( benchmark

13+ ) where

14+

15+import Criterion (Benchmark, bench, bgroup, whnf)

16+import Data.Char (toLower)

17+import Data.List (foldl')

18+import Data.Map (Map)

19+import qualified Data.ByteString.Char8 as B

20+import qualified Data.Map as M

21+import qualified Data.Text as T

22+import qualified Data.Text.IO as T

23+

24+benchmark :: FilePath -> IO Benchmark

25+benchmark fp = do

26+ s <- readFile fp

27+ b <- B.readFile fp

28+ t <- T.readFile fp

29+ return $ bgroup "WordFrequencies"

30+ [ bench "String" $ whnf (frequencies . words . map toLower) s

31+ , bench "ByteString" $ whnf (frequencies . B.words . B.map toLower) b

32+ , bench "Text" $ whnf (frequencies . T.words . T.toLower) t

33+ ]

34+

35+frequencies :: Ord a => [a] -> Map a Int

36+frequencies = foldl' (\m k -> M.insertWith (+) k 1 m) M.empty