f23938f · text · git.iko.soy

git.iko.soy/text

Merge pull request #5 from jaspervdj/master

Further work on benchmarks

commitf23938f81ec4b912f5f7822fef01a4a6b0133fcf authorBryan O'Sullivan <bos@serpentine.com> dateThu 23 Jun 2011, 13:40 (15 years ago) parent419ee9b 7d61b05

40 files changed+610−478

compare with master

unifiedsplit

tests/benchmarks/python/case_map.pydeleted−7

@@ -1,7 +0,0 @@

1−#!/usr/bin/env python

2−

3−import utils, sys

4−

5−for f in sys.argv[1:]:

6− t = utils.benchmark(lambda: utils.with_utf8_file(f, lambda c: c.upper()))

7− sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/cut.pynew file+12

@@ -0,0 +1,12 @@

1+#!/usr/bin/env python

3+import utils, sys, codecs

5+def cut(filename, l, r):

6+ content = open(filename, encoding='utf-8')

7+ for line in content:

8+ print(line[l:r])

10+for f in sys.argv[1:]:

11+ t = utils.benchmark(lambda: cut(f, 20, 40))

12+ sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/file_read.pydeleted−7

@@ -1,7 +0,0 @@

1−#!/usr/bin/env python

2−

3−import utils, sys

4−

5−for f in sys.argv[1:]:

6− t = utils.benchmark(lambda: utils.with_utf8_file(f, lambda c: len(c)))

7− sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/sort.py+6−8

@@ -1,15 +1,13 @@

11#!/usr/bin/env python

3−import utils, sys

3+import utils, sys, codecs

5−def sort(string):

6− lines = string.splitlines()

5+def sort(filename):

6+ content = open(filename, encoding='utf-8').read()

7+ lines = content.splitlines()

78 lines.sort()

8− return '\n'.join(lines)

9+ print('\n'.join(lines))

910

1011for f in sys.argv[1:]:

11− t = utils.benchmark(lambda: sys.stdout.write(

12− utils.with_utf8_file(f,sort).encode('utf-8'))

13− )

12+ t = utils.benchmark(lambda: sort(f))

1413 sys.stderr.write('{0}: {1}\n'.format(f, t))

15−

tests/benchmarks/python/strip_brackets.py → tests/benchmarks/python/strip_tags.pyrenamed+9−6

@@ -2,21 +2,24 @@

33import utils, sys

5−def strip_brackets(string):

5+def strip_tags(filename):

6+ string = open(filename, encoding='utf-8').read()

68 d = 0

7− out = ''

9+ out = []

10+

811 for c in string:

9− if c == '{' or c == '[': d += 1

12+ if c == '<': d += 1

1013

1114 if d > 0:

1215 out += ' '

1316 else:

1417 out += c

1518

16− if c == '}' or c == ']': d -= 1

19+ if c == '>': d -= 1

1720

18− return out

21+ print(''.join(out))

1922

2023for f in sys.argv[1:]:

21− t = utils.benchmark(lambda: utils.with_utf8_file(f, strip_brackets))

24+ t = utils.benchmark(lambda: strip_tags(f))

2225 sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/python/utils.py−4

@@ -16,7 +16,3 @@ def benchmark(f):

1616 sys.stderr.write('Run {0}: {1}\n'.format(i, result))

1717 total += result

1818 return total / runs

19−

20−def with_utf8_file(filename, f):

21− contents = open(filename).read().decode('utf-8')

22− return f(contents)

tests/benchmarks/python/word_count.pydeleted−17

@@ -1,17 +0,0 @@

1−#!/usr/bin/env python

2−

3−import utils, sys

4−

5−def word_count(string):

6− freqs = {}

7− for w in string.split():

8− w = w.lower()

9− if freqs.get(w):

10− freqs[w] += 1

11− else:

12− freqs[w] = 1

13− return freqs

14−

15−for f in sys.argv[1:]:

16− t = utils.benchmark(lambda: utils.with_utf8_file(f, word_count))

17− sys.stderr.write('{0}: {1}\n'.format(f, t))

tests/benchmarks/ruby/case_map.rbdeleted−8

@@ -1,8 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−ARGV.each do |f|

6− t = benchmark { with_utf8_file(f) { |c| c.upcase } }

7− STDERR.puts "#{f}: #{t}"

8−end

tests/benchmarks/ruby/cut.rbnew file+16

@@ -0,0 +1,16 @@

1+#!/usr/bin/env ruby

3+require './utils.rb'

5+def cut(filename, l, r)

6+ File.open(filename, 'r:utf-8') do |file|

7+ file.each_line do |line|

8+ puts line[l, r - l]

9+ end

10+ end

11+end

12+

13+ARGV.each do |f|

14+ t = benchmark { cut(f, 20, 40) }

15+ STDERR.puts "#{f}: #{t}"

16+end

tests/benchmarks/ruby/file_read.rbdeleted−8

@@ -1,8 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−ARGV.each do |f|

6− t = benchmark { with_utf8_file(f) { |c| c.size } }

7− STDERR.puts "#{f}: #{t}"

8−end

tests/benchmarks/ruby/fold.rbnew file+50

@@ -0,0 +1,50 @@

1+#!/usr/bin/env ruby

3+require './utils.rb'

5+def fold(filename, max_width)

6+ File.open(filename, 'r:utf-8') do |file|

7+ # Words in this paragraph

8+ paragraph = []

10+ file.each_line do |line|

11+ # If we encounter an empty line, we reformat and dump the current

12+ # paragraph

13+ if line.strip.empty?

14+ puts fold_paragraph(paragraph, max_width)

15+ puts

16+ paragraph = []

17+ # Otherwise, we append the words found in the line to the paragraph

18+ else

19+ paragraph.concat line.split

20+ end

21+ end

22+

23+ # Last paragraph

24+ puts fold_paragraph(paragraph, max_width) unless paragraph.empty?

25+ end

26+end

27+

28+# Fold a single paragraph to the desired width

29+def fold_paragraph(paragraph, max_width)

30+ # Gradually build our output

31+ str, *rest = paragraph

32+ width = str.length

33+

34+ rest.each do |word|

35+ if width + word.length + 1 <= max_width

36+ str << ' ' << word

37+ width += word.length + 1

38+ else

39+ str << "\n" << word

40+ width = word.length

41+ end

42+ end

43+

44+ str

45+end

46+

47+ARGV.each do |f|

48+ t = benchmark { fold(f, 80) }

49+ STDERR.puts "#{f}: #{t}"

50+end

tests/benchmarks/ruby/sort.rb+6−5

@@ -2,13 +2,14 @@

33require './utils.rb'

5−def sort(str)

6− str.lines.sort.join

5+def sort(filename)

6+ File.open(filename, 'r:utf-8') do |file|

7+ content = file.read

8+ puts content.lines.sort.join

9+ end

710end

811

912ARGV.each do |f|

10− t = benchmark do

11− with_utf8_file(f) { |c| puts sort(c) }

12− end

13+ t = benchmark { sort(f) }

1314 STDERR.puts "#{f}: #{t}"

1415end

tests/benchmarks/ruby/strip_brackets.rbdeleted−21

@@ -1,21 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−def strip_brackets(str)

6− d = 0

7− out = ''

8−

9− str.each_char do |c|

10− d += 1 if c == '{' || c == '['

11− out << if d > 0 then ' ' else c end

12− d -= 1 if c == '}' || c == ']'

13− end

14−

15− out

16−end

17−

18−ARGV.each do |f|

19− t = benchmark { with_utf8_file(f) { |c| strip_brackets(c) } }

20− STDERR.puts "#{f}: #{t}"

21−end

tests/benchmarks/ruby/strip_tags.rbnew file+22

@@ -0,0 +1,22 @@

1+#!/usr/bin/env ruby

3+require './utils.rb'

5+def strip_tags(filename)

6+ File.open(filename, 'r:utf-8') do |file|

7+ str = file.read

9+ d = 0

10+

11+ str.each_char do |c|

12+ d += 1 if c == '<'

13+ putc(if d > 0 then ' ' else c end)

14+ d -= 1 if c == '>'

15+ end

16+ end

17+end

18+

19+ARGV.each do |f|

20+ t = benchmark { strip_tags(f) }

21+ STDERR.puts "#{f}: #{t}"

22+end

tests/benchmarks/ruby/utils.rb−6

@@ -12,9 +12,3 @@ def benchmark(&block)

1212

1313 total / runs

1414end

15−

16−def with_utf8_file(filename)

17− File.open(filename, 'r:utf-8') do |file|

18− yield file.read

19− end

20−end

tests/benchmarks/ruby/word_count.rbdeleted−16

@@ -1,16 +0,0 @@

1−#!/usr/bin/env ruby

2−

3−require './utils.rb'

4−

5−def word_count(str)

6− freqs = Hash.new 0

7− str.split.each do |w|

8− freqs[w.downcase] += 1

9− end

10− freqs

11−end

12−

13−ARGV.each do |f|

14− t = benchmark { with_utf8_file(f) { |c| word_count(c) } }

15− STDERR.puts "#{f}: #{t}"

16−end

tests/benchmarks/src/Data/Text/Benchmarks.hs+29−20

@@ -5,27 +5,28 @@ module Main

55 ( main

66 ) where

8−import Criterion.Main (Benchmark, defaultMain)

8+import Criterion.Main (Benchmark, defaultMain, bgroup)

99import System.FilePath ((</>))

1010import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8)

1111

1212import qualified Data.Text.Benchmarks.Builder as Builder

13−import qualified Data.Text.Benchmarks.CaseMap as CaseMap

14−import qualified Data.Text.Benchmarks.Cut as Cut

1513import qualified Data.Text.Benchmarks.DecodeUtf8 as DecodeUtf8

1614import qualified Data.Text.Benchmarks.EncodeUtf8 as EncodeUtf8

1715import qualified Data.Text.Benchmarks.Equality as Equality

18−import qualified Data.Text.Benchmarks.FileIndices as FileIndices

1916import qualified Data.Text.Benchmarks.FileRead as FileRead

2017import qualified Data.Text.Benchmarks.FoldLines as FoldLines

21−import qualified Data.Text.Benchmarks.HtmlCombinator as HtmlCombinator

22−import qualified Data.Text.Benchmarks.Ordering as Ordering

2318import qualified Data.Text.Benchmarks.Pure as Pure

2419import qualified Data.Text.Benchmarks.ReadNumbers as ReadNumbers

2520import qualified Data.Text.Benchmarks.Replace as Replace

26−import qualified Data.Text.Benchmarks.Sort as Sort

27−import qualified Data.Text.Benchmarks.StripBrackets as StripBrackets

28−import qualified Data.Text.Benchmarks.WordCount as WordCount

21+import qualified Data.Text.Benchmarks.Search as Search

22+import qualified Data.Text.Benchmarks.WordFrequencies as WordFrequencies

23+

24+import qualified Data.Text.Benchmarks.Programs.BigTable as Programs.BigTable

25+import qualified Data.Text.Benchmarks.Programs.Cut as Programs.Cut

26+import qualified Data.Text.Benchmarks.Programs.Fold as Programs.Fold

27+import qualified Data.Text.Benchmarks.Programs.Sort as Programs.Sort

28+import qualified Data.Text.Benchmarks.Programs.StripTags as Programs.StripTags

29+import qualified Data.Text.Benchmarks.Programs.Throughput as Programs.Throughput

2930

3031main :: IO ()

3132main = benchmarks >>= defaultMain

@@ -34,25 +35,33 @@ benchmarks :: IO [Benchmark]

3435benchmarks = do

3536 sink <- openFile "/dev/null" WriteMode

3637 hSetEncoding sink utf8

37− sequence

38+

39+ -- Traditional benchmarks

40+ bs <- sequence

3841 [ Builder.benchmark

39− , CaseMap.benchmark (tf "russian.txt") sink

40− , Cut.benchmark (tf "russian.txt") sink 30 60

4142 , DecodeUtf8.benchmark (tf "russian.txt")

42− , EncodeUtf8.benchmark sink "επανάληψη 竺法蘭共譯"

43+ , EncodeUtf8.benchmark "επανάληψη 竺法蘭共譯"

4344 , Equality.benchmark (tf "japanese.txt")

44− , FileIndices.benchmark (tf "russian.txt") "принимая"

4545 , FileRead.benchmark (tf "russian.txt")

4646 , FoldLines.benchmark (tf "russian.txt")

47− , HtmlCombinator.benchmark sink

48− , Ordering.benchmark (tf "russian.txt")

4947 , Pure.benchmark (tf "japanese.txt")

5048 , ReadNumbers.benchmark (tf "numbers.txt")

51− , Replace.benchmark (tf "russian.txt") sink "принимая" "своем"

52− , Sort.benchmark (tf "russian.txt") sink

53− , StripBrackets.benchmark (tf "russian.txt") sink

54− , WordCount.benchmark (tf "russian.txt")

49+ , Replace.benchmark (tf "russian.txt") "принимая" "своем"

50+ , Search.benchmark (tf "russian.txt") "принимая"

51+ , WordFrequencies.benchmark (tf "russian.txt")

5552 ]

53+

54+ -- Program-like benchmarks

55+ ps <- bgroup "Programs" `fmap` sequence

56+ [ Programs.BigTable.benchmark sink

57+ , Programs.Cut.benchmark (tf "russian.txt") sink 20 40

58+ , Programs.Fold.benchmark (tf "russian.txt") sink

59+ , Programs.Sort.benchmark (tf "russian.txt") sink

60+ , Programs.StripTags.benchmark (tf "yiwiki.xml") sink

61+ , Programs.Throughput.benchmark (tf "russian.txt") sink

62+ ]

63+

64+ return $ bs ++ [ps]

5665 where

5766 -- Location of a test file

5867 tf = ("../text/test" </>)

tests/benchmarks/src/Data/Text/Benchmarks/Builder.hs+4

@@ -1,5 +1,9 @@

11-- | Testing the internal builder monoid

22--

3+-- Tested in this benchmark:

4+--

5+-- * Concatenating many small strings using a builder

6+--

37{-# LANGUAGE OverloadedStrings #-}

48module Data.Text.Benchmarks.Builder

59 ( benchmark

tests/benchmarks/src/Data/Text/Benchmarks/CaseMap.hsdeleted−15

@@ -1,15 +0,0 @@

1−-- | This benchmark converts a number of UTF-8 encoded files to uppercase

2−--

3−module Data.Text.Benchmarks.CaseMap

4− ( benchmark

5− ) where

6−

7−import Criterion (Benchmark, bench)

8−import System.IO (Handle)

9−import qualified Data.ByteString as B

10−import qualified Data.Text as T

11−import qualified Data.Text.Encoding as T

12−

13−benchmark :: FilePath -> Handle -> IO Benchmark

14−benchmark fp sink = return $ bench "CaseMap" $

15− B.readFile fp >>= B.hPutStr sink . T.encodeUtf8 . T.toUpper . T.decodeUtf8

tests/benchmarks/src/Data/Text/Benchmarks/DecodeUtf8.hs+30−82

@@ -1,95 +1,43 @@

1+-- | Test decoding of UTF-8

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Decoding bytes using UTF-8

6+--

7+-- In some tests:

8+--

9+-- * Taking the length of the result

10+--

11+-- * Taking the init of the result

12+--

13+-- The latter are used for testing stream fusion.

14+--

115module Data.Text.Benchmarks.DecodeUtf8

216 ( benchmark

317 ) where

418

5−import Control.DeepSeq (rnf)

6−import Criterion (Benchmark, bgroup, bench)

7−import System.IO (IOMode (ReadMode), openFile, hGetContents, hSetEncoding, utf8)

19+import Criterion (Benchmark, bgroup, bench, nf)

820import qualified Codec.Binary.UTF8.Generic as U8

921import qualified Data.ByteString as B

1022import qualified Data.ByteString.Lazy as BL

1123import qualified Data.Text as T

1224import qualified Data.Text.Encoding as T

13−import qualified Data.Text.IO as T

1425import qualified Data.Text.Lazy as TL

1526import qualified Data.Text.Lazy.Encoding as TL

16−import qualified Data.Text.Lazy.IO as TL

1727

1828benchmark :: FilePath -> IO Benchmark

19−benchmark fp = return $ bgroup "DecodeUtf8"

20− [ bench "Strict" $ do

21− bs <- B.readFile fp

22− rnf (T.decodeUtf8 bs) `seq` return ()

23−

24− , bench "StrictLength" $ do

25− bs <- B.readFile fp

26− rnf (T.length $ T.decodeUtf8 bs) `seq` return ()

27−

28− , bench "StrictInitLength" $ do

29− bs <- B.readFile fp

30− rnf (T.length $ T.init $ T.decodeUtf8 bs) `seq` return ()

31−

32− , bench "StrictIO" $ do

33− h <- openFile fp ReadMode

34− hSetEncoding h utf8

35− t <- T.hGetContents h

36− rnf t `seq` return ()

37−

38− , bench "StrictLengthIO" $ do

39− h <- openFile fp ReadMode

40− hSetEncoding h utf8

41− t <- T.hGetContents h

42− rnf (T.length t) `seq` return ()

43−

44− , bench "Lazy" $ do

45− bs <- BL.readFile fp

46− rnf (TL.decodeUtf8 bs) `seq` return ()

47−

48− , bench "LazyLength" $ do

49− bs <- BL.readFile fp

50− rnf (TL.length $ TL.decodeUtf8 bs) `seq` return ()

51−

52− , bench "LazyInitLength" $ do

53− bs <- BL.readFile fp

54− rnf (TL.length $ TL.init $ TL.decodeUtf8 bs) `seq` return ()

55−

56− , bench "LazyIO" $ do

57− h <- openFile fp ReadMode

58− hSetEncoding h utf8

59− t <- TL.hGetContents h

60− rnf t `seq` return ()

61−

62− , bench "LazyLengthIO" $ do

63− h <- openFile fp ReadMode

64− hSetEncoding h utf8

65− t <- TL.hGetContents h

66− rnf (TL.length t) `seq` return ()

67−

68− , bench "String" $ do

69− h <- openFile fp ReadMode

70− hSetEncoding h utf8

71− t <- hGetContents h

72− rnf t `seq` return ()

73−

74− , bench "StringLength" $ do

75− h <- openFile fp ReadMode

76− hSetEncoding h utf8

77− t <- hGetContents h

78− rnf (length t) `seq` return ()

79−

80− , bench "LazyStringUtf8" $ do

81− s <- U8.toString `fmap` BL.readFile fp

82− rnf s `seq` return ()

83−

84− , bench "LazyStringUtf8Length" $ do

85− s <- U8.toString `fmap` BL.readFile fp

86− rnf (length s) `seq` return ()

87−

88− , bench "StrictStringUtf8" $ do

89− s <- U8.toString `fmap` B.readFile fp

90− rnf s `seq` return ()

91−

92− , bench "StrictStringUtf8Length" $ do

93− s <- U8.toString `fmap` B.readFile fp

94− rnf (length s) `seq` return ()

95− ]

29+benchmark fp = do

30+ bs <- B.readFile fp

31+ lbs <- BL.readFile fp

32+ return $ bgroup "DecodeUtf8"

33+ [ bench "Strict" $ nf T.decodeUtf8 bs

34+ , bench "StrictLength" $ nf (T.length . T.decodeUtf8) bs

35+ , bench "StrictInitLength" $ nf (T.length . T.init . T.decodeUtf8) bs

36+ , bench "Lazy" $ nf TL.decodeUtf8 lbs

37+ , bench "LazyLength" $ nf (TL.length . TL.decodeUtf8) lbs

38+ , bench "LazyInitLength" $ nf (TL.length . TL.init . TL.decodeUtf8) lbs

39+ , bench "StrictStringUtf8" $ nf U8.toString bs

40+ , bench "StrictStringUtf8Length" $ nf (length . U8.toString) bs

41+ , bench "LazyStringUtf8" $ nf U8.toString lbs

42+ , bench "LazyStringUtf8Length" $ nf (length . U8.toString) lbs

43+ ]

tests/benchmarks/src/Data/Text/Benchmarks/EncodeUtf8.hs+17−17

@@ -1,33 +1,33 @@

1+-- | UTF-8 encode a text

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Replicating a string a number of times

6+--

7+-- * UTF-8 encoding it

8+--

19module Data.Text.Benchmarks.EncodeUtf8

210 ( benchmark

311 ) where

412

5−import Criterion (Benchmark, bgroup, bench)

6−import System.IO (Handle, hPutStr)

13+import Criterion (Benchmark, bgroup, bench, whnf)

714import qualified Data.ByteString as B

815import qualified Data.ByteString.Lazy as BL

916import qualified Data.Text as T

1017import qualified Data.Text.Encoding as T

11−import qualified Data.Text.IO as T

1218import qualified Data.Text.Lazy as TL

1319import qualified Data.Text.Lazy.Encoding as TL

14−import qualified Data.Text.Lazy.IO as TL

1520

16−benchmark :: Handle -> String -> IO Benchmark

17−benchmark sink string = return $ bgroup "EncodeUtf8"

18− [ bench "String" $ hPutStr sink $ concat $ replicate k string

19− , bench "Text" $ T.hPutStr sink $ T.replicate k text

20− , bench "TextLazy" $ TL.hPutStr sink $

21− TL.replicate (fromIntegral k) lazyText

22− , bench "TextByteString" $ B.hPutStr sink $

23− T.encodeUtf8 $ T.replicate k text

24− , bench "TextByteStringLazy" $ BL.hPutStr sink $

25− TL.encodeUtf8 $ TL.replicate (fromIntegral k) lazyText

26− ]

21+benchmark :: String -> IO Benchmark

22+benchmark string = do

23+ return $ bgroup "EncodeUtf8"

24+ [ bench "Text" $ whnf (B.length . T.encodeUtf8) text

25+ , bench "LazyText" $ whnf (BL.length . TL.encodeUtf8) lazyText

26+ ]

2727 where

2828 -- The string in different formats

29− text = T.pack string

30− lazyText = TL.pack string

29+ text = T.replicate k $ T.pack string

30+ lazyText = TL.replicate (fromIntegral k) $ TL.pack string

3131

3232 -- Amount

3333 k = 100000

tests/benchmarks/src/Data/Text/Benchmarks/Equality.hs+5−2

@@ -1,9 +1,12 @@

11-- | Compare a string with a copy of itself that is identical except

22-- for the last character.

33--

4+-- Tested in this benchmark:

5+--

6+-- * Comparison of strings (Eq instance)

7+--

48module Data.Text.Benchmarks.Equality

5− (

6− benchmark

9+ ( benchmark

710 ) where

811

912import Criterion (Benchmark, bgroup, bench, whnf)

tests/benchmarks/src/Data/Text/Benchmarks/FileIndices.hsdeleted−28

@@ -1,28 +0,0 @@

1−-- | Search for a pattern in a file, find the number of occurences

2−--

3−module Data.Text.Benchmarks.FileIndices

4− ( benchmark

5− ) where

6−

7−import Control.Exception (evaluate)

8−import Criterion (Benchmark, bench, bgroup)

9−import qualified Data.ByteString as B

10−import qualified Data.ByteString.Lazy as BL

11−import qualified Data.ByteString.Lazy.Search as BL

12−import qualified Data.Text.Lazy as TL

13−import qualified Data.Text.Lazy.Encoding as TL

14−import qualified Data.Text.Lazy.IO as TL

15−

16−benchmark :: FilePath -> TL.Text -> IO Benchmark

17−benchmark fp t = return $ bgroup "FileIndices"

18− [ bench "LazyText" $ TL.readFile fp >>= evaluate . text t

19− , bench "LazyByteString" $ BL.readFile fp >>= evaluate . byteString b

20− ]

21− where

22− b = B.concat $ BL.toChunks $ TL.encodeUtf8 t

23−

24−text :: TL.Text -> TL.Text -> Int

25−text needle = fromIntegral . TL.count needle

26−

27−byteString :: B.ByteString -> BL.ByteString -> Int

28−byteString needle = length . BL.indices needle

tests/benchmarks/src/Data/Text/Benchmarks/FileRead.hs+4

@@ -1,5 +1,9 @@

11-- | Benchmarks simple file reading

22--

3+-- Tested in this benchmark:

4+--

5+-- * Reading a file from the disk

6+--

37module Data.Text.Benchmarks.FileRead

48 ( benchmark

59 ) where

tests/benchmarks/src/Data/Text/Benchmarks/FoldLines.hs+9−4

@@ -1,4 +1,9 @@

1−-- | Fold over the lines in a file

1+-- | Read a file line-by-line using handles, and perform a fold over the lines.

2+-- The fold is used here to calculate the number of lines in the file.

3+--

4+-- Tested in this benchmark:

5+--

6+-- * Buffered, line-based IO

27--

38{-# LANGUAGE BangPatterns #-}

49module Data.Text.Benchmarks.FoldLines

@@ -12,8 +17,8 @@ import qualified Data.Text as T

1217import qualified Data.Text.IO as T

1318

1419benchmark :: FilePath -> IO Benchmark

15−benchmark fp = return $ bgroup "FoldLines"

16− [ bench "Text" $ withHandle $ foldLinesT (\n _ -> n + 1) (0 :: Int)

20+benchmark fp = return $ bgroup "ReadLines"

21+ [ bench "Text" $ withHandle $ foldLinesT (\n _ -> n + 1) (0 :: Int)

1722 , bench "ByteString" $ withHandle $ foldLinesB (\n _ -> n + 1) (0 :: Int)

1823 ]

1924 where

@@ -24,7 +29,7 @@ benchmark fp = return $ bgroup "FoldLines"

2429 hClose h

2530 return x

2631

27−-- Text line fold

32+-- | Text line fold

2833--

2934foldLinesT :: (a -> T.Text -> a) -> a -> Handle -> IO a

3035foldLinesT f z0 h = go z0

tests/benchmarks/src/Data/Text/Benchmarks/Ordering.hsdeleted−58

@@ -1,58 +0,0 @@

1−-- | For every 1000th line of a file, check how many lines in the file are

2−-- lexicographically smaller

3−--

4−module Data.Text.Benchmarks.Ordering

5− ( benchmark

6− ) where

7−

8−import Control.Exception (evaluate)

9−import Criterion (Benchmark, bgroup, bench)

10−import qualified Data.ByteString.Char8 as B

11−import qualified Data.ByteString.Lazy.Char8 as BL

12−import qualified Data.Text as T

13−import qualified Data.Text.Encoding as T

14−import qualified Data.Text.Lazy as TL

15−import qualified Data.Text.Lazy.Encoding as TL

16−

17−benchmark :: FilePath -> IO Benchmark

18−benchmark fp = return $ bgroup "Ordering"

19− [ bench "ByteString" $ B.readFile fp >>=

20− evaluate . numSmaller . B.lines

21− , bench "LazyByteString" $ BL.readFile fp >>=

22− evaluate . numSmaller . BL.lines

23−

24− , bench "Text" $ B.readFile fp >>=

25− evaluate . numSmaller . T.lines . T.decodeUtf8

26− , bench "TextFusion" $ B.readFile fp >>=

27− evaluate . numSmallerFusion . T.lines . T.decodeUtf8

28− , bench "LazyText" $ BL.readFile fp >>=

29− evaluate . numSmaller . TL.lines . TL.decodeUtf8

30−

31− , bench "String" $ readFile fp >>=

32− evaluate . numSmaller . lines

33− ]

34−

35−-- | Take every Nth item from a list

36−--

37−every :: Int -> [a] -> [a]

38−every k = go k

39− where

40− go n (x : xs)

41− | n < k = go (n+1) xs

42− | otherwise = x : go 1 xs

43− go _ _ = []

44−

45−-- | Benchmark logic

46−--

47−numSmaller :: (Ord a) => [a] -> Int

48−numSmaller ls = sum . map f $ every 1000 ls

49− where

50− f x = length . filter ((== GT) . compare x) $ ls

51−

52−-- | Test a comparison that could be fused: compare (toLower a) (toLower b)

53−-- Currently, this funcion performs very badly!

54−--

55−numSmallerFusion :: [T.Text] -> Int

56−numSmallerFusion ls = sum . map f $ every 1000 ls

57− where

58− f x = length . filter ((== GT) . compare (T.toLower x) . T.toLower) $ ls

tests/benchmarks/src/Data/Text/Benchmarks/HtmlCombinator.hs → tests/benchmarks/src/Data/Text/Benchmarks/Programs/BigTable.hsrenamed+8−2

@@ -1,7 +1,13 @@

11-- | Create a large HTML table and dump it to a handle

22--

3+-- Tested in this benchmark:

4+--

5+-- * Creating a large HTML document using a builder

6+--

7+-- * Writing to a handle

8+--

39{-# LANGUAGE OverloadedStrings #-}

4−module Data.Text.Benchmarks.HtmlCombinator

10+module Data.Text.Benchmarks.Programs.BigTable

511 ( benchmark

612 ) where

713

@@ -13,7 +19,7 @@ import System.IO (Handle)

1319import qualified Data.Text as T

1420

1521benchmark :: Handle -> IO Benchmark

16−benchmark sink = return $ bench "HtmlCombinator" $ do

22+benchmark sink = return $ bench "BigTable" $ do

1723 hPutStr sink "Content-Type: text/html\n\n<table>"

1824 hPutStr sink . toLazyText . makeTable =<< rows

1925 hPutStr sink "</table>"

tests/benchmarks/src/Data/Text/Benchmarks/Cut.hs → tests/benchmarks/src/Data/Text/Benchmarks/Programs/Cut.hsrenamed+14−2

@@ -1,6 +1,18 @@

1−-- | Cut into a file, selecting certain columns (e.g. lines 10 to 40)

1+-- | Cut into a file, selecting certain columns (e.g. columns 10 to 40)

22--

3−module Data.Text.Benchmarks.Cut

3+-- Tested in this benchmark:

4+--

5+-- * Reading the file

6+--

7+-- * Splitting into lines

8+--

9+-- * Taking a number of characters from the lines

10+--

11+-- * Joining the lines

12+--

13+-- * Writing back to a handle

14+--

15+module Data.Text.Benchmarks.Programs.Cut

416 ( benchmark

517 ) where

618

tests/benchmarks/src/Data/Text/Benchmarks/Programs/Fold.hsnew file+68

@@ -0,0 +1,68 @@

1+-- | Benchmark which formats paragraph, like the @sort@ unix utility.

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Reading the file

6+--

7+-- * Splitting into paragraphs

8+--

9+-- * Reformatting the paragraphs to a certain line width

10+--

11+-- * Concatenating the results using the text builder

12+--

13+-- * Writing back to a handle

14+--

15+{-# LANGUAGE OverloadedStrings #-}

16+module Data.Text.Benchmarks.Programs.Fold

17+ ( benchmark

18+ ) where

19+

20+import Data.List (foldl')

21+import Data.List (intersperse)

22+import Data.Monoid (mempty, mappend, mconcat)

23+import System.IO (Handle)

24+import Criterion (Benchmark, bench)

25+import qualified Data.Text as T

26+import qualified Data.Text.IO as T

27+import qualified Data.Text.Lazy.Builder as TLB

28+import qualified Data.Text.Lazy as TL

29+import qualified Data.Text.Lazy.IO as TL

30+

31+benchmark :: FilePath -> Handle -> IO Benchmark

32+benchmark i o = return $

33+ bench "Fold" $ T.readFile i >>= TL.hPutStr o . fold 80

34+

35+-- | We represent a paragraph by a word list

36+--

37+type Paragraph = [T.Text]

38+

39+-- | Fold a text

40+--

41+fold :: Int -> T.Text -> TL.Text

42+fold maxWidth = TLB.toLazyText . mconcat .

43+ intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs

44+

45+-- | Fold a paragraph

46+--

47+foldParagraph :: Int -> Paragraph -> TLB.Builder

48+foldParagraph _ [] = mempty

49+foldParagraph max' (w : ws) = fst $ foldl' go (TLB.fromText w, T.length w) ws

50+ where

51+ go (builder, width) word

52+ | width + len + 1 <= max' =

53+ (builder `mappend` " " `mappend` word', width + len + 1)

54+ | otherwise =

55+ (builder `mappend` "\n" `mappend` word', len)

56+ where

57+ word' = TLB.fromText word

58+ len = T.length word

59+

60+-- | Divide a text into paragraphs

61+--

62+paragraphs :: T.Text -> [Paragraph]

63+paragraphs = splitParagraphs . map T.words . T.lines

64+ where

65+ splitParagraphs ls = case break null ls of

66+ ([], []) -> []

67+ (p, []) -> [concat p]

68+ (p, lr) -> concat p : splitParagraphs (dropWhile null lr)

tests/benchmarks/src/Data/Text/Benchmarks/Programs/Sort.hsnew file+63

@@ -0,0 +1,63 @@

1+-- | This benchmark sorts the lines of a file, like the @sort@ unix utility.

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Reading the file

6+--

7+-- * Splitting into lines

8+--

9+-- * Sorting the lines

10+--

11+-- * Joining the lines

12+--

13+-- * Writing back to a handle

14+--

15+{-# LANGUAGE OverloadedStrings #-}

16+module Data.Text.Benchmarks.Programs.Sort

17+ ( benchmark

18+ ) where

19+

20+import Criterion (Benchmark, bgroup, bench)

21+import Data.Monoid (mconcat)

22+import System.IO (Handle, hPutStr)

23+import qualified Data.ByteString as B

24+import qualified Data.ByteString.Lazy as BL

25+import qualified Data.ByteString.Lazy.Char8 as BLC

26+import qualified Data.ByteString.Char8 as BC

27+import qualified Data.List as L

28+import qualified Data.Text as T

29+import qualified Data.Text.IO as T

30+import qualified Data.Text.Lazy as TL

31+import qualified Data.Text.Lazy.IO as TL

32+import qualified Data.Text.Lazy.Builder as TLB

33+

34+benchmark :: FilePath -> Handle -> IO Benchmark

35+benchmark i o = return $ bgroup "Sort"

36+ [ bench "String" $ readFile i >>= hPutStr o . string

37+ , bench "ByteString" $ B.readFile i >>= B.hPutStr o . byteString

38+ , bench "LazyByteString" $ BL.readFile i >>= BL.hPutStr o . lazyByteString

39+ , bench "Text" $ T.readFile i >>= T.hPutStr o . text

40+ , bench "LazyText" $ TL.readFile i >>= TL.hPutStr o . lazyText

41+ , bench "TextBuilder" $ T.readFile i >>= TL.hPutStr o . textBuilder

42+ ]

43+

44+string :: String -> String

45+string = unlines . L.sort . lines

46+

47+byteString :: B.ByteString -> B.ByteString

48+byteString = BC.unlines . L.sort . BC.lines

49+

50+lazyByteString :: BL.ByteString -> BL.ByteString

51+lazyByteString = BLC.unlines . L.sort . BLC.lines

52+

53+text :: T.Text -> T.Text

54+text = T.unlines . L.sort . T.lines

55+

56+lazyText :: TL.Text -> TL.Text

57+lazyText = TL.unlines . L.sort . TL.lines

58+

59+-- | Text variant using a builder monoid for the final concatenation

60+--

61+textBuilder :: T.Text -> TL.Text

62+textBuilder = TLB.toLazyText . mconcat . L.intersperse (TLB.singleton '\n') .

63+ map TLB.fromText . L.sort . T.lines

tests/benchmarks/src/Data/Text/Benchmarks/Programs/StripTags.hsnew file+51

@@ -0,0 +1,51 @@

1+-- | Program to replace HTML tags by whitespace

2+--

3+-- This program was originally contributed by Petr Prokhorenkov.

4+--

5+-- Tested in this benchmark:

6+--

7+-- * Reading the file

8+--

9+-- * Replacing text between HTML tags (<>) with whitespace

10+--

11+-- * Writing back to a handle

12+--

13+{-# OPTIONS_GHC -fspec-constr-count=5 #-}

14+module Data.Text.Benchmarks.Programs.StripTags

15+ ( benchmark

16+ ) where

17+

18+import Criterion (Benchmark, bgroup, bench)

19+import Data.List (mapAccumL)

20+import System.IO (Handle, hPutStr)

21+import qualified Data.ByteString as B

22+import qualified Data.ByteString.Char8 as BC

23+import qualified Data.Text as T

24+import qualified Data.Text.Encoding as T

25+

26+benchmark :: FilePath -> Handle -> IO Benchmark

27+benchmark i o = return $ bgroup "StripTags"

28+ [ bench "String" $ readFile i >>= hPutStr o . string

29+ , bench "ByteString" $ B.readFile i >>= B.hPutStr o . byteString

30+ , bench "Text" $

31+ B.readFile i >>= B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8

32+ ]

33+

34+string :: String -> String

35+string = snd . mapAccumL step 0

36+

37+text :: T.Text -> T.Text

38+text = snd . T.mapAccumL step 0

39+

40+byteString :: B.ByteString -> B.ByteString

41+byteString = snd . BC.mapAccumL step 0

42+

43+step :: Int -> Char -> (Int, Char)

44+step d c

45+ | d > 0 || d' > 0 = (d', ' ')

46+ | otherwise = (d', c)

47+ where

48+ d' = d + depth c

49+ depth '>' = 1

50+ depth '<' = -1

51+ depth _ = 0

tests/benchmarks/src/Data/Text/Benchmarks/Programs/Throughput.hsnew file+35

@@ -0,0 +1,35 @@

1+-- | This benchmark simply reads and writes a file using the various string

2+-- libraries. The point of it is that we can make better estimations on how

3+-- much time the other benchmarks spend doing IO.

4+--

5+-- Note that we expect ByteStrings to be a whole lot faster, since they do not

6+-- do any actual encoding/decoding here, while String and Text do have UTF-8

7+-- encoding/decoding.

8+--

9+-- Tested in this benchmark:

10+--

11+-- * Reading the file

12+--

13+-- * Replacing text between HTML tags (<>) with whitespace

14+--

15+-- * Writing back to a handle

16+--

17+module Data.Text.Benchmarks.Programs.Throughput

18+ ( benchmark

19+ ) where

20+

21+import Criterion (Benchmark, bgroup, bench)

22+import System.IO (Handle, hPutStr)

23+import qualified Data.ByteString as B

24+import qualified Data.ByteString.Lazy as BL

25+import qualified Data.Text.IO as T

26+import qualified Data.Text.Lazy.IO as TL

27+

28+benchmark :: FilePath -> Handle -> IO Benchmark

29+benchmark fp sink = return $ bgroup "Throughput"

30+ [ bench "String" $ readFile fp >>= hPutStr sink

31+ , bench "ByteString" $ B.readFile fp >>= B.hPutStr sink

32+ , bench "LazyByteString" $ BL.readFile fp >>= BL.hPutStr sink

33+ , bench "Text" $ T.readFile fp >>= T.hPutStr sink

34+ , bench "LazyText" $ TL.readFile fp >>= TL.hPutStr sink

35+ ]

tests/benchmarks/src/Data/Text/Benchmarks/Pure.hs+4

@@ -1,5 +1,9 @@

11-- | Benchmarks various pure functions from the Text library

22--

3+-- Tested in this benchmark:

4+--

5+-- * Most pure functions defined the string types

6+--

37{-# LANGUAGE BangPatterns, GADTs, MagicHash #-}

48{-# OPTIONS_GHC -fno-warn-orphans #-}

59module Data.Text.Benchmarks.Pure

tests/benchmarks/src/Data/Text/Benchmarks/ReadNumbers.hs+50−37

@@ -1,12 +1,26 @@

11-- | Read numbers from a file with a just a number on each line, find the

2−-- minimum of those numbers.

2+-- minimum of those numbers. The file contains different kinds of numbers:

3+--

4+-- * Decimals

5+--

6+-- * Hexadecimals

7+--

8+-- * Floating point numbers

9+--

10+-- * Floating point numbers in scientific notation

11+--

12+-- The different benchmarks will only take into account the values they can

13+-- parse.

14+--

15+-- Tested in this benchmark:

16+--

17+-- * Lexing/parsing of different numerical types

318--

419module Data.Text.Benchmarks.ReadNumbers

520 ( benchmark

621 ) where

722

8−import Control.Exception (evaluate)

9−import Criterion (Benchmark, bgroup, bench)

23+import Criterion (Benchmark, bgroup, bench, whnf)

1024import Data.List (foldl')

1125import Numeric (readDec, readFloat, readHex)

1226import qualified Data.ByteString.Char8 as B

@@ -14,49 +28,48 @@ import qualified Data.ByteString.Lazy.Char8 as BL

1428import qualified Data.ByteString.Lex.Double as B

1529import qualified Data.ByteString.Lex.Lazy.Double as BL

1630import qualified Data.Text as T

17−import qualified Data.Text.Encoding as T

31+import qualified Data.Text.IO as T

1832import qualified Data.Text.Lazy as TL

19−import qualified Data.Text.Lazy.Encoding as TL

33+import qualified Data.Text.Lazy.IO as TL

2034import qualified Data.Text.Lazy.Read as TL

2135import qualified Data.Text.Read as T

2236

2337benchmark :: FilePath -> IO Benchmark

24−benchmark fp = return $ bgroup "ReadNumbers"

25− [ bench "DecimalString" $ readFile fp >>= evaluate .

26− int . string readDec . lines

27− , bench "HexadecimalString" $ readFile fp >>= evaluate .

28− int . string readHex . lines

29− , bench "DoubleString" $ readFile fp >>= evaluate .

30− double . string readFloat . lines

38+benchmark fp = do

39+ -- Read all files into lines: string, text, lazy text, bytestring, lazy

40+ -- bytestring

41+ s <- lines `fmap` readFile fp

42+ t <- T.lines `fmap` T.readFile fp

43+ tl <- TL.lines `fmap` TL.readFile fp

44+ b <- B.lines `fmap` B.readFile fp

45+ bl <- BL.lines `fmap` BL.readFile fp

46+ return $ bgroup "ReadNumbers"

47+ [ bench "DecimalString" $ whnf (int . string readDec) s

48+ , bench "HexadecimalString" $ whnf (int . string readHex) s

49+ , bench "DoubleString" $ whnf (double . string readFloat) s

3150

32− , bench "DecimalText" $ B.readFile fp >>= evaluate .

33− int . text (T.signed T.decimal) . T.lines . T.decodeUtf8

34− , bench "HexadecimalText" $ B.readFile fp >>= evaluate .

35− int . text (T.signed T.hexadecimal) . T.lines . T.decodeUtf8

36− , bench "DoubleText" $ B.readFile fp >>= evaluate .

37− double . text T.double . T.lines . T.decodeUtf8

38− , bench "RationalText" $ B.readFile fp >>= evaluate .

39− double . text T.rational . T.lines . T.decodeUtf8

51+ , bench "DecimalText" $ whnf (int . text (T.signed T.decimal)) t

52+ , bench "HexadecimalText" $ whnf (int . text (T.signed T.hexadecimal)) t

53+ , bench "DoubleText" $ whnf (double . text T.double) t

54+ , bench "RationalText" $ whnf (double . text T.rational) t

4055

41− , bench "DecimalLazyText" $ BL.readFile fp >>= evaluate .

42− int . text (TL.signed TL.decimal) . TL.lines . TL.decodeUtf8

43− , bench "HexadecimalLazyText" $ BL.readFile fp >>= evaluate .

44− int . text (TL.signed TL.hexadecimal) . TL.lines . TL.decodeUtf8

45− , bench "DoubleLazyText" $ BL.readFile fp >>= evaluate .

46− double . text TL.double . TL.lines . TL.decodeUtf8

47− , bench "RationalLazyText" $ BL.readFile fp >>= evaluate .

48− double . text TL.rational . TL.lines . TL.decodeUtf8

56+ , bench "DecimalLazyText" $

57+ whnf (int . text (TL.signed TL.decimal)) tl

58+ , bench "HexadecimalLazyText" $

59+ whnf (int . text (TL.signed TL.hexadecimal)) tl

60+ , bench "DoubleLazyText" $

61+ whnf (double . text TL.double) tl

62+ , bench "RationalLazyText" $

63+ whnf (double . text TL.rational) tl

4964

50− , bench "DecimalByteString" $ B.readFile fp >>= evaluate .

51− int . byteString B.readInt . B.lines

52− , bench "DoubleByteString" $ B.readFile fp >>= evaluate .

53− double . byteString B.readDouble . B.lines

65+ , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b

66+ , bench "DoubleByteString" $ whnf (double . byteString B.readDouble) b

5467

55− , bench "DecimalLazyByteString" $ BL.readFile fp >>= evaluate .

56− int . byteString BL.readInt . BL.lines

57− , bench "DoubleLazyByteString" $ BL.readFile fp >>= evaluate .

58− double . byteString BL.readDouble . BL.lines

59− ]

68+ , bench "DecimalLazyByteString" $

69+ whnf (int . byteString BL.readInt) bl

70+ , bench "DoubleLazyByteString" $

71+ whnf (double . byteString BL.readDouble) bl

72+ ]

6073 where

6174 -- Used for fixing types

6275 int :: Int -> Int

tests/benchmarks/src/Data/Text/Benchmarks/Replace.hs+14−16

@@ -1,11 +1,14 @@

1−-- | Replace a string by another string in a file

1+-- | Replace a string by another string

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Search and replace of a pattern in a text

26--

37module Data.Text.Benchmarks.Replace

48 ( benchmark

59 ) where

610

7−import Criterion (Benchmark, bgroup, bench)

8−import System.IO (Handle)

11+import Criterion (Benchmark, bgroup, bench, nf)

912import qualified Data.ByteString.Char8 as B

1013import qualified Data.ByteString.Lazy as BL

1114import qualified Data.ByteString.Lazy.Search as BL

@@ -13,19 +16,14 @@ import qualified Data.Text.Lazy as TL

1316import qualified Data.Text.Lazy.Encoding as TL

1417import qualified Data.Text.Lazy.IO as TL

1518

16−benchmark :: FilePath -> Handle -> String -> String -> IO Benchmark

17−benchmark fp sink pat sub = return $ bgroup "Replace"

18− -- We have benchmarks for lazy text and lazy bytestrings. We also benchmark

19− -- without the acual replacement, so we can get an idea of what time is

20− -- spent on IO and computations.

21− [ bench "LazyText" $ TL.readFile fp >>=

22− TL.hPutStr sink . TL.replace tpat tsub

23− , bench "LazyTextNull" $ TL.readFile fp >>= TL.hPutStr sink

24−

25− , bench "LazyByteString" $ BL.readFile fp >>=

26− BL.hPutStr sink . BL.replace bpat bsub

27− , bench "LazyByteStringNull" $ BL.readFile fp >>= BL.hPutStr sink

28− ]

19+benchmark :: FilePath -> String -> String -> IO Benchmark

20+benchmark fp pat sub = do

21+ tl <- TL.readFile fp

22+ bl <- BL.readFile fp

23+ return $ bgroup "Replace"

24+ [ bench "LazyText" $ nf (TL.length . TL.replace tpat tsub) tl

25+ , bench "LazyByteString" $ nf (BL.length . BL.replace bpat bsub) bl

26+ ]

2927 where

3028 tpat = TL.pack pat

3129 tsub = TL.pack sub

tests/benchmarks/src/Data/Text/Benchmarks/Search.hsnew file+48

@@ -0,0 +1,48 @@

1+-- | Search for a pattern in a file, find the number of occurences

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Searching all occurences of a pattern using library routines

6+--

7+module Data.Text.Benchmarks.Search

8+ ( benchmark

9+ ) where

10+

11+import Criterion (Benchmark, bench, bgroup, whnf)

12+import qualified Data.ByteString as B

13+import qualified Data.ByteString.Lazy as BL

14+import qualified Data.ByteString.Lazy.Search as BL

15+import qualified Data.ByteString.Search as B

16+import qualified Data.Text as T

17+import qualified Data.Text.Encoding as T

18+import qualified Data.Text.IO as T

19+import qualified Data.Text.Lazy as TL

20+import qualified Data.Text.Lazy.IO as TL

21+

22+benchmark :: FilePath -> T.Text -> IO Benchmark

23+benchmark fp needleT = do

24+ b <- B.readFile fp

25+ bl <- BL.readFile fp

26+ t <- T.readFile fp

27+ tl <- TL.readFile fp

28+ return $ bgroup "FileIndices"

29+ [ bench "ByteString" $ whnf (byteString needleB) b

30+ , bench "LazyByteString" $ whnf (lazyByteString needleB) bl

31+ , bench "Text" $ whnf (text needleT) t

32+ , bench "LazyText" $ whnf (lazyText needleTL) tl

33+ ]

34+ where

35+ needleB = T.encodeUtf8 needleT

36+ needleTL = TL.fromChunks [needleT]

37+

38+byteString :: B.ByteString -> B.ByteString -> Int

39+byteString needle = length . B.indices needle

40+

41+lazyByteString :: B.ByteString -> BL.ByteString -> Int

42+lazyByteString needle = length . BL.indices needle

43+

44+text :: T.Text -> T.Text -> Int

45+text = T.count

46+

47+lazyText :: TL.Text -> TL.Text -> Int

48+lazyText needle = fromIntegral . TL.count needle

tests/benchmarks/src/Data/Text/Benchmarks/Sort.hsdeleted−27

@@ -1,27 +0,0 @@

1−-- | Implements the unix @sort@ program

2−--

3−{-# LANGUAGE OverloadedStrings #-}

4−module Data.Text.Benchmarks.Sort

5− ( benchmark

6− ) where

7−

8−import Criterion (Benchmark, bench)

9−import Data.Monoid (mconcat)

10−import System.IO (Handle)

11−import qualified Data.ByteString as B

12−import qualified Data.ByteString.Lazy as BL

13−import qualified Data.List as L

14−import qualified Data.Text as T

15−import qualified Data.Text.Encoding as T

16−import qualified Data.Text.Lazy as TL

17−import qualified Data.Text.Lazy.Builder as TLB

18−import qualified Data.Text.Lazy.Encoding as TL

19−

20−benchmark :: FilePath -> Handle -> IO Benchmark

21−benchmark fp sink = return $ bench "Sort" $ do

22− t <- T.decodeUtf8 `fmap` B.readFile fp

23− BL.hPutStr sink $ TL.encodeUtf8 $ sort t

24−

25−sort :: T.Text -> TL.Text

26−sort = TLB.toLazyText . mconcat . L.intersperse (TLB.fromText "\n") .

27− map TLB.fromText . L.sort . T.lines

tests/benchmarks/src/Data/Text/Benchmarks/StripBrackets.hsdeleted−33

@@ -1,33 +0,0 @@

1−-- | Program to replace everything between brackets by spaces

2−--

3−-- This program was originally contributed by Petr Prokhorenkov.

4−--

5−module Data.Text.Benchmarks.StripBrackets

6− ( benchmark

7− ) where

8−

9−import Criterion (Benchmark, bench)

10−import System.IO (Handle)

11−import qualified Data.ByteString as B

12−import qualified Data.Text as T

13−import qualified Data.Text.Encoding as T

14−

15−benchmark :: FilePath -> Handle -> IO Benchmark

16−benchmark fp sink = return $ bench "StripBrackets" $ do

17− t <- T.decodeUtf8 `fmap` B.readFile fp

18− B.hPutStr sink $ T.encodeUtf8 $ stripBrackets t

19−

20−stripBrackets :: T.Text -> T.Text

21−stripBrackets = snd . T.mapAccumL f (0 :: Int)

22− where

23− f depth c =

24− let depth' = depth + d' c

25− c' | depth > 0 || depth' > 0 = ' '

26− | otherwise = c

27− in (depth', c')

28−

29− d' '{' = 1

30− d' '[' = 1

31− d' '}' = -1

32− d' ']' = -1

33− d' _ = 0

tests/benchmarks/src/Data/Text/Benchmarks/WordCount.hsdeleted−22

@@ -1,22 +0,0 @@

1−-- | A word frequence count program

2−--

3−module Data.Text.Benchmarks.WordCount

4− ( benchmark

5− ) where

6−

7−import Control.Exception (evaluate)

8−import Criterion (Benchmark, bench)

9−import Data.List (foldl')

10−import Data.Map (Map)

11−import qualified Data.Map as M

12−import qualified Data.Text as T

13−import qualified Data.Text.IO as T

14−

15−benchmark :: FilePath -> IO Benchmark

16−benchmark fp = return $ bench "WordCount" $ do

17− t <- T.readFile fp

18− evaluate $ M.size $ wordCount t

19−

20−wordCount :: T.Text -> Map T.Text Int

21−wordCount =

22− foldl' (\m k -> M.insertWith (+) k 1 m) M.empty . map T.toLower . T.words

tests/benchmarks/src/Data/Text/Benchmarks/WordFrequencies.hsnew file+36

@@ -0,0 +1,36 @@

1+-- | A word frequency count using the different string types

2+--

3+-- Tested in this benchmark:

4+--

5+-- * Splitting into words

6+--

7+-- * Converting to lowercase

8+--

9+-- * Comparing: Eq/Ord instances

10+--

11+module Data.Text.Benchmarks.WordFrequencies

12+ ( benchmark

13+ ) where

14+

15+import Criterion (Benchmark, bench, bgroup, whnf)

16+import Data.Char (toLower)

17+import Data.List (foldl')

18+import Data.Map (Map)

19+import qualified Data.ByteString.Char8 as B

20+import qualified Data.Map as M

21+import qualified Data.Text as T

22+import qualified Data.Text.IO as T

23+

24+benchmark :: FilePath -> IO Benchmark

25+benchmark fp = do

26+ s <- readFile fp

27+ b <- B.readFile fp

28+ t <- T.readFile fp

29+ return $ bgroup "WordFrequencies"

30+ [ bench "String" $ whnf (frequencies . words . map toLower) s

31+ , bench "ByteString" $ whnf (frequencies . B.words . B.map toLower) b

32+ , bench "Text" $ whnf (frequencies . T.words . T.toLower) t

33+ ]

34+

35+frequencies :: Ord a => [a] -> Map a Int

36+frequencies = foldl' (\m k -> M.insertWith (+) k 1 m) M.empty