git.iko.soy/distributed-process
on master help sign in
Merge branch 'master' of https://github.com/haskell-distributed/distributed-process
5395dd9Adam Foltzer
benchmarks/Makefile+54−12
compare with master
unifiedsplit
@@ -1,27 +1,69 @@
1
1
2+# Run 100K ping-pongs, for one trial:
3+size = 100000
4+trials = 1
5+# You can run "make run1 size=1000 trials=100" for example.
6+
7+
8+# Disable this to test installed packages only:
9+INCLUDES= -i../network-transport/src -i../network-transport-pipes/src
2
10
3ifeq ($(GHC),)
11ifeq ($(GHC),)
4 GHC=ghc
12 GHC=ghc
5endif
13endif
6
14
7−all: pipes tcp transport
15+BENCHS= PingTCP.exe PingTCPTransport.exe PingPipes.exe SendTransport.exe
8−tcp:
16+
9− $(GHC) -O2 -rtsopts -threaded --make -i../network-transport/src PingTCP.hs -o PingTCP.exe
17+all: $(BENCHS)
10−transport:
18+$(BENCHS): %.exe: %.hs
11− $(GHC) -O2 -rtsopts -threaded --make -i../network-transport/src PingTransport.hs -o PingTransport.exe
19+ $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) $< -o $@
12−pipes:
20+
13− $(GHC) -O2 -rtsopts -threaded --make -i../network-transport/src PingPipes.hs -o PingPipes.exe
21+#-------------------------------------------------------------------------------
22+# Shorthands for running benchmarks:
23+
24+run: run1 run2 run3 run4
14
25
15−run: runBaseline runTransport
26+run1: runBaseline
16runBaseline:
27runBaseline:
17 ./PingTCP.exe server 8080 &
28 ./PingTCP.exe server 8080 &
18 sleep 1
29 sleep 1
19− ./PingTCP.exe client 0.0.0.0 8080 1000
30+ time ./PingTCP.exe client 0.0.0.0 8080 $(size) $(trials)
31+ # Kill the server:
32+ killall PingTCP.exe
20
33
21−runTransport:
34+run2: runTCP
35+runTCP:
22− ./PingTransport.exe server 0.0.0.0 8080 sourceAddr &
36+ ./PingTCPTransport.exe server 0.0.0.0 8080 sourceAddr &
23 sleep 1
37 sleep 1
38+ # Run 100K ping-pongs, for one trial:
24− ./PingTransport.exe client 0.0.0.0 8081 sourceAddr 1000
39+ time ./PingTCPTransport.exe client 0.0.0.0 8081 sourceAddr $(size) $(trials)
40+ # Kill the server:
41+ killall PingTCPTransport.exe
42+
43+run3: runPipes
44+runPipes:
45+ ./PingPipes.exe server sourceAddr &
46+ sleep 1
47+ time ./PingPipes.exe client sourceAddr $(size) $(trials)
48+ # Kill the server:
49+ killall PingPipes.exe
50+
51+# This benchmark measures throughput rather than latency
52+# Here 'size' means message size rather than number of pings:
53+run4: runTPT
54+runTPT:
55+ ./SendTransport.exe server 0.0.0.0 8080 sourceAddr &
56+ sleep 1
57+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr $(size)
58+ killall SendTransport.exe
59+
60+# Run multiple sizes:
61+runTPTs:
62+ $(MAKE) runTPT size=1000
63+ $(MAKE) runTPT size=5000
64+ $(MAKE) runTPT size=10000
65+ $(MAKE) runTPT size=50000
66+ $(MAKE) runTPT size=100000
25
67
26clean:
68clean:
27 rm -f *.hi *.o *.exe
69 rm -f *.hi *.o *.exe
benchmarks/PingPipes.hs+27−9
@@ -1,16 +1,22 @@
1module Main where
1module Main where
2
2
3+-- TODO: Merge with PingTCPTransport and factor differences.
4+
3−import Network.Transport
5+import Network.Transport (newConnection, receive, connect, send,
6+ serialize, deserialize, SourceEnd, TargetEnd)
4import Network.Transport.Pipes (mkTransport)
7import Network.Transport.Pipes (mkTransport)
5
8
6import Control.Monad (forever, replicateM_)
9import Control.Monad (forever, replicateM_)
7−import Criterion.Main (Benchmark, bench, defaultMain, nfIO)
10+import Criterion.Main (Benchmark, bench, defaultMainWith, nfIO)
11+import Criterion.Config (defaultConfig, ljust, Config(cfgSamples))
8−import Data.Binary
12+-- import Data.Binary (encode,decode)
13+import Data.Serialize (encode,decode)
9import Data.Maybe (fromJust)
14import Data.Maybe (fromJust)
10import Data.Int
15import Data.Int
11import System.Environment (getArgs, withArgs)
16import System.Environment (getArgs, withArgs)
12
17
13−import qualified Data.ByteString.Lazy.Char8 as BS
18+-- import qualified Data.ByteString.Lazy.Char8 as LBS
19+import qualified Data.ByteString.Char8 as BS
14
20
15-- | This performs a ping benchmark on the TCP transport. This can be
21-- | This performs a ping benchmark on the TCP transport. This can be
16-- compiled using:
22-- compiled using:
@@ -50,7 +56,7 @@ main = do
50 forever $ pong targetEndPing sourceEndPong
56 forever $ pong targetEndPing sourceEndPong
51
57
52
58
53− "client" : sourceAddrFilePath : pingsStr : args' -> do
59+ "client" : sourceAddrFilePath : pingsStr : reps : args' -> do
54 let pings = read pingsStr
60 let pings = read pingsStr
55 -- establish transport
61 -- establish transport
56 transport <- mkTransport
62 transport <- mkTransport
@@ -64,10 +70,17 @@ main = do
64 send sourceEndPing [serialize sourceAddrPong]
70 send sourceEndPing [serialize sourceAddrPong]
65
71
66 -- benchmark the pings
72 -- benchmark the pings
73+ case (read reps) :: Int of
74+ 0 -> error "What would zero reps mean?"
75+ 1 -> do putStrLn "Because you're timing only one trial, skipping Criterion..."
76+ replicateM_ pings (ping sourceEndPing targetEndPong)
77+ n -> withArgs args' $ defaultMainWith
78+ (defaultConfig{ cfgSamples = ljust n })
79+ (return ()) -- Init action.
67−-- withArgs args' $ defaultMain [ benchPing sourceEndPing targetEndPong pings]
80+ [ benchPing sourceEndPing targetEndPong (fromIntegral pings) ]
68− replicateM_ pings (ping sourceEndPing targetEndPong)
69 putStrLn "Done with all ping/pongs."
81 putStrLn "Done with all ping/pongs."
70
82
83+
71-- | This function takes a `TargetEnd` for the pings, and a `SourceEnd` for
84-- | This function takes a `TargetEnd` for the pings, and a `SourceEnd` for
72-- pongs. Whenever a ping is received from the `TargetEnd`, a pong is sent
85-- pongs. Whenever a ping is received from the `TargetEnd`, a pong is sent
73-- in reply down the `SourceEnd`, repeating whatever was sent.
86-- in reply down the `SourceEnd`, repeating whatever was sent.
@@ -81,9 +94,14 @@ pong targetEndPing sourceEndPong = do
81-- `targetEndPong`, which is then returned.
94-- `targetEndPong`, which is then returned.
82ping :: SourceEnd -> TargetEnd -> IO Int64
95ping :: SourceEnd -> TargetEnd -> IO Int64
83ping sourceEndPing targetEndPong = do
96ping sourceEndPing targetEndPong = do
84− send sourceEndPing [BS.empty]
97+ -- send sourceEndPing [BS.empty]
98+ send sourceEndPing [encode (42 :: Int64)]
99+
85 [bs] <- receive targetEndPong
100 [bs] <- receive targetEndPong
86− return $ decode bs
101+-- return $ decode bs
102+ case decode bs of
103+ Left err -> error$ "decode failure: " ++ err
104+ Right x -> return x
87
105
88-- | The effect of `benchPing sourceEndPing targetEndPong n` is to send
106-- | The effect of `benchPing sourceEndPing targetEndPong n` is to send
89-- `n` pings down `sourceEndPing` using the `ping` function. The time
107-- `n` pings down `sourceEndPing` using the `ping` function. The time
benchmarks/PingTCP.hs+15−4
@@ -3,7 +3,8 @@
3module Main where
3module Main where
4
4
5import Control.Monad
5import Control.Monad
6−import Criterion.Main (Benchmark, bench, defaultMain, nfIO, bgroup)
6+import Criterion.Main (Benchmark, bench, defaultMainWith, nfIO)
7+import Criterion.Config (defaultConfig, ljust, Config(cfgSamples))
7
8
8import Data.Int
9import Data.Int
9import qualified Data.Serialize as Ser
10import qualified Data.Serialize as Ser
@@ -73,7 +74,7 @@ main = do
73 putStrLn "server: listening for pings"
74 putStrLn "server: listening for pings"
74 forever (pong clientSock)
75 forever (pong clientSock)
75
76
76− "client": host : service : pingsStr : args' -> withSocketsDo $ do
77+ "client": host : service : pingsStr : reps : args' -> withSocketsDo $ do
77 let pings = read pingsStr
78 let pings = read pingsStr
78 serverAddrs <- getAddrInfo
79 serverAddrs <- getAddrInfo
79 Nothing
80 Nothing
@@ -85,14 +86,24 @@ main = do
85 N.connect sock (addrAddress serverAddr)
86 N.connect sock (addrAddress serverAddr)
86
87
87 -- benchmark the pings
88 -- benchmark the pings
89+ case (read reps) :: Int of
90+ 0 -> error "What would zero reps mean?"
91+ 1 -> do putStrLn "Because you're timing only one trial, skipping Criterion..."
92+ replicateM_ pings (ping sock)
93+ n -> withArgs args' $ defaultMainWith
94+ (defaultConfig{ cfgSamples = ljust n })
95+ (return ()) -- Init action.
96+ [ benchPing sock (fromIntegral pings) ]
97+ putStrLn "Done with all ping/pongs."
98+
88− withArgs args' $ defaultMain [ benchPing sock pings ]
99+-- withArgs args' $ defaultMain [ benchPing sock pings ]
89-- replicateM_ pings (ping sock)
100-- replicateM_ pings (ping sock)
90
101
91-- | Each `ping` sends a single byte, and expects to receive one
102-- | Each `ping` sends a single byte, and expects to receive one
92-- back in return.
103-- back in return.
93ping :: Socket -> IO Int64
104ping :: Socket -> IO Int64
94ping sock = do
105ping sock = do
95− NBS.send sock $ encode (0 :: Int64)
106+ NBS.send sock $ encode (42 :: Int64)
96 bs <- NBS.recv sock 8
107 bs <- NBS.recv sock 8
97 either error return $ decode bs
108 either error return $ decode bs
98
109
benchmarks/PingTransport.hs → benchmarks/PingTCPTransport.hsrenamed+15−5
@@ -2,11 +2,14 @@
2
2
3module Main where
3module Main where
4
4
5−import Network.Transport
5+import Network.Transport (newConnection, receive, connect, send, defaultHints,
6+ serialize, deserialize, SourceEnd, TargetEnd)
6import Network.Transport.TCP (mkTransport, TCPConfig (..))
7import Network.Transport.TCP (mkTransport, TCPConfig (..))
7
8
8import Control.Monad (forever, replicateM_)
9import Control.Monad (forever, replicateM_)
9−import Criterion.Main (Benchmark, bench, defaultMain, nfIO)
10+import Criterion.Main (Benchmark, bench, defaultMainWith, nfIO)
11+import Criterion.Config (defaultConfig, ljust, Config(cfgSamples))
12+
10import qualified Data.Serialize as Ser
13import qualified Data.Serialize as Ser
11import Data.Maybe (fromJust)
14import Data.Maybe (fromJust)
12import Data.Int
15import Data.Int
@@ -68,7 +71,7 @@ main = do
68 forever $ pong targetEndPing sourceEndPong
71 forever $ pong targetEndPing sourceEndPong
69
72
70
73
71− "client" : host : service : sourceAddrFilePath : pingsStr : args' -> do
74+ "client" : host : service : sourceAddrFilePath : pingsStr : reps : args' -> do
72 let pings = read pingsStr
75 let pings = read pingsStr
73 -- establish transport
76 -- establish transport
74 transport <- mkTransport $ TCPConfig defaultHints host service
77 transport <- mkTransport $ TCPConfig defaultHints host service
@@ -82,8 +85,15 @@ main = do
82 send sourceEndPing [serialize sourceAddrPong]
85 send sourceEndPing [serialize sourceAddrPong]
83
86
84 -- benchmark the pings
87 -- benchmark the pings
88+ case (read reps) :: Int of
89+ 0 -> error "What would zero reps mean?"
90+ 1 -> do putStrLn "Because you're timing only one trial, skipping Criterion..."
91+ replicateM_ pings (ping sourceEndPing targetEndPong 42)
92+ n -> withArgs args' $ defaultMainWith
93+ (defaultConfig{ cfgSamples = ljust n })
94+ (return ()) -- Init action.
85− withArgs args' $ defaultMain [ benchPing sourceEndPing targetEndPong pings]
95+ [ benchPing sourceEndPing targetEndPong (fromIntegral pings)]
86−-- replicateM_ pings (ping sourceEndPing targetEndPong)
96+ putStrLn "Done with all ping/pongs."
87
97
88-- | This function takes a `TargetEnd` for the pings, and a `SourceEnd` for
98-- | This function takes a `TargetEnd` for the pings, and a `SourceEnd` for
89-- pongs. Whenever a ping is received from the `TargetEnd`, a pong is sent
99-- pongs. Whenever a ping is received from the `TargetEnd`, a pong is sent
benchmarks/SendTransport.hs+2−1
@@ -3,7 +3,8 @@
3
3
4module Main where
4module Main where
5
5
6−import Network.Transport
6+import Network.Transport (newConnection, receive, connect, send, defaultHints,
7+ serialize, deserialize, SourceEnd, TargetEnd)
7import Network.Transport.TCP (mkTransport, TCPConfig (..))
8import Network.Transport.TCP (mkTransport, TCPConfig (..))
8
9
9import Control.Applicative
10import Control.Applicative
benchmarks/results_notes.mdnew file+111
@@ -0,0 +1,111 @@
1+
2+
3+[2012.02.22] {File for notes on results}
4+============================================================
5+
6+Here's a run of the PingTCP benchmark on a 3.1 ghz Intel Westmere running RHEL 6.2:
7+
8+ ./PingTCP.exe server 8080 &
9+ sleep 1
10+ server: creating TCP connection
11+ server: awaiting client connection
12+ ./PingTCP.exe client 0.0.0.0 8080 1000
13+ server: listening for pings
14+ warming up
15+ estimating clock resolution...
16+ mean is 1.963031 us (320001 iterations)
17+ found 38180 outliers among 319999 samples (11.9%)
18+ 32741 (10.2%) low severe
19+ 5439 (1.7%) high severe
20+ estimating cost of a clock call...
21+ mean is 43.17696 ns (15 iterations)
22+ found 2 outliers among 15 samples (13.3%)
23+ 1 (6.7%) high mild
24+ 1 (6.7%) high severe
25+
26+ benchmarking PingTCP
27+ collecting 100 samples, 1 iterations each, in estimated 8.073997 s
28+ mean: 77.01177 ms, lb 76.92822 ms, ub 77.14009 ms, ci 0.950
29+ std dev: 526.1251 us, lb 366.2692 us, ub 803.1751 us, ci 0.950
30+
31+(That was without -O2)
32+
33+ -----
34+
35+Running a C benchmark on the same machine it takes 5.6 seconds to do
36+200K ping pongs.
37+
38+ => 28 us avg latency for C
39+
40+I got 72 us avg latency using PingTCP. The named pipes backend took
41+19.8s and was using 100% CPU. That's a 99 us latency.
42+
43+Pipes with unix-bytestring take 11.0 seconds, but still use 100% CPU.
44+
45+ => 55 us latency for Pipes with unix-bytestring
46+
47+
48+
49+[2012.02.24] {After Adam's strictness refactoring.}
50+============================================================
51+
52+100K ping/pongs, 1 trial:
53+
54+ PingTCP: 7.34s >80%CPU
55+ PingTCPTransport: 19s ~60%CPU
56+ PingPipes: 6.36s >=100%CPU
57+
58+Timing instead using 100 trials of 1000 pingpongs via criterion gives
59+results consistent with the above. Here are the mean latencies:
60+
61+ PingTCP: 72 us
62+ PingTCPTransport: 191 us
63+ PingPipes: 65 us
64+
65+And here is my first attempt at running the SendTransport benchmark (TCP backend):
66+
67+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 1000
68+ mean: 38.10901 ms, lb 35.92353 ms, ub 39.30006 ms, ci 0.950
69+ std dev: 8.305210 ms, lb 4.935312 ms, ub 11.97144 ms, ci 0.950
70+ found 5 outliers among 100 samples (5.0%)
71+ 5 (5.0%) low severe
72+ variance introduced by outliers: 94.703%
73+ variance is severely inflated by outliers
74+
75+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 5000
76+ mean: 32.87646 ms, lb 29.69947 ms, ub 35.62485 ms, ci 0.950
77+ std dev: 15.21419 ms, lb 12.39949 ms, ub 17.39441 ms, ci 0.950
78+ found 18 outliers among 100 samples (18.0%)
79+ 18 (18.0%) low severe
80+ variance introduced by outliers: 98.928%
81+ variance is severely inflated by outliers
82+
83+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 10000
84+ mean: 32.87737 ms, lb 29.34305 ms, ub 35.62680 ms, ci 0.950
85+ std dev: 15.22060 ms, lb 12.40398 ms, ub 17.57401 ms, ci 0.950
86+ found 18 outliers among 100 samples (18.0%)
87+ 18 (18.0%) low severe
88+ variance introduced by outliers: 98.929%
89+ variance is severely inflated by outliers
90+
91+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 50000
92+ mean: 1.351308 ms, lb 1.249680 ms, ub 1.548209 ms, ci 0.950
93+ std dev: 696.6768 us, lb 448.3538 us, ub 1.314057 ms, ci 0.950
94+ found 24 outliers among 100 samples (24.0%)
95+ 3 (3.0%) high mild
96+ 21 (21.0%) high severe
97+ variance introduced by outliers: 98.942%
98+ variance is severely inflated by outliers
99+
100+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 100000
101+ mean: 2.184967 ms, lb 2.030858 ms, ub 2.641543 ms, ci 0.950
102+ std dev: 1.235358 ms, lb 518.3995 us, ub 2.666540 ms, ci 0.950
103+ found 15 outliers among 100 samples (15.0%)
104+ 4 (4.0%) low severe
105+ 9 (9.0%) high severe
106+ variance introduced by outliers: 98.952%
107+ variance is severely inflated by outliers
108+
109+This is quite odd. so in addition to the high variance it would seem
110+that messages of size 50K or 100K are much FASTER to send than smaller
111+massages?
install_all.shnew file+18
@@ -0,0 +1,18 @@
1+#!/bin/bash
2+
3+set -e
4+
5+if [ "$GHC" == "" ]; then
6+ GHC=ghc
7+fi
8+
9+cd network-transport/
10+cabal.par install --with-ghc=$GHC
11+
12+echo;echo
13+cd ../network-transport-pipes/
14+cabal.par install --with-ghc=$GHC
15+
16+echo;echo
17+cd ../distributed-process/
18+cabal.par install --with-ghc=$GHC
network-transport-pipes/network-transport-pipes.cabal+5−2
@@ -1,5 +1,8 @@
1Name: network-transport-pipes
1Name: network-transport-pipes
2−Version: 0.0.1
2+Version: 0.0.1.1
3+-- Version History:
4+-- 0.0.1.1 - Initial
5+
3Description: Named Pipes backend for network-transport interface.
6Description: Named Pipes backend for network-transport interface.
4License: BSD3
7License: BSD3
5License-file: LICENSE
8License-file: LICENSE
@@ -14,7 +17,7 @@ Library
14 binary >= 0.5,
17 binary >= 0.5,
15 cereal,
18 cereal,
16 bytestring >= 0.9,
19 bytestring >= 0.9,
17− network-transport,
20+ network-transport >= 0.0.1.1,
18 containers >= 0.4,
21 containers >= 0.4,
19-- network >= 2.3,
22-- network >= 2.3,
20-- safe >= 0.3,
23-- safe >= 0.3,
network-transport/network-transport.cabal+6−1
@@ -1,5 +1,10 @@
1Name: network-transport
1Name: network-transport
2−Version: 0.0.1
2+Version: 0.0.1.1
3+
4+-- Version History:
5+-- 0.0.1 - Initial
6+-- 0.0.1.1 - Bump for Adam's change to strict BS.
7+
3Description: Network transport interface
8Description: Network transport interface
4License: BSD3
9License: BSD3
5License-file: LICENSE
10License-file: LICENSE