git.iko.soy/distributed-process
on master help sign in
Add SendTransport related entries in benchmarks/Makefile and performed some refactoring
commitff8d9c68efa2a5b144abdc6945cb29aabd730f3c authorRyan Newton <rrnewton@gmail.com> dateFri 24 Feb 2012, 11:17 (15 years ago) parent97352da
3 files changed+80−14
compare with master
unifiedsplit
benchmarks/Makefile+31−13 benchmarks/SendTransport.hs+2−1 benchmarks/results_notes.md+47
benchmarks/Makefile+31−13
@@ -1,8 +1,8 @@
11
22# Run 100K ping-pongs, for one trial:
3−pings = 100000
3+size = 100000
44trials = 1
5−# You can run "make run1 pings=1000 trials=100" for example.
5+# You can run "make run1 size=1000 trials=100" for example.
66
77
88# Disable this to test installed packages only:
@@ -12,21 +12,22 @@ ifeq ($(GHC),)
1212 GHC=ghc
1313endif
1414
15−all: pipes tcp baseline
16−baseline:
17− $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) PingTCP.hs -o PingTCP.exe
18−tcp:
19− $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) PingTCPTransport.hs -o PingTCPTransport.exe
20−pipes:
21− $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) PingPipes.hs -o PingPipes.exe
15+BENCHS= PingTCP.exe PingTCPTransport.exe PingPipes.exe SendTransport.exe
2216
23−run: run1 run2
17+all: $(BENCHS)
18+$(BENCHS): %.exe: %.hs
19+ $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) $< -o $@
20+
21+#-------------------------------------------------------------------------------
22+# Shorthands for running benchmarks:
23+
24+run: run1 run2 run3 run4
2425
2526run1: runBaseline
2627runBaseline:
2728 ./PingTCP.exe server 8080 &
2829 sleep 1
29− time ./PingTCP.exe client 0.0.0.0 8080 $(pings) $(trials)
30+ time ./PingTCP.exe client 0.0.0.0 8080 $(size) $(trials)
3031 # Kill the server:
3132 killall PingTCP.exe
3233
@@ -35,7 +36,7 @@ runTCP:
3536 ./PingTCPTransport.exe server 0.0.0.0 8080 sourceAddr &
3637 sleep 1
3738 # Run 100K ping-pongs, for one trial:
38− time ./PingTCPTransport.exe client 0.0.0.0 8081 sourceAddr $(pings) $(trials)
39+ time ./PingTCPTransport.exe client 0.0.0.0 8081 sourceAddr $(size) $(trials)
3940 # Kill the server:
4041 killall PingTCPTransport.exe
4142
@@ -43,9 +44,26 @@ run3: runPipes
4344runPipes:
4445 ./PingPipes.exe server sourceAddr &
4546 sleep 1
46− time ./PingPipes.exe client sourceAddr $(pings) $(trials)
47+ time ./PingPipes.exe client sourceAddr $(size) $(trials)
4748 # Kill the server:
4849 killall PingPipes.exe
4950
51+# This benchmark measures throughput rather than latency
52+# Here 'size' means message size rather than number of pings:
53+run4: runTPT
54+runTPT:
55+ ./SendTransport.exe server 0.0.0.0 8080 sourceAddr &
56+ sleep 1
57+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr $(size)
58+ killall SendTransport.exe
59+
60+# Run multiple sizes:
61+runTPTs:
62+ $(MAKE) runTPT size=1000
63+ $(MAKE) runTPT size=5000
64+ $(MAKE) runTPT size=10000
65+ $(MAKE) runTPT size=50000
66+ $(MAKE) runTPT size=100000
67+
5068clean:
5169 rm -f *.hi *.o *.exe
benchmarks/SendTransport.hs+2−1
@@ -3,7 +3,8 @@
33
44module Main where
55
6−import Network.Transport
6+import Network.Transport (newConnection, receive, connect, send, defaultHints,
7+ serialize, deserialize, SourceEnd, TargetEnd)
78import Network.Transport.TCP (mkTransport, TCPConfig (..))
89
910import Control.Applicative
benchmarks/results_notes.md+47
@@ -62,3 +62,50 @@ results consistent with the above. Here are the mean latencies:
6262 PingTCPTransport: 191 us
6363 PingPipes: 65 us
6464
65+And here is my first attempt at running the SendTransport benchmark (TCP backend):
66+
67+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 1000
68+ mean: 38.10901 ms, lb 35.92353 ms, ub 39.30006 ms, ci 0.950
69+ std dev: 8.305210 ms, lb 4.935312 ms, ub 11.97144 ms, ci 0.950
70+ found 5 outliers among 100 samples (5.0%)
71+ 5 (5.0%) low severe
72+ variance introduced by outliers: 94.703%
73+ variance is severely inflated by outliers
74+
75+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 5000
76+ mean: 32.87646 ms, lb 29.69947 ms, ub 35.62485 ms, ci 0.950
77+ std dev: 15.21419 ms, lb 12.39949 ms, ub 17.39441 ms, ci 0.950
78+ found 18 outliers among 100 samples (18.0%)
79+ 18 (18.0%) low severe
80+ variance introduced by outliers: 98.928%
81+ variance is severely inflated by outliers
82+
83+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 10000
84+ mean: 32.87737 ms, lb 29.34305 ms, ub 35.62680 ms, ci 0.950
85+ std dev: 15.22060 ms, lb 12.40398 ms, ub 17.57401 ms, ci 0.950
86+ found 18 outliers among 100 samples (18.0%)
87+ 18 (18.0%) low severe
88+ variance introduced by outliers: 98.929%
89+ variance is severely inflated by outliers
90+
91+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 50000
92+ mean: 1.351308 ms, lb 1.249680 ms, ub 1.548209 ms, ci 0.950
93+ std dev: 696.6768 us, lb 448.3538 us, ub 1.314057 ms, ci 0.950
94+ found 24 outliers among 100 samples (24.0%)
95+ 3 (3.0%) high mild
96+ 21 (21.0%) high severe
97+ variance introduced by outliers: 98.942%
98+ variance is severely inflated by outliers
99+
100+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 100000
101+ mean: 2.184967 ms, lb 2.030858 ms, ub 2.641543 ms, ci 0.950
102+ std dev: 1.235358 ms, lb 518.3995 us, ub 2.666540 ms, ci 0.950
103+ found 15 outliers among 100 samples (15.0%)
104+ 4 (4.0%) low severe
105+ 9 (9.0%) high severe
106+ variance introduced by outliers: 98.952%
107+ variance is severely inflated by outliers
108+
109+This is quite odd. so in addition to the high variance it would seem
110+that messages of size 50K or 100K are much FASTER to send than smaller
111+massages?