git.iko.soy/distributed-process
on master help sign in
Add SendTransport related entries in benchmarks/Makefile and performed some refactoring
ff8d9c6Ryan Newton
benchmarks/Makefile+31−13
compare with master
unifiedsplit
@@ -1,8 +1,8 @@
1
1
2# Run 100K ping-pongs, for one trial:
2# Run 100K ping-pongs, for one trial:
3−pings = 100000
3+size = 100000
4trials = 1
4trials = 1
5−# You can run "make run1 pings=1000 trials=100" for example.
5+# You can run "make run1 size=1000 trials=100" for example.
6
6
7
7
8# Disable this to test installed packages only:
8# Disable this to test installed packages only:
@@ -12,21 +12,22 @@ ifeq ($(GHC),)
12 GHC=ghc
12 GHC=ghc
13endif
13endif
14
14
15−all: pipes tcp baseline
15+BENCHS= PingTCP.exe PingTCPTransport.exe PingPipes.exe SendTransport.exe
16−baseline:
17− $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) PingTCP.hs -o PingTCP.exe
18−tcp:
19− $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) PingTCPTransport.hs -o PingTCPTransport.exe
20−pipes:
21− $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) PingPipes.hs -o PingPipes.exe
22
16
17+all: $(BENCHS)
18+$(BENCHS): %.exe: %.hs
19+ $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) $< -o $@
20+
21+#-------------------------------------------------------------------------------
22+# Shorthands for running benchmarks:
23+
23−run: run1 run2
24+run: run1 run2 run3 run4
24
25
25run1: runBaseline
26run1: runBaseline
26runBaseline:
27runBaseline:
27 ./PingTCP.exe server 8080 &
28 ./PingTCP.exe server 8080 &
28 sleep 1
29 sleep 1
29− time ./PingTCP.exe client 0.0.0.0 8080 $(pings) $(trials)
30+ time ./PingTCP.exe client 0.0.0.0 8080 $(size) $(trials)
30 # Kill the server:
31 # Kill the server:
31 killall PingTCP.exe
32 killall PingTCP.exe
32
33
@@ -35,7 +36,7 @@ runTCP:
35 ./PingTCPTransport.exe server 0.0.0.0 8080 sourceAddr &
36 ./PingTCPTransport.exe server 0.0.0.0 8080 sourceAddr &
36 sleep 1
37 sleep 1
37 # Run 100K ping-pongs, for one trial:
38 # Run 100K ping-pongs, for one trial:
38− time ./PingTCPTransport.exe client 0.0.0.0 8081 sourceAddr $(pings) $(trials)
39+ time ./PingTCPTransport.exe client 0.0.0.0 8081 sourceAddr $(size) $(trials)
39 # Kill the server:
40 # Kill the server:
40 killall PingTCPTransport.exe
41 killall PingTCPTransport.exe
41
42
@@ -43,9 +44,26 @@ run3: runPipes
43runPipes:
44runPipes:
44 ./PingPipes.exe server sourceAddr &
45 ./PingPipes.exe server sourceAddr &
45 sleep 1
46 sleep 1
46− time ./PingPipes.exe client sourceAddr $(pings) $(trials)
47+ time ./PingPipes.exe client sourceAddr $(size) $(trials)
47 # Kill the server:
48 # Kill the server:
48 killall PingPipes.exe
49 killall PingPipes.exe
49
50
51+# This benchmark measures throughput rather than latency
52+# Here 'size' means message size rather than number of pings:
53+run4: runTPT
54+runTPT:
55+ ./SendTransport.exe server 0.0.0.0 8080 sourceAddr &
56+ sleep 1
57+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr $(size)
58+ killall SendTransport.exe
59+
60+# Run multiple sizes:
61+runTPTs:
62+ $(MAKE) runTPT size=1000
63+ $(MAKE) runTPT size=5000
64+ $(MAKE) runTPT size=10000
65+ $(MAKE) runTPT size=50000
66+ $(MAKE) runTPT size=100000
67+
50clean:
68clean:
51 rm -f *.hi *.o *.exe
69 rm -f *.hi *.o *.exe
benchmarks/SendTransport.hs+2−1
@@ -3,7 +3,8 @@
3
3
4module Main where
4module Main where
5
5
6−import Network.Transport
6+import Network.Transport (newConnection, receive, connect, send, defaultHints,
7+ serialize, deserialize, SourceEnd, TargetEnd)
7import Network.Transport.TCP (mkTransport, TCPConfig (..))
8import Network.Transport.TCP (mkTransport, TCPConfig (..))
8
9
9import Control.Applicative
10import Control.Applicative
benchmarks/results_notes.md+47
@@ -62,3 +62,50 @@ results consistent with the above. Here are the mean latencies:
62 PingTCPTransport: 191 us
62 PingTCPTransport: 191 us
63 PingPipes: 65 us
63 PingPipes: 65 us
64
64
65+And here is my first attempt at running the SendTransport benchmark (TCP backend):
66+
67+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 1000
68+ mean: 38.10901 ms, lb 35.92353 ms, ub 39.30006 ms, ci 0.950
69+ std dev: 8.305210 ms, lb 4.935312 ms, ub 11.97144 ms, ci 0.950
70+ found 5 outliers among 100 samples (5.0%)
71+ 5 (5.0%) low severe
72+ variance introduced by outliers: 94.703%
73+ variance is severely inflated by outliers
74+
75+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 5000
76+ mean: 32.87646 ms, lb 29.69947 ms, ub 35.62485 ms, ci 0.950
77+ std dev: 15.21419 ms, lb 12.39949 ms, ub 17.39441 ms, ci 0.950
78+ found 18 outliers among 100 samples (18.0%)
79+ 18 (18.0%) low severe
80+ variance introduced by outliers: 98.928%
81+ variance is severely inflated by outliers
82+
83+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 10000
84+ mean: 32.87737 ms, lb 29.34305 ms, ub 35.62680 ms, ci 0.950
85+ std dev: 15.22060 ms, lb 12.40398 ms, ub 17.57401 ms, ci 0.950
86+ found 18 outliers among 100 samples (18.0%)
87+ 18 (18.0%) low severe
88+ variance introduced by outliers: 98.929%
89+ variance is severely inflated by outliers
90+
91+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 50000
92+ mean: 1.351308 ms, lb 1.249680 ms, ub 1.548209 ms, ci 0.950
93+ std dev: 696.6768 us, lb 448.3538 us, ub 1.314057 ms, ci 0.950
94+ found 24 outliers among 100 samples (24.0%)
95+ 3 (3.0%) high mild
96+ 21 (21.0%) high severe
97+ variance introduced by outliers: 98.942%
98+ variance is severely inflated by outliers
99+
100+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr 100000
101+ mean: 2.184967 ms, lb 2.030858 ms, ub 2.641543 ms, ci 0.950
102+ std dev: 1.235358 ms, lb 518.3995 us, ub 2.666540 ms, ci 0.950
103+ found 15 outliers among 100 samples (15.0%)
104+ 4 (4.0%) low severe
105+ 9 (9.0%) high severe
106+ variance introduced by outliers: 98.952%
107+ variance is severely inflated by outliers
108+
109+This is quite odd. so in addition to the high variance it would seem
110+that messages of size 50K or 100K are much FASTER to send than smaller
111+massages?