git.iko.soy/distributed-process
on master help sign in
Resurect the simple ping/pong benchmark. This new network-transport-tcp does better, but still twice as slow as raw Network.Socket
commit92ac241444cf694af1be4398c5e9b83d5d44609a authorRyan Newton <rrnewton@gmail.com> dateTue 7 Aug 2012, 12:23 (14 years ago) parenta5daa9a
4 files changed+370
benchmarks/Makefile+76 benchmarks/PingPongTCP.hs+124 benchmarks/PingPongTCPTransport.hs+146 benchmarks/results_notes.md+24
benchmarks/Makefilenew file+76
@@ -0,0 +1,76 @@
1+
2+# Makefile for building and running benchmarks.
3+# Usage:
4+# make all
5+# make run
6+
7+#--------------------------------------------------------------------------------
8+# Settings
9+
10+# By default, run 100K ping-pongs, for one trial:
11+size = 100000
12+trials = 1
13+# Alternatively, you can run "make run1 size=1000 trials=100", for example.
14+
15+ifeq ($(GHC),)
16+ GHC=ghc
17+endif
18+
19+BENCHS= PingPongTCP.exe PingPongTCPTransport.exe
20+# PingPongPipes.exe SendTransport.exe
21+
22+all: $(BENCHS)
23+$(BENCHS): %.exe: %.hs
24+ $(GHC) -O2 -rtsopts -threaded --make $(INCLUDES) $< -o $@
25+
26+#-------------------------------------------------------------------------------
27+# Shorthands for running benchmarks:
28+
29+run: run1 run2
30+# Disabled: run3 run4
31+
32+# Benchmark raw sockets without network-transport:
33+run1: runBaseline
34+runBaseline:
35+ ./PingPongTCP.exe server 8080 &
36+ sleep 1
37+ time ./PingPongTCP.exe client 0.0.0.0 8080 $(size) $(trials)
38+ # Kill the server:
39+ killall PingPongTCP.exe
40+
41+run2: runTCP
42+runTCP:
43+ ./PingPongTCPTransport.exe server 0.0.0.0 8080 sourceAddr.txt &
44+ sleep 1
45+ # Run 100K ping-pongs, for one trial:
46+ time ./PingPongTCPTransport.exe client 0.0.0.0 8081 sourceAddr.txt $(size) $(trials)
47+ # Kill the server:
48+# killall PingPongTCPTransport.exe
49+
50+run3: runPipes
51+runPipes:
52+ ./PingPongPipes.exe server sourceAddr &
53+ sleep 1
54+ time ./PingPongPipes.exe client sourceAddr $(size) $(trials)
55+ # Kill the server:
56+ killall PingPongPipes.exe
57+
58+# This benchmark measures throughput rather than latency
59+# Here 'size' means message size rather than number of pings:
60+run4: runTPT
61+runTPT:
62+ ./SendTransport.exe server 0.0.0.0 8080 sourceAddr &
63+ sleep 1
64+ ./SendTransport.exe client 0.0.0.0 8081 sourceAddr $(size)
65+ killall SendTransport.exe
66+
67+# Run multiple sizes:
68+runTPTs:
69+ $(MAKE) runTPT size=1000
70+ $(MAKE) runTPT size=5000
71+ $(MAKE) runTPT size=10000
72+ $(MAKE) runTPT size=50000
73+ $(MAKE) runTPT size=100000
74+
75+clean:
76+ rm -f *.hi *.o *.exe
benchmarks/PingPongTCP.hsnew file+124
@@ -0,0 +1,124 @@
1+{-# LANGUAGE CPP #-}
2+
3+-- | This performs a ping benchmark on a TCP connection created by
4+-- Network.Socket. To compile this file, you might use:
5+--
6+-- ghc --make -O2 benchmarks/PingTCP.hs
7+--
8+-- To use the compiled binary, first set up the server on the current machine:
9+--
10+-- ./benchmarks/PingTCP server 8080
11+--
12+-- Next, perform the benchmark on a client using the server address, where
13+-- each 1000 pings and one trial:
14+--
15+-- ./benchmarks/PingTCP client 0.0.0.0 8080 1000 1
16+--
17+-- The server must be restarted between benchmarks.
18+
19+--------------------------------------------------------------------------------
20+module Main where
21+
22+import Control.Monad
23+import Criterion.Main (Benchmark, bench, defaultMainWith, nfIO)
24+import Criterion.Config (defaultConfig, ljust, Config(cfgSamples))
25+
26+import Data.Int
27+import qualified Data.Serialize as Ser
28+import Data.Word (Word8)
29+import Network.Socket
30+ ( AddrInfoFlag (AI_PASSIVE), HostName, ServiceName, Socket
31+ , SocketType (Stream), SocketOption (ReuseAddr)
32+ , accept, addrAddress, addrFlags, addrFamily, bindSocket, defaultProtocol
33+ , defaultHints
34+ , getAddrInfo, listen, setSocketOption, socket, sClose, withSocketsDo )
35+import System.Environment (getArgs, withArgs)
36+
37+import qualified Network.Socket as N
38+
39+import Debug.Trace
40+
41+#ifndef LAZY
42+import Data.ByteString (ByteString)
43+import qualified Network.Socket.ByteString as NBS
44+encode = Ser.encode
45+decode = Ser.decode
46+#else
47+import Data.ByteString.Lazy (ByteString)
48+import qualified Network.Socket.ByteString.Lazy as NBS
49+encode = Ser.encodeLazy
50+decode = Ser.decodeLazy
51+#endif
52+{-# INLINE encode #-}
53+{-# INLINE decode #-}
54+encode :: Ser.Serialize a => a -> ByteString
55+decode :: Ser.Serialize a => ByteString -> Either String a
56+
57+--------------------------------------------------------------------------------
58+
59+main :: IO ()
60+main = do
61+ args <- getArgs
62+ case args of
63+ "server" : service : [] -> withSocketsDo $ do
64+ putStrLn "server: creating TCP connection"
65+ serverAddrs <- getAddrInfo
66+ (Just (defaultHints { addrFlags = [AI_PASSIVE] } ))
67+ Nothing
68+ (Just service)
69+ let serverAddr = head serverAddrs
70+ sock <- socket (addrFamily serverAddr) Stream defaultProtocol
71+ setSocketOption sock ReuseAddr 1
72+ bindSocket sock (addrAddress serverAddr)
73+
74+ putStrLn "server: awaiting client connection"
75+ listen sock 1
76+ (clientSock, clientAddr) <- accept sock
77+
78+ putStrLn "server: listening for pings"
79+ forever (pong clientSock)
80+
81+ "client": host : service : pingsStr : reps : args' -> withSocketsDo $ do
82+ let pings = read pingsStr
83+ serverAddrs <- getAddrInfo
84+ Nothing
85+ (Just host)
86+ (Just service)
87+ let serverAddr = head serverAddrs
88+ sock <- socket (addrFamily serverAddr) Stream defaultProtocol
89+
90+ N.connect sock (addrAddress serverAddr)
91+
92+ -- benchmark the pings
93+ case (read reps) :: Int of
94+ 0 -> error "What would zero reps mean?"
95+ 1 -> do putStrLn "Because you're timing only one trial, skipping Criterion..."
96+ replicateM_ pings (ping sock)
97+ n -> withArgs args' $ defaultMainWith
98+ (defaultConfig{ cfgSamples = ljust n })
99+ (return ()) -- Init action.
100+ [ benchPing sock (fromIntegral pings) ]
101+ putStrLn "Done with all ping/pongs."
102+
103+-- withArgs args' $ defaultMain [ benchPing sock pings ]
104+-- replicateM_ pings (ping sock)
105+
106+-- | Each `ping` sends a single byte, and expects to receive one
107+-- back in return.
108+ping :: Socket -> IO Int64
109+ping sock = do
110+ NBS.send sock $ encode (42 :: Int64)
111+ bs <- NBS.recv sock 8
112+ either error return $ decode bs
113+
114+pong :: Socket -> IO ()
115+pong sock = do
116+ bs <- NBS.recv sock 8
117+ NBS.sendAll sock bs
118+ return ()
119+
120+benchPing :: Socket -> Int64 -> Benchmark
121+benchPing sock n =
122+ bench "PingTCP" $
123+ nfIO (replicateM_ (fromIntegral n) (ping sock))
124+
benchmarks/PingPongTCPTransport.hsnew file+146
@@ -0,0 +1,146 @@
1+{-# LANGUAGE CPP #-}
2+
3+
4+
5+-- | This performs a ping benchmark on the TCP transport. If
6+-- network-transport-tcp has been "cabal install"ed, then this
7+-- benchmark can be compiled with:
8+--
9+-- ghc --make -O2 benchmarks/PingTransport.hs
10+--
11+-- To use the compiled binary, first set up a server:
12+--
13+-- ./benchmarks/PingTCPTransport server 0.0.0.0 8080 sourceAddr.dat
14+--
15+-- Once this is established, launch a client to perform the benchmark. The
16+-- following command sends 1000 pings for 1 trial:
17+--
18+-- ./benchmarks/PingTCPTransport client 0.0.0.0 8081 sourceAddr.dat 1000 1
19+--
20+-- The server must be restarted between benchmarks.
21+
22+--------------------------------------------------------------------------------
23+module Main where
24+
25+import Network.Transport (receive, connect, send, defaultConnectHints, Event(..),
26+ Connection, EndPoint, EndPointAddress(EndPointAddress), Reliability(ReliableOrdered),
27+ newEndPoint, address, endPointAddressToByteString)
28+import Network.Transport.TCP (createTransport, defaultTCPParameters, decodeEndPointAddress)
29+
30+import Control.Monad (forever, replicateM_)
31+import Criterion.Main (Benchmark, bench, defaultMainWith, nfIO)
32+import Criterion.Config (defaultConfig, ljust, Config(cfgSamples))
33+
34+import qualified Data.Serialize as Ser
35+import Data.Maybe (fromJust)
36+import Data.Int
37+import System.Environment (getArgs, withArgs)
38+
39+import System.Exit (exitSuccess)
40+
41+#ifndef LAZY
42+import qualified Data.ByteString.Char8 as BS
43+encode = Ser.encode
44+decode = Ser.decode
45+#else
46+import qualified Data.ByteString.Lazy.Char8 as BS
47+encode = Ser.encodeLazy
48+decode = Ser.decodeLazy
49+#endif
50+{-# INLINE encode #-}
51+{-# INLINE decode #-}
52+encode :: Ser.Serialize a => a -> BS.ByteString
53+decode :: Ser.Serialize a => BS.ByteString -> Either String a
54+
55+--------------------------------------------------------------------------------
56+
57+main :: IO ()
58+main = do
59+ args <- getArgs
60+ case args of
61+ "server" : host : port : sourceAddrFilePath : [] -> do
62+ -- establish transport
63+ Right transport <- createTransport host port defaultTCPParameters
64+
65+ -- create ping end
66+ putStrLn "server: creating ping end"
67+-- (sourceAddrPing, targetEndPing)
68+ Right endpoint <- newEndPoint transport
69+ BS.writeFile sourceAddrFilePath $ endPointAddressToByteString $ address endpoint
70+
71+ -- create pong end
72+ putStrLn "server: creating pong end"
73+ -- Establish the connection:
74+ event <- receive endpoint
75+ Right conn <- case event of
76+ ConnectionOpened cid rel addr ->
77+ -- Connect right back, and since this is a single-client
78+ -- benchmark, block this thread to do it:
79+ connect endpoint addr rel defaultConnectHints
80+ oth -> do putStrLn$" server waiting for connection, unexpected event: "++show oth
81+ exitSuccess
82+
83+ putStrLn "server: going into pong loop..."
84+ forever $ pong endpoint conn
85+ return ()
86+
87+ "client" : host : port : sourceAddrFilePath : numPings : reps : args' -> do
88+ let pings = read numPings :: Int
89+ -- establish transport
90+ Right transport <- createTransport host port defaultTCPParameters
91+ Right endpoint <- newEndPoint transport
92+
93+ -- create ping end
94+ bs <- BS.readFile sourceAddrFilePath
95+ Right conn <- connect endpoint (EndPointAddress bs) ReliableOrdered defaultConnectHints
96+-- let Just (host,port,endptID) = decodeEndPointAddress (EndPointAddress bs)
97+ -- create pong end
98+-- send sourceEndPing [serialize sourceAddrPong]
99+
100+ -- benchmark the pings
101+ case (read reps) :: Int of
102+ 0 -> error "Error: What would zero trials mean?"
103+ 1 -> do putStrLn "Because you're timing only one trial, skipping Criterion..."
104+ replicateM_ pings (ping conn endpoint 42)
105+ n -> withArgs args' $ defaultMainWith
106+ (defaultConfig{ cfgSamples = ljust n })
107+ (return ()) -- Init action.
108+ [ benchPing conn endpoint (fromIntegral pings)]
109+
110+ putStrLn$"client: Done with all "++show pings++" ping/pongs."
111+ return ()
112+
113+-- | This function takes an EndPoint to receieve pings, and a
114+-- Connection to send back pongs. It doesn't bother decoding the
115+-- message, rather it sends it right on back.
116+pong :: EndPoint -> Connection -> IO ()
117+pong endpoint conn = do
118+ event <- receive endpoint
119+ case event of
120+ Received cid payloads -> do
121+ Right _ <- send conn payloads
122+ return ()
123+ oth -> error$"while awaiting pings, server received unexpected event: \n"++show oth
124+
125+-- | The effect of `ping conn endpt n` is to send the number `n` on
126+-- the connection and then receive another number from the endpoint,
127+-- which is then returned.
128+ping :: Connection -> EndPoint -> Int64 -> IO Int64
129+ping conn endpt n = do
130+ send conn [encode n]
131+ loop
132+ where
133+ loop = do
134+ event <- receive endpt
135+ case event of
136+ Received _cid [payload] -> do
137+ let (Right n2) = decode payload
138+ return $! n2
139+ ConnectionOpened _ _ _ -> loop -- ignore this
140+ other -> error$"Unexpected event on endpoint during ping process: "++show other
141+
142+-- | The effect of `benchPing conn endpt n` is to send
143+-- `n` pings down `conn` using the `ping` function.
144+benchPing :: Connection -> EndPoint -> Int64 -> Benchmark
145+benchPing conn endpoint n = bench "PingTransport" $
146+ nfIO (replicateM_ (fromIntegral n) (ping conn endpoint 42))
benchmarks/results_notes.mdnew file+24
@@ -0,0 +1,24 @@
1+
2+
3+[2012.08.07] {A new round of testing with the new API}
4+============================================================
5+
6+Here's a run of the ping-pong benchmark on a 3.1 ghz Intel Westmere
7+running RHEL 6.2.
8+
9+ 100K ping/pongs, 1 trial:
10+ PingTCP: 4.987 >90%CPU
11+ PingTCPTransport: 11.2s ~65%CPU
12+
13+That's with 3.9GB allocation and 99.2% productivity.
14+
15+For reference, here are some old results from the previous incarnation
16+of network-transport back in February:
17+
18+ 100K ping/pongs, 1 trial:
19+ PingTCP: 7.34s >80%CPU
20+ PingTCPTransport: 19s ~60%CPU
21+ PingPipes: 6.36s >=100%CPU
22+
23+ (Running a C benchmark on the same machine it takes 5.6 seconds to do
24+ 200K ping pongs. => 28 us avg latency for C)