@@ -14,12 +14,7 @@ module Network.Transport.TCP ( -- * Main API
14 , ControlHeader(..)
14 , ControlHeader(..)
15 , ConnectionRequestResponse(..)
15 , ConnectionRequestResponse(..)
16 -- * Design notes
16 -- * Design notes
17− -- ** Overview
17+ -- $design
25import Network.Transport
20import Network.Transport
@@ -80,10 +75,12 @@ import Data.IORef (newIORef, modifyIORef, readIORef)
80import Data.Lens.Lazy (Lens, lens, intMapLens, mapLens, (^.), (^=), (^+=))
75import Data.Lens.Lazy (Lens, lens, intMapLens, mapLens, (^.), (^=), (^+=))
81import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
76import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
83−-- $overview
78+-- $design
85−-- The TCP transport maps multiple logical connections between A and B
82+-- The TCP transport maps multiple logical connections between /A/ and /B/ (in
86−-- (in either direction) to a single TCP connection:
83+-- either direction) to a single TCP connection:
88-- > +-------+ +-------+
85-- > +-------+ +-------+
89-- > | A |==========================| B |
86-- > | A |==========================| B |
@@ -92,32 +89,41 @@ import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
92-- > | \~~~|~~~~~~~~~~~~~~~~~~~~~~~~~<| |
89-- > | \~~~|~~~~~~~~~~~~~~~~~~~~~~~~~<| |
93-- > | |==========================| |
90-- > | |==========================| |
94-- > +-------+ +-------+
91-- > +-------+ +-------+
93+-- Ignoring the complications detailed below, the TCP connection is created is
94+-- created the first lightweight connection is created (in either direction),
95+-- and torn down when the last lightweight connection (in either direction) is
98-- Let /A/, /B/ be two endpoints without any connections. When /A/ wants to
100-- Let /A/, /B/ be two endpoints without any connections. When /A/ wants to
99-- connect to /B/, it locally records that it is trying to connect to /B/ and
101-- connect to /B/, it locally records that it is trying to connect to /B/ and
100-- sends a request to /B/. As part of the request /A/ sends its own endpoint
102-- sends a request to /B/. As part of the request /A/ sends its own endpoint
101−-- address to /B/.
103+-- address to /B/ (so that /B/ can reuse the connection in the other direction).
103-- When /B/ receives the connection request it first checks if it had not
105-- When /B/ receives the connection request it first checks if it had not
104-- already initiated a connection request to /A/. If it had not, then it will
106-- already initiated a connection request to /A/. If it had not, then it will
107+-- acknowledge the connection request by sending 'ConnectionRequestAccepted' to
105−-- acknowledge the connection request and record that it has a TCP connection to
108+-- /A/ and record that it has a TCP connection to /A/.
108-- The tricky case arises when /A/ sends a connection request to /B/ and /B/
110-- The tricky case arises when /A/ sends a connection request to /B/ and /B/
109-- finds that it had already sent a connection request to /A/. In this case /B/
111-- finds that it had already sent a connection request to /A/. In this case /B/
110-- will accept the connection request from /A/ if /A/s endpoint address is
112-- will accept the connection request from /A/ if /A/s endpoint address is
111−-- smaller (lexicographically) than /B/s, and reject it otherwise.
113+-- smaller (lexicographically) than /B/s, and reject it otherwise. If it rejects
114+-- it, it sends a 'ConnectionRequestCrossed' message to /A/. (The
115+-- lexicographical ordering is an arbitrary but convenient way to break the
118+-- When it receives a 'ConnectionRequestCrossed' message the /A/ thread that
113−-- Upon a rejection the /A/ thread that initiated the request just needs to wait
119+-- initiated the request just needs to wait until the /A/ thread that is dealing
114−-- until the /A/ thread that is dealing with /B/s connection request completes.
120+-- with /B/'s connection request completes.
116−-- $disconnecting
122+-- [Disconnecting]
118−-- The TCP connection is created as soon as the first logical connection
124+-- The TCP connection is created as soon as the first logical connection from
119−-- from A to B (or B to A) is established. At this point a thread (@#@)
125+-- /A/ to /B/ (or /B/ to /A/) is established. At this point a thread (@#@) is
120−-- is spawned that listens for incoming connections from B:
126+-- spawned that listens for incoming connections from /B/:
122-- > +-------+ +-------+
128-- > +-------+ +-------+
123-- > | A |==========================| B |
129-- > | A |==========================| B |
@@ -127,25 +133,37 @@ import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
127-- > | |==========================| |
133-- > | |==========================| |
128-- > +-------+ +-------+
134-- > +-------+ +-------+
130−-- The question is when the TCP connection can be closed again.
136+-- The question is when the TCP connection can be closed again. Conceptually,
131−-- Conceptually, we want to do reference counting: when there are no
137+-- we want to do reference counting: when there are no logical connections left
132−-- logical connections left between A and B we want to close the socket
138+-- between /A/ and /B/ we want to close the socket (possibly after some
133−-- (possibly after some timeout).
139+-- timeout).
135−-- However, A and B need to agree that the refcount has reached zero.
141+-- However, /A/ and /B/ need to agree that the refcount has reached zero. It
136−-- It might happen that B sends a connection request over the existing
142+-- might happen that /B/ sends a connection request over the existing socket at
137−-- socket at the same time that A closes its logical connection to B and
143+-- the same time that /A/ closes its logical connection to /B/ and closes the
138−-- closes the socket. This will cause a failure in B (which will have to
144+-- socket. This will cause a failure in /B/ (which will have to retry) which is
139−-- retry) which is not caused by a network failure, which is
145+-- not caused by a network failure, which is unfortunate. (Note that the
140−-- unfortunate. (Note that the connection request from B might succeed
146+-- connection request from /B/ might succeed even if /A/ closes the socket.)
141−-- even if A closes the socket.)
143−-- Instead, A will first ask B if it's okay to close the connection. If
148+-- Instead, when /A/ is ready to close the socket it sends a 'CloseSocket'
144−-- B has already initiated a logical connection to A then it can simply
149+-- request to /B/ and records that its connection to /B/ is closing. If /A/
145−-- ignore this message, because any message other than 'OkayToClose'
150+-- receives a new connection request from /B/ after having sent the
146−-- will imply to A that the connection should not yet be closed. On the
151+-- 'CloseSocket' request it simply forgets that it sent a 'CloseSocket' request
147−-- other hand, B must guarantee that once it does send an 'OkayToClose'
152+-- and increments the reference count of the connection again.
148−-- message it will not reuse the socket anymore.
153+--
154+-- When /B/ receives a 'CloseSocket' message and it too is ready to close the
155+-- connection, it will respond with a reciprocal 'CloseSocket' request to /A/
156+-- and then actually close the socket. /A/ meanwhile will not send any more
157+-- requests to /B/ after having sent a 'CloseSocket' request, and will actually
158+-- close its end of the socket only when receiving the 'CloseSocket' message
159+-- from /B/. (Since /A/ recorded that its connection to /B/ is in closing state
160+-- after sending a 'CloseSocket' request to /B/, it knows not to reciprocate /B/
161+-- reciprocal 'CloseSocket' message.)
163+-- If there is a concurrent thread in /A/ waiting to connect to /B/ after /A/
164+-- has sent a 'CloseSocket' request then this thread will block until /A/ knows
165+-- whether to reuse the old socket (if /B/ sends a new connection request
166+-- instead of acknowledging the 'CloseSocket') or to set up a new socket.
150--------------------------------------------------------------------------------
168--------------------------------------------------------------------------------
151-- Internal datatypes --
169-- Internal datatypes --
@@ -541,7 +559,8 @@ handleIncomingMessages (ourEndPoint, theirEndPoint) = do
541 didClose <- modifyMVar (ourEndPoint ^. localState) $ \st -> do
559 didClose <- modifyMVar (ourEndPoint ^. localState) $ \st -> do
542 case st ^. connectionTo theirAddr of
560 case st ^. connectionTo theirAddr of
543 Just (Just _, mvar) -> do
561 Just (Just _, mvar) -> do
544− sendTo theirEndPoint [encodeInt32 CloseSocket]
562+ -- The connection is already in "closing" state, which means we
563+ -- don't need to send another CloseSocket request (we already did)
545 N.sClose sock
564 N.sClose sock
546 putMVar mvar EndPointClosed
565 putMVar mvar EndPointClosed
547 return (connectionTo theirAddr ^= Nothing $ st, True)
566 return (connectionTo theirAddr ^= Nothing $ st, True)