@@ -14,12 +14,7 @@ module Network.Transport.TCP ( -- * Main API
1414 , ControlHeader(..)
1515 , ConnectionRequestResponse(..)
1616 -- * Design notes
17− -- ** Overview
18− -- $overview
19− -- ** Connecting
20− -- $connecting
21− -- ** Disconnecting
22− -- $disconnecting
17+ -- $design
2318 ) where
2419
2520import Network.Transport
@@ -80,10 +75,12 @@ import Data.IORef (newIORef, modifyIORef, readIORef)
8075import Data.Lens.Lazy (Lens, lens, intMapLens, mapLens, (^.), (^=), (^+=))
8176import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
8277
83−-- $overview
78+-- $design
8479--
85−-- The TCP transport maps multiple logical connections between A and B
86−-- (in either direction) to a single TCP connection:
80+-- [Goals]
81+--
82+-- The TCP transport maps multiple logical connections between /A/ and /B/ (in
83+-- either direction) to a single TCP connection:
8784--
8885-- > +-------+ +-------+
8986-- > | A |==========================| B |
@@ -92,32 +89,41 @@ import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
9289-- > | \~~~|~~~~~~~~~~~~~~~~~~~~~~~~~<| |
9390-- > | |==========================| |
9491-- > +-------+ +-------+
95−
96−-- $connecting
9792--
93+-- Ignoring the complications detailed below, the TCP connection is created is
94+-- created the first lightweight connection is created (in either direction),
95+-- and torn down when the last lightweight connection (in either direction) is
96+-- closed.
97+--
98+-- [Connecting]
99+--
98100-- Let /A/, /B/ be two endpoints without any connections. When /A/ wants to
99101-- connect to /B/, it locally records that it is trying to connect to /B/ and
100102-- sends a request to /B/. As part of the request /A/ sends its own endpoint
101−-- address to /B/.
103+-- address to /B/ (so that /B/ can reuse the connection in the other direction).
102104--
103105-- When /B/ receives the connection request it first checks if it had not
104106-- already initiated a connection request to /A/. If it had not, then it will
105−-- acknowledge the connection request and record that it has a TCP connection to
106−-- /A/.
107+-- acknowledge the connection request by sending 'ConnectionRequestAccepted' to
108+-- /A/ and record that it has a TCP connection to /A/.
107109--
108110-- The tricky case arises when /A/ sends a connection request to /B/ and /B/
109111-- finds that it had already sent a connection request to /A/. In this case /B/
110112-- will accept the connection request from /A/ if /A/s endpoint address is
111−-- smaller (lexicographically) than /B/s, and reject it otherwise.
113+-- smaller (lexicographically) than /B/s, and reject it otherwise. If it rejects
114+-- it, it sends a 'ConnectionRequestCrossed' message to /A/. (The
115+-- lexicographical ordering is an arbitrary but convenient way to break the
116+-- tie.)
112117--
113−-- Upon a rejection the /A/ thread that initiated the request just needs to wait
114−-- until the /A/ thread that is dealing with /B/s connection request completes.
115−
116−-- $disconnecting
118+-- When it receives a 'ConnectionRequestCrossed' message the /A/ thread that
119+-- initiated the request just needs to wait until the /A/ thread that is dealing
120+-- with /B/'s connection request completes.
121+--
122+-- [Disconnecting]
117123--
118−-- The TCP connection is created as soon as the first logical connection
119−-- from A to B (or B to A) is established. At this point a thread (@#@)
120−-- is spawned that listens for incoming connections from B:
124+-- The TCP connection is created as soon as the first logical connection from
125+-- /A/ to /B/ (or /B/ to /A/) is established. At this point a thread (@#@) is
126+-- spawned that listens for incoming connections from /B/:
121127--
122128-- > +-------+ +-------+
123129-- > | A |==========================| B |
@@ -127,25 +133,37 @@ import Text.Regex.Applicative (RE, few, anySym, sym, many, match)
127133-- > | |==========================| |
128134-- > +-------+ +-------+
129135--
130−-- The question is when the TCP connection can be closed again.
131−-- Conceptually, we want to do reference counting: when there are no
132−-- logical connections left between A and B we want to close the socket
133−-- (possibly after some timeout).
136+-- The question is when the TCP connection can be closed again. Conceptually,
137+-- we want to do reference counting: when there are no logical connections left
138+-- between /A/ and /B/ we want to close the socket (possibly after some
139+-- timeout).
134140--
135−-- However, A and B need to agree that the refcount has reached zero.
136−-- It might happen that B sends a connection request over the existing
137−-- socket at the same time that A closes its logical connection to B and
138−-- closes the socket. This will cause a failure in B (which will have to
139−-- retry) which is not caused by a network failure, which is
140−-- unfortunate. (Note that the connection request from B might succeed
141−-- even if A closes the socket.)
141+-- However, /A/ and /B/ need to agree that the refcount has reached zero. It
142+-- might happen that /B/ sends a connection request over the existing socket at
143+-- the same time that /A/ closes its logical connection to /B/ and closes the
144+-- socket. This will cause a failure in /B/ (which will have to retry) which is
145+-- not caused by a network failure, which is unfortunate. (Note that the
146+-- connection request from /B/ might succeed even if /A/ closes the socket.)
142147--
143−-- Instead, A will first ask B if it's okay to close the connection. If
144−-- B has already initiated a logical connection to A then it can simply
145−-- ignore this message, because any message other than 'OkayToClose'
146−-- will imply to A that the connection should not yet be closed. On the
147−-- other hand, B must guarantee that once it does send an 'OkayToClose'
148−-- message it will not reuse the socket anymore.
148+-- Instead, when /A/ is ready to close the socket it sends a 'CloseSocket'
149+-- request to /B/ and records that its connection to /B/ is closing. If /A/
150+-- receives a new connection request from /B/ after having sent the
151+-- 'CloseSocket' request it simply forgets that it sent a 'CloseSocket' request
152+-- and increments the reference count of the connection again.
153+--
154+-- When /B/ receives a 'CloseSocket' message and it too is ready to close the
155+-- connection, it will respond with a reciprocal 'CloseSocket' request to /A/
156+-- and then actually close the socket. /A/ meanwhile will not send any more
157+-- requests to /B/ after having sent a 'CloseSocket' request, and will actually
158+-- close its end of the socket only when receiving the 'CloseSocket' message
159+-- from /B/. (Since /A/ recorded that its connection to /B/ is in closing state
160+-- after sending a 'CloseSocket' request to /B/, it knows not to reciprocate /B/
161+-- reciprocal 'CloseSocket' message.)
162+--
163+-- If there is a concurrent thread in /A/ waiting to connect to /B/ after /A/
164+-- has sent a 'CloseSocket' request then this thread will block until /A/ knows
165+-- whether to reuse the old socket (if /B/ sends a new connection request
166+-- instead of acknowledging the 'CloseSocket') or to set up a new socket.
149167
150168--------------------------------------------------------------------------------
151169-- Internal datatypes --
@@ -541,7 +559,8 @@ handleIncomingMessages (ourEndPoint, theirEndPoint) = do
541559 didClose <- modifyMVar (ourEndPoint ^. localState) $ \st -> do
542560 case st ^. connectionTo theirAddr of
543561 Just (Just _, mvar) -> do
544− sendTo theirEndPoint [encodeInt32 CloseSocket]
562+ -- The connection is already in "closing" state, which means we
563+ -- don't need to send another CloseSocket request (we already did)
545564 N.sClose sock
546565 putMVar mvar EndPointClosed
547566 return (connectionTo theirAddr ^= Nothing $ st, True)