@@ -3,6 +3,7 @@
4-- * Various errors are still left as "undefined"
4-- * Various errors are still left as "undefined"
5-- * Many possible errors are not yet dealt with
5-- * Many possible errors are not yet dealt with
6module Network.Transport.TCP ( -- * Main API
7module Network.Transport.TCP ( -- * Main API
7 createTransport
8 createTransport
8 , -- TCP specific functionality
9 , -- TCP specific functionality
@@ -12,38 +13,55 @@ module Network.Transport.TCP ( -- * Main API
13import Network.Transport
14import Network.Transport
14import Network.Transport.Internal.TCP ( forkServer
15import Network.Transport.Internal.TCP ( forkServer
16 , recvWithLength
16 , recvWithLength
17− , sendWithLength
17+ , sendMany
19 , recvInt32
18 , recvInt32
22−import Network.Transport.Internal ( encodeInt16
20+import Network.Transport.Internal ( encodeInt32
23− , encodeInt32
21+ , decodeInt32
24 , failWith
23 , failWith
25− , failWithT
24+ , failWithIO
27import qualified Network.Socket as N ( HostName
26import qualified Network.Socket as N ( HostName
28 , ServiceName
27 , ServiceName
38+ , SocketOption(ReuseAddr)
33import Control.Concurrent (forkIO)
41import Control.Concurrent (forkIO)
34import Control.Concurrent.Chan (Chan, newChan, readChan, writeChan)
42import Control.Concurrent.Chan (Chan, newChan, readChan, writeChan)
35−import Control.Concurrent.MVar (MVar, newMVar, modifyMVar, modifyMVar_, readMVar, takeMVar, putMVar)
43+import Control.Concurrent.MVar ( MVar
36import Control.Category ((>>>))
52import Control.Category ((>>>))
37import Control.Applicative ((<*>), (*>), (<$>))
53import Control.Applicative ((<*>), (*>), (<$>))
38import Control.Monad (forever, forM_, void)
54import Control.Monad (forever, forM_, void)
39−import Control.Monad.Error (ErrorT, liftIO, runErrorT)
55+import Control.Monad.Error (MonadError, ErrorT, liftIO, runErrorT, throwError)
40import Control.Monad.Trans.Maybe (MaybeT(MaybeT), runMaybeT)
56import Control.Monad.Trans.Maybe (MaybeT(MaybeT), runMaybeT)
41import Control.Monad.Trans.Writer (WriterT, execWriterT)
57import Control.Monad.Trans.Writer (WriterT, execWriterT)
42import Control.Monad.Writer.Class (tell)
58import Control.Monad.Writer.Class (tell)
43import Control.Monad.IO.Class (MonadIO)
59import Control.Monad.IO.Class (MonadIO)
60+import Control.Exception (IOException)
61+import Data.ByteString (ByteString)
44import qualified Data.ByteString as BS (concat)
62import qualified Data.ByteString as BS (concat)
45import qualified Data.ByteString.Char8 as BSC (pack, unpack)
63import qualified Data.ByteString.Char8 as BSC (pack, unpack)
46−import Data.Int (Int16)
64+import Data.Int (Int32)
47import Data.IntMap (IntMap)
65import Data.IntMap (IntMap)
48import qualified Data.IntMap as IntMap (empty, size)
66import qualified Data.IntMap as IntMap (empty, size)
49import Data.Map (Map)
67import Data.Map (Map)
@@ -64,15 +82,14 @@ data LocalEndPoint = LocalEndPoint { _localAddress :: EndPointAddress
65data RemoteEndPoint = RemoteEndPoint { _remoteAddress :: EndPointAddress
83data RemoteEndPoint = RemoteEndPoint { _remoteAddress :: EndPointAddress
66 , _remoteSocket :: N.Socket
84 , _remoteSocket :: N.Socket
67− , _remoteConnectionIds :: Chan ConnectionId
68 , _remoteLock :: MVar ()
85 , _remoteLock :: MVar ()
70data EndPointState = EndPointState { _nextConnectionId :: ConnectionId
87data EndPointState = EndPointState { _nextConnectionId :: ConnectionId
88+ , _pendingCtrlRequests :: IntMap (MVar [ByteString])
89+ , _nextCtrlRequestId :: ControlRequestId
72−type EndPointId = Int16
91+type EndPointId = Int32
73−
92+type ControlRequestId = Int32
74−#define REQUEST_CONNECTION_ID 0
75−#define RESPOND_CONNECTION_ID 1
77-- | Create a TCP transport
94-- | Create a TCP transport
@@ -81,10 +98,8 @@ type EndPointId = Int16
81-- connects to A two socket pairs will be created that will both be used
98-- connects to A two socket pairs will be created that will both be used
82-- unidirectionally, at degraded performance).
99-- unidirectionally, at degraded performance).
84−-- TODOs:
101+-- TODOs: deal with hints
85−-- * Perhaps we should allow to use something different other than 'defaultHints'
86−-- * Deal with all exceptions that may occur
87−createTransport :: N.HostName -> N.ServiceName -> IO Transport
102+createTransport :: N.HostName -> N.ServiceName -> IO (Either IOException Transport)
88createTransport host port = do
103createTransport host port = do
89 state <- newMVar TransportState { _localEndPoints = IntMap.empty
104 state <- newMVar TransportState { _localEndPoints = IntMap.empty
90 , _remoteEndPoints = Map.empty
105 , _remoteEndPoints = Map.empty
@@ -93,8 +108,9 @@ createTransport host port = do
93 , _transportHost = host
108 , _transportHost = host
94 , _transportPort = port
109 , _transportPort = port
96− forkServer host port (transportServer transport)
112+ forkServer host port (transportServer transport)
97− return Transport { newEndPoint = tcpNewEndPoint transport }
113+ return Transport { newEndPoint = tcpNewEndPoint transport }
99-- | The transport server handles incoming connections for all endpoints
115-- | The transport server handles incoming connections for all endpoints
@@ -104,39 +120,50 @@ transportServer transport sock = forever $ do
104 (clientSock, _) <- N.accept sock
120 (clientSock, _) <- N.accept sock
105 handleConnectionRequest transport clientSock
121 handleConnectionRequest transport clientSock
123+-- | Set up a new TCP connection (at the request of a remote endpoint)
107handleConnectionRequest :: TCPTransport -> N.Socket -> IO ()
124handleConnectionRequest :: TCPTransport -> N.Socket -> IO ()
108handleConnectionRequest transport sock = do
125handleConnectionRequest transport sock = do
109 request <- runMaybeT $ do
126 request <- runMaybeT $ do
110− ourEndPointId <- recvInt16 sock
127+ ourEndPointId <- recvInt32 sock
111 theirAddress <- EndPointAddress . BS.concat <$> recvWithLength sock
128 theirAddress <- EndPointAddress . BS.concat <$> recvWithLength sock
112 ourEndPoint <- MaybeT $ (^. localEndPointAt ourEndPointId) <$> readMVar (transport ^. transportState)
129 ourEndPoint <- MaybeT $ (^. localEndPointAt ourEndPointId) <$> readMVar (transport ^. transportState)
113 return (ourEndPoint, theirAddress)
130 return (ourEndPoint, theirAddress)
114 case request of
131 case request of
115 Just (ourEndPoint, theirAddress) ->
132 Just (ourEndPoint, theirAddress) ->
133+ -- We ignore the return value of 'forkRemoteEndPoint' here because _they_ are
134+ -- connecting to _us_. 'forkRemoteEndPoint' will have stored the remote endpoint
135+ -- in the transport state, however, in case _we_ later connect to _them_.
116 void $ forkRemoteEndPoint transport ourEndPoint theirAddress sock
136 void $ forkRemoteEndPoint transport ourEndPoint theirAddress sock
117 Nothing ->
137 Nothing ->
118 return ()
138 return ()
140+-- | Set up a new remote endpoint (either because we connected to a
141+-- new remote endpoint, or because a remote endpoint connected to us)
120forkRemoteEndPoint :: TCPTransport
142forkRemoteEndPoint :: TCPTransport
121 -> LocalEndPoint
143 -> LocalEndPoint
122 -> EndPointAddress
144 -> EndPointAddress
123 -> N.Socket
145 -> N.Socket
124 -> IO RemoteEndPoint
146 -> IO RemoteEndPoint
125−forkRemoteEndPoint transport ourEndPoint theirAddress theirSock = do
147+forkRemoteEndPoint transport ourEndPoint theirAddress sock = do
126− theirConnectionIds <- newChan
127− theirLock <- newMVar ()
148+ lock <- newMVar ()
128− let theirEndPoint = RemoteEndPoint { _remoteAddress = theirAddress
149+ let theirEndPoint = RemoteEndPoint { _remoteAddress = theirAddress
129− , _remoteSocket = theirSock
150+ , _remoteSocket = sock
130− , _remoteConnectionIds = theirConnectionIds
131− , _remoteLock = theirLock
151+ , _remoteLock = lock
133 modifyMVar_ (transport ^. transportState) $ return . (remoteEndPointAt theirAddress ^= Just theirEndPoint)
153 modifyMVar_ (transport ^. transportState) $ return . (remoteEndPointAt theirAddress ^= Just theirEndPoint)
134 forkIO $ do
154 forkIO $ do
135 unclosedConnections <- handleIncomingMessages ourEndPoint theirEndPoint
155 unclosedConnections <- handleIncomingMessages ourEndPoint theirEndPoint
136 forM_ unclosedConnections $ \cix -> writeChan (ourEndPoint ^. localChannel) (ConnectionClosed cix)
156 forM_ unclosedConnections $ \cix -> writeChan (ourEndPoint ^. localChannel) (ConnectionClosed cix)
137 -- TODO: should we close the socket here? (forkRemoteEndPoint gets called from multiple places)
157 -- TODO: should we close the socket here? (forkRemoteEndPoint gets called from multiple places)
158+ -- TODO: and should we remove it from the transaction state?
138 return theirEndPoint
159 return theirEndPoint
162+data ControlHeaders =
163+ RequestConnectionId -- ^ Request a new connection ID from the remote endpoint
164+ | ControlResponse -- ^ Respond to a control request _from_ the remote endpoint
140-- | Handle requests from a remote endpoint.
167-- | Handle requests from a remote endpoint.
142-- Returns only if the remote party closes the socket or if an error occurs.
169-- Returns only if the remote party closes the socket or if an error occurs.
@@ -145,37 +172,45 @@ forkRemoteEndPoint transport ourEndPoint theirAddress theirSock = do
145handleIncomingMessages :: LocalEndPoint -> RemoteEndPoint -> IO [ConnectionId]
172handleIncomingMessages :: LocalEndPoint -> RemoteEndPoint -> IO [ConnectionId]
146handleIncomingMessages ourEndPoint theirEndPoint = execWriterT . runMaybeT . forever $ do
173handleIncomingMessages ourEndPoint theirEndPoint = execWriterT . runMaybeT . forever $ do
147 connId <- recvInt32 sock
174 connId <- recvInt32 sock
175+ if connId >= firstNonReservedConnectionId
176+ then readMessage connId
178+ reqId <- recvInt32 sock
148− case connId of
179+ case toEnum (fromIntegral connId) of
149− REQUEST_CONNECTION_ID -> createNewConnection
180+ RequestConnectionId -> createNewConnection reqId
150− RESPOND_CONNECTION_ID -> readNewConnectionId
181+ ControlResponse -> readControlResponse reqId
151− _ -> readMessage connId
153 -- A brief note on the types:
183 -- A brief note on the types:
154 -- These functions may fail (connection error): hence MaybeT
184 -- These functions may fail (connection error): hence MaybeT
155 -- We also need to keep track of which connections are still open: hence WriterT
185 -- We also need to keep track of which connections are still open: hence WriterT
157 -- Create a new connection
187 -- Create a new connection
158− createNewConnection :: MaybeT (WriterT [ConnectionId] IO) ()
188+ createNewConnection :: ControlRequestId -> MaybeT (WriterT [ConnectionId] IO) ()
159− createNewConnection = do
189+ createNewConnection reqId = do
160 newId <- liftIO $ getNextConnectionId ourEndPoint
190 newId <- liftIO $ getNextConnectionId ourEndPoint
161 withRemoteLock theirEndPoint $ do
191 withRemoteLock theirEndPoint $ do
162− sendInt32 sock RESPOND_CONNECTION_ID
192+ sendMany sock (encodeInt32 ControlResponse : encodeInt32 reqId : prependLength [encodeInt32 newId])
163− sendInt32 sock newId
164 liftIO $ writeChan ourChannel (ConnectionOpened newId ReliableOrdered (theirEndPoint ^. remoteAddress))
193 liftIO $ writeChan ourChannel (ConnectionOpened newId ReliableOrdered (theirEndPoint ^. remoteAddress))
165 -- We add the new connection ID to the list of open connections only once the
194 -- We add the new connection ID to the list of open connections only once the
166 -- endpoint has been notified of the new connection (sendInt32 may fail)
195 -- endpoint has been notified of the new connection (sendInt32 may fail)
167 tell [newId]
196 tell [newId]
198+ -- Read a control response
199+ readControlResponse :: ControlRequestId -> MaybeT (WriterT [ConnectionId] IO) ()
200+ readControlResponse reqId = do
201+ response <- recvWithLength sock
202+ mmvar <- liftIO $ (^. pendingCtrlRequestsAt reqId) <$> readMVar ourState
204+ Nothing -> return () -- Invalid request ID. TODO: We just ignore it?
205+ Just mvar -> liftIO $ putMVar mvar response
169 -- Read a message and output it on the endPoint's channel
207 -- Read a message and output it on the endPoint's channel
170 readMessage :: ConnectionId -> MaybeT (WriterT [ConnectionId] IO) ()
208 readMessage :: ConnectionId -> MaybeT (WriterT [ConnectionId] IO) ()
171 readMessage connId = recvWithLength sock >>= liftIO . writeChan ourChannel . Received connId
209 readMessage connId = recvWithLength sock >>= liftIO . writeChan ourChannel . Received connId
173− -- Read a new connection ID
174− readNewConnectionId :: MaybeT (WriterT [ConnectionId] IO) ()
175− readNewConnectionId = recvInt32 sock >>= liftIO . writeChan (theirEndPoint ^. remoteConnectionIds)
177 -- Breakdown of the arguments
211 -- Breakdown of the arguments
178 ourChannel = ourEndPoint ^. localChannel
212 ourChannel = ourEndPoint ^. localChannel
213+ ourState = ourEndPoint ^. localState
179 sock = theirEndPoint ^. remoteSocket
214 sock = theirEndPoint ^. remoteSocket
181-- | Get the next connection ID
216-- | Get the next connection ID
@@ -189,7 +224,10 @@ tcpNewEndPoint :: TCPTransport -> IO (Either (FailedWith NewEndPointErrorCode) E
189tcpNewEndPoint transport = do
224tcpNewEndPoint transport = do
190 endPoint <- modifyMVar (transport ^. transportState) $ \st -> do
225 endPoint <- modifyMVar (transport ^. transportState) $ \st -> do
191 chan <- newChan
226 chan <- newChan
192− state <- newMVar EndPointState { _nextConnectionId = firstNonReservedConnectionId }
227+ state <- newMVar EndPointState { _nextConnectionId = firstNonReservedConnectionId
228+ , _pendingCtrlRequests = IntMap.empty
229+ , _nextCtrlRequestId = 0
193 let ix = fromIntegral $ IntMap.size (st ^. localEndPoints)
231 let ix = fromIntegral $ IntMap.size (st ^. localEndPoints)
194 let host = transport ^. transportHost
232 let host = transport ^. transportHost
195 let port = transport ^. transportPort
233 let port = transport ^. transportPort
@@ -220,22 +258,41 @@ tcpConnect :: TCPTransport -- ^ Transport
220 -> Reliability -- ^ Reliability (ignored)
258 -> Reliability -- ^ Reliability (ignored)
221 -> IO (Either (FailedWith ConnectErrorCode) Connection)
259 -> IO (Either (FailedWith ConnectErrorCode) Connection)
222tcpConnect transport ourEndPoint theirAddress _ = runErrorT $ do
260tcpConnect transport ourEndPoint theirAddress _ = runErrorT $ do
223− theirEndPoint <- remoteEndPoint transport ourEndPoint theirAddress
261+ theirEndPoint <- remoteEndPoint transport ourEndPoint theirAddress
224− connId <- failWithT undefined $ requestNewConnection theirEndPoint
262+ connId <- requestNewConnection ourEndPoint theirEndPoint
225− connBs <- encodeInt32 connId
226− let sock = theirEndPoint ^. remoteSocket
263+ let sock = theirEndPoint ^. remoteSocket
227− return Connection { send = runErrorT .
264+ return Connection { send = runErrorT .
228− failWithT (FailedWith SendFailed "Send failed") .
265+ failWithIO (FailedWith SendFailed . show) .
229− withRemoteLock theirEndPoint .
266+ withRemoteLock theirEndPoint .
230− sendWithLength sock (Just connBs)
267+ sendMany sock .
268+ (\payload -> encodeInt32 connId : prependLength payload)
269+ , close = -- TODO: we should not close the TCP connection until all
270+ -- lightweight connections have been closed
231− , close = N.sClose sock
271+ N.sClose sock
234-- | Request a new connection
274-- | Request a new connection
275+requestNewConnection :: (MonadIO m, MonadError (FailedWith ConnectErrorCode) m)
235−requestNewConnection :: RemoteEndPoint -> MaybeT IO ConnectionId
277+ -> RemoteEndPoint
236−requestNewConnection endPoint = do
279+requestNewConnection ourEndPoint theirEndPoint = do
237− withRemoteLock endPoint $ sendInt32 (endPoint ^. remoteSocket) 0
280+ mcid <- failWithIO (failed . show) $ do
238− liftIO $ readChan (endPoint ^. remoteConnectionIds)
281+ reply <- newEmptyMVar
282+ reqId <- modifyMVar state $ \st -> do
283+ let reqId = st ^. nextCtrlRequestId
284+ return ((nextCtrlRequestId ^+= 1) . (pendingCtrlRequestsAt reqId ^= Just reply) $ st, reqId)
285+ withRemoteLock theirEndPoint $ sendMany sock [encodeInt32 RequestConnectionId, encodeInt32 reqId]
286+ response <- decodeInt32 . BS.concat <$> takeMVar reply
287+ modifyMVar_ state $ return . (pendingCtrlRequestsAt reqId ^= Nothing)
290+ Nothing -> throwError (failed "Invalid integer")
291+ Just cid -> return cid
293+ sock = theirEndPoint ^. remoteSocket
294+ failed = FailedWith ConnectFailed
295+ state = ourEndPoint ^. localState
240-- | Find a remote endpoint
297-- | Find a remote endpoint
@@ -250,14 +307,23 @@ remoteEndPoint transport ourEndPoint theirAddress = do
250 Just endpoint ->
307 Just endpoint ->
251 return endpoint
308 return endpoint
252 Nothing -> do
309 Nothing -> do
310+ -- Create a new connection
253 -- liftIO $ putStrLn $ "Creating socket from " ++ show ourAddress ++ " to " ++ show theirAddress
311 -- liftIO $ putStrLn $ "Creating socket from " ++ show ourAddress ++ " to " ++ show theirAddress
254− (host, port, endPointId) <- failWith invalidAddress (decodeEndPointAddress theirAddress)
312+ (host, port, endPointId) <- failWith (invalidAddress "Could not parse") $
255− sock <- failWithT undefined (connectTo host port)
313+ (decodeEndPointAddress theirAddress)
256− endPointBS <- encodeInt16 endPointId
314+ addr:_ <- failWithIO (invalidAddress . show) $
257− failWithT undefined $ sendWithLength sock (Just endPointBS) [ourAddress]
315+ N.getAddrInfo Nothing (Just host) (Just port)
316+ sock <- failWithIO (insufficientResources . show) $
317+ N.socket (N.addrFamily addr) N.Stream N.defaultProtocol
318+ liftIO $ N.setSocketOption sock N.ReuseAddr 1
319+ failWithIO (failed . show) $ N.connect sock (N.addrAddress addr)
320+ -- Send the endpoint ID we're interested in and our own address
321+ failWithIO (failed . show) $ sendMany sock (encodeInt32 endPointId : prependLength [ourAddress])
258 liftIO $ forkRemoteEndPoint transport ourEndPoint theirAddress sock
322 liftIO $ forkRemoteEndPoint transport ourEndPoint theirAddress sock
260− invalidAddress = FailedWith ConnectInvalidAddress "Invalid address"
324+ invalidAddress = FailedWith ConnectInvalidAddress
325+ insufficientResources = FailedWith ConnectInsufficientResources
326+ failed = FailedWith ConnectFailed
261 EndPointAddress ourAddress = ourEndPoint ^. localAddress
327 EndPointAddress ourAddress = ourEndPoint ^. localAddress
263-- | We must avoid scrambling concurrent sends to the same endpoint
329-- | We must avoid scrambling concurrent sends to the same endpoint
@@ -313,17 +379,23 @@ remoteAddress = lens _remoteAddress (\addr ep -> ep { _remoteAddress = addr })
313remoteSocket :: Lens RemoteEndPoint N.Socket
379remoteSocket :: Lens RemoteEndPoint N.Socket
314remoteSocket = lens _remoteSocket (\sock ep -> ep { _remoteSocket = sock })
380remoteSocket = lens _remoteSocket (\sock ep -> ep { _remoteSocket = sock })
316−remoteConnectionIds :: Lens RemoteEndPoint (Chan ConnectionId)
317−remoteConnectionIds = lens _remoteConnectionIds (\ch ep -> ep { _remoteConnectionIds = ch })
319remoteLock :: Lens RemoteEndPoint (MVar ())
382remoteLock :: Lens RemoteEndPoint (MVar ())
320remoteLock = lens _remoteLock (\lock ep -> ep { _remoteLock = lock })
383remoteLock = lens _remoteLock (\lock ep -> ep { _remoteLock = lock })
322nextConnectionId :: Lens EndPointState ConnectionId
385nextConnectionId :: Lens EndPointState ConnectionId
323nextConnectionId = lens _nextConnectionId (\cix st -> st { _nextConnectionId = cix })
386nextConnectionId = lens _nextConnectionId (\cix st -> st { _nextConnectionId = cix })
388+pendingCtrlRequests :: Lens EndPointState (IntMap (MVar [ByteString]))
389+pendingCtrlRequests = lens _pendingCtrlRequests (\rep st -> st { _pendingCtrlRequests = rep })
391+nextCtrlRequestId :: Lens EndPointState ControlRequestId
392+nextCtrlRequestId = lens _nextCtrlRequestId (\cid st -> st { _nextCtrlRequestId = cid })
325localEndPointAt :: EndPointId -> Lens TransportState (Maybe LocalEndPoint)
394localEndPointAt :: EndPointId -> Lens TransportState (Maybe LocalEndPoint)
326localEndPointAt ix = localEndPoints >>> intMapLens (fromIntegral ix)
395localEndPointAt ix = localEndPoints >>> intMapLens (fromIntegral ix)
328remoteEndPointAt :: EndPointAddress -> Lens TransportState (Maybe RemoteEndPoint)
397remoteEndPointAt :: EndPointAddress -> Lens TransportState (Maybe RemoteEndPoint)
329remoteEndPointAt addr = remoteEndPoints >>> mapLens addr
398remoteEndPointAt addr = remoteEndPoints >>> mapLens addr
400+pendingCtrlRequestsAt :: ControlRequestId -> Lens EndPointState (Maybe (MVar [ByteString]))
401+pendingCtrlRequestsAt ix = pendingCtrlRequests >>> intMapLens (fromIntegral ix)