@@ -62,7 +62,9 @@ import Control.Distributed.Process.Internal.StrictMVar
62 , takeMVar
62 , takeMVar
64import Control.Concurrent.Chan (newChan, writeChan, readChan)
64import Control.Concurrent.Chan (newChan, writeChan, readChan)
65−import Control.Concurrent.STM (atomically)
65+import Control.Concurrent.STM
66import Control.Distributed.Process.Internal.CQueue
68import Control.Distributed.Process.Internal.CQueue
@@ -97,6 +99,7 @@ import Control.Distributed.Process.Internal.Types
97 , LocalProcessId(..)
99 , LocalProcessId(..)
98 , ProcessId(..)
100 , ProcessId(..)
99 , LocalNode(..)
101 , LocalNode(..)
102+ , Tracer(InactiveTracer)
100 , LocalNodeState(..)
103 , LocalNodeState(..)
101 , LocalProcess(..)
104 , LocalProcess(..)
102 , LocalProcessState(..)
105 , LocalProcessState(..)
@@ -108,6 +111,7 @@ import Control.Distributed.Process.Internal.Types
108 , localPidUnique
111 , localPidUnique
109 , localProcessWithId
112 , localProcessWithId
110 , localConnections
113 , localConnections
111 , MonitorRef(..)
115 , MonitorRef(..)
112 , ProcessMonitorNotification(..)
116 , ProcessMonitorNotification(..)
113 , NodeMonitorNotification(..)
117 , NodeMonitorNotification(..)
@@ -139,6 +143,12 @@ import Control.Distributed.Process.Internal.Types
139 , firstNonReservedProcessId
143 , firstNonReservedProcessId
140 , ImplicitReconnect(WithImplicitReconnect, NoImplicitReconnect)
144 , ImplicitReconnect(WithImplicitReconnect, NoImplicitReconnect)
146+import Control.Distributed.Process.Internal.Trace
142import Control.Distributed.Process.Serializable (Serializable)
152import Control.Distributed.Process.Serializable (Serializable)
143import Control.Distributed.Process.Internal.Messaging
153import Control.Distributed.Process.Internal.Messaging
144 ( sendBinary
154 ( sendBinary
@@ -195,16 +205,13 @@ createBareLocalNode endPoint rtable = do
195 , localEndPoint = endPoint
205 , localEndPoint = endPoint
196 , localState = state
206 , localState = state
197 , localCtrlChan = ctrlChan
207 , localCtrlChan = ctrlChan
208+ , localTracer = InactiveTracer
198 , remoteTable = rtable
209 , remoteTable = rtable
211+ tracedNode <- startTracing node
200− void . forkIO $ runNodeController node
212+ void . forkIO $ runNodeController tracedNode
201− void . forkIO $ handleIncomingMessages node
213+ void . forkIO $ handleIncomingMessages tracedNode
202− return node
214+ return tracedNode
204−-- Like 'Control.Monad.forever' but sans space leak
205−{-# INLINE forever' #-}
206−forever' :: Monad m => m a -> m b
207−forever' a = let a' = a >> a' in a'
209-- | Start and register the service processes on a node
216-- | Start and register the service processes on a node
210-- (for now, this is only the logger)
217-- (for now, this is only the logger)
@@ -218,6 +225,10 @@ startServiceProcesses node = do
218 [ match $ \((time, pid, string) ::(String, ProcessId, String)) -> do
225 [ match $ \((time, pid, string) ::(String, ProcessId, String)) -> do
219 liftIO . hPutStrLn stderr $ time ++ " " ++ show pid ++ ": " ++ string
226 liftIO . hPutStrLn stderr $ time ++ " " ++ show pid ++ ": " ++ string
228+ , match $ \((time, string) :: (String, String)) -> do
229+ -- this is a 'trace' message from the local node tracer
230+ liftIO . hPutStrLn stderr $ time ++ " [trace] " ++ string
221 , match $ \(ch :: SendPort ()) -> -- a shutdown request
232 , match $ \(ch :: SendPort ()) -> -- a shutdown request
222 sendChan ch ()
233 sendChan ch ()
@@ -267,6 +278,7 @@ forkProcess node proc = modifyMVar (localState node) startProcess
267 reason <- Exception.catch
278 reason <- Exception.catch
268 (runLocalProcess lproc proc >> return DiedNormal)
279 (runLocalProcess lproc proc >> return DiedNormal)
269 (return . DiedException . (show :: SomeException -> String))
280 (return . DiedException . (show :: SomeException -> String))
270 -- [Unified: Table 4, rules termination and exiting]
282 -- [Unified: Table 4, rules termination and exiting]
271 modifyMVar_ (localState node) (cleanupProcess pid)
283 modifyMVar_ (localState node) (cleanupProcess pid)
272 writeChan (localCtrlChan node) NCMsg
284 writeChan (localCtrlChan node) NCMsg
@@ -423,22 +435,27 @@ handleIncomingMessages node = go initConnectionState
425 NT.ErrorEvent (NT.TransportError NT.EventEndPointFailed str) ->
437 NT.ErrorEvent (NT.TransportError NT.EventEndPointFailed str) ->
426− fail $ "Cloud Haskell fatal error: end point failed: " ++ str
438+ fatal $ "Cloud Haskell fatal error: end point failed: " ++ str
427 NT.ErrorEvent (NT.TransportError NT.EventTransportFailed str) ->
439 NT.ErrorEvent (NT.TransportError NT.EventTransportFailed str) ->
428− fail $ "Cloud Haskell fatal error: transport failed: " ++ str
440+ fatal $ "Cloud Haskell fatal error: transport failed: " ++ str
429 NT.EndPointClosed ->
441 NT.EndPointClosed ->
430− return ()
442+ stopTracer (localTracer node) >> return ()
431 NT.ReceivedMulticast _ _ ->
443 NT.ReceivedMulticast _ _ ->
432 -- If we received a multicast message, something went horribly wrong
444 -- If we received a multicast message, something went horribly wrong
433 -- and we just give up
445 -- and we just give up
434− fail "Cloud Haskell fatal error: received unexpected multicast"
446+ fatal "Cloud Haskell fatal error: received unexpected multicast"
448+ fatal :: String -> IO ()
449+ fatal msg = stopTracer (localTracer node) >> fail msg
436 invalidRequest :: NT.ConnectionId -> ConnectionState -> IO ()
451 invalidRequest :: NT.ConnectionId -> ConnectionState -> IO ()
437− invalidRequest cid st =
452+ invalidRequest cid st = do
438 -- TODO: We should treat this as a fatal error on the part of the remote
453 -- TODO: We should treat this as a fatal error on the part of the remote
439 -- node. That is, we should report the remote node as having died, and we
454 -- node. That is, we should report the remote node as having died, and we
440 -- should close incoming connections (this requires a Transport layer
455 -- should close incoming connections (this requires a Transport layer
441 -- extension).
456 -- extension).
457+ traceEventFmtIO node "" [(TraceStr "[network] invalid request: "),
442 go ( incomingAt cid ^= Nothing
459 go ( incomingAt cid ^= Nothing
@@ -490,6 +507,35 @@ instance Show ProcessKillException where
490 show (ProcessKillException pid reason) =
507 show (ProcessKillException pid reason) =
491 "killed-by=" ++ show pid ++ ",reason=" ++ reason
508 "killed-by=" ++ show pid ++ ",reason=" ++ reason
510+--------------------------------------------------------------------------------
511+-- Tracing/Debugging --
512+--------------------------------------------------------------------------------
516+traceNotifyDied :: LocalNode -> Identifier -> DiedReason -> NC ()
517+traceNotifyDied node ident reason =
519+ DiedNormal -> return ()
520+ _ -> traceNcEventFmt node " "
521+ [(TraceStr "[node-controller]"),
525+traceNcEventFmt :: LocalNode -> String -> [TraceArg] -> NC ()
526+traceNcEventFmt node fmt args =
527+ liftIO $ traceEventFmtIO node fmt args
529+traceEventFmtIO :: LocalNode
533+traceEventFmtIO node fmt args =
534+ withLocalTracer node $ \t -> traceFormat t fmt args
536+withLocalTracer :: LocalNode -> (Tracer -> IO ()) -> IO ()
537+withLocalTracer node act = act (localTracer node)
493--------------------------------------------------------------------------------
539--------------------------------------------------------------------------------
494-- Core functionality --
540-- Core functionality --
495--------------------------------------------------------------------------------
541--------------------------------------------------------------------------------
@@ -597,6 +643,7 @@ ncEffectUnmonitor from ref = do
597ncEffectDied :: Identifier -> DiedReason -> NC ()
643ncEffectDied :: Identifier -> DiedReason -> NC ()
598ncEffectDied ident reason = do
644ncEffectDied ident reason = do
599 node <- ask
645 node <- ask
646+ traceNotifyDied node ident reason
600 (affectedLinks, unaffectedLinks) <- gets (splitNotif ident . (^. links))
647 (affectedLinks, unaffectedLinks) <- gets (splitNotif ident . (^. links))
601 (affectedMons, unaffectedMons) <- gets (splitNotif ident . (^. monitors))
648 (affectedMons, unaffectedMons) <- gets (splitNotif ident . (^. monitors))
@@ -639,7 +686,6 @@ ncEffectDied ident reason = do
639 , ctrlMsgSignal = Died ident reason
686 , ctrlMsgSignal = Died ident reason
643-- [Unified: Table 13]
689-- [Unified: Table 13]
644ncEffectSpawn :: ProcessId -> Closure (Process ()) -> SpawnRef -> NC ()
690ncEffectSpawn :: ProcessId -> Closure (Process ()) -> SpawnRef -> NC ()
645ncEffectSpawn pid cProc ref = do
691ncEffectSpawn pid cProc ref = do
@@ -841,18 +887,18 @@ notifyDied dest src reason mRef = do
842-- | [Unified: Table 8]
888-- | [Unified: Table 8]
843destNid :: ProcessSignal -> Maybe NodeId
889destNid :: ProcessSignal -> Maybe NodeId
844−destNid (Link ident) = Just $ nodeOf ident
890+destNid (Link ident) = Just $ nodeOf ident
845−destNid (Unlink ident) = Just $ nodeOf ident
891+destNid (Unlink ident) = Just $ nodeOf ident
846−destNid (Monitor ref) = Just $ nodeOf (monitorRefIdent ref)
892+destNid (Monitor ref) = Just $ nodeOf (monitorRefIdent ref)
847−destNid (Unmonitor ref) = Just $ nodeOf (monitorRefIdent ref)
893+destNid (Unmonitor ref) = Just $ nodeOf (monitorRefIdent ref)
848−destNid (Spawn _ _) = Nothing
894+destNid (Spawn _ _) = Nothing
849−destNid (Register _ _ _ _) = Nothing
895+destNid (Register _ _ _ _) = Nothing
850−destNid (WhereIs _) = Nothing
896+destNid (WhereIs _) = Nothing
851−destNid (NamedSend _ _) = Nothing
897+destNid (NamedSend _ _) = Nothing
852-- We don't need to forward 'Died' signals; if monitoring/linking is setup,
898-- We don't need to forward 'Died' signals; if monitoring/linking is setup,
853-- then when a local process dies the monitoring/linking machinery will take
899-- then when a local process dies the monitoring/linking machinery will take
854-- care of notifying remote nodes
900-- care of notifying remote nodes
855−destNid (Died _ _) = Nothing
901+destNid (Died _ _) = Nothing
856destNid (Kill pid _) = Just $ processNodeId pid
902destNid (Kill pid _) = Just $ processNodeId pid
857destNid (Exit pid _) = Just $ processNodeId pid
903destNid (Exit pid _) = Just $ processNodeId pid
858destNid (GetInfo pid) = Just $ processNodeId pid
904destNid (GetInfo pid) = Just $ processNodeId pid