git.iko.soy/distributed-process
on master help sign in
Another instance of the async exception bug
commit54129b034bdb81ede5627aa24deba04d28f81515 authorEdsko de Vries <edsko@edsko.net> dateThu 24 May 2012, 16:55 (14 years ago) parentc69a77f
5 files changed+86−22
network-transport/network-transport.cabal+7−4 network-transport/src/Network/Transport/TCP.hs+11−12 network-transport/tests/TestAuxiliary.hs+13−2 network-transport/tests/TestTCP.hs+1−1 network-transport/tests/TestTransport.hs+54−3
network-transport/network-transport.cabal+7−4
@@ -37,9 +37,10 @@ Test-Suite TestTCP
3737 network,
3838 mtl,
3939 transformers,
40− ansi-terminal
40+ ansi-terminal,
41+ random
4142 extensions: OverloadedStrings, ForeignFunctionInterface, DeriveDataTypeable, EmptyDataDecls, RankNTypes, ExistentialQuantification, FlexibleInstances, OverlappingInstances
42− ghc-options: -Wall -fno-warn-unused-do-bind
43+ ghc-options: -Wall -fno-warn-unused-do-bind -threaded -rtsopts -with-rtsopts=-N
4344 HS-Source-Dirs: tests src
4445
4546Test-Suite TestMulticastInMemory
@@ -51,7 +52,8 @@ Test-Suite TestMulticastInMemory
5152 data-accessor,
5253 mtl,
5354 transformers,
54− ansi-terminal
55+ ansi-terminal,
56+ random
5557 extensions: OverloadedStrings, ForeignFunctionInterface, DeriveDataTypeable, EmptyDataDecls, RankNTypes, ExistentialQuantification, FlexibleInstances, OverlappingInstances
5658 ghc-options: -Wall -fno-warn-unused-do-bind
5759 HS-Source-Dirs: tests src
@@ -65,7 +67,8 @@ Test-Suite TestInMemory
6567 data-accessor,
6668 mtl,
6769 transformers,
68− ansi-terminal
70+ ansi-terminal,
71+ random
6972 extensions: OverloadedStrings, ForeignFunctionInterface, DeriveDataTypeable, EmptyDataDecls, RankNTypes, ExistentialQuantification, FlexibleInstances, OverlappingInstances
7073 ghc-options: -Wall -fno-warn-unused-do-bind
7174 HS-Source-Dirs: tests src
network-transport/src/Network/Transport/TCP.hs+11−12
@@ -483,7 +483,7 @@ apiSend (ourEndPoint, theirEndPoint) connId connAlive payload =
483483 -- instance if the connection is closed) with the outer exception (which is
484484 -- returned by 'try' when an exception is thrown by 'sendOn', and handled
485485 -- by 'withRemoteState')
486− liftM join . try . mapIOException sendFailed $
486+ liftM join . try . mapIOException sendFailed $
487487 withRemoteState (ourEndPoint, theirEndPoint) RemoteStatePatternMatch
488488 { caseInvalid = \_ ->
489489 relyViolation (ourEndPoint, theirEndPoint) "apiSend"
@@ -679,7 +679,7 @@ requestConnectionTo ourEndPoint theirAddress = go
679679 findRemoteEndPoint ourEndPoint theirAddress
680680
681681 -- If it was new, start listening for incoming messages, too
682− when isNew . void . forkEndPointThread ourEndPoint $
682+ when isNew . void . forkEndPointThread ourEndPoint $
683683 setupRemoteEndPoint (ourEndPoint, theirEndPoint)
684684
685685 let theirState = remoteState theirEndPoint
@@ -703,15 +703,15 @@ requestConnectionTo ourEndPoint theirAddress = go
703703 -- endpoint state, and might have changed in the meantime, these changes
704704 -- won't matter.
705705 case endPointStateSnapshot of
706− RemoteEndPointInvalid err ->
706+ RemoteEndPointInvalid err ->
707707 throwIO err
708708
709− RemoteEndPointClosing resolved _ ->
709+ RemoteEndPointClosing resolved _ ->
710710 -- If the remote endpoint is closing, then we need to block until
711711 -- this is resolved and we then try again
712712 readMVar resolved >> go
713713
714− RemoteEndPointClosed ->
714+ RemoteEndPointClosed ->
715715 -- RemoteEndPointClosed indicates that a concurrent thread was in the
716716 -- process of closing the TCP connection to the remote endpoint when
717717 -- we obtained a reference to it. By INV-CLOSE we can assume that the
@@ -728,9 +728,9 @@ requestConnectionTo ourEndPoint theirAddress = go
728728 reply <- mapIOException connectFailed $
729729 doRemoteRequest (ourEndPoint, theirEndPoint) RequestConnectionId
730730 case decodeInt32 . BS.concat $ reply of
731− Nothing ->
731+ Nothing ->
732732 throwIO (connectFailed $ userError "Invalid integer")
733− Just cid ->
733+ Just cid ->
734734 return (theirEndPoint, cid)
735735
736736 connectFailed = TransportError ConnectFailed . show
@@ -772,15 +772,14 @@ findRemoteEndPoint ourEndPoint theirAddress =
772772setupRemoteEndPoint :: EndPointPair -> IO ()
773773setupRemoteEndPoint (ourEndPoint, theirEndPoint) = do
774774 didAccept <- bracketOnError (socketToEndPoint ourAddress theirAddress)
775− onError $ \result ->
775+ onError $ \result ->
776776 case result of
777777 Right (sock, ConnectionRequestAccepted) -> do
778778 let vst = ValidRemoteEndPointState
779779 { remoteSocket = sock
780780 , _remoteOutgoing = 0
781781 , _remoteIncoming = IntSet.empty
782− , sendOn = (`onException` tryCloseSocket sock)
783− . sendMany sock
782+ , sendOn = sendMany sock
784783 }
785784 putMVar theirState (RemoteEndPointValid vst)
786785 return True
@@ -871,7 +870,7 @@ socketToEndPoint (EndPointAddress ourAddress) theirAddress = try $ do
871870--
872871-- If the local endpoint is closed, do nothing
873872removeRemoteEndPoint :: EndPointPair -> IO ()
874−removeRemoteEndPoint (ourEndPoint, theirEndPoint) =
873+removeRemoteEndPoint (ourEndPoint, theirEndPoint) =
875874 modifyMVar_ ourState $ \st -> case st of
876875 LocalEndPointValid vst ->
877876 case vst ^. localConnectionTo theirAddress of
@@ -956,7 +955,7 @@ doRemoteRequest (ourEndPoint, theirEndPoint) header = do
956955
957956-- | Send a CloseSocket request if the remote endpoint is unused
958957closeIfUnused :: EndPointPair -> IO ()
959−closeIfUnused (ourEndPoint, theirEndPoint) =
958+closeIfUnused (ourEndPoint, theirEndPoint) =
960959 modifyRemoteState_ (ourEndPoint, theirEndPoint) remoteStateIdentity
961960 { caseValid = \vst ->
962961 if vst ^. remoteOutgoing == 0 && IntSet.null (vst ^. remoteIncoming)
network-transport/tests/TestAuxiliary.hs+13−2
@@ -5,10 +5,11 @@ module TestAuxiliary ( -- Running tests
55 -- Writing tests
66 , forkTry
77 , trySome
8+ , randomThreadDelay
89 ) where
910
1011import Prelude hiding (catch)
11−import Control.Concurrent (myThreadId, forkIO, ThreadId, throwTo)
12+import Control.Concurrent (myThreadId, forkIO, ThreadId, throwTo, threadDelay)
1213import Control.Concurrent.Chan (Chan)
1314import Control.Monad (liftM2, unless)
1415import Control.Exception (SomeException, try, catch)
@@ -20,6 +21,7 @@ import System.Console.ANSI ( SGR(SetColor, Reset)
2021 , ColorIntensity(Vivid)
2122 , setSGR
2223 )
24+import System.Random (randomIO)
2325import Network.Transport
2426import Traced (Traceable(..), traceShow)
2527
@@ -38,7 +40,7 @@ runTest :: String -> IO () -> IO Bool
3840runTest description test = do
3941 putStr $ "Running " ++ show description ++ ": "
4042 hFlush stdout
41− done <- try . timeout 20000000 $ test -- 20 seconds
43+ done <- try . timeout 30000000 $ test -- 30 seconds
4244 case done of
4345 Left err -> failed $ "(exception: " ++ show (err :: SomeException) ++ ")"
4446 Right Nothing -> failed $ "(timeout)"
@@ -65,6 +67,12 @@ runTests tests = do
6567 success <- foldr (liftM2 (&&) . uncurry runTest) (return True) $ tests
6668 unless success $ fail "Some tests failed"
6769
70+-- | Random thread delay between 0 and the specified max
71+randomThreadDelay :: Int -> IO ()
72+randomThreadDelay maxDelay = do
73+ delay <- randomIO :: IO Int
74+ threadDelay (delay `mod` maxDelay)
75+
6876--------------------------------------------------------------------------------
6977-- traceShow instances --
7078--------------------------------------------------------------------------------
@@ -95,3 +103,6 @@ instance Traceable ThreadId where
95103
96104instance Traceable (Chan a) where
97105 trace = const Nothing
106+
107+instance Traceable Float where
108+ trace = traceShow
network-transport/tests/TestTCP.hs+1−1
@@ -588,7 +588,7 @@ testReconnect nextPort = do
588588 Right conn1 <- connect endpoint theirAddr ReliableOrdered defaultConnectHints
589589
590590 -- But a send will fail because the server has closed the connection again
591− Left (TransportError SendClosed _) <- send conn1 ["ping"]
591+ Left (TransportError SendFailed _) <- send conn1 ["ping"]
592592 ErrorEvent (TransportError (EventConnectionLost _ []) _) <- receive endpoint
593593
594594 -- But a subsequent call to connect should reestablish the connection
network-transport/tests/TestTransport.hs+54−3
@@ -2,20 +2,23 @@
22module TestTransport where
33
44import Prelude hiding (catch, (>>=), (>>), return, fail)
5−import TestAuxiliary (forkTry, runTests, trySome)
5+import TestAuxiliary (forkTry, runTests, trySome, randomThreadDelay)
6+import Control.Concurrent (forkIO, killThread)
67import Control.Concurrent.MVar (newEmptyMVar, takeMVar, putMVar, readMVar)
78import Control.Exception (evaluate, throw)
8−import Control.Monad (replicateM, replicateM_, when, guard, forM_)
9+import Control.Monad (replicateM, replicateM_, when, guard, forM, forM_)
910import Control.Monad.Error ()
1011import Network.Transport hiding (connect)
1112import qualified Network.Transport as NT
1213import Network.Transport.Internal (tlog)
1314import Network.Transport.Util (spawn)
15+import System.Random (randomIO)
1416import Data.ByteString (ByteString)
1517import Data.ByteString.Char8 (pack)
1618import Data.Map (Map)
1719import qualified Data.Map as Map (empty, insert, delete, findWithDefault, adjust, null, toList, map)
1820import Data.String (fromString)
21+import Data.Maybe (catMaybes)
1922import Traced
2023
2124-- | We overload connect to always pass the default hints
@@ -648,9 +651,9 @@ testCloseTransport newTransport = do
648651
649652 -- Client now closes down its transport. We should receive connection closed messages
650653 -- TODO: this assumes a certain ordering on the messages we receive; that's not guaranteed
654+ ConnectionClosed cid1' <- receive endpoint ; True <- return $ cid1' == cid1
651655 ConnectionClosed cid2'' <- receive endpoint ; True <- return $ cid2'' == cid2
652656 ErrorEvent (TransportError (EventConnectionLost addr'' []) _) <- receive endpoint ; True <- return $ addr'' == theirAddr2
653− ConnectionClosed cid1' <- receive endpoint ; True <- return $ cid1' == cid1
654657
655658 -- An attempt to send to the endpoint should now fail
656659 Left (TransportError SendClosed _) <- send conn ["pong2"]
@@ -755,6 +758,7 @@ testExceptionOnReceive newTransport = do
755758
756759 return ()
757760
761+-- | Test what happens when the argument to 'send' is an exceptional value
758762testSendException :: IO (Either String Transport) -> IO ()
759763testSendException newTransport = do
760764 Right transport <- newTransport
@@ -785,6 +789,52 @@ testSendException newTransport = do
785789
786790 return ()
787791
792+-- | If threads get killed while executing a 'connect', 'send', or 'close', this
793+-- should not affect other threads.
794+--
795+-- The intention of this test is to see what happens when a asynchronous
796+-- exception happes _while executing a send, connect, or close_. This is
797+-- exceedingly difficult to guarantee, however. Hence we run a large number of
798+-- tests and insert random thread delays -- and even then it might not happen.
799+-- Moreover, it will only happen when we run on multiple cores.
800+testKill :: IO (Either String Transport) -> Int -> IO ()
801+testKill newTransport numThreads = do
802+ Right transport1 <- newTransport
803+ Right transport2 <- newTransport
804+ Right endpoint1 <- newEndPoint transport1
805+ Right endpoint2 <- newEndPoint transport2
806+
807+ threads <- forM [1 .. numThreads] $ \_ -> do
808+ done <- newEmptyMVar
809+ tid <- forkIO $ do
810+ randomThreadDelay 10
811+ Right conn <- connect endpoint1 (address endpoint2) ReliableOrdered
812+ randomThreadDelay 10
813+ Right () <- send conn ["ping"]
814+ randomThreadDelay 10
815+ close conn
816+ putMVar done ()
817+
818+ return (tid, done)
819+
820+ -- Kill half of those threads, and wait on the rest
821+ killerDone <- newEmptyMVar
822+ forkIO $ do
823+ wait <- forM threads $ \(tid, done) -> do
824+ shouldKill <- randomIO
825+ if shouldKill
826+ then do
827+ randomThreadDelay 30
828+ killThread tid
829+ return Nothing
830+ else
831+ return (Just done)
832+
833+ mapM_ takeMVar (catMaybes wait)
834+ putMVar killerDone ()
835+
836+ takeMVar killerDone
837+
788838-- Transport tests
789839testTransport :: IO (Either String Transport) -> IO ()
790840testTransport newTransport = do
@@ -806,6 +856,7 @@ testTransport newTransport = do
806856 , ("ConnectClosedEndPoint", testConnectClosedEndPoint transport)
807857 , ("ExceptionOnReceive", testExceptionOnReceive newTransport)
808858 , ("SendException", testSendException newTransport)
859+ , ("Kill", testKill newTransport 100000)
809860 ]
810861 where
811862 numPings = 10000 :: Int