2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Low-Level IPC flows for interacting with the serf process.
|
|
|
|
|
|
|
|
- Serf process can be started and shutdown with `start` and `stop`.
|
|
|
|
- You can ask the serf what it's last event was with
|
|
|
|
`serfLastEventBlocking`.
|
|
|
|
- A running serf can be asked to compact it's heap or take a snapshot.
|
|
|
|
- You can scry into a running serf.
|
|
|
|
- A running serf can be asked to execute a boot sequence, replay from
|
|
|
|
existing events, and run a ship with `boot`, `replay`, and `run`.
|
|
|
|
|
|
|
|
The running and replay flows will do batching of events to keep the
|
|
|
|
IPC pipe full.
|
|
|
|
|
|
|
|
```
|
|
|
|
|%
|
|
|
|
:: +writ: from king to serf
|
|
|
|
::
|
|
|
|
+$ gang (unit (set ship))
|
|
|
|
+$ writ
|
|
|
|
$% $: %live
|
|
|
|
$% [%exit cod=@]
|
|
|
|
[%save eve=@]
|
|
|
|
[%pack eve=@]
|
|
|
|
== ==
|
|
|
|
[%peek now=date lyc=gang pat=path]
|
|
|
|
[%play eve=@ lit=(list ?((pair date ovum) *))]
|
|
|
|
[%work job=(pair date ovum)]
|
|
|
|
==
|
|
|
|
:: +plea: from serf to king
|
|
|
|
::
|
|
|
|
+$ plea
|
|
|
|
$% [%live ~]
|
|
|
|
[%ripe [pro=@ hon=@ nok=@] eve=@ mug=@]
|
|
|
|
[%slog pri=@ ?(cord tank)]
|
|
|
|
[%peek dat=(unit (cask))]
|
|
|
|
$: %play
|
|
|
|
$% [%done mug=@]
|
|
|
|
[%bail eve=@ mug=@ dud=goof]
|
|
|
|
== ==
|
|
|
|
$: %work
|
|
|
|
$% [%done eve=@ mug=@ fec=(list ovum)]
|
|
|
|
[%swap eve=@ mug=@ job=(pair date ovum) fec=(list ovum)]
|
|
|
|
[%bail lud=(list goof)]
|
|
|
|
== ==
|
|
|
|
==
|
|
|
|
```
|
2020-05-21 00:20:01 +03:00
|
|
|
-}
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
module Urbit.Vere.Serf.IPC
|
|
|
|
( Serf
|
|
|
|
, Config(..)
|
|
|
|
, PlayBail(..)
|
|
|
|
, Flag(..)
|
2020-06-01 01:37:27 +03:00
|
|
|
, WorkError(..)
|
2020-06-01 20:51:37 +03:00
|
|
|
, EvErr(..)
|
|
|
|
, RunReq(..)
|
2020-05-28 01:57:34 +03:00
|
|
|
, start
|
2020-06-01 20:51:37 +03:00
|
|
|
, stop
|
2020-05-28 01:57:34 +03:00
|
|
|
, serfLastEventBlocking
|
|
|
|
, snapshot
|
2020-06-01 20:51:37 +03:00
|
|
|
, compact
|
|
|
|
, scry
|
|
|
|
, boot
|
2020-05-28 01:57:34 +03:00
|
|
|
, replay
|
2020-06-01 20:51:37 +03:00
|
|
|
, run
|
2020-06-02 00:31:24 +03:00
|
|
|
, swim
|
2020-05-28 01:57:34 +03:00
|
|
|
)
|
|
|
|
where
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
import Urbit.Prelude hiding ((<|))
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
import Data.Bits
|
2020-05-21 03:35:33 +03:00
|
|
|
import Data.Conduit
|
2020-05-28 01:57:34 +03:00
|
|
|
import System.Process
|
2020-05-21 03:35:33 +03:00
|
|
|
import Urbit.Arvo
|
|
|
|
import Urbit.Vere.Pier.Types hiding (Work)
|
|
|
|
|
2020-06-05 00:10:33 +03:00
|
|
|
import Control.Monad.STM (retry)
|
|
|
|
import Control.Monad.Trans.Resource (MonadResource, allocate, runResourceT)
|
|
|
|
import Data.Sequence (Seq((:<|), (:|>)))
|
|
|
|
import Foreign.Marshal.Alloc (alloca)
|
|
|
|
import Foreign.Ptr (castPtr)
|
|
|
|
import Foreign.Storable (peek, poke)
|
|
|
|
import RIO.Prelude (decodeUtf8Lenient)
|
|
|
|
import System.Posix.Signals (sigKILL, signalProcess)
|
|
|
|
import Urbit.Time (Wen)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-05-22 01:42:00 +03:00
|
|
|
import qualified Data.ByteString as BS
|
|
|
|
import qualified Data.ByteString.Unsafe as BS
|
2020-05-27 02:01:03 +03:00
|
|
|
import qualified System.IO.Error as IO
|
|
|
|
import qualified Urbit.Time as Time
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-05-22 01:42:00 +03:00
|
|
|
|
|
|
|
-- IPC Types -------------------------------------------------------------------
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
data Live
|
2020-05-28 01:57:34 +03:00
|
|
|
= LExit Atom -- exit status code
|
2020-05-21 03:35:33 +03:00
|
|
|
| LSave EventId
|
|
|
|
| LPack EventId
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
type PlayBail = (EventId, Mug, Goof)
|
|
|
|
|
|
|
|
data Play
|
|
|
|
= PDone Mug
|
|
|
|
| PBail PlayBail
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
data Work
|
2020-06-01 01:04:56 +03:00
|
|
|
= WDone EventId Mug FX
|
|
|
|
| WSwap EventId Mug (Wen, Noun) FX
|
2020-05-21 03:35:33 +03:00
|
|
|
| WBail [Goof]
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
data Writ
|
|
|
|
= WLive Live
|
|
|
|
| WPeek Wen Gang Path
|
|
|
|
| WPlay EventId [Noun]
|
|
|
|
| WWork Wen Ev
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
data RipeInfo = RipeInfo
|
|
|
|
{ riProt :: Atom
|
|
|
|
, riHoon :: Atom
|
|
|
|
, riNock :: Atom
|
|
|
|
}
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
data SerfState = SerfState
|
|
|
|
{ ssLast :: EventId
|
|
|
|
, ssHash :: Mug
|
|
|
|
}
|
2020-06-05 00:10:33 +03:00
|
|
|
deriving (Show, Eq)
|
2020-05-27 02:01:03 +03:00
|
|
|
|
2020-05-21 03:35:33 +03:00
|
|
|
data SerfInfo = SerfInfo
|
|
|
|
{ siRipe :: RipeInfo
|
2020-05-27 02:01:03 +03:00
|
|
|
, siStat :: SerfState
|
2020-05-21 03:35:33 +03:00
|
|
|
}
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
|
|
|
|
|
|
|
type Slog = (Atom, Tank)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
data Plea
|
|
|
|
= PLive ()
|
|
|
|
| PRipe SerfInfo
|
2020-05-22 01:42:00 +03:00
|
|
|
| PSlog Slog
|
2020-05-21 03:35:33 +03:00
|
|
|
| PPeek (Maybe (Term, Noun))
|
|
|
|
| PPlay Play
|
|
|
|
| PWork Work
|
2020-05-22 01:42:00 +03:00
|
|
|
deriving (Show)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
deriveNoun ''Live
|
|
|
|
deriveNoun ''Play
|
|
|
|
deriveNoun ''Work
|
|
|
|
deriveNoun ''Writ
|
|
|
|
deriveNoun ''RipeInfo
|
2020-05-27 02:01:03 +03:00
|
|
|
deriveNoun ''SerfState
|
2020-05-21 03:35:33 +03:00
|
|
|
deriveNoun ''SerfInfo
|
|
|
|
deriveNoun ''Plea
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
|
|
|
|
-- Serf API Types --------------------------------------------------------------
|
|
|
|
|
2020-05-21 03:35:33 +03:00
|
|
|
data Serf = Serf
|
2020-05-22 01:42:00 +03:00
|
|
|
{ serfSend :: Handle
|
|
|
|
, serfRecv :: Handle
|
|
|
|
, serfProc :: ProcessHandle
|
|
|
|
, serfSlog :: Slog -> IO ()
|
2020-06-05 00:10:33 +03:00
|
|
|
, serfLock :: MVar (Maybe SerfState)
|
2020-05-21 00:20:01 +03:00
|
|
|
}
|
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
data Flag
|
|
|
|
= DebugRam
|
|
|
|
| DebugCpu
|
|
|
|
| CheckCorrupt
|
|
|
|
| CheckFatal
|
|
|
|
| Verbose
|
|
|
|
| DryRun
|
|
|
|
| Quiet
|
|
|
|
| Hashless
|
|
|
|
| Trace
|
|
|
|
deriving (Eq, Ord, Show, Enum, Bounded)
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
data Config = Config
|
|
|
|
{ scSerf :: FilePath -- Where is the urbit-worker executable?
|
|
|
|
, scPier :: FilePath -- Where is the pier directory?
|
|
|
|
, scFlag :: [Flag] -- Serf execution flags.
|
|
|
|
, scSlog :: Slog -> IO () -- What to do with slogs?
|
|
|
|
, scStdr :: Text -> IO () -- What to do with lines from stderr?
|
|
|
|
, scDead :: IO () -- What to do when the serf process goes down?
|
2020-05-27 02:01:03 +03:00
|
|
|
}
|
2020-05-21 00:20:01 +03:00
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
|
2020-05-22 01:42:00 +03:00
|
|
|
-- Exceptions ------------------------------------------------------------------
|
|
|
|
|
|
|
|
data SerfExn
|
2020-06-01 20:51:37 +03:00
|
|
|
= UnexpectedPlea Plea Text
|
|
|
|
| BadPleaAtom Atom
|
|
|
|
| BadPleaNoun Noun [Text] Text
|
|
|
|
| SerfConnectionClosed
|
|
|
|
| SerfHasShutdown
|
2020-06-02 00:31:24 +03:00
|
|
|
| BailDuringReplay EventId [Goof]
|
|
|
|
| SwapDuringReplay EventId Mug (Wen, Noun) FX
|
2020-06-05 00:10:33 +03:00
|
|
|
| SerfNotRunning
|
2020-06-01 20:51:37 +03:00
|
|
|
deriving (Show, Exception)
|
2020-05-22 01:42:00 +03:00
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
-- Access Current Serf State ---------------------------------------------------
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
serfLastEventBlocking :: Serf -> IO EventId
|
2020-05-28 03:08:53 +03:00
|
|
|
serfLastEventBlocking Serf{serfLock} = readMVar serfLock >>= \case
|
2020-06-05 00:10:33 +03:00
|
|
|
Nothing -> throwIO SerfNotRunning
|
|
|
|
Just ss -> pure (ssLast ss)
|
2020-05-28 01:57:34 +03:00
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
|
2020-05-21 03:35:33 +03:00
|
|
|
-- Low Level IPC Functions -----------------------------------------------------
|
|
|
|
|
2020-05-22 01:42:00 +03:00
|
|
|
fromRightExn :: (Exception e, MonadIO m) => Either a b -> (a -> e) -> m b
|
|
|
|
fromRightExn (Left m) exn = throwIO (exn m)
|
|
|
|
fromRightExn (Right x) _ = pure x
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
-- TODO Support Big Endian
|
2020-05-22 01:42:00 +03:00
|
|
|
sendLen :: Serf -> Int -> IO ()
|
|
|
|
sendLen s i = do
|
|
|
|
w <- evaluate (fromIntegral i :: Word64)
|
2020-05-28 01:57:34 +03:00
|
|
|
withWord64AsByteString w (hPut (serfSend s))
|
|
|
|
where
|
|
|
|
withWord64AsByteString :: Word64 -> (ByteString -> IO a) -> IO a
|
|
|
|
withWord64AsByteString w k = alloca $ \wp -> do
|
|
|
|
poke wp w
|
|
|
|
bs <- BS.unsafePackCStringLen (castPtr wp, 8)
|
|
|
|
k bs
|
2020-05-22 01:42:00 +03:00
|
|
|
|
|
|
|
sendBytes :: Serf -> ByteString -> IO ()
|
|
|
|
sendBytes s bs = handle onIOError $ do
|
|
|
|
sendLen s (length bs)
|
|
|
|
hPut (serfSend s) bs
|
|
|
|
hFlush (serfSend s)
|
|
|
|
where
|
|
|
|
onIOError :: IOError -> IO ()
|
2020-06-01 01:37:27 +03:00
|
|
|
onIOError = const (throwIO SerfConnectionClosed)
|
2020-05-22 01:42:00 +03:00
|
|
|
|
|
|
|
recvBytes :: Serf -> Word64 -> IO ByteString
|
2020-05-28 01:57:34 +03:00
|
|
|
recvBytes serf = BS.hGet (serfRecv serf) . fromIntegral
|
2020-05-22 01:42:00 +03:00
|
|
|
|
|
|
|
recvLen :: Serf -> IO Word64
|
|
|
|
recvLen w = do
|
|
|
|
bs <- BS.hGet (serfRecv w) 8
|
|
|
|
case length bs of
|
2020-05-28 01:57:34 +03:00
|
|
|
8 -> BS.unsafeUseAsCString bs (peek @Word64 . castPtr)
|
2020-06-01 01:37:27 +03:00
|
|
|
_ -> throwIO SerfConnectionClosed
|
2020-05-22 01:42:00 +03:00
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
recvResp :: Serf -> IO ByteString
|
|
|
|
recvResp serf = do
|
|
|
|
len <- recvLen serf
|
|
|
|
recvBytes serf len
|
2020-05-22 01:42:00 +03:00
|
|
|
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-05-22 01:42:00 +03:00
|
|
|
-- Send Writ / Recv Plea -------------------------------------------------------
|
|
|
|
|
|
|
|
sendWrit :: Serf -> Writ -> IO ()
|
2020-05-28 01:57:34 +03:00
|
|
|
sendWrit s = sendBytes s . jamBS . toNoun
|
2020-05-22 01:42:00 +03:00
|
|
|
|
|
|
|
recvPlea :: Serf -> IO Plea
|
|
|
|
recvPlea w = do
|
2020-05-28 01:57:34 +03:00
|
|
|
b <- recvResp w
|
|
|
|
n <- fromRightExn (cueBS b) (const $ BadPleaAtom $ bytesAtom b)
|
|
|
|
p <- fromRightExn (fromNounErr @Plea n) (\(p, m) -> BadPleaNoun n p m)
|
2020-05-22 01:42:00 +03:00
|
|
|
pure p
|
|
|
|
|
|
|
|
recvPleaHandlingSlog :: Serf -> IO Plea
|
|
|
|
recvPleaHandlingSlog serf = loop
|
|
|
|
where
|
|
|
|
loop = recvPlea serf >>= \case
|
|
|
|
PSlog info -> serfSlog serf info >> loop
|
|
|
|
other -> pure other
|
|
|
|
|
|
|
|
|
|
|
|
-- Higher-Level IPC Functions --------------------------------------------------
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
recvRipe :: Serf -> IO SerfInfo
|
|
|
|
recvRipe serf = recvPleaHandlingSlog serf >>= \case
|
|
|
|
PRipe ripe -> pure ripe
|
|
|
|
plea -> throwIO (UnexpectedPlea plea "expecting %play")
|
|
|
|
|
2020-05-21 03:35:33 +03:00
|
|
|
recvPlay :: Serf -> IO Play
|
2020-05-22 01:42:00 +03:00
|
|
|
recvPlay serf = recvPleaHandlingSlog serf >>= \case
|
2020-05-21 03:35:33 +03:00
|
|
|
PPlay play -> pure play
|
2020-05-22 01:42:00 +03:00
|
|
|
plea -> throwIO (UnexpectedPlea plea "expecting %play")
|
2020-05-21 03:35:33 +03:00
|
|
|
|
|
|
|
recvLive :: Serf -> IO ()
|
2020-05-22 01:42:00 +03:00
|
|
|
recvLive serf = recvPleaHandlingSlog serf >>= \case
|
|
|
|
PLive () -> pure ()
|
|
|
|
plea -> throwIO (UnexpectedPlea plea "expecting %live")
|
|
|
|
|
|
|
|
recvWork :: Serf -> IO Work
|
|
|
|
recvWork serf = do
|
|
|
|
recvPleaHandlingSlog serf >>= \case
|
|
|
|
PWork work -> pure work
|
|
|
|
plea -> throwIO (UnexpectedPlea plea "expecting %work")
|
|
|
|
|
|
|
|
recvPeek :: Serf -> IO (Maybe (Term, Noun))
|
|
|
|
recvPeek serf = do
|
|
|
|
recvPleaHandlingSlog serf >>= \case
|
|
|
|
PPeek peek -> pure peek
|
|
|
|
plea -> throwIO (UnexpectedPlea plea "expecting %peek")
|
|
|
|
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
-- Request-Response Points -- These don't touch the lock -----------------------
|
2020-05-22 01:42:00 +03:00
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
sendSnapshotRequest :: Serf -> EventId -> IO ()
|
|
|
|
sendSnapshotRequest serf eve = do
|
2020-05-22 01:42:00 +03:00
|
|
|
sendWrit serf (WLive $ LSave eve)
|
2020-05-21 03:35:33 +03:00
|
|
|
recvLive serf
|
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
sendCompactionRequest :: Serf -> EventId -> IO ()
|
|
|
|
sendCompactionRequest serf eve = do
|
2020-05-22 01:42:00 +03:00
|
|
|
sendWrit serf (WLive $ LPack eve)
|
2020-05-21 03:35:33 +03:00
|
|
|
recvLive serf
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
sendScryRequest :: Serf -> Wen -> Gang -> Path -> IO (Maybe (Term, Noun))
|
|
|
|
sendScryRequest serf w g p = do
|
2020-05-22 01:42:00 +03:00
|
|
|
sendWrit serf (WPeek w g p)
|
2020-05-21 03:35:33 +03:00
|
|
|
recvPeek serf
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
sendShutdownRequest :: Serf -> Atom -> IO ()
|
|
|
|
sendShutdownRequest serf exitCode = do
|
|
|
|
sendWrit serf (WLive $ LExit exitCode)
|
|
|
|
pure ()
|
|
|
|
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
-- Starting the Serf -----------------------------------------------------------
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-05-27 02:01:03 +03:00
|
|
|
compileFlags :: [Flag] -> Word
|
|
|
|
compileFlags = foldl' (\acc flag -> setBit acc (fromEnum flag)) 0
|
|
|
|
|
|
|
|
readStdErr :: Handle -> (Text -> IO ()) -> IO () -> IO ()
|
|
|
|
readStdErr h onLine onClose = loop
|
|
|
|
where
|
|
|
|
loop = do
|
|
|
|
IO.tryIOError (BS.hGetLine h >>= onLine . decodeUtf8Lenient) >>= \case
|
|
|
|
Left exn -> onClose
|
|
|
|
Right () -> loop
|
|
|
|
|
|
|
|
start :: Config -> IO (Serf, SerfInfo)
|
|
|
|
start (Config exePax pierPath flags onSlog onStdr onDead) = do
|
|
|
|
(Just i, Just o, Just e, p) <- createProcess pSpec
|
2020-05-28 01:57:34 +03:00
|
|
|
void $ async (readStdErr e onStdr onDead)
|
2020-05-27 02:01:03 +03:00
|
|
|
vLock <- newEmptyMVar
|
|
|
|
let serf = Serf i o p onSlog vLock
|
2020-05-28 01:57:34 +03:00
|
|
|
info <- recvRipe serf
|
2020-06-05 00:10:33 +03:00
|
|
|
putMVar vLock (Just $ siStat info)
|
2020-05-27 02:01:03 +03:00
|
|
|
pure (serf, info)
|
|
|
|
where
|
|
|
|
diskKey = ""
|
|
|
|
config = show (compileFlags flags)
|
|
|
|
args = [pierPath, diskKey, config]
|
|
|
|
pSpec = (proc exePax args) { std_in = CreatePipe
|
|
|
|
, std_out = CreatePipe
|
|
|
|
, std_err = CreatePipe
|
|
|
|
}
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
|
|
|
|
-- Taking the SerfState Lock ---------------------------------------------------
|
|
|
|
|
2020-06-05 00:10:33 +03:00
|
|
|
takeLock :: MonadIO m => Serf -> m SerfState
|
|
|
|
takeLock serf = io $ do
|
|
|
|
takeMVar (serfLock serf) >>= \case
|
|
|
|
Nothing -> putMVar (serfLock serf) Nothing >> throwIO SerfNotRunning
|
|
|
|
Just ss -> pure ss
|
|
|
|
|
|
|
|
serfLockTaken
|
|
|
|
:: MonadResource m => Serf -> m (IORef (Maybe SerfState), SerfState)
|
|
|
|
serfLockTaken serf = snd <$> allocate take release
|
2020-05-28 03:08:53 +03:00
|
|
|
where
|
2020-06-05 00:10:33 +03:00
|
|
|
take = (,) <$> newIORef Nothing <*> takeLock serf
|
|
|
|
release (rv, _) = do
|
|
|
|
mRes <- readIORef rv
|
|
|
|
when (mRes == Nothing) (forcefullyKillSerf serf)
|
|
|
|
putMVar (serfLock serf) mRes
|
|
|
|
|
|
|
|
withSerfLock
|
|
|
|
:: MonadResource m => Serf -> (SerfState -> m (SerfState, a)) -> m a
|
|
|
|
withSerfLock serf act = do
|
|
|
|
(vState , initialState) <- serfLockTaken serf
|
|
|
|
(newState, result ) <- act initialState
|
|
|
|
writeIORef vState (Just newState)
|
|
|
|
pure result
|
|
|
|
|
|
|
|
withSerfLockIO :: Serf -> (SerfState -> IO (SerfState, a)) -> IO a
|
|
|
|
withSerfLockIO s a = runResourceT (withSerfLock s (io . a))
|
2020-05-28 03:08:53 +03:00
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
|
|
|
|
-- Flows for Interacting with the Serf -----------------------------------------
|
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Ask the serf to write a snapshot to disk.
|
|
|
|
-}
|
2020-05-29 05:14:25 +03:00
|
|
|
snapshot :: Serf -> IO ()
|
2020-06-05 00:10:33 +03:00
|
|
|
snapshot serf = withSerfLockIO serf $ \ss -> do
|
2020-06-01 01:37:27 +03:00
|
|
|
sendSnapshotRequest serf (ssLast ss)
|
|
|
|
pure (ss, ())
|
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Ask the serf to de-duplicate and de-fragment it's heap.
|
|
|
|
-}
|
2020-06-01 01:37:27 +03:00
|
|
|
compact :: Serf -> IO ()
|
2020-06-05 00:10:33 +03:00
|
|
|
compact serf = withSerfLockIO serf $ \ss -> do
|
2020-06-01 01:37:27 +03:00
|
|
|
sendCompactionRequest serf (ssLast ss)
|
|
|
|
pure (ss, ())
|
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Peek into the serf state.
|
|
|
|
-}
|
|
|
|
scry :: Serf -> Wen -> Gang -> Path -> IO (Maybe (Term, Noun))
|
2020-06-05 00:10:33 +03:00
|
|
|
scry serf w g p = withSerfLockIO serf $ \ss -> do
|
2020-06-01 20:51:37 +03:00
|
|
|
(ss,) <$> sendScryRequest serf w g p
|
2020-05-28 01:57:34 +03:00
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Ask the serf to shutdown. If it takes more than 2s, kill it with
|
|
|
|
SIGKILL.
|
|
|
|
-}
|
|
|
|
stop :: HasLogFunc e => Serf -> RIO e ()
|
|
|
|
stop serf = do
|
|
|
|
race_ niceKill (wait2sec >> forceKill)
|
2020-05-28 01:57:34 +03:00
|
|
|
where
|
2020-06-01 20:51:37 +03:00
|
|
|
wait2sec = threadDelay 2_000_000
|
|
|
|
|
|
|
|
niceKill = do
|
|
|
|
logTrace "Asking serf to shut down"
|
|
|
|
io (gracefullyKillSerf serf)
|
|
|
|
logTrace "Serf went down when asked."
|
|
|
|
|
2020-05-28 01:57:34 +03:00
|
|
|
forceKill = do
|
|
|
|
logTrace "Serf taking too long to go down, kill with fire (SIGTERM)."
|
2020-06-01 20:51:37 +03:00
|
|
|
io (forcefullyKillSerf serf)
|
2020-05-28 03:08:53 +03:00
|
|
|
logTrace "Serf process killed with SIGTERM."
|
2020-05-28 01:57:34 +03:00
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Kill the serf by taking the lock, then asking for it to exit.
|
|
|
|
-}
|
|
|
|
gracefullyKillSerf :: Serf -> IO ()
|
|
|
|
gracefullyKillSerf serf@Serf{..} = do
|
|
|
|
finalState <- takeMVar serfLock
|
|
|
|
sendShutdownRequest serf 0
|
|
|
|
waitForProcess serfProc
|
|
|
|
pure ()
|
|
|
|
|
|
|
|
{-|
|
|
|
|
Kill the serf by sending it a SIGKILL.
|
|
|
|
-}
|
|
|
|
forcefullyKillSerf :: Serf -> IO ()
|
|
|
|
forcefullyKillSerf serf = do
|
2020-05-28 03:08:53 +03:00
|
|
|
getPid (serfProc serf) >>= \case
|
|
|
|
Nothing -> pure ()
|
|
|
|
Just pid -> do
|
|
|
|
io $ signalProcess sigKILL pid
|
|
|
|
io $ void $ waitForProcess (serfProc serf)
|
2020-05-28 01:57:34 +03:00
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Given a list of boot events, send them to to the serf in a single
|
|
|
|
%play message. They must all be sent in a single %play event so that
|
|
|
|
the serf can determine the length of the boot sequence.
|
|
|
|
-}
|
|
|
|
boot :: Serf -> [Noun] -> IO (Maybe PlayBail)
|
|
|
|
boot serf@Serf {..} seq = do
|
2020-06-05 00:10:33 +03:00
|
|
|
withSerfLockIO serf $ \ss -> do
|
2020-05-28 03:08:53 +03:00
|
|
|
recvPlay serf >>= \case
|
2020-06-01 01:37:27 +03:00
|
|
|
PBail bail -> pure (ss, Just bail)
|
|
|
|
PDone mug -> pure (SerfState (fromIntegral $ length seq) mug, Nothing)
|
2020-05-27 02:01:03 +03:00
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
|
|
|
Given a stream of nouns (from the event log), feed them into the serf
|
|
|
|
in batches of size `batchSize`.
|
|
|
|
|
|
|
|
- On `%bail` response, return early.
|
|
|
|
- On IPC errors, kill the serf and rethrow.
|
|
|
|
- On success, return `Nothing`.
|
|
|
|
-}
|
2020-05-28 03:08:53 +03:00
|
|
|
replay
|
|
|
|
:: forall m
|
2020-06-05 00:10:33 +03:00
|
|
|
. (MonadResource m, MonadUnliftIO m, MonadIO m)
|
2020-05-29 05:14:25 +03:00
|
|
|
=> Int
|
2020-06-05 02:49:56 +03:00
|
|
|
-> (EventId -> IO ())
|
2020-05-29 05:14:25 +03:00
|
|
|
-> Serf
|
2020-05-28 03:08:53 +03:00
|
|
|
-> ConduitT Noun Void m (Maybe PlayBail)
|
2020-06-05 02:49:56 +03:00
|
|
|
replay batchSize cb serf = do
|
2020-06-05 00:10:33 +03:00
|
|
|
withSerfLock serf $ \ss -> do
|
2020-05-28 03:08:53 +03:00
|
|
|
(r, ss') <- loop ss
|
|
|
|
pure (ss', r)
|
2020-05-21 03:35:33 +03:00
|
|
|
where
|
2020-05-27 02:01:03 +03:00
|
|
|
loop :: SerfState -> ConduitT Noun Void m (Maybe PlayBail, SerfState)
|
2020-05-29 05:14:25 +03:00
|
|
|
loop ss@(SerfState lastEve lastMug) = do
|
2020-06-05 02:49:56 +03:00
|
|
|
io (cb lastEve)
|
2020-05-29 05:14:25 +03:00
|
|
|
awaitBatch batchSize >>= \case
|
2020-06-01 01:37:27 +03:00
|
|
|
[] -> pure (Nothing, SerfState lastEve lastMug)
|
2020-05-29 05:14:25 +03:00
|
|
|
evs -> do
|
|
|
|
let nexEve = lastEve + 1
|
|
|
|
let newEve = lastEve + fromIntegral (length evs)
|
|
|
|
io $ sendWrit serf (WPlay nexEve evs)
|
|
|
|
io (recvPlay serf) >>= \case
|
|
|
|
PBail bail -> pure (Just bail, SerfState lastEve lastMug)
|
|
|
|
PDone newMug -> loop (SerfState newEve newMug)
|
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
2020-06-01 01:37:27 +03:00
|
|
|
TODO If this is slow, use a mutable vector instead of reversing a list.
|
2020-05-29 05:14:25 +03:00
|
|
|
-}
|
|
|
|
awaitBatch :: Monad m => Int -> ConduitT i o m [i]
|
|
|
|
awaitBatch = go []
|
|
|
|
where
|
|
|
|
go acc 0 = pure (reverse acc)
|
|
|
|
go acc n = await >>= \case
|
|
|
|
Nothing -> pure (reverse acc)
|
|
|
|
Just x -> go (x:acc) (n-1)
|
2020-05-21 03:35:33 +03:00
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
|
2020-06-02 00:31:24 +03:00
|
|
|
-- Special Replay for Collecting FX --------------------------------------------
|
|
|
|
|
|
|
|
{-|
|
|
|
|
This does event-log replay using the running IPC flow so that we
|
|
|
|
can collect effects.
|
|
|
|
|
|
|
|
We don't tolerate replacement events or bails since we are actually
|
|
|
|
replaying the log, so we just throw exceptions in those cases.
|
|
|
|
-}
|
|
|
|
swim
|
|
|
|
:: forall m
|
2020-06-05 00:10:33 +03:00
|
|
|
. (MonadIO m, MonadUnliftIO m, MonadResource m)
|
2020-06-02 00:31:24 +03:00
|
|
|
=> Serf
|
|
|
|
-> ConduitT (Wen, Ev) (EventId, FX) m ()
|
|
|
|
swim serf = do
|
2020-06-05 00:10:33 +03:00
|
|
|
withSerfLock serf $ \SerfState {..} -> do
|
2020-06-02 00:31:24 +03:00
|
|
|
(, ()) <$> loop ssHash ssLast
|
|
|
|
where
|
2020-06-05 00:10:33 +03:00
|
|
|
loop
|
|
|
|
:: Mug
|
|
|
|
-> EventId
|
|
|
|
-> ConduitT (Wen, Ev) (EventId, FX) m SerfState
|
2020-06-02 00:31:24 +03:00
|
|
|
loop mug eve = await >>= \case
|
|
|
|
Nothing -> do
|
|
|
|
pure (SerfState eve mug)
|
|
|
|
Just (wen, evn) -> do
|
|
|
|
io (sendWrit serf (WWork wen evn))
|
|
|
|
io (recvWork serf) >>= \case
|
|
|
|
WBail goofs -> do
|
|
|
|
throwIO (BailDuringReplay eve goofs)
|
|
|
|
WSwap eid hash (wen, noun) fx -> do
|
|
|
|
throwIO (SwapDuringReplay eid hash (wen, noun) fx)
|
|
|
|
WDone eid hash fx -> do
|
|
|
|
yield (eid, fx)
|
|
|
|
loop hash eid
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
-- Running Ship Flow -----------------------------------------------------------
|
|
|
|
|
|
|
|
{-
|
|
|
|
- RRWork: Ask the serf to do work, will output (Fact, FX) if work
|
|
|
|
succeeded and call callback on failure.
|
|
|
|
- RRSave: Wait for the serf to finish all pending work
|
|
|
|
-}
|
|
|
|
data RunReq
|
|
|
|
= RRWork EvErr
|
|
|
|
| RRSave ()
|
|
|
|
| RRKill ()
|
|
|
|
| RRPack ()
|
|
|
|
| RRScry Wen Gang Path (Maybe (Term, Noun) -> IO ())
|
|
|
|
|
|
|
|
{-|
|
2020-06-01 01:37:27 +03:00
|
|
|
TODO Don't take snapshot until event log has processed current event.
|
2020-05-21 03:35:33 +03:00
|
|
|
-}
|
2020-06-01 20:51:37 +03:00
|
|
|
run
|
2020-05-29 05:14:25 +03:00
|
|
|
:: Serf
|
2020-06-01 01:37:27 +03:00
|
|
|
-> Int
|
2020-06-03 02:03:04 +03:00
|
|
|
-> STM EventId
|
2020-06-01 20:51:37 +03:00
|
|
|
-> STM RunReq
|
2020-06-01 01:37:27 +03:00
|
|
|
-> ((Fact, FX) -> STM ())
|
2020-06-01 20:51:37 +03:00
|
|
|
-> (Maybe Ev -> STM ())
|
2020-05-29 05:14:25 +03:00
|
|
|
-> IO ()
|
2020-06-03 02:03:04 +03:00
|
|
|
run serf maxBatchSize getLastEvInLog onInput sendOn spin = topLoop
|
2020-05-29 05:14:25 +03:00
|
|
|
where
|
2020-05-29 23:10:17 +03:00
|
|
|
topLoop :: IO ()
|
|
|
|
topLoop = atomically onInput >>= \case
|
2020-06-01 20:51:37 +03:00
|
|
|
RRWork workErr -> doWork workErr
|
|
|
|
RRSave () -> doSave
|
|
|
|
RRKill () -> pure ()
|
|
|
|
RRPack () -> doPack
|
|
|
|
RRScry w g p k -> doScry w g p k
|
2020-05-29 05:14:25 +03:00
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
doPack :: IO ()
|
|
|
|
doPack = compact serf >> topLoop
|
|
|
|
|
2020-06-03 02:03:04 +03:00
|
|
|
waitForLog :: IO ()
|
|
|
|
waitForLog = do
|
|
|
|
serfLast <- serfLastEventBlocking serf
|
|
|
|
atomically $ do
|
|
|
|
logLast <- getLastEvInLog
|
|
|
|
when (logLast < serfLast) retry
|
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
doSave :: IO ()
|
2020-06-03 02:03:04 +03:00
|
|
|
doSave = waitForLog >> snapshot serf >> topLoop
|
2020-06-01 01:37:27 +03:00
|
|
|
|
|
|
|
doScry :: Wen -> Gang -> Path -> (Maybe (Term, Noun) -> IO ()) -> IO ()
|
2020-06-01 20:51:37 +03:00
|
|
|
doScry w g p k = (scry serf w g p >>= k) >> topLoop
|
2020-05-29 05:14:25 +03:00
|
|
|
|
2020-05-29 23:10:17 +03:00
|
|
|
doWork :: EvErr -> IO ()
|
2020-05-29 05:14:25 +03:00
|
|
|
doWork firstWorkErr = do
|
2020-05-29 23:10:17 +03:00
|
|
|
que <- newTBMQueueIO 1
|
|
|
|
() <- atomically (writeTBMQueue que firstWorkErr)
|
2020-06-01 01:37:27 +03:00
|
|
|
tWork <- async (processWork serf maxBatchSize que onWorkResp spin)
|
2020-06-07 00:33:15 +03:00
|
|
|
flip onException (cancel tWork) $ do
|
2020-06-05 00:10:33 +03:00
|
|
|
nexSt <- workLoop que
|
|
|
|
wait tWork
|
|
|
|
nexSt
|
2020-05-29 23:10:17 +03:00
|
|
|
|
|
|
|
workLoop :: TBMQueue EvErr -> IO (IO ())
|
|
|
|
workLoop que = atomically onInput >>= \case
|
2020-06-01 20:51:37 +03:00
|
|
|
RRKill () -> atomically (closeTBMQueue que) >> pure (pure ())
|
|
|
|
RRSave () -> atomically (closeTBMQueue que) >> pure doSave
|
|
|
|
RRPack () -> atomically (closeTBMQueue que) >> pure doPack
|
|
|
|
RRScry w g p k -> atomically (closeTBMQueue que) >> pure (doScry w g p k)
|
|
|
|
RRWork workErr -> atomically (writeTBMQueue que workErr) >> workLoop que
|
2020-05-29 23:10:17 +03:00
|
|
|
|
|
|
|
onWorkResp :: Wen -> EvErr -> Work -> IO ()
|
|
|
|
onWorkResp wen (EvErr evn err) = \case
|
|
|
|
WDone eid hash fx -> do
|
2020-06-01 01:37:27 +03:00
|
|
|
atomically $ sendOn ((Fact eid hash wen (toNoun evn)), fx)
|
2020-05-29 23:10:17 +03:00
|
|
|
WSwap eid hash (wen, noun) fx -> do
|
|
|
|
io $ err (RunSwap eid hash wen noun fx)
|
2020-06-01 01:37:27 +03:00
|
|
|
atomically $ sendOn (Fact eid hash wen noun, fx)
|
2020-05-29 23:10:17 +03:00
|
|
|
WBail goofs -> do
|
|
|
|
io $ err (RunBail goofs)
|
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
2020-06-01 01:37:27 +03:00
|
|
|
Given:
|
|
|
|
|
|
|
|
- A stream of incoming requests
|
|
|
|
- A sequence of in-flight requests that haven't been responded to
|
|
|
|
- A maximum number of in-flight requests.
|
|
|
|
|
|
|
|
Wait until the number of in-fligh requests is smaller than the maximum,
|
|
|
|
and then take the next item from the stream of requests.
|
|
|
|
-}
|
|
|
|
pullFromQueueBounded :: Int -> TVar (Seq a) -> TBMQueue b -> STM (Maybe b)
|
|
|
|
pullFromQueueBounded maxSize vInFlight queue = do
|
2020-05-29 23:10:17 +03:00
|
|
|
inFlight <- length <$> readTVar vInFlight
|
2020-06-01 01:37:27 +03:00
|
|
|
if inFlight >= maxSize
|
2020-05-29 23:10:17 +03:00
|
|
|
then retry
|
|
|
|
else readTBMQueue queue
|
2020-05-29 05:14:25 +03:00
|
|
|
|
2020-06-01 20:51:37 +03:00
|
|
|
{-|
|
2020-06-01 01:37:27 +03:00
|
|
|
Given
|
|
|
|
|
|
|
|
- `maxSize`: The maximum number of jobs to send to the serf before
|
|
|
|
getting a response.
|
|
|
|
- `q`: A bounded queue (which can be closed)
|
|
|
|
- `onResp`: a callback to call for each response from the serf.
|
|
|
|
- `spin`: a callback to tell the terminal driver which event is
|
|
|
|
currently being processed.
|
|
|
|
|
|
|
|
Pull jobs from the queue and send them to the serf (eagerly, up to
|
|
|
|
`maxSize`) and call the callback with each response from the serf.
|
|
|
|
|
|
|
|
When the queue is closed, wait for the serf to respond to all pending
|
|
|
|
work, and then return.
|
|
|
|
|
|
|
|
Whenever the serf is idle, call `spin Nothing` and whenever the serf
|
|
|
|
is working on an event, call `spin (Just ev)`.
|
|
|
|
-}
|
2020-05-29 05:14:25 +03:00
|
|
|
processWork
|
|
|
|
:: Serf
|
2020-06-01 01:37:27 +03:00
|
|
|
-> Int
|
2020-05-29 05:14:25 +03:00
|
|
|
-> TBMQueue EvErr
|
|
|
|
-> (Wen -> EvErr -> Work -> IO ())
|
2020-06-01 20:51:37 +03:00
|
|
|
-> (Maybe Ev -> STM ())
|
2020-05-29 05:14:25 +03:00
|
|
|
-> IO ()
|
2020-06-01 01:37:27 +03:00
|
|
|
processWork serf maxSize q onResp spin = do
|
2020-05-29 23:10:17 +03:00
|
|
|
vDoneFlag <- newTVarIO False
|
|
|
|
vInFlightQueue <- newTVarIO empty
|
|
|
|
recvThread <- async (recvLoop serf vDoneFlag vInFlightQueue)
|
2020-06-05 00:10:33 +03:00
|
|
|
flip onException (print "KILLING: processWork" >> cancel recvThread) $ do
|
|
|
|
loop vInFlightQueue vDoneFlag
|
|
|
|
wait recvThread
|
2020-05-29 23:10:17 +03:00
|
|
|
where
|
|
|
|
loop :: TVar (Seq (Ev, Work -> IO ())) -> TVar Bool -> IO ()
|
|
|
|
loop vInFlight vDone = do
|
2020-06-01 01:37:27 +03:00
|
|
|
atomically (pullFromQueueBounded maxSize vInFlight q) >>= \case
|
2020-05-29 23:10:17 +03:00
|
|
|
Nothing -> do
|
|
|
|
atomically (writeTVar vDone True)
|
|
|
|
Just evErr@(EvErr ev _) -> do
|
|
|
|
now <- Time.now
|
|
|
|
let cb = onRecv (currentEv vInFlight) now evErr
|
|
|
|
atomically $ do
|
|
|
|
modifyTVar' vInFlight (:|> (ev, cb))
|
|
|
|
currentEv vInFlight >>= spin
|
|
|
|
sendWrit serf (WWork now ev)
|
|
|
|
loop vInFlight vDone
|
|
|
|
|
|
|
|
onRecv :: STM (Maybe Ev) -> Wen -> EvErr -> Work -> IO ()
|
|
|
|
onRecv getCurrentEv now evErr work = do
|
|
|
|
atomically (getCurrentEv >>= spin)
|
|
|
|
onResp now evErr work
|
|
|
|
|
|
|
|
currentEv :: TVar (Seq (Ev, a)) -> STM (Maybe Ev)
|
|
|
|
currentEv vInFlight = readTVar vInFlight >>= \case
|
|
|
|
(ev, _) :<| _ -> pure (Just ev)
|
|
|
|
_ -> pure Nothing
|
2020-05-29 05:14:25 +03:00
|
|
|
|
2020-06-01 01:37:27 +03:00
|
|
|
{-|
|
|
|
|
Given:
|
|
|
|
|
|
|
|
- `vDone`: A flag that no more work will be sent to the serf.
|
|
|
|
|
|
|
|
- `vWork`: A list of work requests that have been sent to the serf,
|
|
|
|
haven't been responded to yet.
|
|
|
|
|
|
|
|
If the serf has responded to all work requests, and no more work is
|
|
|
|
going to be sent to the serf, then return.
|
|
|
|
|
|
|
|
If we are going to send more work to the serf, but the queue is empty,
|
|
|
|
then wait.
|
|
|
|
|
|
|
|
If work requests have been sent to the serf, take the first one,
|
|
|
|
wait for a response from the serf, call the associated callback,
|
|
|
|
and repeat the whole process.
|
|
|
|
-}
|
2020-05-29 05:14:25 +03:00
|
|
|
recvLoop :: Serf -> TVar Bool -> TVar (Seq (Ev, Work -> IO ())) -> IO ()
|
2020-05-29 23:10:17 +03:00
|
|
|
recvLoop serf vDone vWork = do
|
2020-06-05 00:10:33 +03:00
|
|
|
withSerfLockIO serf \SerfState {..} -> do
|
2020-05-29 23:10:17 +03:00
|
|
|
loop ssLast ssHash
|
2020-05-29 05:14:25 +03:00
|
|
|
where
|
2020-05-29 23:10:17 +03:00
|
|
|
loop eve mug = do
|
|
|
|
atomically takeCallback >>= \case
|
|
|
|
Nothing -> pure (SerfState eve mug, ())
|
|
|
|
Just cb -> recvWork serf >>= \case
|
|
|
|
work@(WDone eid hash _) -> cb work >> loop eid hash
|
|
|
|
work@(WSwap eid hash _ _) -> cb work >> loop eid hash
|
|
|
|
work@(WBail _) -> cb work >> loop eve mug
|
|
|
|
|
|
|
|
takeCallback :: STM (Maybe (Work -> IO ()))
|
|
|
|
takeCallback = do
|
|
|
|
((,) <$> readTVar vDone <*> readTVar vWork) >>= \case
|
|
|
|
(False, Empty ) -> retry
|
|
|
|
(True , Empty ) -> pure Nothing
|
|
|
|
(_ , (_, x) :<| xs) -> writeTVar vWork xs $> Just x
|
|
|
|
(_ , _ ) -> error "impossible"
|