on master help sign in
attempt to detect if a filesystem is journaled or not

add --disable-fscheck to prevent this from being a fatal error

commitf64726c07f743f864be11fefc96cb9e4d11e6d79 authorAgentM <agentm@themactionfaction.com> dateSat 14 Oct 2017, 23:45 (9 years ago) parent038a30e
7 files changed+203−87
project-m36.cabal+2−1 src/bin/TutorialD/Interpreter.hs+3−2 src/bin/TutorialD/tutd.hs+40−54 src/lib/ProjectM36/FSType.hsc+100 src/lib/ProjectM36/Server.hs+47−26 src/lib/ProjectM36/Server/Config.hs+3−1 src/lib/ProjectM36/Server/ParseArgs.hs+8−3
project-m36.cabal+2−1
@@ -147,7 +147,8 @@ Library
147147 ProjectM36.Transaction.Persist,
148148 ProjectM36.TypeConstructor,
149149 ProjectM36.TypeConstructorDef,
150− ProjectM36.FileLock
150+ ProjectM36.FileLock,
151+ ProjectM36.FSType
151152 GHC-Options: -Wall
152153 if os(windows)
153154 Build-Depends: Win32 >= 2.3
src/bin/TutorialD/Interpreter.hs+3−2
@@ -224,9 +224,10 @@ evalTutorialDInteractive sessionId conn safe interactive expr = case expr of
224224
225225type GhcPkgPath = String
226226type TutorialDExec = String
227+type CheckFS = Bool
227228
228−data InterpreterConfig = LocalInterpreterConfig PersistenceStrategy HeadName (Maybe TutorialDExec) [GhcPkgPath] |
229− RemoteInterpreterConfig C.NodeId C.DatabaseName HeadName (Maybe TutorialDExec)
229+data InterpreterConfig = LocalInterpreterConfig PersistenceStrategy HeadName (Maybe TutorialDExec) [GhcPkgPath] CheckFS |
230+ RemoteInterpreterConfig C.NodeId C.DatabaseName HeadName (Maybe TutorialDExec) CheckFS
230231
231232outputNotificationCallback :: C.NotificationCallback
232233outputNotificationCallback notName evaldNot = hPutStrLn stderr $ "Notification received " ++ show notName ++ ":\n" ++ "\n" ++ prettyEvaluatedNotification evaldNot
src/bin/TutorialD/tutd.hs+40−54
@@ -2,41 +2,25 @@
22import TutorialD.Interpreter
33import ProjectM36.Base
44import ProjectM36.Client
5+import ProjectM36.Server.ParseArgs
6+import ProjectM36.Server
57import System.IO
68import Options.Applicative
79import System.Exit
810import Data.Monoid
11+import Data.Maybe
912import qualified Data.Text as T
1013
1114parseArgs :: Parser InterpreterConfig
12−parseArgs = LocalInterpreterConfig <$> parsePersistenceStrategy <*> parseHeadName <*> parseTutDExec <*> parseGhcPkgPaths <|>
13− RemoteInterpreterConfig <$> parseNodeId <*> parseDatabaseName <*> parseHeadName <*> parseTutDExec
15+parseArgs = LocalInterpreterConfig <$> parsePersistenceStrategy <*> parseHeadName <*> parseTutDExec <*> many parseGhcPkgPath <*> parseCheckFS <|>
16+ RemoteInterpreterConfig <$> parseNodeId <*> parseDatabaseName <*> parseHeadName <*> parseTutDExec <*> parseCheckFS
1417
15−parsePersistenceStrategy :: Parser PersistenceStrategy
16−parsePersistenceStrategy = CrashSafePersistence <$> (dbdirOpt <* fsyncOpt) <|>
17− MinimalPersistence <$> dbdirOpt <|>
18− pure NoPersistence
19− where
20− dbdirOpt = strOption (short 'd' <>
21− long "database-directory" <>
22− metavar "DIRECTORY" <>
23− showDefaultWith show
24− )
25− fsyncOpt = switch (short 'f' <>
26− long "fsync" <>
27− help "Fsync all new transactions.")
28−
2918parseHeadName :: Parser HeadName
3019parseHeadName = option auto (long "head" <>
3120 help "Start session at head name." <>
3221 value "master"
3322 )
34−
35−parseDatabaseName :: Parser DatabaseName
36−parseDatabaseName = strOption (long "database" <>
37− short 'n' <>
38− help "Remote database name")
39−
23+
4024parseNodeId :: Parser NodeId
4125parseNodeId = createNodeId <$>
4226 strOption (long "host" <>
@@ -54,31 +38,29 @@ parseTutDExec = optional $ strOption (long "exec-tutd" <>
5438 short 'e' <>
5539 help "Execute TutorialD expression and exit"
5640 )
57−
58−parseGhcPkgPaths :: Parser [GhcPkgPath]
59−parseGhcPkgPaths = many (strOption (long "ghc-pkg-dir" <>
60− metavar "GHC_PACKAGE_DIRECTORY"))
6141
6242opts :: ParserInfo InterpreterConfig
6343opts = info parseArgs idm
6444
6545connectionInfoForConfig :: InterpreterConfig -> ConnectionInfo
66−connectionInfoForConfig (LocalInterpreterConfig pStrategy _ _ ghcPkgPaths) = InProcessConnectionInfo pStrategy outputNotificationCallback ghcPkgPaths
67−connectionInfoForConfig (RemoteInterpreterConfig remoteNodeId remoteDBName _ _) = RemoteProcessConnectionInfo remoteDBName remoteNodeId outputNotificationCallback
46+connectionInfoForConfig (LocalInterpreterConfig pStrategy _ _ ghcPkgPaths _) = InProcessConnectionInfo pStrategy outputNotificationCallback ghcPkgPaths
47+connectionInfoForConfig (RemoteInterpreterConfig remoteNodeId remoteDBName _ _ _) = RemoteProcessConnectionInfo remoteDBName remoteNodeId outputNotificationCallback
6848
6949headNameForConfig :: InterpreterConfig -> HeadName
70−headNameForConfig (LocalInterpreterConfig _ headn _ _) = headn
71−headNameForConfig (RemoteInterpreterConfig _ _ headn _) = headn
50+headNameForConfig (LocalInterpreterConfig _ headn _ _ _) = headn
51+headNameForConfig (RemoteInterpreterConfig _ _ headn _ _) = headn
7252
7353execTutDForConfig :: InterpreterConfig -> Maybe String
74−execTutDForConfig (LocalInterpreterConfig _ _ t _) = t
75−execTutDForConfig (RemoteInterpreterConfig _ _ _ t) = t
54+execTutDForConfig (LocalInterpreterConfig _ _ t _ _) = t
55+execTutDForConfig (RemoteInterpreterConfig _ _ _ t _) = t
56+
57+checkFSForConfig :: InterpreterConfig -> Bool
58+checkFSForConfig (LocalInterpreterConfig _ _ _ _ c) = c
59+checkFSForConfig (RemoteInterpreterConfig _ _ _ _ c) = c
7660
77−{-
78−ghcPkgPathsForConfig :: InterpreterConfig -> [GhcPkgPath]
79−ghcPkgPathsForConfig (LocalInterpreterConfig _ _ paths) = paths
80−ghcPkgPathsForConfig _ = error "Clients cannot configure remote ghc package paths."
81−-}
61+persistenceStrategyForConfig :: InterpreterConfig -> Maybe PersistenceStrategy
62+persistenceStrategyForConfig (LocalInterpreterConfig strat _ _ _ _) = Just strat
63+persistenceStrategyForConfig RemoteInterpreterConfig{} = Nothing
8264
8365errDie :: String -> IO ()
8466errDie err = hPutStrLn stderr err >> exitFailure
@@ -97,21 +79,25 @@ main :: IO ()
9779main = do
9880 interpreterConfig <- execParser opts
9981 let connInfo = connectionInfoForConfig interpreterConfig
100− dbconn <- connectProjectM36 connInfo
101− case dbconn of
102− Left err ->
103− errDie ("Failed to create database connection: " ++ show err)
104− Right conn -> do
105− let connHeadName = headNameForConfig interpreterConfig
106− eSessionId <- createSessionAtHead conn connHeadName
107− case eSessionId of
108− Left err -> errDie ("Failed to create database session at \"" ++ show connHeadName ++ "\": " ++ show err)
109− Right sessionId ->
110− case execTutDForConfig interpreterConfig of
111− Nothing -> do
112− printWelcome
113− _ <- reprLoop interpreterConfig sessionId conn
114− pure ()
115− Just tutdStr ->
116− runTutorialD sessionId conn (T.pack tutdStr)
82+ fscheck <- checkFSType (checkFSForConfig interpreterConfig) (fromMaybe NoPersistence (persistenceStrategyForConfig interpreterConfig))
83+ if not fscheck then
84+ errDie checkFSErrorMsg
85+ else do
86+ dbconn <- connectProjectM36 connInfo
87+ case dbconn of
88+ Left err ->
89+ errDie ("Failed to create database connection: " ++ show err)
90+ Right conn -> do
91+ let connHeadName = headNameForConfig interpreterConfig
92+ eSessionId <- createSessionAtHead conn connHeadName
93+ case eSessionId of
94+ Left err -> errDie ("Failed to create database session at \"" ++ show connHeadName ++ "\": " ++ show err)
95+ Right sessionId ->
96+ case execTutDForConfig interpreterConfig of
97+ Nothing -> do
98+ printWelcome
99+ _ <- reprLoop interpreterConfig sessionId conn
100+ pure ()
101+ Just tutdStr ->
102+ runTutorialD sessionId conn (T.pack tutdStr)
117103
src/lib/ProjectM36/FSType.hscnew file+100
@@ -0,0 +1,100 @@
1+-- confirm that the filesystem type is a journaled FS type expected by Project:M36
2+-- use statfs on Linux and macOS and GetVolumeInformation on Windows
3+-- this could still be fooled with symlinks or by disabling journaling on filesystems that support that
4+module ProjectM36.FSType where
5+import Foreign
6+import Foreign.C.Types
7+import Foreign.C.String
8+import Foreign.C.Error
9+
10+#ifdef mingw32_HOST_OS
11+import System.Win32.String
12+import System.Win32.File
13+
14+foreign import WINDOWS_CCONV unsafe "windows.h GetVolumeInformationW"
15+ c_GetVolumeInformation :: LPCTSTR -> LPTSTR -> DWORD -> LPDWORD -> LPDWORD -> LPDWORD -> LPTSTR -> DWORD -> IO BOOL
16+
17+data VolumeInformation = VolumeInformation
18+ { volumeName :: String
19+ , volumeSerialNumber :: DWORD
20+ , maximumComponentLength :: DWORD
21+ , fileSystemFlags :: DWORD
22+ , fileSystemName :: String
23+ } deriving Show
24+
25+-- from https://github.com/haskell/win32/blob/master/System/Win32/HardLink.hs
26+getVolumeInformation :: Maybe String -> IO VolumeInformation
27+getVolumeInformation mPath =
28+ maybeWith withTString mPath $ \c_path ->
29+ withTStringBufferLen 256 $ \(vnBuf, vnLen) ->
30+ alloca $ \serialNum ->
31+ alloca $ \maxLen ->
32+ alloca $ \fsFlags ->
33+ withTStringBufferLen 256 $ \(fsBuf, fsLen) -> do
34+ failIfFalse_ (unwords ["GetVolumeInformationW", c_path]) $
35+ c_GetVolumeInformation c_path vnBuf (fromIntegral vnLen)
36+ serialNum maxLen fsFlags
37+ fsBuf (fromIntegral fsLen)
38+ return VolumeInformation
39+ <*> peekTString vnBuf
40+ <*> peek serialNum
41+ <*> peek maxLen
42+ <*> peek fsFlags
43+ <*> peekTString fsBuf
44+
45+fsTypeSupportsJournaling :: FilePath -> IO Bool
46+fsTypeSupportJournaling path = do
47+ info <- getVolumeInformation
48+#define FILE_SUPPORTS_USN_JOURNAL 0x02000000
49+ pure (fileSystemFlags info .&. FILE_SUPPORTS_USN_JOURNAL)
50+
51+#elif darwin_HOST_OS
52+--Darwin reports journaling directly in the fs flags
53+import Data.Bits
54+#include <sys/param.h>
55+#include <sys/mount.h>
56+type CStatFS = ()
57+foreign import ccall unsafe "sys/mount.h statfs"
58+ c_statfs :: CString -> Ptr CStatFS -> IO CInt
59+
60+type CFSFlags = #{type long}
61+
62+fsTypeSupportsJournaling :: FilePath -> IO Bool
63+fsTypeSupportsJournaling path = do
64+ struct_statfs <- mallocForeignPtrBytes #{size statfs}
65+ withCString path $ \c_path -> do
66+ withForeignPtr struct_statfs $ \ptr_statfs -> do
67+ throwErrnoIfMinus1_ "statfs" (c_statfs c_path ptr_statfs)
68+ cfsflags <- peekByteOff ptr_statfs #{offset statfs, f_flags} :: IO CFSFlags
69+ pure (MNT_JOURNALED .&. cfsflags)
70+
71+#elif linux_HOST_OS
72+--Linux cannot report journaling, so we just check the filesystem type as a proxy
73+#include <sys/vfs.h>
74+#include <linux/magic.h>
75+type CStatFS = ()
76+foreign import ccall unsafe "sys/vfs.h statfs"
77+ c_statfs :: CString -> Ptr CStatFS -> IO CInt
78+
79+type CFSType = #{type __SWORD_TYPE}
80+
81+fsTypeSupportsJournaling :: FilePath -> IO Bool
82+fsTypeSupportsJournaling path = do
83+ struct_statfs <- mallocForeignPtrBytes #{size statfs}
84+ withCString path $ \c_path -> do
85+ withForeignPtr struct_statfs $ \ptr_statfs -> do
86+ throwErrnoIfMinus1_ "statfs" (c_statfs c_path ptr_statfs)
87+ cfstype <- peekByteOff ptr_statfs 0 :: IO CFSType
88+ let journaledFS = [#{const EXT3_SUPER_MAGIC},
89+ #{const EXT4_SUPER_MAGIC},
90+ #{ifdef NTFS_SB_MAGIC
91+ NTFS_SB_MAGIC,
92+ #endif}
93+ #{const REISERFS_SUPER_MAGIC},
94+ #{ifdef XFS_SUPER_MAGIC
95+ XFS_SUPER_MAGIC,
96+ #endif}
97+ 0]
98+ pure (elem cfstype journaledFS)
99+#endif
100+
src/lib/ProjectM36/Server.hs+47−26
@@ -5,6 +5,7 @@ import ProjectM36.Client
55import ProjectM36.Server.EntryPoints
66import ProjectM36.Server.RemoteCallTypes
77import ProjectM36.Server.Config (ServerConfig(..))
8+import ProjectM36.FSType
89
910import Control.Monad.IO.Class (liftIO)
1011import Network.Transport.TCP (createTransport, defaultTCPParameters)
@@ -16,6 +17,7 @@ import Control.Distributed.Process.ManagedProcess (defaultProcess, UnhandledMess
1617import Control.Concurrent.MVar (putMVar, MVar)
1718import System.IO (stderr, hPutStrLn)
1819
20+
1921-- the state should be a mapping of remote connection to the disconnected transaction- the graph should be the same, so discon must be removed from the stm tuple
2022--trying to refactor this for less repetition is very challenging because the return type cannot be polymorphic or the distributed-process call gets confused and drops messages
2123serverDefinition :: Bool -> Timeout -> ProcessDefinition Connection
@@ -70,36 +72,55 @@ registerDB dbname = do
7072 let dbname' = remoteDBLookupName dbname
7173 register dbname' self
7274 --liftIO $ putStrLn $ "registered " ++ (show self) ++ " " ++ dbname'
73−
75+
7476-- | A notification callback which logs the notification to stderr and does nothing else.
7577loggingNotificationCallback :: NotificationCallback
7678loggingNotificationCallback notName evaldNot = hPutStrLn stderr $ "Notification received \"" ++ show notName ++ "\": " ++ show evaldNot
7779
80+checkFSType :: Bool -> PersistenceStrategy -> IO Bool
81+checkFSType performCheck strat =
82+ case strat of
83+ NoPersistence -> pure True
84+ MinimalPersistence _ -> pure True
85+ CrashSafePersistence path ->
86+ if performCheck then
87+ fsTypeSupportsJournaling path
88+ else
89+ pure True
90+
91+checkFSErrorMsg :: String
92+checkFSErrorMsg = "The filesystem does not support journaling so writes may not be crash-safe. Use --disable-fscheck to disable this fatal error."
93+
7894-- | A synchronous function to start the project-m36 daemon given an appropriate 'ServerConfig'. Note that this function only returns if the server exits. Returns False if the daemon exited due to an error. If the second argument is not Nothing, the port is put after the server is ready to service the port.
7995launchServer :: ServerConfig -> Maybe (MVar EndPointAddress) -> IO Bool
80−launchServer daemonConfig mAddressMVar = do
81− econn <- connectProjectM36 (InProcessConnectionInfo (persistenceStrategy daemonConfig) loggingNotificationCallback (ghcPkgPaths daemonConfig))
82− case econn of
83− Left err -> do
84− hPutStrLn stderr ("Failed to create database connection: " ++ show err)
85− pure False
86− Right conn -> do
87− let hostname = bindHost daemonConfig
88− port = bindPort daemonConfig
89− etransport <- createTransport hostname (show port) defaultTCPParameters
90− case etransport of
91− Left err -> error ("failed to create transport: " ++ show err)
92− Right transport -> do
93− eEndpoint <- newEndPoint transport
94− case eEndpoint of
95− Left err -> hPutStrLn stderr ("Failed to create transport: " ++ show err) >> pure False
96− Right endpoint -> do
97− localTCPNode <- newLocalNode transport initRemoteTable
98− --traceShowM ("newLocalNode in Server " ++ show (localNodeId localTCPNode))
99− runProcess localTCPNode $ do
100− let testBool = testMode daemonConfig
101− reqTimeout = perRequestTimeout daemonConfig
102− serve (conn, databaseName daemonConfig, mAddressMVar, address endpoint) initServer (serverDefinition testBool reqTimeout)
103− liftIO $ putStrLn "serve returned"
104− pure True
96+launchServer daemonConfig mAddressMVar = do
97+ checkFSResult <- checkFSType (checkFS daemonConfig) (persistenceStrategy daemonConfig)
98+ if not checkFSResult then do
99+ hPutStrLn stderr checkFSErrorMsg
100+ pure False
101+ else do
102+ econn <- connectProjectM36 (InProcessConnectionInfo (persistenceStrategy daemonConfig) loggingNotificationCallback (ghcPkgPaths daemonConfig))
103+ case econn of
104+ Left err -> do
105+ hPutStrLn stderr ("Failed to create database connection: " ++ show err)
106+ pure False
107+ Right conn -> do
108+ let hostname = bindHost daemonConfig
109+ port = bindPort daemonConfig
110+ etransport <- createTransport hostname (show port) defaultTCPParameters
111+ case etransport of
112+ Left err -> error ("failed to create transport: " ++ show err)
113+ Right transport -> do
114+ eEndpoint <- newEndPoint transport
115+ case eEndpoint of
116+ Left err -> hPutStrLn stderr ("Failed to create transport: " ++ show err) >> pure False
117+ Right endpoint -> do
118+ localTCPNode <- newLocalNode transport initRemoteTable
119+ --traceShowM ("newLocalNode in Server " ++ show (localNodeId localTCPNode))
120+ runProcess localTCPNode $ do
121+ let testBool = testMode daemonConfig
122+ reqTimeout = perRequestTimeout daemonConfig
123+ serve (conn, databaseName daemonConfig, mAddressMVar, address endpoint) initServer (serverDefinition testBool reqTimeout)
124+ liftIO $ putStrLn "serve returned"
125+ pure True
105126
src/lib/ProjectM36/Server/Config.hs+3−1
@@ -1,7 +1,8 @@
11module ProjectM36.Server.Config where
22import ProjectM36.Client
33
4−data ServerConfig = ServerConfig { persistenceStrategy :: PersistenceStrategy,
4+data ServerConfig = ServerConfig { persistenceStrategy :: PersistenceStrategy,
5+ checkFS :: Bool,
56 databaseName :: DatabaseName,
67 bindHost :: Hostname,
78 bindPort :: Port,
@@ -13,6 +14,7 @@ data ServerConfig = ServerConfig { persistenceStrategy :: PersistenceStrategy,
1314
1415defaultServerConfig :: ServerConfig
1516defaultServerConfig = ServerConfig { persistenceStrategy = NoPersistence,
17+ checkFS = True,
1618 databaseName = "base",
1719 bindHost = "127.0.0.1",
1820 bindPort = 6543,
src/lib/ProjectM36/Server/ParseArgs.hs+8−3
@@ -8,10 +8,11 @@ import Data.Monoid
88parseArgsWithDefaults :: ServerConfig -> Parser ServerConfig
99parseArgsWithDefaults defaults = ServerConfig <$>
1010 parsePersistenceStrategy <*>
11+ parseCheckFS <*>
1112 parseDatabaseName <*>
1213 parseHostname (bindHost defaults) <*>
1314 parsePort (bindPort defaults) <*>
14− many parseGhcPkgPaths <*>
15+ many parseGhcPkgPath <*>
1516 parseTimeout (perRequestTimeout defaults) <*>
1617 pure False
1718
@@ -29,6 +30,10 @@ parsePersistenceStrategy = CrashSafePersistence <$> (dbdirOpt <* fsyncOpt) <|>
2930 long "fsync" <>
3031 help "Fsync all new transactions.")
3132
33+parseCheckFS :: Parser Bool
34+parseCheckFS = flag True False (long "disable-fscheck" <>
35+ help "Disable filesystem check for journaling.")
36+
3237parseDatabaseName :: Parser DatabaseName
3338parseDatabaseName = strOption (short 'n' <>
3439 long "database" <>
@@ -46,8 +51,8 @@ parsePort defPort = option auto (short 'p' <>
4651 metavar "PORT_NUMBER" <>
4752 value defPort)
4853
49−parseGhcPkgPaths :: Parser String
50−parseGhcPkgPaths = strOption (long "ghc-pkg-dir" <>
54+parseGhcPkgPath :: Parser String
55+parseGhcPkgPath = strOption (long "ghc-pkg-dir" <>
5156 metavar "GHC_PACKAGE_DIRECTORY")
5257
5358parseTimeout :: Int -> Parser Int