diff --git a/meshcentral.js b/meshcentral.js index 95323f8d..0579da8b 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -435,7 +435,7 @@ function CreateMeshCentralServer(config, args) { try { var errlogpath = null; if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); } - obj.fs.appendFileSync(obj.getConfigFilePath('mesherrors.txt'), '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n'); + obj.fs.appendFileSync(errlogpath, '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n'); } catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); } }); childProcess.on('close', function (code) { if ((code != 0) && (code != 123)) { /* console.log("Exited with code " + code); */ } }); @@ -1431,6 +1431,34 @@ function CreateMeshCentralServer(config, args) { } } } + + // Start watchdog timer if needed + // This is used to monitor if NodeJS is servicing IO correctly or getting held up a lot. Add this line to the settings section of config.json + // "watchDog": { "interval": 100, "timeout": 150 } + // This will check every 100ms, if the timer is more than 150ms late, it will warn. + if ((typeof config.settings.watchdog == 'object') && (typeof config.settings.watchdog.interval == 'number') && (typeof config.settings.watchdog.timeout == 'number') && (config.settings.watchdog.interval >= 50) && (config.settings.watchdog.timeout >= 50)) { + obj.watchdogtime = Date.now(); + obj.watchdogmax = 0; + obj.watchdogmaxtime = null; + obj.watchdogtable = []; + obj.watchdog = setInterval(function () { + var now = Date.now(), delta = now - obj.watchdogtime - config.settings.watchdog.interval; + if (delta > obj.watchdogmax) { obj.watchdogmax = delta; obj.watchdogmaxtime = new Date().toLocaleString(); } + if (delta > config.settings.watchdog.timeout) { + const msg = obj.common.format("Watchdog timer timeout, {0}ms.", delta); + obj.watchdogtable.push(new Date().toLocaleString() + ', ' + delta + 'ms'); + while (obj.watchdogtable.length > 10) { obj.watchdogtable.shift(); } + obj.debug('main', msg); + try { + var errlogpath = null; + if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); } + obj.fs.appendFileSync(errlogpath, new Date().toLocaleString() + ': ' + msg + '\r\n'); + } catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); } + } + obj.watchdogtime = now; + }, config.settings.watchdog.interval); + obj.debug('main', "Started watchdog timer."); + } }); }); }; diff --git a/meshuser.js b/meshuser.js index 8c1e93b7..fa44000f 100644 --- a/meshuser.js +++ b/meshuser.js @@ -775,7 +775,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use switch (cmd) { case 'help': { - var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes'; + var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes,watchdog'; if (parent.parent.config.settings.heapdump === true) { availcommands += ',heapdump'; } availcommands = availcommands.split(',').sort(); while (availcommands.length > 0) { if (f.length > 80) { fin += (f + ',\r\n'); f = ''; } f += (((f != '') ? ', ' : ' ') + availcommands.shift()); } @@ -931,6 +931,16 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use } break; } + case 'watchdog': { + if (parent.parent.watchdog == null) { + r = 'Server watchdog not active.'; + } else { + r = 'Server watchdog active.\r\n'; + if (parent.parent.watchdogmaxtime != null) { r += 'Largest timeout was ' + parent.parent.watchdogmax + 'ms on ' + parent.parent.watchdogmaxtime + '\r\n'; } + for (var i in parent.parent.watchdogtable) { r += parent.parent.watchdogtable[i] + '\r\n'; } + } + break; + } case 'acceleratorsstats': { var stats = parent.parent.certificateOperations.getAcceleratorStats(); for (var i in stats) { @@ -1091,23 +1101,6 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use if (parent.parent.webViewsOverridePath) { r += 'XWebViews: ' + parent.parent.webPublicOverridePath + '\r\n'; } break; } - case 'showconfig': { - // Make a copy of the configuration and hide any secrets - var config = common.Clone(parent.parent.config); - if (config.settings) { - if (config.settings.configkey) { config.settings.configkey = '(present)'; } - if (config.settings.sessionkey) { config.settings.sessionkey = '(present)'; } - if (config.settings.dbencryptkey) { config.settings.dbencryptkey = '(present)'; } - } - if (config.domains) { - for (var i in config.domains) { - if (config.domains[i].yubikey && config.domains[i].yubikey.secret) { config.domains[i].yubikey.secret = '(present)'; } - } - } - - r = JSON.stringify(removeAllUnderScore(config), null, 4); - break; - } case 'migrationagents': { if (parent.parent.swarmserver == null) { r = 'Swarm server not running.'; diff --git a/public/x.pdf b/public/x.pdf new file mode 100644 index 00000000..79c69046 Binary files /dev/null and b/public/x.pdf differ diff --git a/sample-config-advanced.json b/sample-config-advanced.json index c481ff08..0d817a64 100644 --- a/sample-config-advanced.json +++ b/sample-config-advanced.json @@ -96,6 +96,7 @@ "count": 10, "coolofftime": 10 }, + "watchDog": { "interval": 100, "timeout": 400 }, "_plugins": { "enabled": true } }, "_domaindefaults": { @@ -194,7 +195,7 @@ "x-frame-options": "SAMEORIGIN", "Content-Security-Policy": "default-src 'none'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-src 'self'; media-src 'self'" }, - "_agentConfig": [ "webSocketMaskOverride=1" ], + "_agentConfig": [ "webSocketMaskOverride=1", "coreDumpEnabled=1" ], "_sessionRecording": { "_filepath": "C:\\temp", "_index": true,