Added server watchdog feature.

This commit is contained in:
Ylian Saint-Hilaire 2020-07-03 11:03:15 -07:00
parent f4cb0f2d72
commit fbabddee79
4 changed files with 42 additions and 20 deletions

View File

@ -435,7 +435,7 @@ function CreateMeshCentralServer(config, args) {
try {
var errlogpath = null;
if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); }
obj.fs.appendFileSync(obj.getConfigFilePath('mesherrors.txt'), '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n');
obj.fs.appendFileSync(errlogpath, '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n');
} catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); }
});
childProcess.on('close', function (code) { if ((code != 0) && (code != 123)) { /* console.log("Exited with code " + code); */ } });
@ -1431,6 +1431,34 @@ function CreateMeshCentralServer(config, args) {
}
}
}
// Start watchdog timer if needed
// This is used to monitor if NodeJS is servicing IO correctly or getting held up a lot. Add this line to the settings section of config.json
// "watchDog": { "interval": 100, "timeout": 150 }
// This will check every 100ms, if the timer is more than 150ms late, it will warn.
if ((typeof config.settings.watchdog == 'object') && (typeof config.settings.watchdog.interval == 'number') && (typeof config.settings.watchdog.timeout == 'number') && (config.settings.watchdog.interval >= 50) && (config.settings.watchdog.timeout >= 50)) {
obj.watchdogtime = Date.now();
obj.watchdogmax = 0;
obj.watchdogmaxtime = null;
obj.watchdogtable = [];
obj.watchdog = setInterval(function () {
var now = Date.now(), delta = now - obj.watchdogtime - config.settings.watchdog.interval;
if (delta > obj.watchdogmax) { obj.watchdogmax = delta; obj.watchdogmaxtime = new Date().toLocaleString(); }
if (delta > config.settings.watchdog.timeout) {
const msg = obj.common.format("Watchdog timer timeout, {0}ms.", delta);
obj.watchdogtable.push(new Date().toLocaleString() + ', ' + delta + 'ms');
while (obj.watchdogtable.length > 10) { obj.watchdogtable.shift(); }
obj.debug('main', msg);
try {
var errlogpath = null;
if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); }
obj.fs.appendFileSync(errlogpath, new Date().toLocaleString() + ': ' + msg + '\r\n');
} catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); }
}
obj.watchdogtime = now;
}, config.settings.watchdog.interval);
obj.debug('main', "Started watchdog timer.");
}
});
});
};

View File

@ -775,7 +775,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
switch (cmd) {
case 'help': {
var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes';
var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes,watchdog';
if (parent.parent.config.settings.heapdump === true) { availcommands += ',heapdump'; }
availcommands = availcommands.split(',').sort();
while (availcommands.length > 0) { if (f.length > 80) { fin += (f + ',\r\n'); f = ''; } f += (((f != '') ? ', ' : ' ') + availcommands.shift()); }
@ -931,6 +931,16 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
}
break;
}
case 'watchdog': {
if (parent.parent.watchdog == null) {
r = 'Server watchdog not active.';
} else {
r = 'Server watchdog active.\r\n';
if (parent.parent.watchdogmaxtime != null) { r += 'Largest timeout was ' + parent.parent.watchdogmax + 'ms on ' + parent.parent.watchdogmaxtime + '\r\n'; }
for (var i in parent.parent.watchdogtable) { r += parent.parent.watchdogtable[i] + '\r\n'; }
}
break;
}
case 'acceleratorsstats': {
var stats = parent.parent.certificateOperations.getAcceleratorStats();
for (var i in stats) {
@ -1091,23 +1101,6 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
if (parent.parent.webViewsOverridePath) { r += 'XWebViews: ' + parent.parent.webPublicOverridePath + '\r\n'; }
break;
}
case 'showconfig': {
// Make a copy of the configuration and hide any secrets
var config = common.Clone(parent.parent.config);
if (config.settings) {
if (config.settings.configkey) { config.settings.configkey = '(present)'; }
if (config.settings.sessionkey) { config.settings.sessionkey = '(present)'; }
if (config.settings.dbencryptkey) { config.settings.dbencryptkey = '(present)'; }
}
if (config.domains) {
for (var i in config.domains) {
if (config.domains[i].yubikey && config.domains[i].yubikey.secret) { config.domains[i].yubikey.secret = '(present)'; }
}
}
r = JSON.stringify(removeAllUnderScore(config), null, 4);
break;
}
case 'migrationagents': {
if (parent.parent.swarmserver == null) {
r = 'Swarm server not running.';

BIN
public/x.pdf Normal file

Binary file not shown.

View File

@ -96,6 +96,7 @@
"count": 10,
"coolofftime": 10
},
"watchDog": { "interval": 100, "timeout": 400 },
"_plugins": { "enabled": true }
},
"_domaindefaults": {
@ -194,7 +195,7 @@
"x-frame-options": "SAMEORIGIN",
"Content-Security-Policy": "default-src 'none'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-src 'self'; media-src 'self'"
},
"_agentConfig": [ "webSocketMaskOverride=1" ],
"_agentConfig": [ "webSocketMaskOverride=1", "coreDumpEnabled=1" ],
"_sessionRecording": {
"_filepath": "C:\\temp",
"_index": true,