From f8c310d39ffc0744f237f187abeeb8b4a5b1705a Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Wed, 1 May 2019 15:02:03 -0700 Subject: [PATCH] Added server instrumentation --- certoperations.js | 59 ++++++++++++++++++++++++++++++------- meshagent.js | 51 +++++++++++++++++++++++++++----- meshcentral.js | 2 +- meshuser.js | 49 +++++++++++++++++++++++++++++-- mpsserver.js | 74 ++++++++++++++++++++++++++++++++++++++++++++--- package.json | 2 +- webserver.js | 48 ++++++++++++++++++++++++++++++ 7 files changed, 258 insertions(+), 27 deletions(-) diff --git a/certoperations.js b/certoperations.js index 8c97601d..d3ddc782 100644 --- a/certoperations.js +++ b/certoperations.js @@ -549,14 +549,47 @@ module.exports.CertificateOperations = function (parent) { // Accelerators, used to dispatch work to other processes const fork = require("child_process").fork; const program = require("path").join(__dirname, "meshaccelerator.js"); - const acceleratorTotalCount = require("os").cpus().length; + const acceleratorTotalCount = 1; //require("os").cpus().length; // TODO: Check if this accelerator can scale. var acceleratorCreateCount = acceleratorTotalCount; var freeAccelerators = []; var pendingAccelerator = []; obj.acceleratorCertStore = null; + // Accelerator Stats + var getAcceleratorFuncCalls = 0; + var acceleratorStartFuncCall = 0; + var acceleratorPerformSignatureFuncCall = 0; + var acceleratorPerformSignaturePushFuncCall = 0; + var acceleratorPerformSignatureRunFuncCall = 0; + var acceleratorMessage = 0; + var acceleratorMessageException = 0; + var acceleratorMessageLastException = null; + var acceleratorException = 0; + var acceleratorLastException = null; + + // Get stats about the accelerators + obj.getAcceleratorStats = function () { + return { + acceleratorTotalCount: acceleratorTotalCount, + acceleratorCreateCount: acceleratorCreateCount, + freeAccelerators: freeAccelerators.length, + pendingAccelerator: pendingAccelerator.length, + getAcceleratorFuncCalls: getAcceleratorFuncCalls, + startFuncCall: acceleratorStartFuncCall, + performSignatureFuncCall: acceleratorPerformSignatureFuncCall, + performSignaturePushFuncCall: acceleratorPerformSignaturePushFuncCall, + performSignatureRunFuncCall: acceleratorPerformSignatureRunFuncCall, + message: acceleratorMessage, + messageException: acceleratorMessageException, + messageLastException: acceleratorMessageLastException, + exception: acceleratorException, + lastException: acceleratorLastException + }; + } + // Create a new accelerator module obj.getAccelerator = function () { + getAcceleratorFuncCalls++; if (obj.acceleratorCertStore == null) { return null; } if (freeAccelerators.length > 0) { return freeAccelerators.pop(); } if (acceleratorCreateCount > 0) { @@ -564,23 +597,26 @@ module.exports.CertificateOperations = function (parent) { var accelerator = fork(program, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] }); accelerator.accid = acceleratorCreateCount; accelerator.on("message", function (message) { - this.func(this.tag, message); - delete this.tag; - if (pendingAccelerator.length > 0) { - var x = pendingAccelerator.shift(); - if (x.tag) { this.tag = x.tag; delete x.tag; } - accelerator.send(x); - } else { freeAccelerators.push(this); } + acceleratorMessage++; + try { this.func(this.tag, message); } catch (ex) { acceleratorMessageException++; acceleratorMessageLastException = ex; } + try { + delete this.tag; + if (pendingAccelerator.length > 0) { + var x = pendingAccelerator.shift(); + if (x.tag) { this.tag = x.tag; delete x.tag; } + accelerator.send(x); + } else { freeAccelerators.push(this); } + } catch (ex) { acceleratorException++; acceleratorLastException = ex; } }); accelerator.send({ action: "setState", certs: obj.acceleratorCertStore }); return accelerator; - } return null; }; // Set the state of the accelerators. This way, we don"t have to send certificate & keys to them each time. obj.acceleratorStart = function (certificates) { + acceleratorStartFuncCall++; if (obj.acceleratorCertStore != null) { console.error("ERROR: Accelerators can only be started once."); return; } obj.acceleratorCertStore = [{ cert: certificates.agent.cert, key: certificates.agent.key }]; if (certificates.swarmserver != null) { obj.acceleratorCertStore.push({ cert: certificates.swarmserver.cert, key: certificates.swarmserver.key }); } @@ -588,19 +624,22 @@ module.exports.CertificateOperations = function (parent) { // Perform any RSA signature, just pass in the private key and data. obj.acceleratorPerformSignature = function (privatekey, data, tag, func) { + acceleratorPerformSignatureFuncCall++; if (acceleratorTotalCount <= 1) { // No accelerators available if (typeof privatekey == "number") { privatekey = obj.acceleratorCertStore[privatekey].key; } const sign = obj.crypto.createSign("SHA384"); sign.end(Buffer.from(data, "binary")); - func(tag, sign.sign(privatekey).toString("binary")); + try { func(tag, sign.sign(privatekey).toString("binary")); } catch (ex) { acceleratorMessageException++; acceleratorMessageLastException = ex; } } else { var acc = obj.getAccelerator(); if (acc == null) { // Add to pending accelerator workload + acceleratorPerformSignaturePushFuncCall++; pendingAccelerator.push({ action: "sign", key: privatekey, data: data, tag: tag }); } else { // Send to accelerator now + acceleratorPerformSignatureRunFuncCall++; acc.func = func; acc.tag = tag; acc.send({ action: "sign", key: privatekey, data: data }); diff --git a/meshagent.js b/meshagent.js index 8bd468b0..ae95f1a3 100644 --- a/meshagent.js +++ b/meshagent.js @@ -19,6 +19,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { const forge = parent.parent.certificateOperations.forge; const common = parent.parent.common; const agentUpdateBlockSize = 65531; + parent.agentStats.createMeshAgentCount++; var obj = {}; obj.domain = domain; @@ -155,6 +156,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { if (meshcorehash == null) { // Clear the core obj.send(common.ShortToStr(10) + common.ShortToStr(0)); // MeshCommand_CoreModule, ask mesh agent to clear the core + parent.agentStats.clearingCoreCount++; parent.parent.debug(1, 'Clearing core'); } else { // Update new core @@ -168,7 +170,8 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Send the updated code. delete obj.agentCoreUpdatePending; obj.send(common.ShortToStr(10) + common.ShortToStr(0) + argument.hash + argument.core, function () { parent.parent.taskLimiter.completed(taskid); }); // MeshCommand_CoreModule, start core update - parent.parent.debug(1, 'Updating code ' + argument.name); + parent.agentStats.updatingCoreCount++; + parent.parent.debug(1, 'Updating core ' + argument.name); agentCoreIsStable(); } else { // This agent is probably disconnected, nothing to do. @@ -359,6 +362,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { } else { // Check that the server hash matches our own web certificate hash (SHA384) if ((getWebCertHash(domain) != msg.substring(2, 50)) && (getWebCertFullHash(domain) != msg.substring(2, 50))) { + parent.agentStats.agentBadWebCertHashCount++; console.log('Agent bad web cert hash (Agent:' + (Buffer.from(msg.substring(2, 50), 'binary').toString('hex').substring(0, 10)) + ' != Server:' + (Buffer.from(getWebCertHash(domain), 'binary').toString('hex').substring(0, 10)) + ' or ' + (new Buffer(getWebCertFullHash(domain), 'binary').toString('hex').substring(0, 10)) + '), holding connection (' + obj.remoteaddrport + ').'); console.log('Agent reported web cert hash:' + (Buffer.from(msg.substring(2, 50), 'binary').toString('hex')) + '.'); return; @@ -388,7 +392,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Check the agent signature if we can if (obj.unauthsign != null) { - if (processAgentSignature(obj.unauthsign) == false) { console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; } else { completeAgentConnection(); } + if (processAgentSignature(obj.unauthsign) == false) { + parent.agentStats.agentBadSignature1Count++; + console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; + } else { completeAgentConnection(); } } } else if (cmd == 2) { @@ -403,7 +410,12 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { obj.unauth.nodeCertPem = '-----BEGIN CERTIFICATE-----\r\n' + Buffer.from(msg.substring(4, 4 + certlen), 'binary').toString('base64') + '\r\n-----END CERTIFICATE-----'; // Check the agent signature if we can - if (obj.agentnonce == null) { obj.unauthsign = msg.substring(4 + certlen); } else { if (processAgentSignature(msg.substring(4 + certlen)) == false) { console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; } } + if (obj.agentnonce == null) { obj.unauthsign = msg.substring(4 + certlen); } else { + if (processAgentSignature(msg.substring(4 + certlen)) == false) { + parent.agentStats.agentBadSignature2Count++; + console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; + } + } completeAgentConnection(); } else if (cmd == 3) { @@ -541,7 +553,11 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { for (var i in parent.wsagents) { if (parent.wsagents[i].domain.id == domain.id) { domainAgentSessionCount++; } } // Check if we have too many user sessions - if (domainAgentSessionCount >= domain.limits.maxagentsessions) { return; } // Too many, hold the connection. + if (domainAgentSessionCount >= domain.limits.maxagentsessions) { + // Too many, hold the connection. + parent.agentStats.agentMaxSessionHoldCount++; + return; + } } /* @@ -593,6 +609,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Check if the mesh exists if (mesh == null) { // If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours. + parent.agentStats.invalidDomainMeshCount++; console.log('Agent connected with invalid domain/mesh, holding connection (' + obj.remoteaddrport + ', ' + obj.dbMeshKey + ').'); return; } @@ -600,6 +617,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Check if the mesh is the right type if (mesh.mtype != 2) { // If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours. + parent.agentStats.invalidMeshTypeCount++; console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').'); return; } @@ -634,6 +652,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Check if the mesh exists if (mesh == null) { // If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours. + parent.agentStats.invalidDomainMesh2Count++; console.log('Agent connected with invalid domain/mesh, holding connection (' + obj.remoteaddrport + ', ' + obj.dbMeshKey + ').'); return; } @@ -641,6 +660,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Check if the mesh is the right type if (mesh.mtype != 2) { // If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours. + parent.agentStats.invalidMeshType2Count++; console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').'); return; } @@ -686,6 +706,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { parent.wsagents[obj.dbNodeKey] = obj; if (dupAgent) { // Close the duplicate agent + parent.agentStats.duplicateAgentCount++; if (obj.nodeid != null) { parent.parent.debug(1, 'Duplicate agent ' + obj.nodeid + ' (' + obj.remoteaddrport + ')'); } dupAgent.close(3); } else { @@ -769,6 +790,8 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { } function recoveryAgentCoreIsStable(mesh) { + parent.agentStats.recoveryCoreIsStableCount++; + // Recovery agent is doing ok, lets perform main agent checking. //console.log('recoveryAgentCoreIsStable()'); @@ -793,9 +816,12 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { } function agentCoreIsStable() { + parent.agentStats.coreIsStableCount++; + // Check that the mesh exists const mesh = parent.meshes[obj.dbMeshKey]; if (mesh == null) { + parent.agentStats.meshDoesNotExistCount++; // TODO: Mark this agent as part of a mesh that does not exists. return; // Probably not worth doing anything else. Hold this agent. } @@ -806,7 +832,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { return; } - // Fetch the the real agent nodeid + // Fetch the the diagnostic agent nodeid db.Get('ra' + obj.dbNodeKey, function (err, nodes) { if (nodes.length == 1) { obj.diagnosticNodeKey = nodes[0].daid; @@ -902,7 +928,11 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { verifier.update(buf); verified = verifier.verify(obj.unauth.nodeCertPem, sig, 'binary'); } - if (verified == false) { return false; } // Not a valid signature + if (verified == false) { + // Not a valid signature + parent.agentStats.invalidPkcsSignatureCount++; + return false; + } } catch (ex) { }; } } @@ -914,7 +944,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { if (verify.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) { const verify2 = parent.crypto.createVerify('SHA384'); verify2.end(Buffer.from(getWebCertFullHash(domain) + obj.nonce + obj.agentnonce, 'binary')); // Test using the full cert hash - if (verify2.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) { return false; } + if (verify2.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) { + parent.agentStats.invalidRsaSignatureCount++; + return false; + } } } } @@ -927,6 +960,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { delete obj.unauth; delete obj.receivedCommands; if (obj.unauthsign) delete obj.unauthsign; + parent.agentStats.verifiedAgentConnectionCount++; parent.parent.debug(1, 'Verified agent connection to ' + obj.nodeid + ' (' + obj.remoteaddrport + ').'); obj.authenticated = 1; return true; @@ -936,7 +970,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { function processAgentData(msg) { var i, str = msg.toString('utf8'), command = null; if (str[0] == '{') { - try { command = JSON.parse(str); } catch (ex) { console.log('Unable to parse agent JSON (' + obj.remoteaddrport + '): ' + str, ex); return; } // If the command can't be parsed, ignore it. + try { command = JSON.parse(str); } catch (ex) { parent.agentStats.invalidJsonCount++; console.log('Unable to parse agent JSON (' + obj.remoteaddrport + '): ' + str, ex); return; } // If the command can't be parsed, ignore it. if (typeof command != 'object') { return; } switch (command.action) { case 'msg': @@ -1159,6 +1193,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { break; } default: { + parent.agentStats.unknownAgentActionCount++; console.log('Unknown agent action (' + obj.remoteaddrport + '): ' + command.action + '.'); break; } diff --git a/meshcentral.js b/meshcentral.js index 15d8677d..03853584 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -216,7 +216,7 @@ function CreateMeshCentralServer(config, args) { if (code == 0) { try { latestVer = xprocess.data.split(' ').join('').split('\r').join('').split('\n').join(''); } catch (e) { } } callback(obj.currentVer, latestVer); }); - } catch (ex) { callback(obj.currentVer, null); } // If the system is running out of memory, an exception here can easily happen. + } catch (ex) { callback(obj.currentVer, null, ex); } // If the system is running out of memory, an exception here can easily happen. }; // Initiate server self-update diff --git a/meshuser.js b/meshuser.js index cfa6f01c..8216deb5 100644 --- a/meshuser.js +++ b/meshuser.js @@ -520,7 +520,50 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use switch (cmd) { case 'help': { r = 'Available commands: help, info, versions, args, resetserver, showconfig, usersessions, tasklimiter, setmaxtasks, cores,\r\n' - r += 'migrationagents, swarmstats, nodeconfig, heapdump, relays.'; + r += 'migrationagents, agentstats, webstats, mpsstats, swarmstats, acceleratorsstats, updatecheck, serverupdate, nodeconfig, heapdump, relays.'; + break; + } + case 'agentstats': { + var stats = parent.getAgentStats(); + for (var i in stats) { + if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); } + } + break; + } + case 'webstats': { + var stats = parent.getStats(); + for (var i in stats) { + if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); } + } + break; + } + case 'acceleratorsstats': { + var stats = parent.parent.certificateOperations.getAcceleratorStats(); + for (var i in stats) { + if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); } + } + break; + } + case 'mpsstats': { + var stats = parent.parent.mpsserver.getStats(); + for (var i in stats) { + if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); } + } + break; + } + case 'serverupdate': { + r = 'Performing server update...'; + parent.parent.performServerUpdate(); + break; + } + case 'updatecheck': { + parent.parent.getLatestServerVersion(function (currentVer, newVer, error) { + var r2 = 'Current Version: ' + currentVer + '\r\n'; + if (newVer != null) { r2 += 'Available Version: ' + newVer + '\r\n'; } + if (error != null) { r2 += 'Exception: ' + ex + '\r\n'; } + try { ws.send(JSON.stringify({ action: 'serverconsole', value: r2, tag: command.tag })); } catch (ex) { } + }); + r = 'Checking server update...'; break; } case 'info': { @@ -622,9 +665,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use } else { for (var i in parent.parent.swarmserver.stats) { if (typeof parent.parent.swarmserver.stats[i] == 'object') { - r += i + ' ' + JSON.stringify(parent.parent.swarmserver.stats[i]) + '
'; + r += i + ': ' + JSON.stringify(parent.parent.swarmserver.stats[i]) + '\r\n'; } else { - r += i + ' ' + parent.parent.swarmserver.stats[i] + '
'; + r += i + ': ' + parent.parent.swarmserver.stats[i] + '\r\n'; } } } diff --git a/mpsserver.js b/mpsserver.js index 40e71ced..f5fa14ab 100644 --- a/mpsserver.js +++ b/mpsserver.js @@ -106,7 +106,58 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { ResourceShortage: 4, }; + // Stat counters + var connectionCount = 0; + var userAuthRequestCount = 0; + var incorrectPasswordCount = 0; + var meshNotFoundCount = 0; + var unknownTlsNodeCount = 0; + var unknownTlsMeshIdCount = 0; + var addedTlsDeviceCount = 0; + var unknownNodeCount = 0; + var unknownMeshIdCount = 0; + var addedDeviceCount = 0; + var ciraTimeoutCount = 0; + var protocolVersionCount = 0; + var badUserNameLengthCount = 0; + var channelOpenCount = 0; + var channelOpenConfirmCount = 0; + var channelOpenFailCount = 0; + var channelCloseCount = 0; + var disconnectCommandCount = 0; + var socketClosedCount = 0; + var socketErrorCount = 0; + + // Return statistics about this MPS server + obj.getStats = function () { + return { + ciraConnections: Object.keys(obj.ciraConnections).length, + tlsSessionStore: Object.keys(tlsSessionStore).length, + connectionCount: connectionCount, + userAuthRequestCount: userAuthRequestCount, + incorrectPasswordCount: incorrectPasswordCount, + meshNotFoundCount: meshNotFoundCount, + unknownTlsNodeCount: unknownTlsNodeCount, + unknownTlsMeshIdCount: unknownTlsMeshIdCount, + addedTlsDeviceCount: addedTlsDeviceCount, + unknownNodeCount: unknownNodeCount, + unknownMeshIdCount: unknownMeshIdCount, + addedDeviceCount: addedDeviceCount, + ciraTimeoutCount: ciraTimeoutCount, + protocolVersionCount: protocolVersionCount, + badUserNameLengthCount: badUserNameLengthCount, + channelOpenCount: channelOpenCount, + channelOpenConfirmCount: channelOpenConfirmCount, + channelOpenFailCount: channelOpenFailCount, + channelCloseCount: channelCloseCount, + disconnectCommandCount: disconnectCommandCount, + socketClosedCount: socketClosedCount, + socketErrorCount: socketErrorCount + }; + } + function onConnection(socket) { + connectionCount++; if (obj.args.mpstlsoffload) { socket.tag = { first: true, clientCert: null, accumulator: "", activetunnels: 0, boundPorts: [], socket: socket, host: null, nextchannelid: 4, channels: {}, nextsourceport: 0 }; } else { @@ -117,7 +168,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { // Setup the CIRA keep alive timer socket.setTimeout(MAX_IDLE); - socket.on("timeout", () => { Debug(1, "MPS:CIRA timeout, disconnecting."); try { socket.end(); } catch (e) { } }); + socket.on("timeout", () => { ciraTimeoutCount++; Debug(1, "MPS:CIRA timeout, disconnecting."); try { socket.end(); } catch (e) { } }); socket.addListener("data", function (data) { if (args.mpsdebug) { var buf = Buffer.from(data, "binary"); console.log("MPS <-- (" + buf.length + "):" + buf.toString('hex')); } // Print out received bytes @@ -156,12 +207,14 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { obj.db.Set(device); // Event the new node + addedTlsDeviceCount++; var device2 = common.Clone(device); if (device2.intelamt.pass != null) delete device2.intelamt.pass; // Remove the Intel AMT password before eventing this. var change = 'CIRA added device ' + socket.tag.name + ' to mesh ' + mesh.name; obj.parent.DispatchEvent(['*', socket.tag.meshid], obj, { etype: 'node', action: 'addnode', node: device2, msg: change, domain: domainid }); } else { // New CIRA connection for unknown node, disconnect. + unknownTlsNodeCount++; console.log('CIRA connection for unknown node with incorrect group type. meshid: ' + socket.tag.meshid); socket.end(); return; @@ -177,6 +230,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { obj.parent.SetConnectivityState(socket.tag.meshid, socket.tag.nodeid, socket.tag.connectTime, 2, 7); // TODO: Right now report power state as "present" (7) until we can poll. }); } else { + unknownTlsMeshIdCount++; console.log('ERROR: Intel AMT CIRA connected with unknown groupid: ' + socket.tag.meshid); socket.end(); return; @@ -219,6 +273,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { } case APFProtocol.PROTOCOLVERSION: { if (len < 93) return 0; + protocolVersionCount++; socket.tag.MajorVersion = common.ReadInt(data, 1); socket.tag.MinorVersion = common.ReadInt(data, 5); socket.tag.SystemId = guidToStr(common.rstr2hex(data.substring(13, 29))).toLowerCase(); @@ -227,6 +282,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { } case APFProtocol.USERAUTH_REQUEST: { if (len < 13) return 0; + userAuthRequestCount++; var usernameLen = common.ReadInt(data, 1); var username = data.substring(5, 5 + usernameLen); var serviceNameLen = common.ReadInt(data, 5 + usernameLen); @@ -242,13 +298,13 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { Debug(3, 'MPS:USERAUTH_REQUEST user=' + username + ', service=' + serviceName + ', method=' + methodName + ', password=' + password); // Check the CIRA password - if ((args.mpspass != null) && (password != args.mpspass)) { Debug(1, 'MPS:Incorrect password', username, password); SendUserAuthFail(socket); return -1; } + if ((args.mpspass != null) && (password != args.mpspass)) { incorrectPasswordCount++; Debug(1, 'MPS:Incorrect password', username, password); SendUserAuthFail(socket); return -1; } // Check the CIRA username, which should be the start of the MeshID. - if (usernameLen != 16) { Debug(1, 'MPS:Username length not 16', username, password); SendUserAuthFail(socket); return -1; } + if (usernameLen != 16) { badUserNameLengthCount++; Debug(1, 'MPS:Username length not 16', username, password); SendUserAuthFail(socket); return -1; } var meshIdStart = '/' + username, mesh = null; if (obj.parent.webserver.meshes) { for (var i in obj.parent.webserver.meshes) { if (obj.parent.webserver.meshes[i]._id.replace(/\@/g, 'X').replace(/\$/g, 'X').indexOf(meshIdStart) > 0) { mesh = obj.parent.webserver.meshes[i]; break; } } } - if (mesh == null) { Debug(1, 'MPS:Mesh not found', username, password); SendUserAuthFail(socket); return -1; } + if (mesh == null) { meshNotFoundCount++; Debug(1, 'MPS:Mesh not found', username, password); SendUserAuthFail(socket); return -1; } // If this is a agent-less mesh, use the device guid 3 times as ID. if (mesh.mtype == 1) { @@ -267,6 +323,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { obj.db.Set(device); // Event the new node + addedDeviceCount++; var device2 = common.Clone(device); if (device2.intelamt.pass != null) delete device2.intelamt.pass; // Remove the Intel AMT password before eventing this. var change = 'CIRA added device ' + socket.tag.name + ' to group ' + mesh.name; @@ -287,6 +344,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { obj.db.getAmtUuidNode(mesh._id, socket.tag.SystemId, function (err, nodes) { // TODO: May need to optimize this request with indexes if (nodes.length !== 1) { // New CIRA connection for unknown node, disconnect. + unknownNodeCount++; console.log('CIRA connection for unknown node. groupid: ' + mesh._id + ', uuid: ' + socket.tag.SystemId); socket.end(); return; @@ -306,6 +364,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { }); } else { // Unknown mesh type // New CIRA connection for unknown node, disconnect. + unknownMeshIdCount++; console.log('CIRA connection to a unknown group type. groupid: ' + socket.tag.meshid); socket.end(); return; @@ -393,6 +452,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { var Source = data.substring(29 + ChannelTypeLength + TargetLen, 29 + ChannelTypeLength + TargetLen + SourceLen); var SourcePort = common.ReadInt(data, 29 + ChannelTypeLength + TargetLen + SourceLen); + channelOpenCount++; Debug(3, 'MPS:CHANNEL_OPEN', ChannelType, SenderChannel, WindowSize, Target + ':' + TargetPort, Source + ':' + SourcePort); // Check if we understand this channel type @@ -423,6 +483,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { if (cirachannel == null) { /*console.log("MPS Error in CHANNEL_OPEN_CONFIRMATION: Unable to find channelid " + RecipientChannel);*/ return 17; } cirachannel.amtchannelid = SenderChannel; cirachannel.sendcredits = cirachannel.amtCiraWindow = WindowSize; + channelOpenConfirmCount++; Debug(3, 'MPS:CHANNEL_OPEN_CONFIRMATION', RecipientChannel, SenderChannel, WindowSize); if (cirachannel.closing == 1) { // Close this channel @@ -454,6 +515,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { if (len < 17) return 0; var RecipientChannel = common.ReadInt(data, 1); var ReasonCode = common.ReadInt(data, 5); + channelOpenFailCount++; Debug(3, 'MPS:CHANNEL_OPEN_FAILURE', RecipientChannel, ReasonCode); var cirachannel = socket.tag.channels[RecipientChannel]; if (cirachannel == null) { console.log("MPS Error in CHANNEL_OPEN_FAILURE: Unable to find channelid " + RecipientChannel); return 17; } @@ -468,6 +530,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { { if (len < 5) return 0; var RecipientChannel = common.ReadInt(data, 1); + channelCloseCount++; Debug(3, 'MPS:CHANNEL_CLOSE', RecipientChannel); var cirachannel = socket.tag.channels[RecipientChannel]; if (cirachannel == null) { console.log("MPS Error in CHANNEL_CLOSE: Unable to find channelid " + RecipientChannel); return 5; } @@ -526,6 +589,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { { if (len < 7) return 0; var ReasonCode = common.ReadInt(data, 1); + disconnectCommandCount++; Debug(3, 'MPS:DISCONNECT', ReasonCode); try { delete obj.ciraConnections[socket.tag.nodeid]; } catch (e) { } obj.parent.ClearConnectivityState(socket.tag.meshid, socket.tag.nodeid, 2); @@ -540,12 +604,14 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) { } socket.addListener("close", function () { + socketClosedCount++; Debug(1, 'MPS:CIRA connection closed'); try { delete obj.ciraConnections[socket.tag.nodeid]; } catch (e) { } obj.parent.ClearConnectivityState(socket.tag.meshid, socket.tag.nodeid, 2); }); socket.addListener("error", function () { + socketErrorCount++; //console.log("MPS Error: " + socket.remoteAddress); }); diff --git a/package.json b/package.json index 67b035ba..eb461130 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meshcentral", - "version": "0.3.3-n", + "version": "0.3.3-o", "keywords": [ "Remote Management", "Intel AMT", diff --git a/webserver.js b/webserver.js index 9cb3c9a4..2c4e2c66 100644 --- a/webserver.js +++ b/webserver.js @@ -214,6 +214,54 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { }); }); + // Return statistics about this web server + obj.getStats = function () { + return { + users: Object.keys(obj.users).length, + meshes: Object.keys(obj.meshes).length, + dnsDomains: Object.keys(obj.dnsDomains).length, + relaySessionCount: obj.relaySessionCount, + relaySessionErrorCount: obj.relaySessionErrorCount, + wsagents: Object.keys(obj.wsagents).length, + wsagentsDisconnections: Object.keys(obj.wsagentsDisconnections).length, + wsagentsDisconnectionsTimer: Object.keys(obj.wsagentsDisconnectionsTimer).length, + wssessions: Object.keys(obj.wssessions).length, + wssessions2: Object.keys(obj.wssessions2).length, + wsPeerSessions: Object.keys(obj.wsPeerSessions).length, + wsPeerSessions2: Object.keys(obj.wsPeerSessions2).length, + wsPeerSessions3: Object.keys(obj.wsPeerSessions3).length, + sessionsCount: Object.keys(obj.sessionsCount).length, + wsrelays: Object.keys(obj.wsrelays).length, + wsPeerRelays: Object.keys(obj.wsPeerRelays).length, + tlsSessionStore: Object.keys(tlsSessionStore).length + }; + } + + // Agent counters + obj.agentStats = { + createMeshAgentCount: 0, + coreIsStableCount: 0, + verifiedAgentConnectionCount: 0, + clearingCoreCount: 0, + updatingCoreCount: 0, + recoveryCoreIsStableCount: 0, + meshDoesNotExistCount: 0, + invalidPkcsSignatureCount: 0, + invalidRsaSignatureCount: 0, + invalidJsonCount: 0, + unknownAgentActionCount: 0, + agentBadWebCertHashCount: 0, + agentBadSignature1Count: 0, + agentBadSignature2Count: 0, + agentMaxSessionHoldCount: 0, + invalidDomainMeshCount: 0, + invalidMeshTypeCount: 0, + invalidDomainMesh2Count: 0, + invalidMeshType2Count: 0, + duplicateAgentCount: 0 + } + obj.getAgentStats = function () { return obj.agentStats; } + // Authenticate the user obj.authenticate = function (name, pass, domain, fn) { if ((typeof (name) != 'string') || (typeof (pass) != 'string') || (typeof (domain) != 'object')) { fn(new Error('invalid fields')); return; }