Improved agent core dump collection system.

db.js

@@ -669,7 +669,7 @@ module.exports.CreateDB = function (parent, func) {
 
         // Start NeDB main collection and setup indexes
         obj.file = new Datastore(datastoreOptions);
-        obj.file.persistence.setAutocompactionInterval(36000);
+        obj.file.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.file.ensureIndex({ fieldName: 'type' });
         obj.file.ensureIndex({ fieldName: 'domain' });
         obj.file.ensureIndex({ fieldName: 'meshid', sparse: true });
@@ -678,14 +678,14 @@ module.exports.CreateDB = function (parent, func) {
 
         // Setup the events collection and setup indexes
         obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true });
-        obj.eventsfile.persistence.setAutocompactionInterval(36000);
+        obj.eventsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field.
         obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true });
         obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 20 }); // Limit the power event log to 20 days (Seconds * Minutes * Hours * Days)
 
         // Setup the power collection and setup indexes
         obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true });
-        obj.powerfile.persistence.setAutocompactionInterval(36000);
+        obj.powerfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.powerfile.ensureIndex({ fieldName: 'nodeid' });
         obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 10 }); // Limit the power event log to 10 days (Seconds * Minutes * Hours * Days)
 
@@ -694,14 +694,14 @@ module.exports.CreateDB = function (parent, func) {
 
         // Setup the server stats collection and setup indexes
         obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true });
-        obj.serverstatsfile.persistence.setAutocompactionInterval(36000);
+        obj.serverstatsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days)
         obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events
 
         // Setup plugin info collection
         if (parent.config.settings != null) {
             obj.pluginsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-plugins.db'), autoload: true });
-            obj.pluginsfile.persistence.setAutocompactionInterval(36000);
+            obj.pluginsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         }
 
         setupFunctions(func); // Completed setup of NeDB

meshagent.js

@@ -978,7 +978,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
         // Set agent core dump
         if ((parent.parent.config.settings != null) && ((parent.parent.config.settings.agentcoredump === true) || (parent.parent.config.settings.agentcoredump === false))) {
             obj.send(JSON.stringify({ action: 'coredump', value: parent.parent.config.settings.agentcoredump }));
-            if (parent.parent.config.settings.agentcoredump === true) { obj.send(JSON.stringify({ action: 'getcoredump' })); }
+            if (parent.parent.config.settings.agentcoredump === true) {
+                // Check if we requested a core dump file in the last minute, if not, ask if one is present.
+                if ((parent.lastCoreDumpRequest == null) || ((Date.now() - parent.lastCoreDumpRequest) >= 60000)) { obj.send(JSON.stringify({ action: 'getcoredump' })); }
+            }
         }
 
         // Do this if IP location is enabled on this domain TODO: Set IP location per device group?
@@ -1370,12 +1373,27 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
                     break;
                 }
                 case 'getcoredump': {
+                    // Check if we requested a core dump file in the last minute, if so, ignore this.
+                    if ((parent.lastCoreDumpRequest != null) && ((Date.now() - parent.lastCoreDumpRequest) < 60000)) break;
+
                     // Indicates if the agent has a coredump available
-                    if (command.exists === true) {
+                    if ((command.exists === true) && (typeof command.agenthashhex == 'string') && (command.agenthashhex.length == 96)) {
-                        //console.log('CoreDump for agent ' + obj.remoteaddrport);
+                        // Check if we already have this exact dump file
-                        obj.coreDumpPresent = true;
+                        const coreDumpFile = parent.path.join(parent.parent.datapath, 'coredumps', obj.agentInfo.agentId + '-' + command.agenthashhex + '-' + obj.nodeid + '.dmp');
-                        // TODO: We need to look at getting the dump uploaded to the server.
+                        parent.fs.stat(coreDumpFile, function (err, stats) {
-                        if (typeof command.agenthashhex == 'string') { obj.RequestCoreDump(command.agenthashhex); }
+                            if (stats != null) return;
+                            obj.coreDumpPresent = true;
+
+                            // Check how many files are in the coredumps folder
+                            const coreDumpPath = parent.path.join(parent.parent.datapath, 'coredumps');
+                            parent.fs.readdir(coreDumpPath, function (err, files) {
+                                if ((files != null) && (files.length >= 20)) return; // Don't get more than 20 core dump files.
+
+                                // Get the core dump uploaded to the server.
+                                parent.lastCoreDumpRequest = Date.now();
+                                obj.RequestCoreDump(command.agenthashhex);
+                            });
+                        });
                     }
                     break;
                 }

meshcentral-config-schema.json

@@ -39,6 +39,7 @@
         "agentAliasPort": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "When set, indicates the actual publically visible agent-only port. If not set, the AgentPort value is used." },
         "agentAliasDNS": { "type": "string", "format": "hostname", "description": "When set, specified the DNS name used by agents to connect to the agent-only port." },
         "agentPortTls": { "type": "boolean", "default": true, "description": "Indicates if the agent-only port must perform TLS, this should be set to false if TLS is performed in front of this server." },
+        "agentCoreDump": { "type": "boolean", "default": false, "description": "Automatically activates and transfers any agent crash dump files to the server in meshcentral-data/coredumps." },
         "exactPorts": { "type": "boolean", "default": false },
         "allowLoginToken": { "type": "boolean", "default": false },
         "allowFraming": { "type": "boolean", "default": false },
@@ -53,6 +54,7 @@
         "agentPing": { "type": "integer", "minimum": 1, "description": "When specified, sends data to the agent at x seconds interval and expects a response from the agent." },
         "agentPong": { "type": "integer", "minimum": 1, "description": "When specified, sends data to the agent at x seconds interval." },
         "agentIdleTimeout": { "type": "integer", "minimum": 1 },
+        "compression": { "type": "boolean", "default": true, "description": "Enables GZIP compression for web requests." },
         "meshErrorLogPath": { "type": "string" },
         "npmPath": { "type": "string" },
         "npmProxy": { "type": "string", "format": "uri" },