Improved NeDB expire of old records.

db.js

@@ -37,6 +37,7 @@ module.exports.CreateDB = function (parent, func) {
     obj.dbRecordsEncryptKey = null;
     obj.dbRecordsDecryptKey = null;
     obj.changeStream = false;
+    obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
 
     obj.SetupDatabase = function (func) {
         // Check if the database unique identifier is present
@@ -65,6 +66,15 @@ module.exports.CreateDB = function (parent, func) {
         });
     };
 
+    // Perform database maintenance
+    obj.maintenance = function () {
+        if (obj.databaseType == 1) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
+            obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
+            obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
+            obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
+        }
+    }
+
     obj.cleanup = function (func) {
         // TODO: Remove all mesh links to invalid users
         // TODO: Remove all meshes that dont have any links
@@ -544,7 +554,7 @@ module.exports.CreateDB = function (parent, func) {
             });
 
             // Setup plugin info collection
-            if (parent.config.settings != null) { obj.pluginsfile = db.collection('plugins'); }
+            if (obj.pluginsActive) { obj.pluginsfile = db.collection('plugins'); }
 
             setupFunctions(func); // Completed setup of MongoDB
         });
@@ -648,7 +658,7 @@ module.exports.CreateDB = function (parent, func) {
         });
 
         // Setup plugin info collection
-        if (parent.config.settings != null) { obj.pluginsfile = db.collection('plugins'); }
+        if (obj.pluginsActive) { obj.pluginsfile = db.collection('plugins'); }
 
         setupFunctions(func); // Completed setup of MongoJS
     } else {
@@ -689,29 +699,32 @@ module.exports.CreateDB = function (parent, func) {
         obj.file.ensureIndex({ fieldName: 'email', sparse: true });
 
         // Setup the events collection and setup indexes
-        obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true });
+        obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 });
         obj.eventsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field.
         obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true });
         obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expireEventsSeconds });
+        obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
 
         // Setup the power collection and setup indexes
-        obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true });
+        obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 });
         obj.powerfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.powerfile.ensureIndex({ fieldName: 'nodeid' });
         obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expirePowerEventsSeconds });
+        obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
 
         // Setup the SMBIOS collection
-        obj.smbiosfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true });
+        obj.smbiosfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true, corruptAlertThreshold: 1 });
 
         // Setup the server stats collection and setup indexes
-        obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true });
+        obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 });
         obj.serverstatsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expireServerStatsSeconds });
         obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events
+        obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
 
         // Setup plugin info collection
-        if (parent.config.settings != null) {
+        if (obj.pluginsActive) {
             obj.pluginsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-plugins.db'), autoload: true });
             obj.pluginsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
         }
@@ -938,7 +951,7 @@ module.exports.CreateDB = function (parent, func) {
             }
 
             // Plugin operations
-            if (parent.config.settings.plugins != null) {
+            if (obj.pluginsActive) {
                 obj.addPlugin = function (plugin, func) { sqlDbQuery('INSERT INTO meshcentral.plugin VALUE (?, ?)', [null, JSON.stringify(value)], func); }; // Add a plugin
                 obj.getPlugins = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.plugin', null, func); }; // Get all plugins
                 obj.getPlugin = function (id, func) { sqlDbQuery('SELECT doc FROM meshcentral.plugin WHERE id = ?', [id], func); }; // Get plugin
@@ -1085,7 +1098,7 @@ module.exports.CreateDB = function (parent, func) {
             }
 
             // Plugin operations
-            if (parent.config.settings.plugins != null) {
+            if (obj.pluginsActive) {
                 obj.addPlugin = function (plugin, func) { plugin.type = 'plugin'; obj.pluginsfile.insertOne(plugin, func); }; // Add a plugin
                 obj.getPlugins = function (func) { obj.pluginsfile.find({ type: 'plugin' }).project({ type: 0 }).sort({ name: 1 }).toArray(func); }; // Get all plugins
                 obj.getPlugin = function (id, func) { id = require('mongodb').ObjectID(id); obj.pluginsfile.find({ _id: id }).sort({ name: 1 }).toArray(func); }; // Get plugin
@@ -1234,7 +1247,7 @@ module.exports.CreateDB = function (parent, func) {
             }
 
             // Plugin operations
-            if (parent.config.settings.plugins != null) {
+            if (obj.pluginsActive) {
                 obj.addPlugin = function (plugin, func) { plugin.type = 'plugin'; obj.pluginsfile.insert(plugin, func); }; // Add a plugin
                 obj.getPlugins = function (func) { obj.pluginsfile.find({ 'type': 'plugin' }, { 'type': 0 }).sort({ name: 1 }).exec(func); }; // Get all plugins
                 obj.getPlugin = function (id, func) { obj.pluginsfile.find({ _id: id }).sort({ name: 1 }).exec(func); }; // Get plugin

meshcentral.js

@@ -130,7 +130,7 @@ function CreateMeshCentralServer(config, args) {
         try { require('./pass').hash('test', function () { }, 0); } catch (e) { console.log('Old version of node, must upgrade.'); return; } // TODO: Not sure if this test works or not.
 
         // Check for invalid arguments
-        var validArguments = ['_', 'notls', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email'];
+        var validArguments = ['_', 'notls', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email'];
         for (var arg in obj.args) { obj.args[arg.toLocaleLowerCase()] = obj.args[arg]; if (validArguments.indexOf(arg.toLocaleLowerCase()) == -1) { console.log('Invalid argument "' + arg + '", use --help.'); return; } }
         if (obj.args.mongodb == true) { console.log('Must specify: --mongodb [connectionstring] \r\nSee https://docs.mongodb.com/manual/reference/connection-string/ for MongoDB connection string.'); return; }
         for (i in obj.config.settings) { obj.args[i] = obj.config.settings[i]; } // Place all settings into arguments, arguments have already been placed into settings so arguments take precedence.
@@ -163,6 +163,20 @@ function CreateMeshCentralServer(config, args) {
             return;
         }
 
+        // Fix a NeDB database
+        if (obj.args.dbfix) {
+            var lines = null, badJsonCount = 0, feildNames = [], fixedDb = [];
+            try { lines = obj.fs.readFileSync(obj.getConfigFilePath(obj.args.dbfix), { encoding: 'utf8' }).split('\n'); } catch (e) { console.log('Invalid file: ' + obj.args.dbfix + ': ' + e); process.exit(); }
+            for (var i = 0; i < lines.length; i++) {
+                var x = null;
+                try { x = JSON.parse(lines[i]); } catch (ex) { badJsonCount++; }
+                if (x != null) { fixedDb.push(lines[i]); for (var j in x) { if (feildNames.indexOf(j) == -1) { feildNames.push(j); } } }
+            }
+            console.log('Lines: ' + lines.length + ', badJSON: ' + badJsonCount + ', Feilds: ' + feildNames);
+            obj.fs.writeFileSync(obj.getConfigFilePath(obj.args.dbfix) + '-fixed', fixedDb.join('\n'), { encoding: 'utf8' });
+            return;
+        }
+
         // Perform web site translations into different languages
         if (obj.args.translate) {
             // Check NodeJS version
@@ -1525,6 +1539,9 @@ function CreateMeshCentralServer(config, args) {
 
     // Perform maintenance operations (called every hour)
     obj.maintenanceActions = function () {
+        // Perform database maintenance
+        obj.db.maintenance();
+
         // Check for self-update that targets a specific version
         if ((typeof obj.args.selfupdate == 'string') && (getCurrentVerion() === obj.args.selfupdate)) { obj.args.selfupdate = false; }
 

translate/translate.json

@@ -6635,9 +6635,6 @@
       "tr": "Çince (Traditioneel)",
       "zh-chs": "中国传统的",
       "zh-cht": "中國傳統的）"
-      "xloc": [
-        "default.handlebars->27->1214"
-      ]
     },
     {
       "cs": "ChromeOS",
@@ -35708,7 +35705,7 @@
     {
       "en": "You can setup MeshCentral to automatically send a server backup to Google Drive. Start by entering a desktop Google API ClientID and ClientSecret for your account.",
       "nl": "U kunt MeshCentral zo instellen dat er automatisch een serverback-up naar Google Drive wordt verzonden.Begin met het invoeren van een desktop Google API ClientID en ClientSecret voor uw account.",
-      "tr": "MeshCentral'ı Google Drive'a otomatik olarak bir sunucu yedeği gönderecek şekilde kurabilirsiniz. Hesabınız için bir masaüstü Google API İstemci Kimliği ve ClientSecret girerek başlayın.",
+      "tr": "MeshCentral'ı Google Drive'a otomatik olarak bir sunucu yedeği gönderecek şekilde kurabilirsiniz. Hesabınız için bir masaüstü Google API İstemci Kimliği ve ClientSecret girerek başlayın."
     },
     {
       "en": "You have been invited to install an application that will allow a remote operator to securely access your computer including the desktop and files. Only follow the instructions below if this invitation was expected and you know who will be accessing your computer. Select your operating system and follow the instructions below for installation.",
@@ -38217,4 +38214,4 @@
       ]
     }
   ]
-}
+}