Add sqlite config options (#6517)

db.js

@@ -39,6 +39,17 @@ module.exports.CreateDB = function (parent, func) {
     let databaseName = 'meshcentral';
     let datapathParentPath = path.dirname(parent.datapath);
     let datapathFoldername = path.basename(parent.datapath);
+    const SQLITE_AUTOVACUUM = ['none', 'full', 'incremental'];
+    const SQLITE_SYNCHRONOUS = ['off', 'normal', 'full', 'extra'];
+    obj.sqliteConfig = {
+        maintenance: '',
+        startupVacuum: false,
+        autoVacuum: 'full',
+        incrementalVacuum: 100,
+        journalMode: 'delete',
+        journalSize: 4096000,
+        synchronous: 'full',
+    };
     obj.performingBackup = false;
     const BACKUPFAIL_ZIPCREATE = 0x0001;
     const BACKUPFAIL_ZIPMODULE = 0x0010;
@@ -119,6 +130,7 @@ module.exports.CreateDB = function (parent, func) {
 
     // Perform database maintenance
     obj.maintenance = function () {
+        parent.debug('db', 'Entering database maintenance');
         if (obj.databaseType == DB_NEDB) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
             obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
             obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
@@ -138,12 +150,21 @@ module.exports.CreateDB = function (parent, func) {
                 });
             });
         } else if (obj.databaseType == DB_SQLITE) { // SQLite3
-            // TODO: Combine with others?
+            //sqlite does not return rows affected for INSERT, UPDATE or DELETE statements, see https://www.sqlite.org/pragma.html#pragma_count_changes
-            sqlDbQuery('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))], function (doc, err) { }); 
+            obj.file.serialize(function () {
-            sqlDbQuery('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))], function (doc, err) { });
+                obj.file.run('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))]); 
-            sqlDbQuery('DELETE FROM serverstats WHERE expire < ?', [new Date()], function (doc, err) { });
+                obj.file.run('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))]);
-            sqlDbQuery('DELETE FROM smbios WHERE expire < ?', [new Date()], function (doc, err) { });
+                obj.file.run('DELETE FROM serverstats WHERE expire < ?', [new Date()]);
-            obj.file.run( 'PRAGMA optimize;' ); //see https://sqlite.org/pragma.html#pragma_optimize
+                obj.file.run('DELETE FROM smbios WHERE expire < ?', [new Date()]);
+                obj.file.exec(obj.sqliteConfig.maintenance, function (err) {
+                    if (err) {console.log('Maintenance error: ' + err.message)};
+                    if (parent.config.settings.debug) {
+                        sqliteGetPragmas(['freelist_count', 'page_size', 'page_count', 'cache_size' ], function (pragma, pragmaValue) {
+                            parent.debug('db', 'SQLite Maintenance: ' + pragma + '=' + pragmaValue);
+                        });
+                    };
+                });
+            });
         }
         obj.removeInactiveDevices();
     }
@@ -742,13 +763,29 @@ module.exports.CreateDB = function (parent, func) {
         // SQLite3 database setup
         obj.databaseType = DB_SQLITE;
         const sqlite3 = require('sqlite3');
-        if (typeof parent.config.settings.sqlite3 == 'string') {databaseName = parent.config.settings.sqlite3};
+        let configParams = parent.config.settings.sqlite3;
-        //use sqlite3 cache mode https://github.com/TryGhost/node-sqlite3/wiki/Caching#caching
+        if (typeof configParams == 'string') {databaseName = configParams} else {databaseName = configParams.name ? configParams.name : 'meshcentral';};
-        obj.file = new sqlite3.cached.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
+        obj.sqliteConfig.startupVacuum = configParams.startupvacuum ? configParams.startupvacuum : false;
+        obj.sqliteConfig.autoVacuum = configParams.autovacuum ? configParams.autovacuum.toLowerCase() : 'incremental';
+        obj.sqliteConfig.incrementalVacuum = configParams.incrementalvacuum ? configParams.incrementalvacuum : 100;
+        obj.sqliteConfig.journalMode = configParams.journalmode ? configParams.journalmode.toLowerCase() : 'delete';
+        //allowed modes, 'none' excluded because not usefull for this app, maybe also remove 'memory'?
+        if (!(['delete', 'truncate', 'persist', 'memory', 'wal'].includes(obj.sqliteConfig.journalMode))) { obj.sqliteConfig.journalMode = 'delete'};
+        obj.sqliteConfig.journalSize = configParams.journalsize ? configParams.journalsize : 409600;
+        //wal can use the more performant 'normal' mode, see https://www.sqlite.org/pragma.html#pragma_synchronous
+        obj.sqliteConfig.synchronous = (obj.sqliteConfig.journalMode == 'wal') ? 'normal' : 'full';
+        if (obj.sqliteConfig.journalMode == 'wal') {obj.sqliteConfig.maintenance += 'PRAGMA wal_checkpoint(PASSIVE);'};
+        if (obj.sqliteConfig.autoVacuum == 'incremental') {obj.sqliteConfig.maintenance += 'PRAGMA incremental_vacuum(' + obj.sqliteConfig.incrementalVacuum + ');'};
+        obj.sqliteConfig.maintenance += 'PRAGMA optimize;';
+        
+        parent.debug('db', 'SQlite config options: ' + JSON.stringify(obj.sqliteConfig, null, 4));
+        if (obj.sqliteConfig.journalMode == 'memory') { console.log('[WARNING] journal_mode=memory: this can lead to database corruption if there is a crash during a transaction. See https://www.sqlite.org/pragma.html#pragma_journal_mode') };
+        //.cached not usefull
+        obj.file = new sqlite3.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
             if (err && (err.code == 'SQLITE_CANTOPEN')) {
                 // Database needs to be created
                 obj.file = new sqlite3.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), function (err) {
-                    if (err) { console.log("SQLite Error: " + err); process.exit(1);; return; }
+                    if (err) { console.log("SQLite Error: " + err); process.exit(1); }
                     obj.file.exec(`
                         CREATE TABLE main (id VARCHAR(256) PRIMARY KEY NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON);
                         CREATE TABLE events(id INTEGER PRIMARY KEY, time TIMESTAMP, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON);
@@ -771,23 +808,18 @@ module.exports.CreateDB = function (parent, func) {
                         CREATE INDEX ndxsmbiosexpire ON smbios (expire);
                         `, function (err) {
                             // Completed DB creation of SQLite3
-                            //WAL mode instead of roll-back/delete
+                            sqliteSetOptions(func);
-                            obj.file.run( 'PRAGMA journal_mode=WAL;' );
+                            //setupFunctions could be put in the sqliteSetupOptions, but left after it for clarity
-                            //Together with the optimize in the maintenance run, see https://sqlite.org/pragma.html#pragma_optimize
-                            obj.file.run( 'PRAGMA optimize=0x10002;' ); 
                             setupFunctions(func);
                         }
                     );
                 });
                 return;
-            } else if (err) { console.log("SQLite Error: " + err); process.exit(0); return; }
+            } else if (err) { console.log("SQLite Error: " + err); process.exit(0); }
 
-            // Completed setup of SQLite3
             //for existing db's
-            //WAL mode instead of roll-back/delete
+            sqliteSetOptions();
-            obj.file.run( 'PRAGMA journal_mode=WAL;' );
+            //setupFunctions could be put in the sqliteSetupOptions, but left after it for clarity
-            //Together with the optimize in the maintenance run, see https://sqlite.org/pragma.html#pragma_optimize
-            obj.file.run( 'PRAGMA optimize=0x10002;' ); 
             setupFunctions(func);
         });
     } else if (parent.args.acebase) {
@@ -1277,6 +1309,45 @@ module.exports.CreateDB = function (parent, func) {
         setupFunctions(func); // Completed setup of NeDB
     }
 
+    function sqliteSetOptions(func) {
+        //get current auto_vacuum mode for comparison
+        obj.file.get('PRAGMA auto_vacuum;', function(err, current){
+            let pragma = 'PRAGMA journal_mode=' + obj.sqliteConfig.journalMode + ';' + 
+                'PRAGMA synchronous='+ obj.sqliteConfig.synchronous + ';' +
+                'PRAGMA journal_size_limit=' + obj.sqliteConfig.journalSize + ';' +
+                'PRAGMA auto_vacuum=' + obj.sqliteConfig.autoVacuum + ';' +
+                'PRAGMA incremental_vacuum=' + obj.sqliteConfig.incrementalVacuum + ';' +
+                'PRAGMA optimize=0x10002;';
+            //check new autovacuum mode, if changing from or to 'none', a VACUUM needs to be done to activate it. See https://www.sqlite.org/pragma.html#pragma_auto_vacuum
+            if ( obj.sqliteConfig.startupVacuum
+                || (current.auto_vacuum == 0 && obj.sqliteConfig.autoVacuum !='none')
+                || (current.auto_vacuum != 0 && obj.sqliteConfig.autoVacuum =='none'))
+                {
+                    pragma += 'VACUUM;';
+                };
+            parent.debug ('db', 'Config statement: ' + pragma);
+            
+            obj.file.exec( pragma,
+                function (err) {
+                if (err) { parent.debug('db', 'Config pragma error: ' + (err.message)) };
+                sqliteGetPragmas(['journal_mode', 'journal_size_limit', 'freelist_count', 'auto_vacuum', 'page_size', 'wal_autocheckpoint', 'synchronous'], function (pragma, pragmaValue) {
+                    parent.debug('db', 'PRAGMA: ' + pragma + '=' + pragmaValue);
+                });
+            });
+        });
+        //setupFunctions(func);
+    }
+
+    function sqliteGetPragmas (pragmas, func){
+        //pragmas can only be gotting one by one
+        pragmas.forEach (function (pragma) {
+            obj.file.get('PRAGMA ' + pragma + ';', function(err, res){
+                if (pragma == 'auto_vacuum') { res[pragma] = SQLITE_AUTOVACUUM[res[pragma]] };
+                if (pragma == 'synchronous') { res[pragma] = SQLITE_SYNCHRONOUS[res[pragma]] };
+                if (func) { func (pragma, res[pragma]); }
+            });
+        });
+    }
     // Create the PostgreSQL tables
     function postgreSqlCreateTables(func) {
         // Database was created, create the tables

meshcentral-config-schema.json

@@ -94,9 +94,41 @@
           }
         },
         "sqlite3": {
-          "type": [ "boolean", "string" ],
+          "type": [ "boolean", "string", "object" ],
           "default": false,
-          "description": "Set boolean true to use SQLite3 as a local MeshCentral database with default db filename 'meshcentral' or enter a string for a different db filename. Extension .sqlite is appended"
+          "description": "Set boolean true to use SQLite3 as a local MeshCentral database with default db filename 'meshcentral' or enter a string for a different db filename. Extension .sqlite is appended",
+          "properties":{
+            "name": {
+              "type": "string",
+              "default": "meshcentral",
+              "description": "Database filename. '.sqlite' is appended"
+            },
+            "journalMode": {
+              "type": "string",
+              "default": "delete",
+              "description": "DELETE, TRUNCATE, PERSIST, MEMORY, WAL. NONE not allowed. See: https://www.sqlite.org/pragma.html#pragma_journal_mode"
+            },
+            "journalSize": {
+              "type": "integer",
+              "default": 4096000,
+              "description": "Maximum size of the journal file in bytes. Can grow larger if needed, but will shrink to this size. -1 is unlimited growth. See: https://www.sqlite.org/pragma.html#pragma_journal_size_limit"
+            },
+            "autoVacuum": {
+              "type": "string",
+              "default": "incremental",
+              "description": "none, full, incremental. Removes unused pages and shrinks databasefile during maintenance. See: https://www.sqlite.org/pragma.html#pragma_auto_vacuum"
+            },
+            "incrementalVacuum": {
+              "type": "integer",
+              "default": 100,
+              "description": "Maximum amount of pages to free during maintenance. Default page size is 4k, so default frees up to 400k from the databasefile. See: https://www.sqlite.org/pragma.html#pragma_incremental_vacuum"
+            },
+            "startupVacuum": {
+              "type": "boolean",
+              "default": false,
+              "description": "Do a full VACUUM at startup. Shrinks the db file and optimizes it. This can take some time with a large database and can temporarily take up to double the database size on disk. See: https://www.sqlite.org/lang_vacuum.html"
+            }
+          }
         },
         "mySQL": {
           "type": "object",