Migration improvements

refs #2600

- fixed issue with defaults not being populated on upgrade
- added logging to all actions in the migration process to help debugging
  in future
- did a little bit of refactoring

core/server/data/fixtures/index.js

@@ -8,18 +8,27 @@
 var when        = require('when'),
     sequence    = require('when/sequence'),
     _           = require('lodash'),
+    errors      = require('../../errors'),
     utils       = require('../../utils'),
     models      = require('../../models'),
     fixtures    = require('./fixtures'),
     permissions = require('./permissions'),
 
-    populate,
+    // Private
-    update,
+    logInfo,
     to003,
     fetchAdmin,
     convertAdminToOwner,
-    createOwner;
+    createOwner,
 
+    // Public
+    populate,
+    update;
+
+
+logInfo = function logInfo(message) {
+    errors.logInfo('Fixtures', message);
+};
 
 /**
  * Fetch Admin
@@ -78,6 +87,8 @@ populate = function () {
         Role = models.Role,
         Client = models.Client;
 
+    logInfo('Populating fixtures');
+
     _.each(fixtures.posts, function (post) {
         ops.push(function () { return Post.add(post); });
     });
@@ -154,6 +165,7 @@ to003 = function () {
 };
 
 update = function (fromVersion, toVersion) {
+    logInfo('Updating fixtures');
     if (fromVersion < '003' && toVersion >= '003') {
         return to003();
     }

core/server/data/fixtures/permissions/index.js

@@ -3,17 +3,23 @@
 var when        = require('when'),
     sequence    = require('when/sequence'),
     _           = require('lodash'),
-
+    errors      = require('../../../errors'),
     models      = require('../../../models'),
     fixtures    = require('./permissions'),
 
-    populate,
+    // private
-    to003,
+    logInfo,
-
     addAllPermissions,
     addAllRolesPermissions,
-    addRolesPermissionsForRole;
+    addRolesPermissionsForRole,
 
+    // public
+    populate,
+    to003;
+
+logInfo = function logInfo(message) {
+    errors.logInfo('Permissions Fixtures', message);
+};
 
 addRolesPermissionsForRole = function (roleName) {
     var fixturesForRole = fixtures.permissions_roles[roleName],

core/server/data/migration/commands.js

@@ -0,0 +1,77 @@
+var  _               = require('lodash'),
+    errors          = require('../../errors'),
+    utils           = require('../utils'),
+    schema          = require('../schema').tables,
+
+
+    // private
+    logInfo,
+
+    // public
+    getDeleteCommands,
+    getAddCommands,
+    addColumnCommands,
+    modifyUniqueCommands;
+
+
+logInfo = function logInfo(message) {
+    errors.logInfo('Migrations', message);
+};
+
+getDeleteCommands = function getDeleteCommands(oldTables, newTables) {
+    var deleteTables = _.difference(oldTables, newTables);
+    return _.map(deleteTables, function (table) {
+        return function () {
+            logInfo('Deleting table: ' + table);
+            return utils.deleteTable(table);
+        };
+    });
+
+};
+getAddCommands = function getAddCommands(oldTables, newTables) {
+    var addTables = _.difference(newTables, oldTables);
+    return _.map(addTables, function (table) {
+        return function () {
+            logInfo('Creating table: ' + table);
+            return utils.createTable(table);
+        };
+    });
+};
+addColumnCommands = function addColumnCommands(table, columns) {
+    var columnKeys = _.keys(schema[table]),
+        addColumns = _.difference(columnKeys, columns);
+
+    return _.map(addColumns, function (column) {
+        return function () {
+            logInfo('Adding column: ' + table + '.' + column);
+            utils.addColumn(table, column);
+        };
+    });
+};
+modifyUniqueCommands = function modifyUniqueCommands(table, indexes) {
+    var columnKeys = _.keys(schema[table]);
+    return _.map(columnKeys, function (column) {
+        if (schema[table][column].unique && schema[table][column].unique === true) {
+            if (!_.contains(indexes, table + '_' + column + '_unique')) {
+                return function () {
+                    logInfo('Adding unique on: ' + table + '.' + column);
+                    return utils.addUnique(table, column);
+                };
+            }
+        } else if (!schema[table][column].unique) {
+            if (_.contains(indexes, table + '_' + column + '_unique')) {
+                return function () {
+                    logInfo('Dropping unique on: ' + table + '.' + column);
+                    return utils.dropUnique(table, column);
+                };
+            }
+        }
+    });
+};
+
+module.exports = {
+    getDeleteCommands: getDeleteCommands,
+    getAddCommands: getAddCommands,
+    addColumnCommands: addColumnCommands,
+    modifyUniqueCommands: modifyUniqueCommands
+};

core/server/data/migration/index.js

@@ -6,6 +6,7 @@ var _               = require('lodash'),
     errors          = require('../../errors'),
     sequence        = require('when/sequence'),
 
+    commands      = require('./commands'),
     versioning      = require('../versioning'),
     models          = require('../../models'),
     fixtures        = require('../fixtures'),
@@ -16,62 +17,40 @@ var _               = require('lodash'),
 
     schemaTables    = _.keys(schema),
 
+    // private
+    logInfo,
+    populateDefaultSettings,
+    backupDatabase,
+
+    // public
     init,
     reset,
     migrateUp,
     migrateUpFreshDb;
 
-function getDeleteCommands(oldTables, newTables) {
+logInfo = function logInfo(message) {
-    var deleteTables = _.difference(oldTables, newTables);
+    errors.logInfo('Migrations', message);
-    if (!_.isEmpty(deleteTables)) {
+};
-        return _.map(deleteTables, function (table) {
-            return function () {
-                return utils.deleteTable(table);
-            };
-        });
-    }
-}
 
-function getAddCommands(oldTables, newTables) {
+populateDefaultSettings = function populateDefaultSettings() {
-    var addTables = _.difference(newTables, oldTables);
+    // Initialise the default settings
-    if (!_.isEmpty(addTables)) {
+    logInfo('Populating default settings');
-        return _.map(addTables, function (table) {
+    return models.Settings.populateDefaults().then(function () {
-            return function () {
+        logInfo('Complete');
-                return utils.createTable(table);
-            };
-        });
-    }
-}
-
-function addColumnCommands(table, columns) {
-    var columnKeys = _.keys(schema[table]),
-        addColumns = _.difference(columnKeys, columns);
-
-    return _.map(addColumns, function (column) {
-        return function () {
-            utils.addColumn(table, column);
-        };
     });
-}
+};
 
-function modifyUniqueCommands(table, indexes) {
+backupDatabase = function backupDatabase() {
-    var columnKeys = _.keys(schema[table]);
+    logInfo('Creating database backup');
-    return _.map(columnKeys, function (column) {
+    return dataExport().then(function (exportedData) {
-        if (schema[table][column].unique && schema[table][column].unique === true) {
+        // Save the exported data to the file system for download
-            if (!_.contains(indexes, table + '_' + column + '_unique')) {
+        var fileName = path.resolve(config().paths.contentPath + '/data/exported-' + (new Date().getTime()) + '.json');
-                return function () {
+
-                    return utils.addUnique(table, column);
+        return nodefn.call(fs.writeFile, fileName, JSON.stringify(exportedData)).then(function () {
-                };
+            logInfo('Database backup written to: ' + fileName);
-            }
+        });
-        } else if (!schema[table][column].unique) {
-            if (_.contains(indexes, table + '_' + column + '_unique')) {
-                return function () {
-                    return utils.dropUnique(table, column);
-                };
-            }
-        }
     });
-}
+};
 
 // Check for whether data is needed to be bootstrapped or not
 init = function () {
@@ -85,11 +64,13 @@ init = function () {
         var defaultVersion = versioning.getDefaultDatabaseVersion();
         if (databaseVersion === defaultVersion) {
             // 1. The database exists and is up-to-date
+            logInfo('Up to date at version ' + databaseVersion);
             return when.resolve();
         }
         if (databaseVersion < defaultVersion) {
             // 2. The database exists but is out of date
             // Migrate to latest version
+            logInfo('Database upgrade required from version ' + databaseVersion + ' to ' +  defaultVersion);
             return self.migrateUp(databaseVersion, defaultVersion).then(function () {
                 // Finally update the databases current version
                 return versioning.setDatabaseVersion();
@@ -107,6 +88,7 @@ init = function () {
         if (err.message || err === 'Settings table does not exist') {
             // 4. The database has not yet been created
             // Bring everything up from initial version.
+            logInfo('Database initialisation required for version ' + versioning.getDefaultDatabaseVersion());
             return self.migrateUpFreshDb();
         }
         // 3. The database exists but the currentVersion setting does not or cannot be understood
@@ -118,8 +100,7 @@ init = function () {
 // ### Reset
 // Delete all tables from the database in reverse order
 reset = function () {
-    var tables = [];
+    var tables = _.map(schemaTables, function (table) {
-    tables = _.map(schemaTables, function (table) {
         return function () {
             return utils.deleteTable(table);
         };
@@ -130,75 +111,41 @@ reset = function () {
 
 // Only do this if we have no database at all
 migrateUpFreshDb = function () {
-    var tables = [];
+    var tables = _.map(schemaTables, function (table) {
-    tables = _.map(schemaTables, function (table) {
         return function () {
+            logInfo('Creating table: ' + table);
             return utils.createTable(table);
         };
     });
-
+    logInfo('Creating tables...');
     return sequence(tables).then(function () {
         // Load the fixtures
-        return fixtures.populate().then(function () {
+        return fixtures.populate();
-            // Initialise the default settings
+    }).then(function () {
-            return models.Settings.populateDefaults();
+        return populateDefaultSettings();
-        });
     });
 };
 
-// This function changes the type of posts.html and posts.markdown columns to mediumtext. Due to
-// a wrong datatype in schema.js some installations using mysql could have been created using the
-// data type text instead of mediumtext.
-// For details see: https://github.com/TryGhost/Ghost/issues/1947
-function checkMySQLPostTable() {
-    var knex = config().database.knex;
-
-    return knex.raw("SHOW FIELDS FROM posts where Field ='html' OR Field = 'markdown'").then(function (response) {
-        return _.flatten(_.map(response[0], function (entry) {
-            if (entry.Type.toLowerCase() !== 'mediumtext') {
-                return knex.raw("ALTER TABLE posts MODIFY " + entry.Field + " MEDIUMTEXT").then(function () {
-                    return when.resolve();
-                });
-            }
-        }));
-    });
-}
-
-function backupDatabase() {
-    return dataExport().then(function (exportedData) {
-        // Save the exported data to the file system for download
-        var fileName = path.resolve(config().paths.contentPath + '/data/exported-' + (new Date().getTime()) + '.json');
-
-        return nodefn.call(fs.writeFile, fileName, JSON.stringify(exportedData));
-    });
-}
-
 // Migrate from a specific version to the latest
 migrateUp = function (fromVersion, toVersion) {
-    var deleteCommands,
+    var oldTables,
-        addCommands,
-        oldTables,
-        client = config().database.client,
-        addColumns = [],
         modifyUniCommands = [],
-        commands = [];
+        migrateOps = [];
 
     return backupDatabase().then(function () {
-        return utils.getTables().then(function (tables) {
+        return utils.getTables();
-            oldTables = tables;
+    }).then(function (tables) {
-        });
+        oldTables = tables;
-    }).then(function () {
+        if (!_.isEmpty(oldTables)) {
-        // if tables exist and client is mysqls check if posts table is okay
+            return utils.checkTables();
-        if (!_.isEmpty(oldTables) && client === 'mysql') {
-            return checkMySQLPostTable();
         }
     }).then(function () {
-        deleteCommands = getDeleteCommands(oldTables, schemaTables);
+        migrateOps = migrateOps.concat(commands.getDeleteCommands(oldTables, schemaTables));
-        addCommands = getAddCommands(oldTables, schemaTables);
+        migrateOps = migrateOps.concat(commands.getAddCommands(oldTables, schemaTables));
         return when.all(
             _.map(oldTables, function (table) {
                 return utils.getIndexes(table).then(function (indexes) {
-                    modifyUniCommands = modifyUniCommands.concat(modifyUniqueCommands(table, indexes));
+                    modifyUniCommands = modifyUniCommands.concat(commands.modifyUniqueCommands(table, indexes));
                 });
             })
         );
@@ -206,37 +153,24 @@ migrateUp = function (fromVersion, toVersion) {
         return when.all(
             _.map(oldTables, function (table) {
                 return utils.getColumns(table).then(function (columns) {
-                    addColumns = addColumns.concat(addColumnCommands(table, columns));
+                    migrateOps = migrateOps.concat(commands.addColumnCommands(table, columns));
                 });
             })
         );
 
     }).then(function () {
-        modifyUniCommands = _.compact(modifyUniCommands);
+        migrateOps = migrateOps.concat(_.compact(modifyUniCommands));
 
-        // delete tables
-        if (!_.isEmpty(deleteCommands)) {
-            commands = commands.concat(deleteCommands);
-        }
-        // add tables
-        if (!_.isEmpty(addCommands)) {
-            commands = commands.concat(addCommands);
-        }
-        // add columns if needed
-        if (!_.isEmpty(addColumns)) {
-            commands = commands.concat(addColumns);
-        }
-        // add/drop unique constraint
-        if (!_.isEmpty(modifyUniCommands)) {
-            commands = commands.concat(modifyUniCommands);
-        }
         // execute the commands in sequence
-        if (!_.isEmpty(commands)) {
+        if (!_.isEmpty(migrateOps)) {
-            return sequence(commands);
+            logInfo('Running migrations');
+            return sequence(migrateOps);
         }
         return;
     }).then(function () {
         return fixtures.update(fromVersion, toVersion);
+    }).then(function () {
+        return populateDefaultSettings();
     });
 };
 
@@ -245,4 +179,4 @@ module.exports = {
     reset: reset,
     migrateUp: migrateUp,
     migrateUpFreshDb: migrateUpFreshDb
-};
+};

core/server/data/utils/clients/index.js

@@ -0,0 +1,10 @@
+var sqlite3 = require('./sqlite3'),
+    mysql   = require('./mysql'),
+    pgsql   = require('./pgsql');
+
+
+module.exports = {
+    sqlite3: sqlite3,
+    mysql: mysql,
+    pgsql: pgsql
+};

core/server/data/utils/clients/mysql.js

@@ -0,0 +1,59 @@
+var _       = require('lodash'),
+    when    = require('when'),
+    config  = require('../../../config/index'),
+
+    //private
+    doRawAndFlatten,
+
+    // public
+    getTables,
+    getIndexes,
+    getColumns,
+    checkPostTable;
+
+doRawAndFlatten = function doRaw(query, flattenFn) {
+    return config().database.knex.raw(query).then(function (response) {
+        return _.flatten(flattenFn(response));
+    });
+};
+
+getTables = function getTables() {
+    return doRawAndFlatten('show tables', function (response) {
+        return _.map(response[0], function (entry) { return _.values(entry); });
+    });
+};
+
+getIndexes = function getIndexes(table) {
+    return doRawAndFlatten('SHOW INDEXES from ' + table, function (response) {
+        return _.pluck(response[0], 'Key_name');
+    });
+};
+
+getColumns = function getColumns(table) {
+    return doRawAndFlatten('SHOW COLUMNS FROM ' + table, function (response) {
+        return _.pluck(response[0], 'Field');
+    });
+};
+
+// This function changes the type of posts.html and posts.markdown columns to mediumtext. Due to
+// a wrong datatype in schema.js some installations using mysql could have been created using the
+// data type text instead of mediumtext.
+// For details see: https://github.com/TryGhost/Ghost/issues/1947
+checkPostTable = function checkPostTable() {
+    return config().database.knex.raw('SHOW FIELDS FROM posts where Field ="html" OR Field = "markdown"').then(function (response) {
+        return _.flatten(_.map(response[0], function (entry) {
+            if (entry.Type.toLowerCase() !== 'mediumtext') {
+                return config().database.knex.raw('ALTER TABLE posts MODIFY ' + entry.Field + ' MEDIUMTEXT').then(function () {
+                    return when.resolve();
+                });
+            }
+        }));
+    });
+};
+
+module.exports = {
+    checkPostTable: checkPostTable,
+    getTables:  getTables,
+    getIndexes: getIndexes,
+    getColumns: getColumns
+};

core/server/data/utils/clients/pgsql.js

@@ -0,0 +1,44 @@
+var _       = require('lodash'),
+    config  = require('../../../config/index'),
+
+    // private
+    doRawFlattenAndPluck,
+
+    // public
+    getTables,
+    getIndexes,
+    getColumns;
+
+
+doRawFlattenAndPluck = function doRaw(query, name) {
+    return config().database.knex.raw(query).then(function (response) {
+        return _.flatten(_.pluck(response.rows, name));
+    });
+};
+
+getTables = function getTables() {
+    return doRawFlattenAndPluck(
+        'SELECT table_name FROM information_schema.tables WHERE table_schema = "public"', 'table_name'
+    );
+};
+
+getIndexes = function getIndexes(table) {
+    var selectIndexes = 'SELECT t.relname as table_name, i.relname as index_name, a.attname as column_name' +
+        ' FROM pg_class t, pg_class i, pg_index ix, pg_attribute a' +
+        ' WHERE t.oid = ix.indrelid and i.oid = ix.indexrelid and' +
+        ' a.attrelid = t.oid and a.attnum = ANY(ix.indkey) and t.relname = "' + table + '"';
+
+    return doRawFlattenAndPluck(selectIndexes, 'index_name');
+};
+
+getColumns = function getColumns(table) {
+    var selectIndexes = 'SELECT column_name FROM information_schema.columns WHERE table_name = "' + table + '"';
+
+    return doRawFlattenAndPluck(selectIndexes, 'column_name');
+};
+
+module.exports = {
+    getTables:  getTables,
+    getIndexes: getIndexes,
+    getColumns: getColumns
+};

core/server/data/utils/clients/sqlite3.js

@@ -0,0 +1,43 @@
+var _       = require('lodash'),
+    config  = require('../../../config/index'),
+
+        //private
+        doRaw,
+
+        // public
+        getTables,
+        getIndexes,
+        getColumns;
+
+
+doRaw = function doRaw(query, fn) {
+    return config().database.knex.raw(query).then(function (response) {
+        return fn(response);
+    });
+};
+
+getTables = function getTables() {
+    return doRaw('select * from sqlite_master where type = "table"', function (response) {
+        return _.reject(_.pluck(response, 'tbl_name'), function (name) {
+            return name === 'sqlite_sequence';
+        });
+    });
+};
+
+getIndexes = function getIndexes(table) {
+    return doRaw('pragma index_list("' + table + '")', function (response) {
+        return _.flatten(_.pluck(response, 'name'));
+    });
+};
+
+getColumns = function getColumns(table) {
+    return doRaw('pragma table_info("' + table + '")', function (response) {
+        return _.flatten(_.pluck(response, 'name'));
+    });
+};
+
+module.exports = {
+    getTables:  getTables,
+    getIndexes: getIndexes,
+    getColumns: getColumns
+};

core/server/data/utils/index.js

@@ -2,9 +2,8 @@ var _       = require('lodash'),
     when    = require('when'),
     config  = require('../../config'),
     schema  = require('../schema').tables,
-    sqlite3 = require('./sqlite3'),
+    clients = require('./clients');
-    mysql   = require('./mysql'),
+
-    pgsql   = require('./pgsql');
 
 function addTableColumn(tablename, table, columnname) {
     var column;
@@ -74,49 +73,43 @@ function deleteTable(table) {
 function getTables() {
     var client = config().database.client;
 
-    if (client === 'sqlite3') {
+    if (_.contains(_.keys(clients), client)) {
-        return sqlite3.getTables();
+        return clients[client].getTables();
     }
-    if (client === 'mysql') {
+
-        return mysql.getTables();
+    return when.reject('No support for database client ' + client);
-    }
-    if (client === 'pg') {
-        return pgsql.getTables();
-    }
-    return when.reject("No support for database client " + client);
 }
 
 function getIndexes(table) {
     var client = config().database.client;
 
-    if (client === 'sqlite3') {
+    if (_.contains(_.keys(clients), client)) {
-        return sqlite3.getIndexes(table);
+        return clients[client].getIndexes(table);
     }
-    if (client === 'mysql') {
+
-        return mysql.getIndexes(table);
+    return when.reject('No support for database client ' + client);
-    }
-    if (client === 'pg') {
-        return pgsql.getIndexes(table);
-    }
-    return when.reject("No support for database client " + client);
 }
 
 function getColumns(table) {
     var client = config().database.client;
 
-    if (client === 'sqlite3') {
+    if (_.contains(_.keys(clients), client)) {
-        return sqlite3.getColumns(table);
+        return clients[client].getColumns(table);
     }
+
+    return when.reject('No support for database client ' + client);
+}
+
+function checkTables() {
+    var client = config().database.client;
+
     if (client === 'mysql') {
-        return mysql.getColumns(table);
+        return clients[client].checkPostTable();
     }
-    if (client === 'pg') {
-        return pgsql.getColumns(table);
-    }
-    return when.reject("No support for database client " + client);
 }
 
 module.exports = {
+    checkTables: checkTables,
     createTable: createTable,
     deleteTable: deleteTable,
     getTables: getTables,

core/server/data/utils/mysql.js

@@ -1,28 +0,0 @@
-var _       = require('lodash'),
-    config  = require('../../config');
-
-function getTables() {
-    return config().database.knex.raw('show tables').then(function (response) {
-        return _.flatten(_.map(response[0], function (entry) {
-            return _.values(entry);
-        }));
-    });
-}
-
-function getIndexes(table) {
-    return config().database.knex.raw('SHOW INDEXES from ' + table).then(function (response) {
-        return _.flatten(_.pluck(response[0], 'Key_name'));
-    });
-}
-
-function getColumns(table) {
-    return config().database.knex.raw('SHOW COLUMNS FROM ' + table).then(function (response) {
-        return _.flatten(_.pluck(response[0], 'Field'));
-    });
-}
-
-module.exports = {
-    getTables:  getTables,
-    getIndexes: getIndexes,
-    getColumns: getColumns
-};

core/server/data/utils/pgsql.js

@@ -1,33 +0,0 @@
-var _       = require('lodash'),
-    config  = require('../../config');
-
-function getTables() {
-    return config().database.knex.raw("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'").then(function (response) {
-        return _.flatten(_.pluck(response.rows, 'table_name'));
-    });
-}
-
-function getIndexes(table) {
-    var selectIndexes = "SELECT t.relname as table_name, i.relname as index_name, a.attname as column_name"
-        + " FROM pg_class t, pg_class i, pg_index ix, pg_attribute a"
-        + " WHERE t.oid = ix.indrelid and i.oid = ix.indexrelid and"
-        + " a.attrelid = t.oid and a.attnum = ANY(ix.indkey) and t.relname = '" + table + "'";
-
-    return config().database.knex.raw(selectIndexes).then(function (response) {
-        return _.flatten(_.pluck(response.rows, 'index_name'));
-    });
-}
-
-function getColumns(table) {
-    var selectIndexes = "SELECT column_name FROM information_schema.columns WHERE table_name = '" + table + "'";
-
-    return config().database.knex.raw(selectIndexes).then(function (response) {
-        return _.flatten(_.pluck(response.rows, 'column_name'));
-    });
-}
-
-module.exports = {
-    getTables:  getTables,
-    getIndexes: getIndexes,
-    getColumns: getColumns
-};

core/server/data/utils/sqlite3.js

@@ -1,29 +0,0 @@
-var _       = require('lodash'),
-    config  = require('../../config');
-
-function getTables() {
-    return config().database.knex.raw("select * from sqlite_master where type = 'table'").then(function (response) {
-        return _.reject(_.pluck(response, 'tbl_name'), function (name) {
-            return name === 'sqlite_sequence';
-        });
-    });
-}
-
-function getIndexes(table) {
-    return config().database.knex.raw("pragma index_list('" + table + "')").then(function (response) {
-
-        return _.flatten(_.pluck(response, 'name'));
-    });
-}
-
-function getColumns(table) {
-    return config().database.knex.raw("pragma table_info('" + table + "')").then(function (response) {
-        return _.flatten(_.pluck(response, 'name'));
-    });
-}
-
-module.exports = {
-    getTables:  getTables,
-    getIndexes: getIndexes,
-    getColumns: getColumns
-};

core/server/errors/index.js

@@ -49,6 +49,17 @@ errors = {
         return when.reject(err);
     },
 
+    logInfo: function (component, info) {
+        if ((process.env.NODE_ENV === 'development' ||
+            process.env.NODE_ENV === 'staging' ||
+            process.env.NODE_ENV === 'production')) {
+
+            var msg = [component.cyan + ':'.cyan, info.cyan];
+
+            console.info.apply(console, msg);
+        }
+    },
+
     logWarn: function (warn, context, help) {
         if ((process.env.NODE_ENV === 'development' ||
             process.env.NODE_ENV === 'staging' ||