diff --git a/core/server/data/fixtures/index.js b/core/server/data/fixtures/index.js index a0991aaaf4..55432eded8 100644 --- a/core/server/data/fixtures/index.js +++ b/core/server/data/fixtures/index.js @@ -8,18 +8,27 @@ var when = require('when'), sequence = require('when/sequence'), _ = require('lodash'), + errors = require('../../errors'), utils = require('../../utils'), models = require('../../models'), fixtures = require('./fixtures'), permissions = require('./permissions'), - populate, - update, + // Private + logInfo, to003, fetchAdmin, convertAdminToOwner, - createOwner; + createOwner, + // Public + populate, + update; + + +logInfo = function logInfo(message) { + errors.logInfo('Fixtures', message); +}; /** * Fetch Admin @@ -78,6 +87,8 @@ populate = function () { Role = models.Role, Client = models.Client; + logInfo('Populating fixtures'); + _.each(fixtures.posts, function (post) { ops.push(function () { return Post.add(post); }); }); @@ -154,6 +165,7 @@ to003 = function () { }; update = function (fromVersion, toVersion) { + logInfo('Updating fixtures'); if (fromVersion < '003' && toVersion >= '003') { return to003(); } diff --git a/core/server/data/fixtures/permissions/index.js b/core/server/data/fixtures/permissions/index.js index 36635d90d9..553bdad029 100644 --- a/core/server/data/fixtures/permissions/index.js +++ b/core/server/data/fixtures/permissions/index.js @@ -3,17 +3,23 @@ var when = require('when'), sequence = require('when/sequence'), _ = require('lodash'), - + errors = require('../../../errors'), models = require('../../../models'), fixtures = require('./permissions'), - populate, - to003, - + // private + logInfo, addAllPermissions, addAllRolesPermissions, - addRolesPermissionsForRole; + addRolesPermissionsForRole, + // public + populate, + to003; + +logInfo = function logInfo(message) { + errors.logInfo('Permissions Fixtures', message); +}; addRolesPermissionsForRole = function (roleName) { var fixturesForRole = fixtures.permissions_roles[roleName], diff --git a/core/server/data/migration/commands.js b/core/server/data/migration/commands.js new file mode 100644 index 0000000000..3880539ff0 --- /dev/null +++ b/core/server/data/migration/commands.js @@ -0,0 +1,77 @@ +var _ = require('lodash'), + errors = require('../../errors'), + utils = require('../utils'), + schema = require('../schema').tables, + + + // private + logInfo, + + // public + getDeleteCommands, + getAddCommands, + addColumnCommands, + modifyUniqueCommands; + + +logInfo = function logInfo(message) { + errors.logInfo('Migrations', message); +}; + +getDeleteCommands = function getDeleteCommands(oldTables, newTables) { + var deleteTables = _.difference(oldTables, newTables); + return _.map(deleteTables, function (table) { + return function () { + logInfo('Deleting table: ' + table); + return utils.deleteTable(table); + }; + }); + +}; +getAddCommands = function getAddCommands(oldTables, newTables) { + var addTables = _.difference(newTables, oldTables); + return _.map(addTables, function (table) { + return function () { + logInfo('Creating table: ' + table); + return utils.createTable(table); + }; + }); +}; +addColumnCommands = function addColumnCommands(table, columns) { + var columnKeys = _.keys(schema[table]), + addColumns = _.difference(columnKeys, columns); + + return _.map(addColumns, function (column) { + return function () { + logInfo('Adding column: ' + table + '.' + column); + utils.addColumn(table, column); + }; + }); +}; +modifyUniqueCommands = function modifyUniqueCommands(table, indexes) { + var columnKeys = _.keys(schema[table]); + return _.map(columnKeys, function (column) { + if (schema[table][column].unique && schema[table][column].unique === true) { + if (!_.contains(indexes, table + '_' + column + '_unique')) { + return function () { + logInfo('Adding unique on: ' + table + '.' + column); + return utils.addUnique(table, column); + }; + } + } else if (!schema[table][column].unique) { + if (_.contains(indexes, table + '_' + column + '_unique')) { + return function () { + logInfo('Dropping unique on: ' + table + '.' + column); + return utils.dropUnique(table, column); + }; + } + } + }); +}; + +module.exports = { + getDeleteCommands: getDeleteCommands, + getAddCommands: getAddCommands, + addColumnCommands: addColumnCommands, + modifyUniqueCommands: modifyUniqueCommands +}; \ No newline at end of file diff --git a/core/server/data/migration/index.js b/core/server/data/migration/index.js index ac677f70e7..e42f8e4343 100644 --- a/core/server/data/migration/index.js +++ b/core/server/data/migration/index.js @@ -6,6 +6,7 @@ var _ = require('lodash'), errors = require('../../errors'), sequence = require('when/sequence'), + commands = require('./commands'), versioning = require('../versioning'), models = require('../../models'), fixtures = require('../fixtures'), @@ -16,62 +17,40 @@ var _ = require('lodash'), schemaTables = _.keys(schema), + // private + logInfo, + populateDefaultSettings, + backupDatabase, + + // public init, reset, migrateUp, migrateUpFreshDb; -function getDeleteCommands(oldTables, newTables) { - var deleteTables = _.difference(oldTables, newTables); - if (!_.isEmpty(deleteTables)) { - return _.map(deleteTables, function (table) { - return function () { - return utils.deleteTable(table); - }; - }); - } -} +logInfo = function logInfo(message) { + errors.logInfo('Migrations', message); +}; -function getAddCommands(oldTables, newTables) { - var addTables = _.difference(newTables, oldTables); - if (!_.isEmpty(addTables)) { - return _.map(addTables, function (table) { - return function () { - return utils.createTable(table); - }; - }); - } -} - -function addColumnCommands(table, columns) { - var columnKeys = _.keys(schema[table]), - addColumns = _.difference(columnKeys, columns); - - return _.map(addColumns, function (column) { - return function () { - utils.addColumn(table, column); - }; +populateDefaultSettings = function populateDefaultSettings() { + // Initialise the default settings + logInfo('Populating default settings'); + return models.Settings.populateDefaults().then(function () { + logInfo('Complete'); }); -} +}; -function modifyUniqueCommands(table, indexes) { - var columnKeys = _.keys(schema[table]); - return _.map(columnKeys, function (column) { - if (schema[table][column].unique && schema[table][column].unique === true) { - if (!_.contains(indexes, table + '_' + column + '_unique')) { - return function () { - return utils.addUnique(table, column); - }; - } - } else if (!schema[table][column].unique) { - if (_.contains(indexes, table + '_' + column + '_unique')) { - return function () { - return utils.dropUnique(table, column); - }; - } - } +backupDatabase = function backupDatabase() { + logInfo('Creating database backup'); + return dataExport().then(function (exportedData) { + // Save the exported data to the file system for download + var fileName = path.resolve(config().paths.contentPath + '/data/exported-' + (new Date().getTime()) + '.json'); + + return nodefn.call(fs.writeFile, fileName, JSON.stringify(exportedData)).then(function () { + logInfo('Database backup written to: ' + fileName); + }); }); -} +}; // Check for whether data is needed to be bootstrapped or not init = function () { @@ -85,11 +64,13 @@ init = function () { var defaultVersion = versioning.getDefaultDatabaseVersion(); if (databaseVersion === defaultVersion) { // 1. The database exists and is up-to-date + logInfo('Up to date at version ' + databaseVersion); return when.resolve(); } if (databaseVersion < defaultVersion) { // 2. The database exists but is out of date // Migrate to latest version + logInfo('Database upgrade required from version ' + databaseVersion + ' to ' + defaultVersion); return self.migrateUp(databaseVersion, defaultVersion).then(function () { // Finally update the databases current version return versioning.setDatabaseVersion(); @@ -107,6 +88,7 @@ init = function () { if (err.message || err === 'Settings table does not exist') { // 4. The database has not yet been created // Bring everything up from initial version. + logInfo('Database initialisation required for version ' + versioning.getDefaultDatabaseVersion()); return self.migrateUpFreshDb(); } // 3. The database exists but the currentVersion setting does not or cannot be understood @@ -118,8 +100,7 @@ init = function () { // ### Reset // Delete all tables from the database in reverse order reset = function () { - var tables = []; - tables = _.map(schemaTables, function (table) { + var tables = _.map(schemaTables, function (table) { return function () { return utils.deleteTable(table); }; @@ -130,75 +111,41 @@ reset = function () { // Only do this if we have no database at all migrateUpFreshDb = function () { - var tables = []; - tables = _.map(schemaTables, function (table) { + var tables = _.map(schemaTables, function (table) { return function () { + logInfo('Creating table: ' + table); return utils.createTable(table); }; }); - + logInfo('Creating tables...'); return sequence(tables).then(function () { // Load the fixtures - return fixtures.populate().then(function () { - // Initialise the default settings - return models.Settings.populateDefaults(); - }); + return fixtures.populate(); + }).then(function () { + return populateDefaultSettings(); }); }; -// This function changes the type of posts.html and posts.markdown columns to mediumtext. Due to -// a wrong datatype in schema.js some installations using mysql could have been created using the -// data type text instead of mediumtext. -// For details see: https://github.com/TryGhost/Ghost/issues/1947 -function checkMySQLPostTable() { - var knex = config().database.knex; - - return knex.raw("SHOW FIELDS FROM posts where Field ='html' OR Field = 'markdown'").then(function (response) { - return _.flatten(_.map(response[0], function (entry) { - if (entry.Type.toLowerCase() !== 'mediumtext') { - return knex.raw("ALTER TABLE posts MODIFY " + entry.Field + " MEDIUMTEXT").then(function () { - return when.resolve(); - }); - } - })); - }); -} - -function backupDatabase() { - return dataExport().then(function (exportedData) { - // Save the exported data to the file system for download - var fileName = path.resolve(config().paths.contentPath + '/data/exported-' + (new Date().getTime()) + '.json'); - - return nodefn.call(fs.writeFile, fileName, JSON.stringify(exportedData)); - }); -} - // Migrate from a specific version to the latest migrateUp = function (fromVersion, toVersion) { - var deleteCommands, - addCommands, - oldTables, - client = config().database.client, - addColumns = [], + var oldTables, modifyUniCommands = [], - commands = []; + migrateOps = []; return backupDatabase().then(function () { - return utils.getTables().then(function (tables) { - oldTables = tables; - }); - }).then(function () { - // if tables exist and client is mysqls check if posts table is okay - if (!_.isEmpty(oldTables) && client === 'mysql') { - return checkMySQLPostTable(); + return utils.getTables(); + }).then(function (tables) { + oldTables = tables; + if (!_.isEmpty(oldTables)) { + return utils.checkTables(); } }).then(function () { - deleteCommands = getDeleteCommands(oldTables, schemaTables); - addCommands = getAddCommands(oldTables, schemaTables); + migrateOps = migrateOps.concat(commands.getDeleteCommands(oldTables, schemaTables)); + migrateOps = migrateOps.concat(commands.getAddCommands(oldTables, schemaTables)); return when.all( _.map(oldTables, function (table) { return utils.getIndexes(table).then(function (indexes) { - modifyUniCommands = modifyUniCommands.concat(modifyUniqueCommands(table, indexes)); + modifyUniCommands = modifyUniCommands.concat(commands.modifyUniqueCommands(table, indexes)); }); }) ); @@ -206,37 +153,24 @@ migrateUp = function (fromVersion, toVersion) { return when.all( _.map(oldTables, function (table) { return utils.getColumns(table).then(function (columns) { - addColumns = addColumns.concat(addColumnCommands(table, columns)); + migrateOps = migrateOps.concat(commands.addColumnCommands(table, columns)); }); }) ); }).then(function () { - modifyUniCommands = _.compact(modifyUniCommands); + migrateOps = migrateOps.concat(_.compact(modifyUniCommands)); - // delete tables - if (!_.isEmpty(deleteCommands)) { - commands = commands.concat(deleteCommands); - } - // add tables - if (!_.isEmpty(addCommands)) { - commands = commands.concat(addCommands); - } - // add columns if needed - if (!_.isEmpty(addColumns)) { - commands = commands.concat(addColumns); - } - // add/drop unique constraint - if (!_.isEmpty(modifyUniCommands)) { - commands = commands.concat(modifyUniCommands); - } // execute the commands in sequence - if (!_.isEmpty(commands)) { - return sequence(commands); + if (!_.isEmpty(migrateOps)) { + logInfo('Running migrations'); + return sequence(migrateOps); } return; }).then(function () { return fixtures.update(fromVersion, toVersion); + }).then(function () { + return populateDefaultSettings(); }); }; @@ -245,4 +179,4 @@ module.exports = { reset: reset, migrateUp: migrateUp, migrateUpFreshDb: migrateUpFreshDb -}; +}; \ No newline at end of file diff --git a/core/server/data/utils/clients/index.js b/core/server/data/utils/clients/index.js new file mode 100644 index 0000000000..85ea9bcd73 --- /dev/null +++ b/core/server/data/utils/clients/index.js @@ -0,0 +1,10 @@ +var sqlite3 = require('./sqlite3'), + mysql = require('./mysql'), + pgsql = require('./pgsql'); + + +module.exports = { + sqlite3: sqlite3, + mysql: mysql, + pgsql: pgsql +}; \ No newline at end of file diff --git a/core/server/data/utils/clients/mysql.js b/core/server/data/utils/clients/mysql.js new file mode 100644 index 0000000000..94b5453845 --- /dev/null +++ b/core/server/data/utils/clients/mysql.js @@ -0,0 +1,59 @@ +var _ = require('lodash'), + when = require('when'), + config = require('../../../config/index'), + + //private + doRawAndFlatten, + + // public + getTables, + getIndexes, + getColumns, + checkPostTable; + +doRawAndFlatten = function doRaw(query, flattenFn) { + return config().database.knex.raw(query).then(function (response) { + return _.flatten(flattenFn(response)); + }); +}; + +getTables = function getTables() { + return doRawAndFlatten('show tables', function (response) { + return _.map(response[0], function (entry) { return _.values(entry); }); + }); +}; + +getIndexes = function getIndexes(table) { + return doRawAndFlatten('SHOW INDEXES from ' + table, function (response) { + return _.pluck(response[0], 'Key_name'); + }); +}; + +getColumns = function getColumns(table) { + return doRawAndFlatten('SHOW COLUMNS FROM ' + table, function (response) { + return _.pluck(response[0], 'Field'); + }); +}; + +// This function changes the type of posts.html and posts.markdown columns to mediumtext. Due to +// a wrong datatype in schema.js some installations using mysql could have been created using the +// data type text instead of mediumtext. +// For details see: https://github.com/TryGhost/Ghost/issues/1947 +checkPostTable = function checkPostTable() { + return config().database.knex.raw('SHOW FIELDS FROM posts where Field ="html" OR Field = "markdown"').then(function (response) { + return _.flatten(_.map(response[0], function (entry) { + if (entry.Type.toLowerCase() !== 'mediumtext') { + return config().database.knex.raw('ALTER TABLE posts MODIFY ' + entry.Field + ' MEDIUMTEXT').then(function () { + return when.resolve(); + }); + } + })); + }); +}; + +module.exports = { + checkPostTable: checkPostTable, + getTables: getTables, + getIndexes: getIndexes, + getColumns: getColumns +}; \ No newline at end of file diff --git a/core/server/data/utils/clients/pgsql.js b/core/server/data/utils/clients/pgsql.js new file mode 100644 index 0000000000..1639a25e2a --- /dev/null +++ b/core/server/data/utils/clients/pgsql.js @@ -0,0 +1,44 @@ +var _ = require('lodash'), + config = require('../../../config/index'), + + // private + doRawFlattenAndPluck, + + // public + getTables, + getIndexes, + getColumns; + + +doRawFlattenAndPluck = function doRaw(query, name) { + return config().database.knex.raw(query).then(function (response) { + return _.flatten(_.pluck(response.rows, name)); + }); +}; + +getTables = function getTables() { + return doRawFlattenAndPluck( + 'SELECT table_name FROM information_schema.tables WHERE table_schema = "public"', 'table_name' + ); +}; + +getIndexes = function getIndexes(table) { + var selectIndexes = 'SELECT t.relname as table_name, i.relname as index_name, a.attname as column_name' + + ' FROM pg_class t, pg_class i, pg_index ix, pg_attribute a' + + ' WHERE t.oid = ix.indrelid and i.oid = ix.indexrelid and' + + ' a.attrelid = t.oid and a.attnum = ANY(ix.indkey) and t.relname = "' + table + '"'; + + return doRawFlattenAndPluck(selectIndexes, 'index_name'); +}; + +getColumns = function getColumns(table) { + var selectIndexes = 'SELECT column_name FROM information_schema.columns WHERE table_name = "' + table + '"'; + + return doRawFlattenAndPluck(selectIndexes, 'column_name'); +}; + +module.exports = { + getTables: getTables, + getIndexes: getIndexes, + getColumns: getColumns +}; \ No newline at end of file diff --git a/core/server/data/utils/clients/sqlite3.js b/core/server/data/utils/clients/sqlite3.js new file mode 100644 index 0000000000..d0ada875e0 --- /dev/null +++ b/core/server/data/utils/clients/sqlite3.js @@ -0,0 +1,43 @@ +var _ = require('lodash'), + config = require('../../../config/index'), + + //private + doRaw, + + // public + getTables, + getIndexes, + getColumns; + + +doRaw = function doRaw(query, fn) { + return config().database.knex.raw(query).then(function (response) { + return fn(response); + }); +}; + +getTables = function getTables() { + return doRaw('select * from sqlite_master where type = "table"', function (response) { + return _.reject(_.pluck(response, 'tbl_name'), function (name) { + return name === 'sqlite_sequence'; + }); + }); +}; + +getIndexes = function getIndexes(table) { + return doRaw('pragma index_list("' + table + '")', function (response) { + return _.flatten(_.pluck(response, 'name')); + }); +}; + +getColumns = function getColumns(table) { + return doRaw('pragma table_info("' + table + '")', function (response) { + return _.flatten(_.pluck(response, 'name')); + }); +}; + +module.exports = { + getTables: getTables, + getIndexes: getIndexes, + getColumns: getColumns +}; \ No newline at end of file diff --git a/core/server/data/utils/index.js b/core/server/data/utils/index.js index ac776b21e5..46ea3af853 100644 --- a/core/server/data/utils/index.js +++ b/core/server/data/utils/index.js @@ -2,9 +2,8 @@ var _ = require('lodash'), when = require('when'), config = require('../../config'), schema = require('../schema').tables, - sqlite3 = require('./sqlite3'), - mysql = require('./mysql'), - pgsql = require('./pgsql'); + clients = require('./clients'); + function addTableColumn(tablename, table, columnname) { var column; @@ -74,49 +73,43 @@ function deleteTable(table) { function getTables() { var client = config().database.client; - if (client === 'sqlite3') { - return sqlite3.getTables(); + if (_.contains(_.keys(clients), client)) { + return clients[client].getTables(); } - if (client === 'mysql') { - return mysql.getTables(); - } - if (client === 'pg') { - return pgsql.getTables(); - } - return when.reject("No support for database client " + client); + + return when.reject('No support for database client ' + client); } function getIndexes(table) { var client = config().database.client; - if (client === 'sqlite3') { - return sqlite3.getIndexes(table); + if (_.contains(_.keys(clients), client)) { + return clients[client].getIndexes(table); } - if (client === 'mysql') { - return mysql.getIndexes(table); - } - if (client === 'pg') { - return pgsql.getIndexes(table); - } - return when.reject("No support for database client " + client); + + return when.reject('No support for database client ' + client); } function getColumns(table) { var client = config().database.client; - if (client === 'sqlite3') { - return sqlite3.getColumns(table); + if (_.contains(_.keys(clients), client)) { + return clients[client].getColumns(table); } + + return when.reject('No support for database client ' + client); +} + +function checkTables() { + var client = config().database.client; + if (client === 'mysql') { - return mysql.getColumns(table); + return clients[client].checkPostTable(); } - if (client === 'pg') { - return pgsql.getColumns(table); - } - return when.reject("No support for database client " + client); } module.exports = { + checkTables: checkTables, createTable: createTable, deleteTable: deleteTable, getTables: getTables, diff --git a/core/server/data/utils/mysql.js b/core/server/data/utils/mysql.js deleted file mode 100644 index 73f1e24282..0000000000 --- a/core/server/data/utils/mysql.js +++ /dev/null @@ -1,28 +0,0 @@ -var _ = require('lodash'), - config = require('../../config'); - -function getTables() { - return config().database.knex.raw('show tables').then(function (response) { - return _.flatten(_.map(response[0], function (entry) { - return _.values(entry); - })); - }); -} - -function getIndexes(table) { - return config().database.knex.raw('SHOW INDEXES from ' + table).then(function (response) { - return _.flatten(_.pluck(response[0], 'Key_name')); - }); -} - -function getColumns(table) { - return config().database.knex.raw('SHOW COLUMNS FROM ' + table).then(function (response) { - return _.flatten(_.pluck(response[0], 'Field')); - }); -} - -module.exports = { - getTables: getTables, - getIndexes: getIndexes, - getColumns: getColumns -}; \ No newline at end of file diff --git a/core/server/data/utils/pgsql.js b/core/server/data/utils/pgsql.js deleted file mode 100644 index b6f9d6d6a9..0000000000 --- a/core/server/data/utils/pgsql.js +++ /dev/null @@ -1,33 +0,0 @@ -var _ = require('lodash'), - config = require('../../config'); - -function getTables() { - return config().database.knex.raw("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'").then(function (response) { - return _.flatten(_.pluck(response.rows, 'table_name')); - }); -} - -function getIndexes(table) { - var selectIndexes = "SELECT t.relname as table_name, i.relname as index_name, a.attname as column_name" - + " FROM pg_class t, pg_class i, pg_index ix, pg_attribute a" - + " WHERE t.oid = ix.indrelid and i.oid = ix.indexrelid and" - + " a.attrelid = t.oid and a.attnum = ANY(ix.indkey) and t.relname = '" + table + "'"; - - return config().database.knex.raw(selectIndexes).then(function (response) { - return _.flatten(_.pluck(response.rows, 'index_name')); - }); -} - -function getColumns(table) { - var selectIndexes = "SELECT column_name FROM information_schema.columns WHERE table_name = '" + table + "'"; - - return config().database.knex.raw(selectIndexes).then(function (response) { - return _.flatten(_.pluck(response.rows, 'column_name')); - }); -} - -module.exports = { - getTables: getTables, - getIndexes: getIndexes, - getColumns: getColumns -}; \ No newline at end of file diff --git a/core/server/data/utils/sqlite3.js b/core/server/data/utils/sqlite3.js deleted file mode 100644 index 3fc79f16db..0000000000 --- a/core/server/data/utils/sqlite3.js +++ /dev/null @@ -1,29 +0,0 @@ -var _ = require('lodash'), - config = require('../../config'); - -function getTables() { - return config().database.knex.raw("select * from sqlite_master where type = 'table'").then(function (response) { - return _.reject(_.pluck(response, 'tbl_name'), function (name) { - return name === 'sqlite_sequence'; - }); - }); -} - -function getIndexes(table) { - return config().database.knex.raw("pragma index_list('" + table + "')").then(function (response) { - - return _.flatten(_.pluck(response, 'name')); - }); -} - -function getColumns(table) { - return config().database.knex.raw("pragma table_info('" + table + "')").then(function (response) { - return _.flatten(_.pluck(response, 'name')); - }); -} - -module.exports = { - getTables: getTables, - getIndexes: getIndexes, - getColumns: getColumns -}; \ No newline at end of file diff --git a/core/server/errors/index.js b/core/server/errors/index.js index e61edb2627..75205dc3d6 100644 --- a/core/server/errors/index.js +++ b/core/server/errors/index.js @@ -49,6 +49,17 @@ errors = { return when.reject(err); }, + logInfo: function (component, info) { + if ((process.env.NODE_ENV === 'development' || + process.env.NODE_ENV === 'staging' || + process.env.NODE_ENV === 'production')) { + + var msg = [component.cyan + ':'.cyan, info.cyan]; + + console.info.apply(console, msg); + } + }, + logWarn: function (warn, context, help) { if ((process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'staging' ||