Added routes.yaml content checksum storage to the db

closes #11999

- When the routes.yaml file changes (manually or through API) we need
to store a checksum to be able to optimize routes reloads in the future
- Added mechanism to detect differences between stored and current routes.yaml hash value
- Added routes.yaml sync on server boot
- Added routes.yaml handling in controllers
- Added routes hash synchronization method in core settings. It lives in core settings
as it needs access to model layer. To avoid coupling with the frontend settings it accepts
a function which has to resolve to a routes hash
- Added note about settings validation side-effect. It mutates input!
- Added async check for currently loaded routes hash
- Extended frontend settings loader with async loader. The default behavior of the loader is
to load settings syncronously for reasons spelled in 0ac19dcf84
To avoid blocking the eventloop added async loading method
- Refactored frontend setting loader for reusability of  settings file path
- Added integrity check test for routes.yaml file

core/frontend/services/settings/ensure-settings.js

@@ -27,10 +27,11 @@ module.exports = function ensureSettingsFiles(knownSettings) {
 
         return fs.readFile(filePath, 'utf8')
             .catch({code: 'ENOENT'}, () => {
+                const defaultFilePath = path.join(defaultSettingsPath, defaultFileName);
                 // CASE: file doesn't exist, copy it from our defaults
                 return fs.copy(
-                    path.join(defaultSettingsPath, defaultFileName),
-                    path.join(contentPath, fileName)
+                    defaultFilePath,
+                    filePath
                 ).then(() => {
                     debug(`'${defaultFileName}' copied to ${contentPath}.`);
                 });

core/frontend/services/settings/index.js

@@ -1,10 +1,24 @@
 const _ = require('lodash');
+const crypto = require('crypto');
 const debug = require('ghost-ignition').debug('frontend:services:settings:index');
 const SettingsLoader = require('./loader');
 const ensureSettingsFiles = require('./ensure-settings');
 
 const errors = require('@tryghost/errors');
 
+/**
+ * md5 hashes of default settings
+ */
+const defaultHashes = {
+    routes: '3d180d52c663d173a6be791ef411ed01'
+};
+
+const calculateHash = (data) => {
+    return crypto.createHash('md5')
+        .update(data, 'binary')
+        .digest('hex');
+};
+
 module.exports = {
     init: function () {
         const knownSettings = this.knownSettings();
@@ -76,5 +90,15 @@ module.exports = {
         });
 
         return settingsToReturn;
+    },
+
+    getDefaulHash: (setting) => {
+        return defaultHashes[setting];
+    },
+
+    getCurrentHash: async (setting) => {
+        const data = await SettingsLoader.loadSettings(setting);
+
+        return calculateHash(JSON.stringify(data));
     }
 };

core/frontend/services/settings/loader.js

@@ -7,18 +7,63 @@ const config = require('../../../shared/config');
 const yamlParser = require('./yaml-parser');
 const validate = require('./validate');
 
-/**
- * Reads the desired settings YAML file and passes the
- * file to the YAML parser which then returns a JSON object.
- * @param {String} setting the requested settings as defined in setting knownSettings
- * @returns {Object} settingsFile
- */
-module.exports = function loadSettings(setting) {
+const getSettingFilePath = (setting) => {
     // we only support the `yaml` file extension. `yml` will be ignored.
     const fileName = `${setting}.yaml`;
     const contentPath = config.getContentPath('settings');
     const filePath = path.join(contentPath, fileName);
 
+    return {
+        fileName,
+        contentPath,
+        filePath
+    };
+};
+
+/**
+ * Functionally same as loadSettingsSync with exception of loading
+ * settigs asyncronously. This method is used at new places to read settings
+ * to prevent blocking the eventloop
+ *
+ * @param {String} setting the requested settings as defined in setting knownSettings
+ * @returns {Object} settingsFile
+ */
+const loadSettings = async (setting) => {
+    const {fileName, contentPath, filePath} = getSettingFilePath(setting);
+
+    try {
+        const file = await fs.readFile(filePath, 'utf8');
+        debug('settings file found for', setting);
+
+        const object = yamlParser(file, fileName);
+        return validate(object);
+    } catch (err) {
+        if (errors.utils.isIgnitionError(err)) {
+            throw err;
+        }
+
+        throw new errors.GhostError({
+            message: i18n.t('errors.services.settings.loader', {
+                setting: setting,
+                path: contentPath
+            }),
+            context: filePath,
+            err: err
+        });
+    }
+};
+
+/**
+ * Reads the desired settings YAML file and passes the
+ * file to the YAML parser which then returns a JSON object.
+ * NOTE: loading happens syncronously
+ *
+ * @param {String} setting the requested settings as defined in setting knownSettings
+ * @returns {Object} settingsFile
+ */
+module.exports = function loadSettingsSync(setting) {
+    const {fileName, contentPath, filePath} = getSettingFilePath(setting);
+
     try {
         const file = fs.readFileSync(filePath, 'utf8');
         debug('settings file found for', setting);
@@ -31,9 +76,14 @@ module.exports = function loadSettings(setting) {
         }
 
         throw new errors.GhostError({
-            message: i18n.t('errors.services.settings.loader', {setting: setting, path: contentPath}),
+            message: i18n.t('errors.services.settings.loader', {
+                setting: setting,
+                path: contentPath
+            }),
             context: filePath,
             err: err
         });
     }
 };
+
+module.exports.loadSettings = loadSettings;

core/frontend/services/settings/validate.js

@@ -402,6 +402,7 @@ _private.validateTaxonomies = function validateTaxonomies(taxonomies) {
 
 /**
  * Validate and sanitize the routing object.
+ * NOTE: mutates the object even if it's a valid configuration
  */
 module.exports = function validate(object) {
     if (!object) {

core/server/api/canary/settings.js

@@ -2,9 +2,11 @@ const Promise = require('bluebird');
 const _ = require('lodash');
 const validator = require('validator');
 const models = require('../../models');
-const routing = require('../../../frontend/services/routing');
+const frontendRouting = require('../../../frontend/services/routing');
+const frontendSettings = require('../../../frontend/services/settings');
 const {i18n} = require('../../lib/common');
 const {BadRequestError, NoPermissionError, NotFoundError} = require('@tryghost/errors');
+const settingsService = require('../../services/settings');
 const settingsCache = require('../../services/settings/cache');
 const membersService = require('../../services/members');
 
@@ -291,8 +293,10 @@ module.exports = {
         permissions: {
             method: 'edit'
         },
-        query(frame) {
-            return routing.settings.setFromFilePath(frame.file.path);
+        async query(frame) {
+            await frontendRouting.settings.setFromFilePath(frame.file.path);
+            const getRoutesHash = () => frontendSettings.getCurrentHash('routes');
+            await settingsService.syncRoutesHash(getRoutesHash);
         }
     },
 
@@ -310,7 +314,7 @@ module.exports = {
             method: 'browse'
         },
         query() {
-            return routing.settings.get();
+            return frontendRouting.settings.get();
         }
     }
 };

core/server/api/v2/settings.js

@@ -1,9 +1,11 @@
 const Promise = require('bluebird');
 const _ = require('lodash');
 const models = require('../../models');
-const routing = require('../../../frontend/services/routing');
+const frontendRouting = require('../../../frontend/services/routing');
+const frontendSettings = require('../../../frontend/services/settings');
 const {i18n} = require('../../lib/common');
 const {NoPermissionError, NotFoundError} = require('@tryghost/errors');
+const settingsService = require('../../services/settings');
 const settingsCache = require('../../services/settings/cache');
 
 module.exports = {
@@ -143,8 +145,10 @@ module.exports = {
         permissions: {
             method: 'edit'
         },
-        query(frame) {
-            return routing.settings.setFromFilePath(frame.file.path);
+        async query(frame) {
+            await frontendRouting.settings.setFromFilePath(frame.file.path);
+            const getRoutesHash = () => frontendSettings.getCurrentHash('routes');
+            await settingsService.syncRoutesHash(getRoutesHash);
         }
     },
 
@@ -162,7 +166,7 @@ module.exports = {
             method: 'browse'
         },
         query() {
-            return routing.settings.get();
+            return frontendRouting.settings.get();
         }
     }
 };

core/server/index.js

@@ -21,6 +21,7 @@ let parentApp;
 // Frontend Components
 const themeService = require('../frontend/services/themes');
 const appService = require('../frontend/services/apps');
+const frontendSettings = require('../frontend/services/settings');
 
 function initialiseServices() {
     // CASE: When Ghost is ready with bootstrapping (db migrations etc.), we can trigger the router creation.
@@ -31,6 +32,7 @@ function initialiseServices() {
     // We pass the themeService API version here, so that the frontend services are less tightly-coupled
     routing.bootstrap.start(themeService.getApiVersion());
 
+    const settings = require('./services/settings');
     const permissions = require('./services/permissions');
     const xmlrpc = require('./services/xmlrpc');
     const slack = require('./services/slack');
@@ -39,6 +41,7 @@ function initialiseServices() {
     const scheduling = require('./adapters/scheduling');
 
     debug('`initialiseServices` Start...');
+    const getRoutesHash = () => frontendSettings.getCurrentHash('routes');
 
     return Promise.join(
         // Initialize the permissions actions and objects
@@ -47,6 +50,7 @@ function initialiseServices() {
         slack.listen(),
         mega.listen(),
         webhooks.listen(),
+        settings.syncRoutesHash(getRoutesHash),
         appService.init(),
         scheduling.init({
             // NOTE: When changing API version need to consider how to migrate custom scheduling adapters
@@ -80,9 +84,6 @@ const minimalRequiredSetupToStartGhost = (dbState) => {
     const models = require('./models');
     const GhostServer = require('./ghost-server');
 
-    // Frontend
-    const frontendSettings = require('../frontend/services/settings');
-
     let ghostServer;
 
     // Initialize Ghost core internationalization

core/server/services/settings/index.js

@@ -18,5 +18,23 @@ module.exports = {
         for (const model of settingsCollection.models) {
             model.emitChange(model.attributes.key + '.' + 'edited', {});
         }
+    },
+
+    /**
+     * Handles syncronization of routes.yaml hash loaded in the frontend with
+     * the value stored in the settings table.
+     * getRoutesHash is a function to allow keeping "frontend" decoupled from settings
+     *
+     * @param {function} getRoutesHash function fetching currently loaded routes file hash
+     */
+    async syncRoutesHash(getRoutesHash) {
+        const currentRoutesHash = await getRoutesHash();
+
+        if (SettingsCache.get('routes_hash') !== currentRoutesHash) {
+            return await models.Settings.edit([{
+                key: 'routes_hash',
+                value: currentRoutesHash
+            }], {context: {internal: true}});
+        }
     }
 };

test/regression/site/dynamic_routing_spec.js

@@ -700,6 +700,8 @@ describe('Dynamic Routing', function () {
                 return testUtils.initData();
             }).then(function () {
                 return testUtils.fixtures.overrideOwnerUser('ghost-owner');
+            }).then(function () {
+                return testUtils.initFixtures('settings');
             }).then(function () {
                 done();
             }).catch(done);

test/unit/data/schema/integrity_spec.js

@@ -1,14 +1,25 @@
 const should = require('should');
 const _ = require('lodash');
+const yaml = require('js-yaml');
 const crypto = require('crypto');
+const fs = require('fs-extra');
+const path = require('path');
+const {config} = require('../../../utils/configUtils');
 const schema = require('../../../../core/server/data/schema');
 const fixtures = require('../../../../core/server/data/schema/fixtures');
+const frontendSettings = require('../../../../core/frontend/services/settings');
+const validateFrontendSettings = require('../../../../core/frontend/services/settings/validate');
 const defaultSettings = require('../../../../core/server/data/schema/default-settings');
 
 /**
  * @NOTE
  *
- * If this test fails for you, you have modified the database schema or fixtures or default settings.
+ * If this test fails for you, you have modified one of:
+ * - the database schema
+ * - fixtures
+ * - default settings
+ * - routes.yaml
+ *
  * When you make a change, please test that:
  *
  * 1. A new blog get's installed and the database looks correct and complete.
@@ -24,14 +35,19 @@ describe('DB version integrity', function () {
     const currentSchemaHash = '42a966364eb4b5851e807133374821da';
     const currentFixturesHash = '29148c40dfaf4f828c5fca95666f6545';
     const currentSettingsHash = 'c8daa2c9632bb75f9d60655de09ae3bd';
+    const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';
 
     // If this test is failing, then it is likely a change has been made that requires a DB version bump,
     // and the values above will need updating as confirmation
     it('should not change without fixing this test', function () {
+        const routesPath = path.join(config.getContentPath('settings'), 'routes.yaml');
+        const defaultRoutes = validateFrontendSettings(yaml.safeLoad(fs.readFileSync(routesPath, 'utf-8')));
+
         const tablesNoValidation = _.cloneDeep(schema.tables);
         let schemaHash;
         let fixturesHash;
         let settingsHash;
+        let routesHash;
 
         _.each(tablesNoValidation, function (table) {
             return _.each(table, function (column, name) {
@@ -42,9 +58,12 @@ describe('DB version integrity', function () {
         schemaHash = crypto.createHash('md5').update(JSON.stringify(tablesNoValidation), 'binary').digest('hex');
         fixturesHash = crypto.createHash('md5').update(JSON.stringify(fixtures), 'binary').digest('hex');
         settingsHash = crypto.createHash('md5').update(JSON.stringify(defaultSettings), 'binary').digest('hex');
+        routesHash = crypto.createHash('md5').update(JSON.stringify(defaultRoutes), 'binary').digest('hex');
 
         schemaHash.should.eql(currentSchemaHash);
         fixturesHash.should.eql(currentFixturesHash);
         settingsHash.should.eql(currentSettingsHash);
+        routesHash.should.eql(currentRoutesHash);
+        routesHash.should.eql(frontendSettings.getDefaulHash('routes'));
     });
 });