YAML settings loader and parser

closes #9528

These code changes introduce a YAML parser which will load and parse YAML files from the `/content/settings` directory. There are three major parts involved:

1. `ensure-settings.js`: this fn takes care that on bootstrap, the supported files are present in the `/content/settings` directory. If the files are not present, they get copied back from our default files. The default files to copy from are located in `core/server/services/settings`.

2. `loader.js`: the settings loader reads the requested `yaml` file from the disk and passes it to the yaml parser, which returns a `json` object of the file. The settings loader throws an error, if the file is not accessible, e. g. because of permission errors.

3. `yaml-parser`: gets passed a `yaml` file and returns a `json` object. If the file is not parseable, it returns a clear error that contains the information, what and where the parsing error occurred (e. g. line number and reason).

- added a `get()` fn to settings services, that returns the settings object that's asked for. e. g. `settings.get('routes').then(()...` will return the `routes` settings.
- added a `getAll()` fn to settings services, that returns all available settings in an object. The object looks like: `{routes: {routes: {}, collections: {}, resources: {}}, globals: {value: {}}`, assuming that we have to supported settings `routes` and `globals`.

Further additions:
- config `contentPath` for `settings`
- config overrides for default `yaml` files location in `/core/server/services/settings`

**Important**: These code changes are in preparation for Dynamic Routing and not yet used. The process of copying the supported `yaml` files (in this first step, the `routes.yaml` file) is not yet activated.
This commit is contained in:
Aileen Nowak 2018-04-13 09:34:03 +08:00 committed by Katharina Irrgang
parent c8b29724e0
commit 63642fd8ad
18 changed files with 622 additions and 2 deletions

View File

@ -6,6 +6,7 @@
"helperTemplates": "core/server/helpers/tpl/",
"adminViews": "core/server/web/admin/views/",
"defaultViews": "core/server/views/",
"defaultSettings": "core/server/services/settings/",
"internalAppPath": "core/server/apps/",
"internalStoragePath": "core/server/adapters/storage/",
"internalSchedulingPath": "core/server/adapters/scheduling/",

View File

@ -63,6 +63,8 @@ exports.getContentPath = function getContentPath(type) {
return path.join(this.get('paths:contentPath'), 'logs/');
case 'data':
return path.join(this.get('paths:contentPath'), 'data/');
case 'settings':
return path.join(this.get('paths:contentPath'), 'settings/');
default:
throw new Error('getContentPath was called with: ' + type);
}

View File

@ -0,0 +1,12 @@
routes:
collections:
/:
route: '{globals.permalinks}'
template:
- home
- index
resources:
tag: /tag/{slug}/
author: /author/{slug}/

View File

@ -0,0 +1,47 @@
'use strict';
const fs = require('fs-extra'),
Promise = require('bluebird'),
path = require('path'),
debug = require('ghost-ignition').debug('services:settings:ensure-settings'),
common = require('../../lib/common'),
config = require('../../config');
/**
* Makes sure that all supported settings files are in the
* `/content/settings` directory. If not, copy the default files
* over.
* @param {Array} knownSettings
* @returns {Promise}
* @description Reads the `/settings` folder of the content path and makes
* sure that the associated yaml file for each setting exists. If it doesn't
* copy the default yaml file over.
*/
module.exports = function ensureSettingsFiles(knownSettings) {
const contentPath = config.getContentPath('settings'),
defaultSettingsPath = config.get('paths').defaultSettings;
return Promise.each(knownSettings, function (setting) {
const fileName = `${setting}.yaml`,
defaultFileName = `default-${fileName}`,
filePath = path.join(contentPath, fileName);
return fs.access(filePath)
.catch({code: 'ENOENT'}, () => {
// CASE: file doesn't exist, copy it from our defaults
return fs.copy(
path.join(defaultSettingsPath, defaultFileName),
path.join(contentPath, fileName)
).then(() => {
debug(`'${defaultFileName}' copied to ${contentPath}.`);
});
}).catch((error) => {
// CASE: we might have a permission error, as we can't access the directory
throw new common.errors.GhostError({
message: common.i18n.t('errors.services.settings.ensureSettings', {path: contentPath}),
err: error,
context: error.path
});
});
});
};

View File

@ -4,10 +4,28 @@
*/
var SettingsModel = require('../../models/settings').Settings,
SettingsCache = require('./cache');
SettingsCache = require('./cache'),
SettingsLoader = require('./loader'),
// EnsureSettingsFiles = require('./ensure-settings'),
_ = require('lodash'),
common = require('../../lib/common'),
debug = require('ghost-ignition').debug('services:settings:index');
module.exports = {
init: function init() {
const knownSettings = this.knownSettings();
debug('init settings service for:', knownSettings);
// TODO: uncomment this section, once we want to
// copy the default routes.yaml file into the /content/settings
// folder
// Make sure that supported settings files are available
// inside of the `content/setting` directory
// return EnsureSettingsFiles(knownSettings)
// .then(() => {
// Update the defaults
return SettingsModel.populateDefaults()
.then(function (settingsCollection) {
@ -15,5 +33,78 @@ module.exports = {
// This will bind to events for further updates
SettingsCache.init(settingsCollection);
});
// });
},
/**
* Global place to switch on more available settings.
*/
knownSettings: function knownSettings() {
return ['routes'];
},
/**
* Getter for YAML settings.
* Example: `settings.get('routes').then(...)`
* will return an Object like this:
* {routes: {}, collections: {}, resources: {}}
* @param {String} setting type of supported setting.
* @returns {Promise<Object>} settingsFile
* @description Returns settings object as defined per YAML files in
* `/content/settings` directory.
*/
get: function get(setting) {
const knownSettings = this.knownSettings();
// CASE: this should be an edge case and only if internal usage of the
// getter is incorrect.
if (!setting || _.indexOf(knownSettings, setting) < 0) {
return Promise.reject(new common.errors.IncorrectUsageError({
message: `Requested setting is not supported: '${setting}'.`,
help: `Please use only the supported settings: ${knownSettings}.`
}));
}
return SettingsLoader(setting)
.then((settingsFile) => {
debug('setting loaded and parsed:', settingsFile);
return settingsFile;
});
},
/**
* Getter for all YAML settings.
* Example: `settings.getAll().then(...)`
* will return an Object like this (assuming we're supporting `routes`
* and `globals`):
* {
* routes: {
* routes: null,
* collections: { '/': [Object] },
* resources: { tag: '/tag/{slug}/', author: '/author/{slug}/' }
* },
* globals: {
* config: { url: 'testblog.com' }
* }
* }
* @returns {Promise<Object>} settingsObject
* @description Returns all settings object as defined per YAML files in
* `/content/settings` directory.
*/
getAll: function getAll() {
const knownSettings = this.knownSettings(),
props = {};
_.each(knownSettings, function (setting) {
props[setting] = SettingsLoader(setting);
});
return Promise.props(props)
.then((settingsFile) => {
debug('all settings loaded and parsed:', settingsFile);
return settingsFile;
});
}
};

View File

@ -0,0 +1,41 @@
'use strict';
const fs = require('fs-extra'),
path = require('path'),
debug = require('ghost-ignition').debug('services:settings:settings-loader'),
common = require('../../lib/common'),
config = require('../../config'),
yamlParser = require('./yaml-parser');
/**
* Reads the desired settings YAML file and passes the
* file to the YAML parser which then returns a JSON object.
* @param {String} setting the requested settings as defined in setting knownSettings
* @returns {Promise<Object>} settingsFile
*/
module.exports = function loadSettings(setting) {
// we only support the `yaml` file extension. `yml` will be ignored.
const fileName = `${setting}.yaml`;
const contentPath = config.getContentPath('settings');
const filePath = path.join(contentPath, fileName);
return fs.readFile(filePath, 'utf8')
.then((file) => {
debug('settings file found for', setting);
// yamlParser returns a JSON object
const parsed = yamlParser(file, fileName);
return parsed;
}).catch((error) => {
if (common.errors.utils.isIgnitionError(error)) {
throw error;
}
throw new common.errors.GhostError({
message: common.i18n.t('errors.services.settings.loader', {setting: setting, path: contentPath}),
context: filePath,
err: error
});
});
};

View File

@ -0,0 +1,30 @@
'use strict';
const yaml = require('js-yaml'),
debug = require('ghost-ignition').debug('services:settings:yaml-parser'),
common = require('../../lib/common');
/**
* Takes a YAML file, parses it and returns a JSON Object
* @param {YAML} file the YAML file utf8 encoded
* @param {String} fileName the name of the file incl. extension
* @returns {Object} parsed
*/
module.exports = function parseYaml(file, fileName) {
try {
const parsed = yaml.safeLoad(file);
debug('YAML settings file parsed:', fileName);
return parsed;
} catch (error) {
// CASE: parsing failed, `js-yaml` tells us exactly what and where in the
// `reason` property as well as in the message.
throw new common.errors.GhostError({
message: common.i18n.t('errors.services.settings.yaml.error', {file: fileName, context: error.reason}),
context: error.message,
err: error,
help: common.i18n.t('errors.services.settings.yaml.help', {file: fileName})
});
}
};

View File

@ -467,6 +467,14 @@
"error": "The {service} service was unable to send a ping request, your blog will continue to function.",
"help": "If you get this error repeatedly, please seek help on {url}."
}
},
"settings": {
"yaml": {
"error": "Could not parse {file}: {context}.",
"help": "Check your {file} file for typos and fix the named issues."
},
"loader": "Error trying to load YAML setting for {setting} from '{path}'.",
"ensureSettings": "Error trying to access settings files in {path}."
}
},
"errors": {

View File

@ -0,0 +1,83 @@
'use strict';
const sinon = require('sinon'),
should = require('should'),
fs = require('fs-extra'),
yaml = require('js-yaml'),
path = require('path'),
configUtils = require('../../../utils/configUtils'),
common = require('../../../../server/lib/common'),
ensureSettings = require('../../../../server/services/settings/ensure-settings'),
sandbox = sinon.sandbox.create();
describe('UNIT > Settings Service:', function () {
beforeEach(function () {
configUtils.set('paths:contentPath', path.join(__dirname, '../../../utils/fixtures/'));
});
afterEach(function () {
sandbox.restore();
configUtils.restore();
});
describe('Ensure settings files', function () {
it('returns yaml file from settings folder if it exists', function () {
const fsAccessSpy = sandbox.spy(fs, 'access');
return ensureSettings(['goodroutes', 'badroutes']).then(() => {
fsAccessSpy.callCount.should.be.eql(2);
});
});
it('copies default settings file if not found but does not overwrite existing files', function () {
const expectedDefaultSettingsPath = path.join(__dirname, '../../../../server/services/settings/default-globals.yaml');
const expectedContentPath = path.join(__dirname, '../../../utils/fixtures/settings/globals.yaml');
const fsError = new Error('not found');
fsError.code = 'ENOENT';
const fsAccessStub = sandbox.stub(fs, 'access');
const fsCopyStub = sandbox.stub(fs, 'copy').resolves();
fsAccessStub.onFirstCall().resolves();
// route file in settings directotry is not found
fsAccessStub.onSecondCall().rejects(fsError);
return ensureSettings(['routes', 'globals'])
.then(() => {
fsAccessStub.calledTwice.should.be.true();
}).then(() => {
fsCopyStub.calledWith(expectedDefaultSettingsPath, expectedContentPath).should.be.true();
fsCopyStub.calledOnce.should.be.true();
});
});
it('copies default settings file if no file found', function () {
const expectedDefaultSettingsPath = path.join(__dirname, '../../../../server/services/settings/default-routes.yaml');
const expectedContentPath = path.join(__dirname, '../../../utils/fixtures/settings/routes.yaml');
const fsError = new Error('not found');
fsError.code = 'ENOENT';
const fsAccessStub = sandbox.stub(fs, 'access').rejects(fsError);
const fsCopyStub = sandbox.stub(fs, 'copy').resolves();
return ensureSettings(['routes']).then(() => {
fsAccessStub.calledOnce.should.be.true();
fsCopyStub.calledWith(expectedDefaultSettingsPath, expectedContentPath).should.be.true();
fsCopyStub.calledOnce.should.be.true();
});
});
it('rejects, if error is not a not found error', function () {
const expectedContentPath = path.join(__dirname, '../../../utils/fixtures/settings/');
const fsError = new Error('no permission');
fsError.code = 'EPERM';
const fsAccessStub = sandbox.stub(fs, 'access').rejects(new Error('Oopsi!'));
return ensureSettings(['routes']).catch((error) => {
should.exist(error);
error.message.should.be.eql(`Error trying to access settings files in ${expectedContentPath}.`);
fsAccessStub.calledOnce.should.be.true();
});
});
});
});

View File

@ -0,0 +1,94 @@
'use strict';
const sinon = require('sinon'),
should = require('should'),
rewire = require('rewire'),
fs = require('fs-extra'),
yaml = require('js-yaml'),
path = require('path'),
configUtils = require('../../../utils/configUtils'),
common = require('../../../../server/lib/common'),
loadSettings = rewire('../../../../server/services/settings/loader'),
sandbox = sinon.sandbox.create();
describe('UNIT > Settings Service:', function () {
beforeEach(function () {
configUtils.set('paths:contentPath', path.join(__dirname, '../../../utils/fixtures/'));
});
afterEach(function () {
sandbox.restore();
configUtils.restore();
});
describe('Settings Loader', function () {
const yamlStubFile = {
routes: null,
collections: {
'/': {
route: '{globals.permalinks}',
template: ['home', 'index']
}
},
resources: {tag: '/tag/{slug}/', author: '/author/{slug}/'}
};
let yamlParserStub;
beforeEach(function () {
yamlParserStub = sinon.stub();
});
it('can find yaml settings file and returns a settings object', function () {
const fsReadFileSpy = sandbox.spy(fs, 'readFile');
const expectedSettingsFile = path.join(__dirname, '../../../utils/fixtures/settings/goodroutes.yaml');
yamlParserStub.returns(yamlStubFile);
loadSettings.__set__('yamlParser', yamlParserStub);
return loadSettings('goodroutes').then((setting) => {
should.exist(setting);
setting.should.be.an.Object().with.properties('routes', 'collections', 'resources');
// There are 4 files in the fixtures folder, but only 1 supported and valid yaml files
fsReadFileSpy.calledOnce.should.be.true();
fsReadFileSpy.calledWith(expectedSettingsFile).should.be.true();
yamlParserStub.callCount.should.be.eql(1);
});
});
it('can handle errors from YAML parser', function () {
yamlParserStub.rejects(new common.errors.GhostError({
message: 'could not parse yaml file',
context: 'bad indentation of a mapping entry at line 5, column 10'
}));
loadSettings.__set__('yamlParser', yamlParserStub);
return loadSettings('goodroutes').then((setting) => {
should.not.exist(setting);
}).catch((error) => {
should.exist(error);
error.message.should.be.eql('could not parse yaml file');
error.context.should.be.eql('bad indentation of a mapping entry at line 5, column 10');
yamlParserStub.calledOnce.should.be.true();
});
});
it('throws error if file can\'t be accessed', function () {
const expectedSettingsFile = path.join(__dirname, '../../../utils/fixtures/settings/routes.yaml');
const fsError = new Error('no permission');
fsError.code = 'EPERM';
const fsReadFileStub = sandbox.stub(fs, 'readFile').rejects(fsError);
yamlParserStub = sinon.spy();
loadSettings.__set__('yamlParser', yamlParserStub);
return loadSettings('routes').then((settings) => {
should.not.exist(settings);
}).catch((error) => {
should.exist(error);
error.message.should.match(/Error trying to load YAML setting for routes from/);
fsReadFileStub.calledWith(expectedSettingsFile).should.be.true();
yamlParserStub.calledOnce.should.be.false();
});
});
});
});

View File

@ -0,0 +1,137 @@
'use strict';
const sinon = require('sinon'),
should = require('should'),
rewire = require('rewire'),
common = require('../../../../server/lib/common'),
settings = rewire('../../../../server/services/settings/index'),
sandbox = sinon.sandbox.create();
describe('UNIT > Settings Service:', function () {
afterEach(function () {
sandbox.restore();
});
describe('knownSettings', function () {
it('returns supported settings files', function () {
const files = settings.knownSettings();
// This test will fail when new settings are added without
// changing this test as well.
files.should.be.an.Array().with.length(1);
});
});
describe('get', function () {
let settingsLoaderStub;
const settingsStubFile = {
routes: null,
collections: {
'/': {
route: '{globals.permalinks}',
template: [ 'home', 'index' ]
}
},
resources: {tag: '/tag/{slug}/', author: '/author/{slug}/'}
};
beforeEach(function () {
settingsLoaderStub = sandbox.stub();
});
it('returns settings object for `routes`', function () {
settingsLoaderStub.resolves(settingsStubFile);
settings.__set__('SettingsLoader', settingsLoaderStub);
settings.get('routes').then((result) => {
should.exist(result);
result.should.be.an.Object().with.properties('routes', 'collections', 'resources');
settingsLoaderStub.calledOnce.should.be.true();
});
});
it('rejects when requested settings type is not supported', function () {
settingsLoaderStub.resolves(settingsStubFile);
settings.__set__('SettingsLoader', settingsLoaderStub);
return settings.get('something').then((result) => {
should.not.exist(result);
}).catch((error) => {
should.exist(error);
error.message.should.be.eql('Requested setting is not supported: \'something\'.');
settingsLoaderStub.callCount.should.be.eql(0);
});
});
it('passes SettingsLoader error through', function () {
settingsLoaderStub.rejects(new common.errors.GhostError({message: 'oops'}));
settings.__set__('SettingsLoader', settingsLoaderStub);
return settings.get('routes').then((result) => {
should.not.exist(result);
}).catch((error) => {
should.exist(error);
error.message.should.be.eql('oops');
settingsLoaderStub.calledOnce.should.be.true();
});
});
});
describe('getAll', function () {
let settingsLoaderStub,
knownSettingsStub;
const settingsStubFile1 = {
routes: null,
collections: {
'/': {
route: '{globals.permalinks}',
template: [ 'home', 'index' ]
}
},
resources: {tag: '/tag/{slug}/', author: '/author/{slug}/'}
},
settingsStubFile2 = {
config: {
url: 'https://testblog.com'
}
};
beforeEach(function () {
knownSettingsStub = sandbox.stub().returns(['routes', 'globals']);
settings.__set__('this.knownSettings', knownSettingsStub);
settingsLoaderStub = sandbox.stub();
});
it('returns settings object for all known settings', function () {
settingsLoaderStub.onFirstCall().resolves(settingsStubFile1);
settingsLoaderStub.onSecondCall().resolves(settingsStubFile2);
settings.__set__('SettingsLoader', settingsLoaderStub);
return settings.getAll().then((result) => {
should.exist(result);
result.should.be.an.Object().with.properties('routes', 'globals');
result.routes.should.be.an.Object().with.properties('routes', 'collections', 'resources');
result.globals.should.be.an.Object().with.properties('config');
settingsLoaderStub.calledTwice.should.be.true();
});
});
it('passes SettinsLoader error through', function () {
settingsLoaderStub.onFirstCall().resolves(settingsStubFile1);
settingsLoaderStub.onSecondCall().rejects(new common.errors.GhostError({message: 'oops'}));
settings.__set__('SettingsLoader', settingsLoaderStub);
return settings.getAll().then((result) => {
should.not.exist(result);
}).catch((error) => {
should.exist(error);
error.message.should.be.eql('oops');
settingsLoaderStub.calledTwice.should.be.true();
});
});
});
});

View File

@ -0,0 +1,49 @@
'use strict';
const sinon = require('sinon'),
should = require('should'),
fs = require('fs-extra'),
yaml = require('js-yaml'),
path = require('path'),
yamlParser = require('../../../../server/services/settings/yaml-parser'),
sandbox = sinon.sandbox.create();
describe('UNIT > Settings Service:', function () {
let yamlSpy;
beforeEach(function () {
yamlSpy = sandbox.spy(yaml, 'safeLoad');
});
afterEach(function () {
sandbox.restore();
});
describe('Yaml Parser', function () {
it('parses correct yaml file', function () {
const file = fs.readFileSync(path.join(__dirname, '../../../utils/fixtures/settings/', 'goodroutes.yaml'), 'utf8');
const result = yamlParser(file, 'goodroutes.yaml');
should.exist(result);
result.should.be.an.Object().with.properties('routes', 'collections', 'resources');
yamlSpy.calledOnce.should.be.true();
});
it('rejects with clear error when parsing fails', function () {
const file = fs.readFileSync(path.join(__dirname, '../../../utils/fixtures/settings/', 'badroutes.yaml'), 'utf8');
try {
const result = yamlParser(file, 'badroutes.yaml');
should.not.exist(result);
} catch (error) {
should.exist(error);
error.message.should.eql('Could not parse badroutes.yaml: bad indentation of a mapping entry.');
error.context.should.eql('bad indentation of a mapping entry at line 5, column 10:\n route: \'{globals.permalinks}\'\n ^');
error.help.should.eql('Check your badroutes.yaml file for typos and fix the named issues.');
yamlSpy.calledOnce.should.be.true();
}
});
});
});

View File

@ -0,0 +1,12 @@
routes:
collections:
/
route: '{globals.permalinks}'
template:
- home
- index
resources:
tag: /tag/{slug}/
author: /author/{slug}/

View File

@ -0,0 +1,12 @@
routes:
collections:
/:
route: '{globals.permalinks}'
template:
- home
- index
resources:
tag: /tag/{slug}/
author: /author/{slug}/

View File

@ -62,6 +62,7 @@
"image-size": "0.6.2",
"intl": "1.2.5",
"intl-messageformat": "1.3.0",
"js-yaml": "3.11.0",
"jsonpath": "1.0.0",
"knex": "0.14.4",
"knex-migrator": "3.1.5",

View File

@ -3198,7 +3198,7 @@ js-tokens@^3.0.0, js-tokens@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
js-yaml@3.x, js-yaml@^3.9.1:
js-yaml@3.11.0, js-yaml@3.x, js-yaml@^3.9.1:
version "3.11.0"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.11.0.tgz#597c1a8bd57152f26d622ce4117851a51f5ebaef"
dependencies: