Ghost/core/server/api/utils.js

338 lines
12 KiB
JavaScript
Raw Normal View History

// # API Utils
// Shared helpers for working with the API
var Promise = require('bluebird'),
_ = require('lodash'),
path = require('path'),
permissions = require('../permissions'),
validation = require('../data/validation'),
errors = require('../errors'),
i18n = require('../i18n'),
utils;
utils = {
// ## Default Options
// Various default options for different types of endpoints
// ### Auto Default Options
// Handled / Added automatically by the validate function
// globalDefaultOptions - valid for every api endpoint
globalDefaultOptions: ['context', 'include'],
// dataDefaultOptions - valid for all endpoints which take object as well as options
dataDefaultOptions: ['data'],
// ### Manual Default Options
// These must be provided by the endpoint
// browseDefaultOptions - valid for all browse api endpoints
browseDefaultOptions: ['page', 'limit', 'fields', 'filter', 'order', 'debug'],
// idDefaultOptions - valid whenever an id is valid
idDefaultOptions: ['id'],
/**
* ## Validate
* Prepare to validate the object and options passed to an endpoint
* @param {String} docName
* @param {Object} extras
* @returns {Function} doValidate
*/
validate: function validate(docName, extras) {
/**
* ### Do Validate
* Validate the object and options passed to an endpoint
* @argument {...*} [arguments] object or object and options hash
*/
return function doValidate() {
var object, options, permittedOptions;
if (arguments.length === 2) {
object = arguments[0];
options = _.clone(arguments[1]) || {};
} else if (arguments.length === 1) {
options = _.clone(arguments[0]) || {};
} else {
options = {};
}
// Setup permitted options, starting with the global defaults
permittedOptions = utils.globalDefaultOptions;
// Add extra permitted options if any are passed in
if (extras && extras.opts) {
permittedOptions = permittedOptions.concat(extras.opts);
}
// This request will have a data key added during validation
if ((extras && extras.attrs) || object) {
permittedOptions = permittedOptions.concat(utils.dataDefaultOptions);
}
// If an 'attrs' object is passed, we use this to pick from options and convert them to data
if (extras && extras.attrs) {
options.data = _.pick(options, extras.attrs);
options = _.omit(options, extras.attrs);
}
/**
* ### Check Options
* Ensure that the options provided match exactly with what is permitted
* - incorrect option keys are sanitized
* - incorrect option values are validated
* @param {object} options
* @returns {Promise<options>}
*/
function checkOptions(options) {
// @TODO: should we throw an error if there are incorrect options provided?
options = _.pick(options, permittedOptions);
var validationErrors = utils.validateOptions(options);
if (_.isEmpty(validationErrors)) {
return Promise.resolve(options);
}
// For now, we can only handle showing the first validation error
🎨 configurable logging with bunyan (#7431) - 🛠 add bunyan and prettyjson, remove morgan - ✨ add logging module - GhostLogger class that handles setup of bunyan - PrettyStream for stdout - ✨ config for logging - @TODO: testing level fatal? - ✨ log each request via GhostLogger (express middleware) - @TODO: add errors to output - 🔥 remove errors.updateActiveTheme - we can read the value from config - 🔥 remove 15 helper functions in core/server/errors/index.js - all these functions get replaced by modules: 1. logging 2. error middleware handling for html/json 3. error creation (which will be part of PR #7477) - ✨ add express error handler for html/json - one true error handler for express responses - contains still some TODO's, but they are not high priority for first implementation/integration - this middleware only takes responsibility of either rendering html responses or return json error responses - 🎨 use new express error handler in middleware/index - 404 and 500 handling - 🎨 return error instead of error message in permissions/index.js - the rule for error handling should be: if you call a unit, this unit should return a custom Ghost error - 🎨 wrap serve static module - rule: if you call a module/unit, you should always wrap this error - it's always the same rule - so the caller never has to worry about what comes back - it's always a clear error instance - in this case: we return our notfounderror if serve static does not find the resource - this avoid having checks everywhere - 🎨 replace usages of errors/index.js functions and adapt tests - use logging.error, logging.warn - make tests green - remove some usages of logging and throwing api errors -> because when a request is involved, logging happens automatically - 🐛 return errorDetails to Ghost-Admin - errorDetails is used for Theme error handling - 🎨 use 500er error for theme is missing error in theme-handler - 🎨 extend file rotation to 1w
2016-10-04 18:33:43 +03:00
return Promise.reject(validationErrors[0]);
}
// If we got an object, check that too
if (object) {
return utils.checkObject(object, docName, options.id).then(function (data) {
options.data = data;
return checkOptions(options);
});
}
// Otherwise just check options and return
return checkOptions(options);
};
},
validateOptions: function validateOptions(options) {
var globalValidations = {
✨ replace auto increment id's by object id (#7495) * 🛠 bookshelf tarball, bson-objectid * 🎨 schema changes - change increment type to string - add a default fallback for string length 191 (to avoid adding this logic to every single column which uses an ID) - remove uuid, because ID now represents a global resource identifier - keep uuid for post, because we are using this as preview id - keep uuid for clients for now - we are using this param for Ghost-Auth * ✨ base model: generate ObjectId on creating event - each new resource get's a auto generate ObjectId - this logic won't work for attached models, this commit comes later * 🎨 centralised attach method When attaching models there are two things important two know 1. To be able to attach an ObjectId, we need to register the `onCreating` event the fetched model!This is caused by the Bookshelf design in general. On this target model we are attaching the new model. 2. We need to manually fetch the target model, because Bookshelf has a weird behaviour (which is known as a bug, see see https://github.com/tgriesser/bookshelf/issues/629). The most important property when attaching a model is `parentFk`, which is the foreign key. This can be null when fetching the model with the option `withRelated`. To ensure quality and consistency, the custom attach wrapper always fetches the target model manual. By fetching the target model (again) is a little performance decrease, but it also has advantages: we can register the event, and directly unregister the event again. So very clean code. Important: please only use the custom attach wrapper in the future. * 🎨 token model had overriden the onCreating function because of the created_at field - we need to ensure that the base onCreating hook get's triggered for ALL models - if not, they don't get an ObjectId assigned - in this case: be smart and check if the target model has a created_at field * 🎨 we don't have a uuid field anymore, remove the usages - no default uuid creation in models - i am pretty sure we have some more definitions in our tests (for example in the export json files), but that is too much work to delete them all * 🎨 do not parse ID to Number - we had various occurances of parsing all ID's to numbers - we don't need this behaviour anymore - ID is string - i will adapt the ID validation in the next commit * 🎨 change ID regex for validation - we only allow: ID as ObjectId, ID as 1 and ID as me - we need to keep ID 1, because our whole software relies on ID 1 (permissions etc) * 🎨 owner fixture - roles: [4] does not work anymore - 4 means -> static id 4 - this worked in an auto increment system (not even in a system with distributed writes) - with ObjectId we generate each ID automatically (for static and dynamic resources) - it is possible to define all id's for static resources still, but that means we need to know which ID is already used and for consistency we have to define ObjectId's for these static resources - so no static id's anymore, except of: id 1 for owner and id 0 for external usage (because this is required from our permission system) - NOTE: please read through the comment in the user model * 🎨 tests: DataGenerator and test utils First of all: we need to ensure using ObjectId's in the tests. When don't, we can't ensure that ObjectId's work properly. This commit brings lot's of dynamic into all the static defined id's. In one of the next commits, i will adapt all the tests. * 🚨 remove counter in Notification API - no need to add a counter - we simply generate ObjectId's (they are auto incremental as well) - our id validator does only allow ObjectId as id,1 and me * 🎨 extend contextUser in Base Model - remove isNumber check, because id's are no longer numbers, except of id 0/1 - use existing isExternalUser - support id 0/1 as string or number * ✨ Ghost Owner has id 1 - ensure we define this id in the fixtures.json - doesn't matter if number or string * 🎨 functional tests adaptions - use dynamic id's * 🎨 fix unit tests * 🎨 integration tests adaptions * 🎨 change importer utils - all our export examples (test/fixtures/exports) contain id's as numbers - fact: but we ignore them anyway when inserting into the database, see https://github.com/TryGhost/Ghost/blob/master/core/server/data/import/utils.js#L249 - in https://github.com/TryGhost/Ghost/pull/7495/commits/0e6ed957cd54dc02a25cf6fb1ab7d7e723295e2c#diff-70f514a06347c048648be464819503c4L67 i removed parsing id's to integers - i realised that this ^ check just existed, because the userIdToMap was an object key and object keys are always strings! - i think this logic is a little bit complicated, but i don't want to refactor this now - this commit ensures when trying to find the user, the id comparison works again - i've added more documentation to understand this logic ;) - plus i renamed an attribute to improve readability * 🎨 Data-Generator: add more defaults to createUser - if i use the function DataGenerator.forKnex.createUser i would like to get a full set of defaults * 🎨 test utils: change/extend function set for functional tests - functional tests work a bit different - they boot Ghost and seed the database - some functional tests have mis-used the test setup - the test setup needs two sections: integration/unit and functional tests - any functional test is allowed to either add more data or change data in the existing Ghost db - but what it should not do is: add test fixtures like roles or users from our DataGenerator and cross fingers it will work - this commit adds a clean method for functional tests to add extra users * 🎨 functional tests adaptions - use last commit to insert users for functional tests clean - tidy up usage of testUtils.setup or testUtils.doAuth * 🐛 test utils: reset database before init - ensure we don't have any left data from other tests in the database when starting ghost * 🐛 fix test (unrelated to this PR) - fixes a random failure - return statement was missing * 🎨 make changes for invites
2016-11-17 12:09:11 +03:00
id: {matches: /^[a-f\d]{24}$|^1$|me/i},
uuid: {isUUID: true},
slug: {isSlug: true},
page: {matches: /^\d+$/},
limit: {matches: /^\d+|all$/},
from: {isDate: true},
to: {isDate: true},
fields: {matches: /^[\w, ]+$/},
order: {matches: /^[a-z0-9_,\. ]+$/i},
name: {}
},
// these values are sanitised/validated separately
noValidation = ['data', 'context', 'include', 'filter', 'forUpdate', 'transacting', 'formats'],
errors = [];
_.each(options, function (value, key) {
// data is validated elsewhere
if (noValidation.indexOf(key) === -1) {
if (globalValidations[key]) {
errors = errors.concat(validation.validate(value, key, globalValidations[key]));
} else {
// all other keys should be alpha-numeric with dashes/underscores, like tag, author, status, etc
errors = errors.concat(validation.validate(value, key, globalValidations.slug));
}
}
});
return errors;
},
/**
* ## Detect Public Context
* Calls parse context to expand the options.context object
* @param {Object} options
* @returns {Boolean}
*/
detectPublicContext: function detectPublicContext(options) {
options.context = permissions.parseContext(options.context);
return options.context.public;
},
/**
* ## Apply Public Permissions
* Update the options object so that the rules reflect what is permitted to be retrieved from a public request
* @param {String} docName
* @param {String} method (read || browse)
* @param {Object} options
* @returns {Object} options
*/
applyPublicPermissions: function applyPublicPermissions(docName, method, options) {
return permissions.applyPublicRules(docName, method, options);
},
/**
* ## Handle Public Permissions
* @param {String} docName
* @param {String} method (read || browse)
* @returns {Function}
*/
handlePublicPermissions: function handlePublicPermissions(docName, method) {
var singular = docName.replace(/s$/, '');
/**
* Check if this is a public request, if so use the public permissions, otherwise use standard canThis
* @param {Object} options
* @returns {Object} options
*/
return function doHandlePublicPermissions(options) {
var permsPromise;
if (utils.detectPublicContext(options)) {
permsPromise = utils.applyPublicPermissions(docName, method, options);
} else {
permsPromise = permissions.canThis(options.context)[method][singular](options.data);
}
return permsPromise.then(function permissionGranted() {
return options;
});
};
},
/**
* ## Handle Permissions
* @param {String} docName
* @param {String} method (browse || read || edit || add || destroy)
* * @param {Array} unsafeAttrNames - attribute names (e.g. post.status) that could change the outcome
* @returns {Function}
*/
handlePermissions: function handlePermissions(docName, method, unsafeAttrNames) {
var singular = docName.replace(/s$/, '');
/**
* ### Handle Permissions
* We need to be an authorised user to perform this action
* @param {Object} options
* @returns {Object} options
*/
return function doHandlePermissions(options) {
var unsafeAttrObject = unsafeAttrNames && _.has(options, 'data.[' + docName + '][0]') ? _.pick(options.data[docName][0], unsafeAttrNames) : {},
permsPromise = permissions.canThis(options.context)[method][singular](options.id, unsafeAttrObject);
return permsPromise.then(function permissionGranted() {
return options;
}).catch(function handleNoPermissionError(err) {
if (err instanceof errors.NoPermissionError) {
err.message = i18n.t('errors.api.utils.noPermissionToCall', {method: method, docName: docName});
return Promise.reject(err);
}
return Promise.reject(new errors.GhostError({
err: err
}));
});
};
},
trimAndLowerCase: function trimAndLowerCase(params) {
params = params || '';
if (_.isString(params)) {
params = params.split(',');
}
return _.map(params, function (item) {
return item.trim().toLowerCase();
});
},
prepareInclude: function prepareInclude(include, allowedIncludes) {
return _.intersection(this.trimAndLowerCase(include), allowedIncludes);
},
prepareFields: function prepareFields(fields) {
return this.trimAndLowerCase(fields);
},
prepareFormats: function prepareFormats(formats, allowedFormats) {
return _.intersection(this.trimAndLowerCase(formats), allowedFormats);
},
/**
* ## Convert Options
* @param {Array} allowedIncludes
* @returns {Function} doConversion
*/
convertOptions: function convertOptions(allowedIncludes, allowedFormats) {
/**
* Convert our options from API-style to Model-style
* @param {Object} options
* @returns {Object} options
*/
return function doConversion(options) {
if (options.include) {
options.include = utils.prepareInclude(options.include, allowedIncludes);
}
if (options.fields) {
options.columns = utils.prepareFields(options.fields);
delete options.fields;
}
✨ post update collision detection (#8328) (#8362) closes #5599 If two users edit the same post, it can happen that they override each others content or post settings. With this change this won't happen anymore. ✨ Update collision for posts - add a new bookshelf plugin to detect these changes - use the `changed` object of bookshelf -> we don't have to create our own diff - compare client and server updated_at field - run editing posts in a transaction (see comments in code base) 🙀 update collision for tags - `updateTags` for adding posts on `onCreated` - happens after the post was inserted --> it's "okay" to attach the tags afterwards on insert --> there is no need to add collision for inserting data --> it's very hard to move the updateTags call to `onCreating`, because the `updateTags` function queries the database to look up the affected post - `updateTags` while editing posts on `onSaving` - all operations run in a transactions and are rolled back if something get's rejected - Post model edit: if we push a transaction from outside, take this one ✨ introduce options.forUpdate - if two queries happening in a transaction we have to signalise knex/mysql that we select for an update - otherwise the following case happens: >> you fetch posts for an update >> a user requests comes in and updates the post (e.g. sets title to "X") >> you update the fetched posts, title would get overriden to the old one use options.forUpdate and protect internal post updates: model listeners - use a transaction for listener updates - signalise forUpdate - write a complex test use options.forUpdate and protect internal post updates: scheduling - publish endpoint runs in a transaction - add complex test - @TODO: right now scheduling api uses posts api, therefor we had to extend the options for api's >> allowed to pass transactions through it >> but these are only allowed if defined from outside {opts: [...]} >> so i think this is fine and not dirty >> will wait for opinions >> alternatively we have to re-write the scheduling endpoint to use the models directly
2017-04-19 16:53:23 +03:00
if (options.formats) {
options.formats = utils.prepareFormats(options.formats, allowedFormats);
}
if (options.formats && options.columns) {
options.columns = options.columns.concat(options.formats);
}
return options;
};
},
/**
* ### Check Object
* Check an object passed to the API is in the correct format
*
* @param {Object} object
* @param {String} docName
* @returns {Promise(Object)} resolves to the original object if it checks out
*/
checkObject: function checkObject(object, docName, editId) {
if (_.isEmpty(object) || _.isEmpty(object[docName]) || _.isEmpty(object[docName][0])) {
return Promise.reject(new errors.BadRequestError({
message: i18n.t('errors.api.utils.noRootKeyProvided', {docName: docName})
}));
}
// convert author property to author_id to match the name in the database
if (docName === 'posts') {
if (object.posts[0].hasOwnProperty('author')) {
object.posts[0].author_id = object.posts[0].author;
delete object.posts[0].author;
}
}
// will remove unwanted null values
_.each(object[docName], function (value, index) {
if (!_.isObject(object[docName][index])) {
return;
}
object[docName][index] = _.omitBy(object[docName][index], _.isNull);
});
✨ replace auto increment id's by object id (#7495) * 🛠 bookshelf tarball, bson-objectid * 🎨 schema changes - change increment type to string - add a default fallback for string length 191 (to avoid adding this logic to every single column which uses an ID) - remove uuid, because ID now represents a global resource identifier - keep uuid for post, because we are using this as preview id - keep uuid for clients for now - we are using this param for Ghost-Auth * ✨ base model: generate ObjectId on creating event - each new resource get's a auto generate ObjectId - this logic won't work for attached models, this commit comes later * 🎨 centralised attach method When attaching models there are two things important two know 1. To be able to attach an ObjectId, we need to register the `onCreating` event the fetched model!This is caused by the Bookshelf design in general. On this target model we are attaching the new model. 2. We need to manually fetch the target model, because Bookshelf has a weird behaviour (which is known as a bug, see see https://github.com/tgriesser/bookshelf/issues/629). The most important property when attaching a model is `parentFk`, which is the foreign key. This can be null when fetching the model with the option `withRelated`. To ensure quality and consistency, the custom attach wrapper always fetches the target model manual. By fetching the target model (again) is a little performance decrease, but it also has advantages: we can register the event, and directly unregister the event again. So very clean code. Important: please only use the custom attach wrapper in the future. * 🎨 token model had overriden the onCreating function because of the created_at field - we need to ensure that the base onCreating hook get's triggered for ALL models - if not, they don't get an ObjectId assigned - in this case: be smart and check if the target model has a created_at field * 🎨 we don't have a uuid field anymore, remove the usages - no default uuid creation in models - i am pretty sure we have some more definitions in our tests (for example in the export json files), but that is too much work to delete them all * 🎨 do not parse ID to Number - we had various occurances of parsing all ID's to numbers - we don't need this behaviour anymore - ID is string - i will adapt the ID validation in the next commit * 🎨 change ID regex for validation - we only allow: ID as ObjectId, ID as 1 and ID as me - we need to keep ID 1, because our whole software relies on ID 1 (permissions etc) * 🎨 owner fixture - roles: [4] does not work anymore - 4 means -> static id 4 - this worked in an auto increment system (not even in a system with distributed writes) - with ObjectId we generate each ID automatically (for static and dynamic resources) - it is possible to define all id's for static resources still, but that means we need to know which ID is already used and for consistency we have to define ObjectId's for these static resources - so no static id's anymore, except of: id 1 for owner and id 0 for external usage (because this is required from our permission system) - NOTE: please read through the comment in the user model * 🎨 tests: DataGenerator and test utils First of all: we need to ensure using ObjectId's in the tests. When don't, we can't ensure that ObjectId's work properly. This commit brings lot's of dynamic into all the static defined id's. In one of the next commits, i will adapt all the tests. * 🚨 remove counter in Notification API - no need to add a counter - we simply generate ObjectId's (they are auto incremental as well) - our id validator does only allow ObjectId as id,1 and me * 🎨 extend contextUser in Base Model - remove isNumber check, because id's are no longer numbers, except of id 0/1 - use existing isExternalUser - support id 0/1 as string or number * ✨ Ghost Owner has id 1 - ensure we define this id in the fixtures.json - doesn't matter if number or string * 🎨 functional tests adaptions - use dynamic id's * 🎨 fix unit tests * 🎨 integration tests adaptions * 🎨 change importer utils - all our export examples (test/fixtures/exports) contain id's as numbers - fact: but we ignore them anyway when inserting into the database, see https://github.com/TryGhost/Ghost/blob/master/core/server/data/import/utils.js#L249 - in https://github.com/TryGhost/Ghost/pull/7495/commits/0e6ed957cd54dc02a25cf6fb1ab7d7e723295e2c#diff-70f514a06347c048648be464819503c4L67 i removed parsing id's to integers - i realised that this ^ check just existed, because the userIdToMap was an object key and object keys are always strings! - i think this logic is a little bit complicated, but i don't want to refactor this now - this commit ensures when trying to find the user, the id comparison works again - i've added more documentation to understand this logic ;) - plus i renamed an attribute to improve readability * 🎨 Data-Generator: add more defaults to createUser - if i use the function DataGenerator.forKnex.createUser i would like to get a full set of defaults * 🎨 test utils: change/extend function set for functional tests - functional tests work a bit different - they boot Ghost and seed the database - some functional tests have mis-used the test setup - the test setup needs two sections: integration/unit and functional tests - any functional test is allowed to either add more data or change data in the existing Ghost db - but what it should not do is: add test fixtures like roles or users from our DataGenerator and cross fingers it will work - this commit adds a clean method for functional tests to add extra users * 🎨 functional tests adaptions - use last commit to insert users for functional tests clean - tidy up usage of testUtils.setup or testUtils.doAuth * 🐛 test utils: reset database before init - ensure we don't have any left data from other tests in the database when starting ghost * 🐛 fix test (unrelated to this PR) - fixes a random failure - return statement was missing * 🎨 make changes for invites
2016-11-17 12:09:11 +03:00
if (editId && object[docName][0].id && editId !== object[docName][0].id) {
return Promise.reject(new errors.BadRequestError({
message: i18n.t('errors.api.utils.invalidIdProvided')
}));
}
return Promise.resolve(object);
},
checkFileExists: function checkFileExists(fileData) {
return !!(fileData.mimetype && fileData.path);
},
checkFileIsValid: function checkFileIsValid(fileData, types, extensions) {
var type = fileData.mimetype,
ext = path.extname(fileData.name).toLowerCase();
if (_.includes(types, type) && _.includes(extensions, ext)) {
return true;
}
return false;
}
};
module.exports = utils;