Allow Upload/Download of redirects.json (#9029)

refs #9028

- add two new endpoints for uploading/downloading the redirects (file based)
- reload/re-register redirects on runtime
- migration for 1.9 to add permissions for redirects download/upload
This commit is contained in:
Katharina Irrgang 2017-09-21 17:01:03 +02:00 committed by Kevin Ansfield
parent 0fbf5e12b8
commit d943fc7cc9
16 changed files with 617 additions and 38 deletions

View File

@ -19,6 +19,7 @@ var _ = require('lodash'),
settings = require('./settings'),
tags = require('./tags'),
invites = require('./invites'),
redirects = require('./redirects'),
clients = require('./clients'),
users = require('./users'),
slugs = require('./slugs'),
@ -34,7 +35,8 @@ var _ = require('lodash'),
cacheInvalidationHeader,
locationHeader,
contentDispositionHeaderExport,
contentDispositionHeaderSubscribers;
contentDispositionHeaderSubscribers,
contentDispositionHeaderRedirects;
function isActiveThemeUpdate(method, endpoint, result) {
if (endpoint === 'themes') {
@ -169,6 +171,10 @@ contentDispositionHeaderSubscribers = function contentDispositionHeaderSubscribe
return Promise.resolve('Attachment; filename="subscribers.' + datetime + '.csv"');
};
contentDispositionHeaderRedirects = function contentDispositionHeaderRedirects() {
return Promise.resolve('Attachment; filename="redirects.json"');
};
addHeaders = function addHeaders(apiMethod, req, res, result) {
var cacheInvalidation,
location,
@ -210,6 +216,18 @@ addHeaders = function addHeaders(apiMethod, req, res, result) {
});
}
// Add Redirects Content-Disposition Header
if (apiMethod === redirects.download) {
contentDisposition = contentDispositionHeaderRedirects()
.then(function contentDispositionHeaderRedirects(header) {
res.set({
'Content-Disposition': header,
'Content-Type': 'application/json',
'Content-Length': JSON.stringify(result).length
});
});
}
return contentDisposition;
};
@ -293,7 +311,8 @@ module.exports = {
uploads: uploads,
slack: slack,
themes: themes,
invites: invites
invites: invites,
redirects: redirects
};
/**

View File

@ -0,0 +1,82 @@
'use strict';
const fs = require('fs'),
Promise = require('bluebird'),
path = require('path'),
config = require('../config'),
errors = require('../errors'),
i18n = require('../i18n'),
globalUtils = require('../utils'),
apiUtils = require('./utils'),
customRedirectsMiddleware = require('../middleware/custom-redirects');
let redirectsAPI,
_private = {};
_private.readRedirectsFile = function readRedirectsFile(customRedirectsPath) {
let redirectsPath = customRedirectsPath || path.join(config.getContentPath('data'), 'redirects.json');
return Promise.promisify(fs.readFile)(redirectsPath, 'utf-8')
.then(function serveContent(content) {
try {
content = JSON.parse(content);
} catch (err) {
throw new errors.BadRequestError({
message: i18n.t('errors.general.jsonParse', {context: err.message})
});
}
return content;
})
.catch(function handleError(err) {
if (err.code === 'ENOENT') {
return Promise.resolve([]);
}
if (errors.utils.isIgnitionError(err)) {
throw err;
}
throw new errors.NotFoundError({
err: err
});
});
};
redirectsAPI = {
download: function download(options) {
return apiUtils.handlePermissions('redirects', 'download')(options)
.then(function () {
return _private.readRedirectsFile();
});
},
upload: function upload(options) {
let redirectsPath = path.join(config.getContentPath('data'), 'redirects.json');
return apiUtils.handlePermissions('redirects', 'upload')(options)
.then(function () {
return Promise.promisify(fs.unlink)(redirectsPath)
.catch(function handleError(err) {
// CASE: ignore file not found
if (err.code === 'ENOENT') {
return Promise.resolve();
}
throw err;
})
.finally(function overrideFile() {
return _private.readRedirectsFile(options.path)
.then(function (content) {
globalUtils.validateRedirects(content);
return Promise.promisify(fs.writeFile)(redirectsPath, JSON.stringify(content), 'utf-8');
})
.then(function () {
// CASE: trigger that redirects are getting re-registered
customRedirectsMiddleware.reload();
});
});
});
}
};
module.exports = redirectsAPI;

View File

@ -191,5 +191,14 @@ module.exports = function apiRoutes() {
apiRouter.post('/invites', mw.authenticatePrivate, api.http(api.invites.add));
apiRouter.del('/invites/:id', mw.authenticatePrivate, api.http(api.invites.destroy));
// ## Redirects (JSON based)
apiRouter.get('/redirects/json', mw.authenticatePrivate, api.http(api.redirects.download));
apiRouter.post('/redirects/json',
mw.authenticatePrivate,
upload.single('redirects'),
validation.upload({type: 'redirects'}),
api.http(api.redirects.upload)
);
return apiRouter;
};

View File

@ -40,7 +40,7 @@ module.exports = function setupBlogApp() {
// you can extend Ghost with a custom redirects file
// see https://github.com/TryGhost/Ghost/issues/7707
customRedirects(blogApp);
customRedirects.use(blogApp);
// Static content/assets
// @TODO make sure all of these have a local 404 error handler

View File

@ -57,6 +57,10 @@
"themes": {
"extensions": [".zip"],
"contentTypes": ["application/zip", "application/x-zip-compressed", "application/octet-stream"]
},
"redirects": {
"extensions": [".json"],
"contentTypes": ["application/json"]
}
},
"times": {

View File

@ -0,0 +1,39 @@
var _ = require('lodash'),
utils = require('../../../schema/fixtures/utils'),
permissions = require('../../../../permissions'),
logging = require('../../../../logging'),
resource = 'redirect',
_private = {};
_private.getPermissions = function getPermissions() {
return utils.findModelFixtures('Permission', {object_type: resource});
};
_private.getRelations = function getRelations() {
return utils.findPermissionRelationsForObject(resource);
};
_private.printResult = function printResult(result, message) {
if (result.done === result.expected) {
logging.info(message);
} else {
logging.warn('(' + result.done + '/' + result.expected + ') ' + message);
}
};
module.exports = function addRedirectsPermissions(options) {
var modelToAdd = _private.getPermissions(),
relationToAdd = _private.getRelations(),
localOptions = _.merge({
context: {internal: true}
}, options);
return utils.addFixturesForModel(modelToAdd, localOptions).then(function (result) {
_private.printResult(result, 'Adding permissions fixtures for ' + resource + 's');
return utils.addFixturesForRelation(relationToAdd, localOptions);
}).then(function (result) {
_private.printResult(result, 'Adding permissions_roles fixtures for ' + resource + 's');
}).then(function () {
return permissions.init(localOptions);
});
};

View File

@ -411,6 +411,16 @@
"name": "Delete invites",
"action_type": "destroy",
"object_type": "invite"
},
{
"name": "Download redirects",
"action_type": "download",
"object_type": "redirect"
},
{
"name": "Upload redirects",
"action_type": "upload",
"object_type": "redirect"
}
]
},
@ -460,7 +470,8 @@
"role": "all",
"client": "all",
"subscriber": "all",
"invite": "all"
"invite": "all",
"redirect": "all"
},
"Editor": {
"post": "all",

View File

@ -1,34 +1,28 @@
var fs = require('fs-extra'),
_ = require('lodash'),
express = require('express'),
url = require('url'),
path = require('path'),
debug = require('ghost-ignition').debug('custom-redirects'),
config = require('../config'),
errors = require('../errors'),
logging = require('../logging');
logging = require('../logging'),
i18n = require('../i18n'),
globalUtils = require('../utils'),
customRedirectsRouter,
_private = {};
/**
* you can extend Ghost with a custom redirects file
* see https://github.com/TryGhost/Ghost/issues/7707
* file loads synchronously, because we need to register the routes before anything else
*/
module.exports = function redirects(blogApp) {
_private.registerRoutes = function registerRoutes() {
debug('redirects loading');
customRedirectsRouter = express.Router();
try {
var redirects = fs.readFileSync(config.getContentPath('data') + '/redirects.json', 'utf-8');
var redirects = fs.readFileSync(path.join(config.getContentPath('data'), 'redirects.json'), 'utf-8');
redirects = JSON.parse(redirects);
globalUtils.validateRedirects(redirects);
_.each(redirects, function (redirect) {
if (!redirect.from || !redirect.to) {
logging.warn(new errors.IncorrectUsageError({
message: 'One of your custom redirects is in a wrong format.',
level: 'normal',
help: JSON.stringify(redirect),
context: 'redirects.json'
}));
return;
}
/**
* always delete trailing slashes, doesn't matter if regex or not
* Example:
@ -43,7 +37,8 @@ module.exports = function redirects(blogApp) {
redirect.from += '\/?$';
}
blogApp.get(new RegExp(redirect.from), function (req, res) {
debug('register', redirect.from);
customRedirectsRouter.get(new RegExp(redirect.from), function (req, res) {
var maxAge = redirect.permanent ? config.get('caching:customRedirects:maxAge') : 0,
parsedUrl = url.parse(req.originalUrl);
@ -58,13 +53,35 @@ module.exports = function redirects(blogApp) {
});
});
} catch (err) {
if (err.code !== 'ENOENT') {
if (errors.utils.isIgnitionError(err)) {
logging.error(err);
} else if (err.code !== 'ENOENT') {
logging.error(new errors.IncorrectUsageError({
message: 'Your redirects.json is broken.',
help: 'Check if your JSON is valid.'
message: i18n.t('errors.middleware.redirects.register'),
context: err.message,
help: 'https://docs.ghost.org/docs/redirects'
}));
}
}
debug('redirects loaded');
};
/**
* - you can extend Ghost with a custom redirects file
* - see https://github.com/TryGhost/Ghost/issues/7707 and https://docs.ghost.org/v1/docs/redirects
* - file loads synchronously, because we need to register the routes before anything else
*/
exports.use = function use(blogApp) {
_private.registerRoutes();
// Recommended approach by express, see https://github.com/expressjs/express/issues/2596#issuecomment-81353034.
// As soon as the express router get's re-instantiated, the old router instance is not used anymore.
blogApp.use(function (req, res, next) {
customRedirectsRouter(req, res, next);
});
};
exports.reload = function reload() {
_private.registerRoutes();
};

View File

@ -108,6 +108,9 @@
"missingTheme": "The currently active theme \"{theme}\" is missing.",
"invalidTheme": "The currently active theme \"{theme}\" is invalid.",
"themeHasErrors": "The currently active theme \"{theme}\" has errors, but will still work."
},
"redirects": {
"register": "Could not register custom redirects."
}
},
"utils": {
@ -119,7 +122,8 @@
},
"blogIcon": {
"error": "Could not fetch icon dimensions."
}
},
"redirectsWrongFormat": "Incorrect redirects file format."
},
"config": {
"couldNotLocateConfigFile": {
@ -168,7 +172,8 @@
"general": {
"maintenance": "Ghost is currently undergoing maintenance, please wait a moment then retry.",
"requiredOnFuture": "This will be required in future. Please see {link}",
"internalError": "Something went wrong."
"internalError": "Something went wrong.",
"jsonParse": "Could not parse JSON: {context}."
},
"httpServer": {
"addressInUse": {

View File

@ -1,6 +1,8 @@
var unidecode = require('unidecode'),
_ = require('lodash'),
config = require('../config'),
errors = require('../errors'),
i18n = require('../i18n'),
utils,
getRandomInt;
@ -113,6 +115,28 @@ utils = {
res.redirect(301, path);
},
/**
* NOTE: No separate utils file, because redirects won't live forever in a JSON file, see V2 of https://github.com/TryGhost/Ghost/issues/7707
*/
validateRedirects: function validateRedirects(redirects) {
if (!_.isArray(redirects)) {
throw new errors.ValidationError({
message: i18n.t('errors.utils.redirectsWrongFormat'),
help: 'https://docs.ghost.org/docs/redirects'
});
}
_.each(redirects, function (redirect) {
if (!redirect.from || !redirect.to) {
throw new errors.ValidationError({
message: i18n.t('errors.utils.redirectsWrongFormat'),
context: JSON.stringify(redirect),
help: 'https://docs.ghost.org/docs/redirects'
});
}
});
},
readCSV: require('./read-csv'),
removeOpenRedirectFromUrl: require('./remove-open-redirect-from-url'),
zipFolder: require('./zip-folder'),

View File

@ -0,0 +1,250 @@
var should = require('should'),
supertest = require('supertest'),
fs = require('fs-extra'),
Promise = require('bluebird'),
path = require('path'),
testUtils = require('../../../utils'),
configUtils = require('../../../utils/configUtils'),
config = require('../../../../../core/server/config'),
ghost = testUtils.startGhost,
request, accesstoken;
should.equal(true, true);
describe('Redirects API', function () {
var ghostServer;
afterEach(function () {
configUtils.restore();
});
describe('Download', function () {
beforeEach(function () {
return ghost().then(function (_ghostServer) {
ghostServer = _ghostServer;
return ghostServer.start();
}).then(function () {
request = supertest.agent(config.get('url'));
}).then(function () {
return testUtils.doAuth(request, 'client:trusted-domain');
}).then(function (token) {
accesstoken = token;
});
});
afterEach(function () {
return testUtils.clearData()
.then(function () {
return ghostServer.stop();
});
});
it('file does not exist', function (done) {
// Just set any content folder, which does not contain a redirects file.
configUtils.set('paths:contentPath', path.join(__dirname, '../../../utils/fixtures/data'));
request
.get(testUtils.API.getApiQuery('redirects/json/?client_id=ghost-admin&client_secret=not_available'))
.set('Authorization', 'Bearer ' + accesstoken)
.set('Origin', testUtils.API.getURL())
.expect(200)
.end(function (err, res) {
if (err) {
return done(err);
}
res.headers['content-disposition'].should.eql('Attachment; filename="redirects.json"');
res.headers['content-type'].should.eql('application/json; charset=utf-8');
// API returns an empty file with the correct file structure (empty [])
res.headers['content-length'].should.eql('2');
done();
});
});
it('file exists', function (done) {
request
.get(testUtils.API.getApiQuery('redirects/json/?client_id=ghost-admin&client_secret=not_available'))
.set('Authorization', 'Bearer ' + accesstoken)
.set('Origin', testUtils.API.getURL())
.expect('Content-Type', /application\/json/)
.expect('Content-Disposition', 'Attachment; filename="redirects.json"')
.expect(200)
.end(function (err, res) {
if (err) {
return done(err);
}
res.headers['content-disposition'].should.eql('Attachment; filename="redirects.json"');
res.headers['content-type'].should.eql('application/json; charset=utf-8');
res.headers['content-length'].should.eql('463');
done();
});
});
});
describe('Upload', function () {
describe('Ensure re-registering redirects works', function () {
var startGhost = function () {
return ghost().then(function (_ghostServer) {
ghostServer = _ghostServer;
return ghostServer.start();
}).then(function () {
request = supertest.agent(config.get('url'));
}).then(function () {
return testUtils.doAuth(request, 'client:trusted-domain');
}).then(function (token) {
accesstoken = token;
});
},
stopGhost = function () {
return testUtils.clearData()
.then(function () {
return ghostServer.stop();
});
};
afterEach(stopGhost);
it('override', function (done) {
return startGhost()
.then(function () {
return new Promise(function (resolve) {
setTimeout(resolve, 100);
});
})
.then(function () {
return request
.get('/my-old-blog-post/')
.expect(301);
})
.then(function (response) {
response.headers.location.should.eql('/revamped-url/');
return stopGhost();
})
.then(function () {
return new Promise(function (resolve) {
setTimeout(resolve, 100);
});
})
.then(function () {
return startGhost();
})
.then(function () {
// Provide a second redirects file in the root directory of the content test folder
fs.writeFileSync(path.join(config.get('paths:contentPath'), 'redirects.json'), JSON.stringify([{from: 'c', to: 'd'}]));
return new Promise(function (resolve) {
setTimeout(resolve, 100);
});
})
.then(function () {
// Override redirects file
return request
.post(testUtils.API.getApiQuery('redirects/json/?client_id=ghost-admin&client_secret=not_available'))
.set('Authorization', 'Bearer ' + accesstoken)
.set('Origin', testUtils.API.getURL())
.attach('redirects', path.join(config.get('paths:contentPath'), 'redirects.json'))
.expect('Content-Type', /application\/json/)
.expect(200);
})
.then(function () {
return request
.get('/my-old-blog-post/')
.expect(404);
})
.then(function () {
return request
.get('/c/')
.expect(302);
})
.then(function (response) {
response.headers.location.should.eql('/d');
done();
})
.catch(done);
});
});
describe('Error cases', function () {
beforeEach(function () {
return ghost().then(function (_ghostServer) {
ghostServer = _ghostServer;
return ghostServer.start();
}).then(function () {
request = supertest.agent(config.get('url'));
}).then(function () {
return testUtils.doAuth(request, 'client:trusted-domain');
}).then(function (token) {
accesstoken = token;
});
});
afterEach(function () {
return testUtils.clearData()
.then(function () {
return ghostServer.stop();
});
});
it('syntax error', function (done) {
fs.writeFileSync(path.join(config.get('paths:contentPath'), 'redirects.json'), 'something');
request
.post(testUtils.API.getApiQuery('redirects/json/?client_id=ghost-admin&client_secret=not_available'))
.set('Authorization', 'Bearer ' + accesstoken)
.set('Origin', testUtils.API.getURL())
.attach('redirects', path.join(config.get('paths:contentPath'), 'redirects.json'))
.expect('Content-Type', /application\/json/)
.expect(400)
.end(function (err) {
if (err) {
return done(err);
}
done();
});
});
it('wrong format: no array', function (done) {
fs.writeFileSync(path.join(config.get('paths:contentPath'), 'redirects.json'), JSON.stringify({from: 'c', to: 'd'}));
request
.post(testUtils.API.getApiQuery('redirects/json/?client_id=ghost-admin&client_secret=not_available'))
.set('Authorization', 'Bearer ' + accesstoken)
.set('Origin', testUtils.API.getURL())
.attach('redirects', path.join(config.get('paths:contentPath'), 'redirects.json'))
.expect('Content-Type', /application\/json/)
.expect(422)
.end(function (err) {
if (err) {
return done(err);
}
done();
});
});
it('wrong format: no from/to', function (done) {
fs.writeFileSync(path.join(config.get('paths:contentPath'), 'redirects.json'), JSON.stringify([{to: 'd'}]));
request
.post(testUtils.API.getApiQuery('redirects/json/?client_id=ghost-admin&client_secret=not_available'))
.set('Authorization', 'Bearer ' + accesstoken)
.set('Origin', testUtils.API.getURL())
.attach('redirects', path.join(config.get('paths:contentPath'), 'redirects.json'))
.expect('Content-Type', /application\/json/)
.expect(422)
.end(function (err) {
if (err) {
return done(err);
}
done();
});
});
});
});
});

View File

@ -0,0 +1,112 @@
var should = require('should'),
sinon = require('sinon'),
testUtils = require('../../utils'),
Promise = require('bluebird'),
RedirectsAPI = require('../../../server/api/redirects'),
mail = require('../../../server/api/mail'),
sandbox = sinon.sandbox.create();
should.equal(true, true);
describe('Redirects API', function () {
beforeEach(testUtils.teardown);
beforeEach(testUtils.setup('settings', 'users:roles', 'perms:redirect', 'perms:init'));
beforeEach(function () {
sandbox.stub(mail, 'send', function () {
return Promise.resolve();
});
});
afterEach(function () {
sandbox.restore();
});
after(testUtils.teardown);
describe('Permissions', function () {
describe('Owner', function () {
it('Can upload', function (done) {
RedirectsAPI.upload(testUtils.context.owner)
.then(function () {
done();
})
.catch(done);
});
it('Can download', function (done) {
RedirectsAPI.download(testUtils.context.owner)
.then(function () {
done();
})
.catch(done);
});
});
describe('Admin', function () {
it('Can upload', function (done) {
RedirectsAPI.upload(testUtils.context.admin)
.then(function () {
done();
})
.catch(done);
});
it('Can download', function (done) {
RedirectsAPI.download(testUtils.context.admin)
.then(function () {
done();
})
.catch(done);
});
});
describe('Editor', function () {
it('Can\'t upload', function (done) {
RedirectsAPI.upload(testUtils.context.editor)
.then(function () {
done(new Error('Editor is not allowed to upload redirects.'));
})
.catch(function (err) {
err.statusCode.should.eql(403);
done();
});
});
it('Can\'t download', function (done) {
RedirectsAPI.upload(testUtils.context.editor)
.then(function () {
done(new Error('Editor is not allowed to download redirects.'));
})
.catch(function (err) {
err.statusCode.should.eql(403);
done();
});
});
});
describe('Author', function () {
it('Can\'t upload', function (done) {
RedirectsAPI.upload(testUtils.context.author)
.then(function () {
done(new Error('Author is not allowed to upload redirects.'));
})
.catch(function (err) {
err.statusCode.should.eql(403);
done();
});
});
it('Can\'t download', function (done) {
RedirectsAPI.upload(testUtils.context.author)
.then(function () {
done(new Error('Author is not allowed to download redirects.'));
})
.catch(function (err) {
err.statusCode.should.eql(403);
done();
});
});
});
});
});

View File

@ -157,6 +157,12 @@ describe('Database Migration (special functions)', function () {
permissions[47].should.be.AssignedToRoles(['Administrator', 'Editor']);
permissions[48].name.should.eql('Delete invites');
permissions[48].should.be.AssignedToRoles(['Administrator', 'Editor']);
// Redirects
permissions[49].name.should.eql('Download redirects');
permissions[49].should.be.AssignedToRoles(['Administrator']);
permissions[50].name.should.eql('Upload redirects');
permissions[50].should.be.AssignedToRoles(['Administrator']);
});
describe('Populate', function () {
@ -218,7 +224,7 @@ describe('Database Migration (special functions)', function () {
result.roles.at(3).get('name').should.eql('Owner');
// Permissions
result.permissions.length.should.eql(49);
result.permissions.length.should.eql(51);
result.permissions.toJSON().should.be.CompletePermissions();
done();

View File

@ -151,19 +151,19 @@ describe('Migration Fixture Utils', function () {
fixtureUtils.addFixturesForRelation(fixtures.relations[0]).then(function (result) {
should.exist(result);
result.should.be.an.Object();
result.should.have.property('expected', 30);
result.should.have.property('done', 30);
result.should.have.property('expected', 31);
result.should.have.property('done', 31);
// Permissions & Roles
permsAllStub.calledOnce.should.be.true();
rolesAllStub.calledOnce.should.be.true();
dataMethodStub.filter.callCount.should.eql(30);
dataMethodStub.filter.callCount.should.eql(31);
dataMethodStub.find.callCount.should.eql(3);
baseUtilAttachStub.callCount.should.eql(30);
baseUtilAttachStub.callCount.should.eql(31);
fromItem.related.callCount.should.eql(30);
fromItem.findWhere.callCount.should.eql(30);
toItem[0].get.callCount.should.eql(60);
fromItem.related.callCount.should.eql(31);
fromItem.findWhere.callCount.should.eql(31);
toItem[0].get.callCount.should.eql(62);
done();
}).catch(done);

View File

@ -20,7 +20,7 @@ var should = require('should'), // jshint ignore:line
describe('DB version integrity', function () {
// Only these variables should need updating
var currentSchemaHash = 'af4028653a7c0804f6bf7b98c50db5dc',
currentFixturesHash = 'a8ccedee7058e68eafd268b73458e954';
currentFixturesHash = '00e9b37f49b8eed5591ec2d381afb9e3';
// If this test is failing, then it is likely a change has been made that requires a DB version bump,
// and the values above will need updating as confirmation

View File

@ -835,6 +835,7 @@ startGhost = function startGhost() {
// Copy all themes into the new test content folder. Default active theme is always casper. If you want to use a different theme, you have to set the active theme (e.g. stub)
fs.copySync(path.join(__dirname, 'fixtures', 'themes'), path.join(contentFolderForTests, 'themes'));
fs.copySync(path.join(__dirname, 'fixtures', 'data'), path.join(contentFolderForTests, 'data'));
return knexMigrator.reset()
.then(function initialiseDatabase() {