✨ Added ability to upload/reload routes.yaml

refs #9744

- added two new endpoints to upload/download routes.yaml
- reload site express app on successful/valid upload
- reload url service on sucessfuly upload
- force clear cache of pages
- ensure we keep a backup of the routes.yaml file
- this feature was mostly tested manually
- @TODO: i have to write unit tests - will do later
- @TODO: do a memory test to ensure we haven't introduced any memory leaks with this feature

core/server/api/index.js

@@ -37,7 +37,8 @@ var _ = require('lodash'),
     locationHeader,
     contentDispositionHeaderExport,
     contentDispositionHeaderSubscribers,
-    contentDispositionHeaderRedirects;
+    contentDispositionHeaderRedirects,
+    contentDispositionHeaderRoutes;
 
 function isActiveThemeUpdate(method, endpoint, result) {
     if (endpoint === 'themes') {
@@ -180,6 +181,10 @@ contentDispositionHeaderRedirects = function contentDispositionHeaderRedirects()
     return Promise.resolve('Attachment; filename="redirects.json"');
 };
 
+contentDispositionHeaderRoutes = () => {
+    return Promise.resolve('Attachment; filename="routes.yaml"');
+};
+
 addHeaders = function addHeaders(apiMethod, req, res, result) {
     var cacheInvalidation,
         location,
@@ -233,6 +238,18 @@ addHeaders = function addHeaders(apiMethod, req, res, result) {
             });
     }
 
+    // Add Routes Content-Disposition Header
+    if (apiMethod === settings.download) {
+        contentDisposition = contentDispositionHeaderRoutes()
+            .then((header) => {
+                res.set({
+                    'Content-Disposition': header,
+                    'Content-Type': 'application/yaml',
+                    'Content-Length': JSON.stringify(result).length
+                });
+            });
+    }
+
     return contentDisposition;
 };
 
@@ -273,8 +290,10 @@ http = function http(apiMethod) {
             if (req.method === 'DELETE') {
                 return res.status(204).end();
             }
-            // Keep CSV header and formatting
-            if (res.get('Content-Type') && res.get('Content-Type').indexOf('text/csv') === 0) {
+
+            // Keep CSV, yaml formatting
+            if (res.get('Content-Type') && res.get('Content-Type').indexOf('text/csv') === 0 ||
+                res.get('Content-Type') && res.get('Content-Type').indexOf('application/yaml') === 0) {
                 return res.status(200).send(response);
             }
 

core/server/api/settings.js

@@ -2,9 +2,14 @@
 // RESTful API for the Setting resource
 var Promise = require('bluebird'),
     _ = require('lodash'),
+    moment = require('moment-timezone'),
+    fs = require('fs-extra'),
+    path = require('path'),
+    config = require('../config'),
     models = require('../models'),
     canThis = require('../services/permissions').canThis,
     localUtils = require('./utils'),
+    urlService = require('../services/url'),
     common = require('../lib/common'),
     settingsCache = require('../services/settings/cache'),
     docName = 'settings',
@@ -236,6 +241,70 @@ settings = {
                 return settingsResult(settingsKeyedJSON, type);
             });
         });
+    },
+
+    /**
+     * The `routes.yaml` file offers a way to configure your Ghost blog. It's currently a setting feature
+     * we have added. That's why the `routes.yaml` file is treated as a "setting" right now.
+     * If we want to add single permissions for this file (e.g. upload/download routes.yaml), we can add later.
+     *
+     * How does it work?
+     *
+     * - we first reset all url generators (each url generator belongs to one express router)
+     *   - we don't destroy the resources, we only release them (this avoids reloading all resources from the db again)
+     * - then we reload the whole site app, which will reset all routers and re-create the url generators
+     */
+    upload: (options) => {
+        const backupRoutesPath = path.join(config.getContentPath('settings'), `routes-${moment().format('YYYY-MM-DD-HH-mm-ss')}.yaml`);
+
+        return localUtils.handlePermissions('settings', 'edit')(options)
+            .then(() => {
+                return fs.copy(config.getContentPath('settings') + '/routes.yaml', backupRoutesPath);
+            })
+            .then(() => {
+                return fs.copy(options.path, config.getContentPath('settings') + '/routes.yaml');
+            })
+            .then(() => {
+                urlService.resetGenerators({releaseResourcesOnly: true});
+            })
+            .then(() => {
+                const siteApp = require('../web/site/app');
+
+                try {
+                    return siteApp.reload();
+                } catch (err) {
+                    // bring back backup, otherwise your Ghost blog is broken
+                    return fs.copy(backupRoutesPath, config.getContentPath('settings') + '/routes.yaml')
+                        .then(() => {
+                            return siteApp.reload();
+                        })
+                        .then(() => {
+                            throw err;
+                        });
+                }
+            });
+    },
+
+    download: (options) => {
+        const routesPath = path.join(config.getContentPath('settings'), 'routes.yaml');
+
+        return localUtils.handlePermissions('settings', 'browse')(options)
+            .then(() => {
+                return fs.readFile(routesPath, 'utf-8');
+            })
+            .catch(function handleError(err) {
+                if (err.code === 'ENOENT') {
+                    return Promise.resolve([]);
+                }
+
+                if (common.errors.utils.isIgnitionError(err)) {
+                    throw err;
+                }
+
+                throw new common.errors.NotFoundError({
+                    err: err
+                });
+            });
     }
 };
 

core/server/api/upload.js

@@ -1,7 +1,8 @@
-var Promise = require('bluebird'),
+const Promise = require('bluebird'),
     fs = require('fs-extra'),
-    storage = require('../adapters/storage'),
-    upload;
+    storage = require('../adapters/storage');
+
+let upload;
 
 /**
  * ## Upload API Methods

core/server/config/overrides.json

@@ -51,6 +51,10 @@
         "redirects": {
             "extensions": [".json"],
             "contentTypes": ["text/plain", "application/octet-stream", "application/json"]
+        },
+        "routes": {
+            "extensions": [".yaml"],
+            "contentTypes": ["text/plain", "text/yaml", "application/octet-stream", "application/yaml"]
         }
     },
     "times": {

core/server/services/routing/bootstrap.js

@@ -9,6 +9,7 @@ const PreviewRouter = require('./PreviewRouter');
 const ParentRouter = require('./ParentRouter');
 
 const registry = require('./registry');
+let siteRouter;
 
 /**
  * Create a set of default and dynamic routers defined in the routing yaml.
@@ -22,7 +23,7 @@ module.exports = function bootstrap() {
     registry.resetAllRouters();
     registry.resetAllRoutes();
 
-    const siteRouter = new ParentRouter('SiteRouter');
+    siteRouter = new ParentRouter('SiteRouter');
     const previewRouter = new PreviewRouter();
 
     siteRouter.mountRouter(previewRouter.router());

core/server/services/url/Queue.js

@@ -143,6 +143,8 @@ class Queue extends EventEmitter {
                 debug('ended (2)', event, action);
                 this.emit('ended', event);
             } else {
+                debug('retry', event, action, this.toNotify[action].timeoutInMS);
+
                 this.toNotify[action].timeoutInMS = this.toNotify[action].timeoutInMS * 1.1;
 
                 this.toNotify[action].timeout = setTimeout(() => {

core/server/services/url/Resources.js

@@ -406,6 +406,14 @@ class Resources {
             this.data[resourceConfig.type] = [];
         });
     }
+
+    releaseAll() {
+        _.each(this.data, (resources, type) => {
+            _.each(this.data[type], (resource) => {
+                resource.release();
+            });
+        });
+    }
 }
 
 module.exports = Resources;

core/server/services/url/UrlService.js

@@ -228,13 +228,18 @@ class UrlService {
         this._onRouterAddedListener && common.events.removeListener('router.created', this._onRouterAddedListener);
     }
 
-    resetGenerators() {
+    resetGenerators(options = {}) {
         debug('resetGenerators');
         this.finished = false;
         this.urlGenerators = [];
         this.urls.reset();
         this.queue.reset();
-        this.resources.softReset();
+
+        if (options.releaseResourcesOnly) {
+            this.resources.releaseAll();
+        } else {
+            this.resources.softReset();
+        }
     }
 
     softReset() {

core/server/translations/en.json

@@ -344,6 +344,10 @@
                 "missingFile": "Please select a JSON file.",
                 "invalidFile": "Please select a valid JSON file to import."
             },
+            "routes": {
+                "missingFile": "Please select a YAML file.",
+                "invalidFile": "Please select a valid YAML file to import."
+            },
             "settings": {
                 "problemFindingSetting": "Problem finding setting: {key}",
                 "accessCoreSettingFromExtReq": "Attempted to access core setting from external request",

core/server/web/api/routes.js

@@ -49,6 +49,14 @@ module.exports = function apiRoutes() {
     ], api.http(api.schedules.publishPost));
 
     // ## Settings
+    apiRouter.get('/settings/routes/yaml', mw.authenticatePrivate, api.http(api.settings.download));
+    apiRouter.post('/settings/routes/yaml',
+        mw.authenticatePrivate,
+        upload.single('routes'),
+        validation.upload({type: 'routes'}),
+        api.http(api.settings.upload)
+    );
+
     apiRouter.get('/settings', mw.authenticatePrivate, api.http(api.settings.browse));
     apiRouter.get('/settings/:key', mw.authenticatePrivate, api.http(api.settings.read));
     apiRouter.put('/settings', mw.authenticatePrivate, api.http(api.settings.edit));

core/server/web/site/app.js

@@ -1,9 +1,11 @@
 var debug = require('ghost-ignition').debug('blog'),
     path = require('path'),
     express = require('express'),
+    setPrototypeOf = require('setprototypeof'),
 
     // App requires
     config = require('../../config'),
+    apps = require('../../services/apps'),
     constants = require('../../lib/constants'),
     storage = require('../../adapters/storage'),
     urlService = require('../../services/url'),
@@ -32,6 +34,8 @@ var debug = require('ghost-ignition').debug('blog'),
     // middleware for themes
     themeMiddleware = require('../../services/themes').middleware;
 
+let router;
+
 module.exports = function setupSiteApp() {
     debug('Site setup start');
 
@@ -123,8 +127,16 @@ module.exports = function setupSiteApp() {
 
     debug('General middleware done');
 
+    router = siteRoutes();
+
+    function SiteRouter(req, res, next) {
+        router(req, res, next);
+    }
+
+    setPrototypeOf(SiteRouter, router);
+
     // Set up Frontend routes (including private blogging routes)
-    siteApp.use(siteRoutes());
+    siteApp.use(SiteRouter);
 
     // ### Error handlers
     siteApp.use(errorHandler.pageNotFound);
@@ -134,3 +146,18 @@ module.exports = function setupSiteApp() {
 
     return siteApp;
 };
+
+module.exports.reload = () => {
+    // https://github.com/expressjs/express/issues/2596
+    router = siteRoutes();
+
+    // re-initialse apps (register app routers, because we have re-initialised the site routers)
+    apps.init();
+
+    // connect routers and resources again
+    urlService.queue.start({
+        event: 'init',
+        tolerance: 100,
+        requiredSubscriberCount: 1
+    });
+};

core/test/functional/routes/api/settings_spec.js

@@ -1,6 +1,8 @@
 var should = require('should'),
     _ = require('lodash'),
     supertest = require('supertest'),
+    os = require('os'),
+    fs = require('fs-extra'),
     testUtils = require('../../../utils'),
     config = require('../../../../../core/server/config'),
     ghost = testUtils.startGhost,
@@ -201,4 +203,37 @@ describe('Settings API', function () {
                     });
             });
     });
+
+    it('can download routes.yaml', ()=> {
+        return request.get(testUtils.API.getApiQuery('settings/routes/yaml/'))
+            .set('Authorization', 'Bearer ' + accesstoken)
+            .set('Accept', 'application/yaml')
+            .expect(200)
+            .then((res)=> {
+                res.headers['content-disposition'].should.eql('Attachment; filename="routes.yaml"');
+                res.headers['content-type'].should.eql('application/yaml; charset=utf-8');
+                res.headers['content-length'].should.eql('152');
+            });
+    });
+
+    it('can upload routes.yaml', ()=> {
+        const newRoutesYamlPath = `${os.tmpdir()}routes.yaml`;
+
+        return fs.writeFile(newRoutesYamlPath, 'routes:\ncollections:\ntaxonomies:\n')
+            .then(()=> {
+                return request
+                    .post(testUtils.API.getApiQuery('settings/routes/yaml/'))
+                    .set('Authorization', 'Bearer ' + accesstoken)
+                    .set('Origin', testUtils.API.getURL())
+                    .attach('routes', newRoutesYamlPath)
+                    .expect('Content-Type', /application\/json/)
+                    .expect(200);
+            })
+            .then((res)=> {
+                res.headers['x-cache-invalidate'].should.eql('/*');
+            })
+            .finally(()=> {
+                return ghostServer.stop();
+            });
+    });
 });