Added new, simpler, linear boot process

Background:
- Ghosts existing boot process is split across multiple files, has affordances for outdated ways of running Ghost and is generally non-linear making it nigh-impossible to follow
- The web of dependencies that are loaded on boot are also impossible to unpick, which makes it really hard to decouple Ghost
- With 4.0 we want to introduce a new, linear, simpler, clearer way to boot up Ghost to unlock decoupling Ghost into much smaller pieces

This commit:
- adds a new ghost.js file which switches between boot mode with `node index` or `node index old` so that if we find bugs we can work around them this week
   - Note: the old boot process will go away very soon, but ghost.js will remain as the interface between the command to start Ghost and the application code
- reworks the database migration process into a standalone utility, so that the DB is handled as one simple step of the boot process, decoupled from everything else
- is missing tests for this new db utility
- leaves a lot of work to do around loading core code, services, express apps in a sensible order, as work to fix this would start to break the old boot process
- doesn't use the new maintenance app because we aren't restarting the server here, instead we have the concept of a "core app" that starts in maintenance mode - need to think about how apps will be decoupled in the near future

core/app.js

@@ -0,0 +1,22 @@
+const express = require('./shared/express');
+const rootApp = express('root');
+
+const fs = require('fs');
+const path = require('path');
+
+// We never want middleware functions to be anonymous
+const maintenanceMiddleware = (req, res, next) => {
+    if (!req.app.get('maintenance')) {
+        return next();
+    }
+    res.set({
+        'Cache-Control': 'no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0'
+    });
+    res.writeHead(503, {'content-type': 'text/html'});
+    fs.createReadStream(path.resolve(__dirname, './server/views/maintenance.html')).pipe(res);
+};
+
+rootApp.enable('maintenance');
+rootApp.use(maintenanceMiddleware);
+
+module.exports = rootApp;

core/boot.js

@@ -0,0 +1,184 @@
+// # The Ghost Boot Sequence
+
+// IMPORTANT: The only global requires here should be debug + overrides
+const debug = require('ghost-ignition').debug('boot');
+require('./server/overrides');
+// END OF GLOBAL REQUIRES
+
+const initCore = async ({ghostServer}) => {
+    const settings = require('./server/services/settings');
+    const jobService = require('./server/services/jobs');
+    const models = require('./server/models');
+    const {events, i18n} = require('./server/lib/common');
+
+    ghostServer.registerCleanupTask(async () => {
+        await jobService.shutdown();
+    });
+
+    // Initialize Ghost core internationalization
+    i18n.init();
+    debug('Default i18n done for core');
+
+    models.init();
+    debug('Models done');
+
+    await settings.init();
+
+    // @TODO: fix this - has to happen before db.ready is emitted
+    debug('Begin: Url Service');
+    require('./frontend/services/url');
+    debug('End: Url Service');
+
+    // @TODO: fix this location
+    events.emit('db.ready');
+};
+
+const initExpressApps = async () => {
+    debug('Begin: initExpressApps');
+    const themeService = require('./frontend/services/themes');
+    const frontendSettings = require('./frontend/services/settings');
+
+    await frontendSettings.init();
+    debug('Frontend settings done');
+
+    await themeService.init();
+    debug('Themes done');
+
+    const parentApp = require('./server/web/parent/app')();
+
+    debug('End: initExpressApps');
+    return parentApp;
+};
+
+const initServices = async ({config}) => {
+    debug('Begin: initialiseServices');
+    const themeService = require('./frontend/services/themes');
+    const frontendSettings = require('./frontend/services/settings');
+    const appService = require('./frontend/services/apps');
+    const urlUtils = require('./shared/url-utils');
+
+    // CASE: When Ghost is ready with bootstrapping (db migrations etc.), we can trigger the router creation.
+    //       Reason is that the routers access the routes.yaml, which shouldn't and doesn't have to be validated to
+    //       start Ghost in maintenance mode.
+    // Routing is a bridge between the frontend and API
+    const routing = require('./frontend/services/routing');
+    // We pass the themeService API version here, so that the frontend services are less tightly-coupled
+    routing.bootstrap.start(themeService.getApiVersion());
+
+    const settings = require('./server/services/settings');
+    const permissions = require('./server/services/permissions');
+    const xmlrpc = require('./server/services/xmlrpc');
+    const slack = require('./server/services/slack');
+    const {mega} = require('./server/services/mega');
+    const webhooks = require('./server/services/webhooks');
+    const scheduling = require('./server/adapters/scheduling');
+    const getRoutesHash = () => frontendSettings.getCurrentHash('routes');
+
+    await Promise.all([
+        // Initialize the permissions actions and objects
+        permissions.init(),
+        xmlrpc.listen(),
+        slack.listen(),
+        mega.listen(),
+        webhooks.listen(),
+        settings.syncRoutesHash(getRoutesHash),
+        appService.init(),
+        scheduling.init({
+            // NOTE: When changing API version need to consider how to migrate custom scheduling adapters
+            //       that rely on URL to lookup persisted scheduled records (jobs, etc.). Ref: https://github.com/TryGhost/Ghost/pull/10726#issuecomment-489557162
+            apiUrl: urlUtils.urlFor('api', {version: 'v3', versionType: 'admin'}, true)
+        })
+    ]);
+
+    debug('XMLRPC, Slack, MEGA, Webhooks, Scheduling, Permissions done');
+
+    // Initialise analytics events
+    if (config.get('segment:key')) {
+        require('./analytics-events').init();
+    }
+
+    debug('End: initialiseServices');
+};
+
+const mountGhost = (rootApp, ghostApp) => {
+    const urlService = require('./frontend/services/url');
+    rootApp.disable('maintenance');
+    rootApp.use(urlService.utils.getSubdir(), ghostApp);
+};
+
+const bootGhost = async () => {
+    // Metrics & debugging
+    const startTime = Date.now();
+    let ghostServer;
+
+    try {
+        // Config is the absolute first thing to do!
+        debug('Begin: Load config');
+        const config = require('./shared/config');
+        debug('End: Load config');
+
+        debug('Begin: Load version info');
+        const version = require('./server/lib/ghost-version');
+        config.set('version', version);
+        debug('End: Load version info');
+
+        debug('Begin: load server + minimal app');
+        process.env.NODE_ENV = process.env.NODE_ENV || 'development';
+
+        // Get minimal application in maintenance mode
+        const rootApp = require('./app');
+
+        // Start server with minimal App
+        const GhostServer = require('./server/ghost-server');
+        ghostServer = new GhostServer();
+        await ghostServer.start(rootApp);
+
+        const logging = require('./shared/logging');
+        logging.info('Ghost server start', (Date.now() - startTime) / 1000 + 's');
+        debug('End: load server + minimal app');
+
+        debug('Begin: Get DB ready');
+        // Get the DB ready
+        await require('./db').ready();
+        debug('End: Get DB ready');
+
+        // Load Ghost with all its services
+        debug('Begin: Load Ghost Core Services');
+        await initCore({ghostServer});
+
+        const ghostApp = await initExpressApps({});
+        await initServices({config});
+        debug('End: Load Ghost Core Services');
+
+        // Mount the full Ghost app onto the minimal root app & disable maintenance mode
+        mountGhost(rootApp, ghostApp);
+
+        // Announce Server Readiness
+        logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's');
+        GhostServer.announceServerReadiness();
+    } catch (error) {
+        const errors = require('@tryghost/errors');
+        // @TODO: fix these extra requires
+        const GhostServer = require('./server/ghost-server');
+        const logging = require('./shared/logging');
+
+        let serverStartError = error;
+
+        if (!errors.utils.isIgnitionError(serverStartError)) {
+            serverStartError = new errors.GhostError({message: serverStartError.message, err: serverStartError});
+        }
+
+        logging.error(serverStartError);
+        GhostServer.announceServerReadiness(serverStartError);
+
+        if (ghostServer) {
+            ghostServer.shutdown(2);
+        } else {
+            setTimeout(() => {
+                process.exit(2);
+            }, 100);
+        }
+    }
+};
+
+module.exports = bootGhost;

core/db.js

@@ -0,0 +1,8 @@
+const config = require('./shared/config');
+const logging = require('./shared/logging');
+
+module.exports.ready = async () => {
+    const DatabaseStateManager = require('./server/data/db/state-manager');
+    const dbStateManager = new DatabaseStateManager({knexMigratorFilePath: config.get('paths:appRoot')});
+    await dbStateManager.makeReady({logging});
+};

core/server/data/db/state-manager.js

@@ -0,0 +1,106 @@
+const KnexMigrator = require('knex-migrator');
+const errors = require('@tryghost/errors');
+
+const states = {
+    READY: 0,
+    NEEDS_INITIALISATION: 1,
+    NEEDS_MIGRATION: 2,
+    ERROR: 3
+};
+
+const printState = ({state, logging}) => {
+    if (state === states.READY) {
+        logging.info('Database is in a ready state.');
+    }
+
+    if (state === states.NEEDS_INITIALISATION) {
+        logging.warn('Database state requires initialisation.');
+    }
+
+    if (state === states.NEEDS_MIGRATION) {
+        logging.warn('Database state requires migration.');
+    }
+
+    if (state === states.ERROR) {
+        logging.error('Database is in an error state.');
+    }
+};
+
+class DatabaseStateManager {
+    constructor({knexMigratorFilePath}) {
+        this.knexMigrator = new KnexMigrator({
+            knexMigratorFilePath
+        });
+    }
+
+    async getState() {
+        let state = states.READY;
+        try {
+            await this.knexMigrator.isDatabaseOK();
+            return state;
+        } catch (error) {
+            // CASE: database has not yet been initialised
+            if (error.code === 'DB_NOT_INITIALISED') {
+                state = states.NEEDS_INITIALISATION;
+                return state;
+            }
+
+            // CASE: there's no migration table so we can't understand
+            if (error.code === 'MIGRATION_TABLE_IS_MISSING') {
+                state = states.NEEDS_INITIALISATION;
+                return state;
+            }
+
+            // CASE: database needs migrations
+            if (error.code === 'DB_NEEDS_MIGRATION') {
+                state = states.NEEDS_MIGRATION;
+                return state;
+            }
+
+            // CASE: database connection errors, unknown cases
+            let errorToThrow = error;
+            if (!errors.utils.isIgnitionError(errorToThrow)) {
+                errorToThrow = new errors.GhostError({message: errorToThrow.message, err: errorToThrow});
+            }
+
+            throw errorToThrow;
+        }
+    }
+
+    async makeReady({logging}) {
+        try {
+            let state = await this.getState();
+
+            if (logging) {
+                printState({state, logging});
+            }
+
+            if (state === states.READY) {
+                return;
+            }
+
+            if (state === states.NEEDS_INITIALISATION) {
+                await this.knexMigrator.init();
+            }
+
+            if (state === states.NEEDS_MIGRATION) {
+                await this.knexMigrator.migrate();
+            }
+
+            state = await this.getState();
+
+            if (logging) {
+                printState({state, logging});
+            }
+        } catch (error) {
+            let errorToThrow = error;
+            if (!errors.utils.isIgnitionError(error)) {
+                errorToThrow = new errors.GhostError({message: errorToThrow.message, err: errorToThrow});
+            }
+
+            throw errorToThrow;
+        }
+    }
+}
+
+module.exports = DatabaseStateManager;

core/server/ghost-server.js

@@ -159,14 +159,18 @@ class GhostServer {
      * Stops the server, handles cleanup and exits the process = a full shutdown
      * Called on SIGINT or SIGTERM
      */
-    async shutdown() {
+    async shutdown(code = 0) {
         try {
             logging.warn(i18n.t('notices.httpServer.ghostIsShuttingDown'));
             await this.stop();
-            process.exit(0);
+            setTimeout(() => {
+                process.exit(code);
+            }, 100);
         } catch (error) {
             logging.error(error);
-            process.exit(1);
+            setTimeout(() => {
+                process.exit(1);
+            }, 100);
         }
     }
 

core/server/views/maintenance.html

@@ -0,0 +1,86 @@
+<!DOCTYPE html>
+<head>
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>We'll be right back</title>
+<style type="text/css">
+* {
+    box-sizing: border-box;
+}
+html {
+    font-size: 62.5%;
+    background: #f1f2f3;
+    -ms-text-size-adjust: 100%;
+    -webkit-text-size-adjust: 100%;
+    -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+}
+body {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+    height: 100vh;
+    width: 100vw;
+    margin: 0;
+    padding: 4vmin;
+    color: #15171A;
+    font-size: 2rem;
+    line-height: 1.4em;
+    font-family: sans-serif;
+    background: #f1f2f3;
+    scroll-behavior: smooth;
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+}
+
+::selection {
+    text-shadow: none;
+    background: #cbeafb;
+}
+
+.content {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    max-width: 500px;
+    min-height: 500px;
+    margin: 0 0 4vmin;
+    padding: 40px;
+    text-align: center;
+    background: #fff;
+    border-radius: 20px;
+    box-shadow:
+        0 50px 100px -20px rgb(50 50 93 / 8%),
+        0 30px 60px -30px rgb(0 0 0 / 13%),
+        0 10px 20px -10px rgb(0 0 0 / 8%);
+}
+h1 {
+    margin: 0 0 0.3em;
+    font-size: 4rem;
+    line-height: 1em;
+    font-weight: 700;
+    letter-spacing: -0.02em;
+}
+p {
+    margin: 0 0 40px;
+    opacity: 0.7;
+    font-weight: 400;
+}
+img {
+    display: block;
+    margin: 0 auto 40px;
+}
+@media (max-width: 500px) {
+    body { font-size: 1.8rem; }
+    h1 { font-size: 3.4rem; }
+}
+</style>
+</head>
+<body>
+<div class="content">
+    <h1>We'll be right back.</h1>
+    <p>We're busy updating our site to give you the best experience, and will be back soon.</p>
+</div>
+</body>
+</html>

ghost.js

@@ -0,0 +1,15 @@
+const argv = process.argv;
+
+const mode = argv[2] || 'new';
+
+// Switch between boot modes
+switch (mode) {
+case 'old':
+case '3':
+    // Old boot sequence
+    require('./startup');
+    break;
+default:
+    // New boot sequence
+    require('./core/boot')();
+}

index.js

@@ -1,42 +1 @@
-// # Ghost Startup
-// Orchestrates the startup of Ghost when run from command line.
-
-const startTime = Date.now();
-const debug = require('ghost-ignition').debug('boot:index');
-// Sentry must be initialised early on
-const sentry = require('./core/shared/sentry');
-
-debug('First requires...');
-
-const ghost = require('./core');
-
-debug('Required ghost');
-
-const express = require('./core/shared/express');
-const logging = require('./core/shared/logging');
-const urlService = require('./core/frontend/services/url');
-// This is what listen gets called on, it needs to be a full Express App
-const ghostApp = express('ghost');
-
-// Use the request handler at the top level
-// @TODO: decide if this should be here or in parent App - should it come after request id mw?
-ghostApp.use(sentry.requestHandler);
-
-debug('Initialising Ghost');
-
-ghost().then(function (ghostServer) {
-    // Mount our Ghost instance on our desired subdirectory path if it exists.
-    ghostApp.use(urlService.utils.getSubdir(), ghostServer.rootApp);
-
-    debug('Starting Ghost');
-    // Let Ghost handle starting our server instance.
-    return ghostServer.start(ghostApp)
-        .then(function afterStart() {
-            logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's');
-        });
-}).catch(function (err) {
-    logging.error(err);
-    setTimeout(() => {
-        process.exit(1);
-    }, 100);
-});
+require('./ghost');

startup.js

@@ -0,0 +1,44 @@
+// # Ghost Startup
+// Orchestrates the startup of Ghost when run from command line.
+
+const startTime = Date.now();
+const debug = require('ghost-ignition').debug('boot:index');
+// Sentry must be initialised early on
+const sentry = require('./core/shared/sentry');
+
+debug('First requires...');
+
+const ghost = require('./core');
+
+debug('Required ghost');
+
+const express = require('./core/shared/express');
+const logging = require('./core/shared/logging');
+const urlService = require('./core/frontend/services/url');
+
+logging.info('Boot Mode: 3.0');
+// This is what listen gets called on, it needs to be a full Express App
+const ghostApp = express('ghost');
+
+// Use the request handler at the top level
+// @TODO: decide if this should be here or in parent App - should it come after request id mw?
+ghostApp.use(sentry.requestHandler);
+
+debug('Initialising Ghost');
+
+ghost().then(function (ghostServer) {
+    // Mount our Ghost instance on our desired subdirectory path if it exists.
+    ghostApp.use(urlService.utils.getSubdir(), ghostServer.rootApp);
+
+    debug('Starting Ghost');
+    // Let Ghost handle starting our server instance.
+    return ghostServer.start(ghostApp)
+        .then(function afterStart() {
+            logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's');
+        });
+}).catch(function (err) {
+    logging.error(err);
+    setTimeout(() => {
+        process.exit(-1);
+    }, 100);
+});