From 0b79abf5b252ab561a6024368d663eeff8da8727 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Tue, 2 Feb 2021 14:47:16 +0000 Subject: [PATCH] Added new, simpler, linear boot process Background: - Ghosts existing boot process is split across multiple files, has affordances for outdated ways of running Ghost and is generally non-linear making it nigh-impossible to follow - The web of dependencies that are loaded on boot are also impossible to unpick, which makes it really hard to decouple Ghost - With 4.0 we want to introduce a new, linear, simpler, clearer way to boot up Ghost to unlock decoupling Ghost into much smaller pieces This commit: - adds a new ghost.js file which switches between boot mode with `node index` or `node index old` so that if we find bugs we can work around them this week - Note: the old boot process will go away very soon, but ghost.js will remain as the interface between the command to start Ghost and the application code - reworks the database migration process into a standalone utility, so that the DB is handled as one simple step of the boot process, decoupled from everything else - is missing tests for this new db utility - leaves a lot of work to do around loading core code, services, express apps in a sensible order, as work to fix this would start to break the old boot process - doesn't use the new maintenance app because we aren't restarting the server here, instead we have the concept of a "core app" that starts in maintenance mode - need to think about how apps will be decoupled in the near future --- core/app.js | 22 ++++ core/boot.js | 184 +++++++++++++++++++++++++++ core/db.js | 8 ++ core/server/data/db/state-manager.js | 106 +++++++++++++++ core/server/ghost-server.js | 10 +- core/server/views/maintenance.html | 86 +++++++++++++ ghost.js | 15 +++ index.js | 43 +------ startup.js | 44 +++++++ 9 files changed, 473 insertions(+), 45 deletions(-) create mode 100644 core/app.js create mode 100644 core/boot.js create mode 100644 core/db.js create mode 100644 core/server/data/db/state-manager.js create mode 100644 core/server/views/maintenance.html create mode 100644 ghost.js create mode 100644 startup.js diff --git a/core/app.js b/core/app.js new file mode 100644 index 0000000000..c79f3bb3b9 --- /dev/null +++ b/core/app.js @@ -0,0 +1,22 @@ +const express = require('./shared/express'); +const rootApp = express('root'); + +const fs = require('fs'); +const path = require('path'); + +// We never want middleware functions to be anonymous +const maintenanceMiddleware = (req, res, next) => { + if (!req.app.get('maintenance')) { + return next(); + } + res.set({ + 'Cache-Control': 'no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0' + }); + res.writeHead(503, {'content-type': 'text/html'}); + fs.createReadStream(path.resolve(__dirname, './server/views/maintenance.html')).pipe(res); +}; + +rootApp.enable('maintenance'); +rootApp.use(maintenanceMiddleware); + +module.exports = rootApp; diff --git a/core/boot.js b/core/boot.js new file mode 100644 index 0000000000..dffe572e4d --- /dev/null +++ b/core/boot.js @@ -0,0 +1,184 @@ +// # The Ghost Boot Sequence + +// IMPORTANT: The only global requires here should be debug + overrides +const debug = require('ghost-ignition').debug('boot'); +require('./server/overrides'); +// END OF GLOBAL REQUIRES + +const initCore = async ({ghostServer}) => { + const settings = require('./server/services/settings'); + const jobService = require('./server/services/jobs'); + const models = require('./server/models'); + const {events, i18n} = require('./server/lib/common'); + + ghostServer.registerCleanupTask(async () => { + await jobService.shutdown(); + }); + + // Initialize Ghost core internationalization + i18n.init(); + debug('Default i18n done for core'); + + models.init(); + debug('Models done'); + + await settings.init(); + + // @TODO: fix this - has to happen before db.ready is emitted + debug('Begin: Url Service'); + require('./frontend/services/url'); + debug('End: Url Service'); + + // @TODO: fix this location + events.emit('db.ready'); +}; + +const initExpressApps = async () => { + debug('Begin: initExpressApps'); + const themeService = require('./frontend/services/themes'); + const frontendSettings = require('./frontend/services/settings'); + + await frontendSettings.init(); + debug('Frontend settings done'); + + await themeService.init(); + debug('Themes done'); + + const parentApp = require('./server/web/parent/app')(); + + debug('End: initExpressApps'); + return parentApp; +}; + +const initServices = async ({config}) => { + debug('Begin: initialiseServices'); + const themeService = require('./frontend/services/themes'); + const frontendSettings = require('./frontend/services/settings'); + const appService = require('./frontend/services/apps'); + const urlUtils = require('./shared/url-utils'); + + // CASE: When Ghost is ready with bootstrapping (db migrations etc.), we can trigger the router creation. + // Reason is that the routers access the routes.yaml, which shouldn't and doesn't have to be validated to + // start Ghost in maintenance mode. + // Routing is a bridge between the frontend and API + const routing = require('./frontend/services/routing'); + // We pass the themeService API version here, so that the frontend services are less tightly-coupled + routing.bootstrap.start(themeService.getApiVersion()); + + const settings = require('./server/services/settings'); + const permissions = require('./server/services/permissions'); + const xmlrpc = require('./server/services/xmlrpc'); + const slack = require('./server/services/slack'); + const {mega} = require('./server/services/mega'); + const webhooks = require('./server/services/webhooks'); + const scheduling = require('./server/adapters/scheduling'); + const getRoutesHash = () => frontendSettings.getCurrentHash('routes'); + + await Promise.all([ + // Initialize the permissions actions and objects + permissions.init(), + xmlrpc.listen(), + slack.listen(), + mega.listen(), + webhooks.listen(), + settings.syncRoutesHash(getRoutesHash), + appService.init(), + scheduling.init({ + // NOTE: When changing API version need to consider how to migrate custom scheduling adapters + // that rely on URL to lookup persisted scheduled records (jobs, etc.). Ref: https://github.com/TryGhost/Ghost/pull/10726#issuecomment-489557162 + apiUrl: urlUtils.urlFor('api', {version: 'v3', versionType: 'admin'}, true) + }) + ]); + + debug('XMLRPC, Slack, MEGA, Webhooks, Scheduling, Permissions done'); + + // Initialise analytics events + if (config.get('segment:key')) { + require('./analytics-events').init(); + } + + debug('End: initialiseServices'); +}; + +const mountGhost = (rootApp, ghostApp) => { + const urlService = require('./frontend/services/url'); + rootApp.disable('maintenance'); + rootApp.use(urlService.utils.getSubdir(), ghostApp); +}; + +const bootGhost = async () => { + // Metrics & debugging + const startTime = Date.now(); + let ghostServer; + + try { + // Config is the absolute first thing to do! + debug('Begin: Load config'); + const config = require('./shared/config'); + debug('End: Load config'); + + debug('Begin: Load version info'); + const version = require('./server/lib/ghost-version'); + config.set('version', version); + debug('End: Load version info'); + + debug('Begin: load server + minimal app'); + process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + // Get minimal application in maintenance mode + const rootApp = require('./app'); + + // Start server with minimal App + const GhostServer = require('./server/ghost-server'); + ghostServer = new GhostServer(); + await ghostServer.start(rootApp); + + const logging = require('./shared/logging'); + logging.info('Ghost server start', (Date.now() - startTime) / 1000 + 's'); + debug('End: load server + minimal app'); + + debug('Begin: Get DB ready'); + // Get the DB ready + await require('./db').ready(); + debug('End: Get DB ready'); + + // Load Ghost with all its services + debug('Begin: Load Ghost Core Services'); + await initCore({ghostServer}); + + const ghostApp = await initExpressApps({}); + await initServices({config}); + debug('End: Load Ghost Core Services'); + + // Mount the full Ghost app onto the minimal root app & disable maintenance mode + mountGhost(rootApp, ghostApp); + + // Announce Server Readiness + logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's'); + GhostServer.announceServerReadiness(); + } catch (error) { + const errors = require('@tryghost/errors'); + // @TODO: fix these extra requires + const GhostServer = require('./server/ghost-server'); + const logging = require('./shared/logging'); + + let serverStartError = error; + + if (!errors.utils.isIgnitionError(serverStartError)) { + serverStartError = new errors.GhostError({message: serverStartError.message, err: serverStartError}); + } + + logging.error(serverStartError); + GhostServer.announceServerReadiness(serverStartError); + + if (ghostServer) { + ghostServer.shutdown(2); + } else { + setTimeout(() => { + process.exit(2); + }, 100); + } + } +}; + +module.exports = bootGhost; diff --git a/core/db.js b/core/db.js new file mode 100644 index 0000000000..bbcf5ea340 --- /dev/null +++ b/core/db.js @@ -0,0 +1,8 @@ +const config = require('./shared/config'); +const logging = require('./shared/logging'); + +module.exports.ready = async () => { + const DatabaseStateManager = require('./server/data/db/state-manager'); + const dbStateManager = new DatabaseStateManager({knexMigratorFilePath: config.get('paths:appRoot')}); + await dbStateManager.makeReady({logging}); +}; diff --git a/core/server/data/db/state-manager.js b/core/server/data/db/state-manager.js new file mode 100644 index 0000000000..efa7f19e94 --- /dev/null +++ b/core/server/data/db/state-manager.js @@ -0,0 +1,106 @@ +const KnexMigrator = require('knex-migrator'); +const errors = require('@tryghost/errors'); + +const states = { + READY: 0, + NEEDS_INITIALISATION: 1, + NEEDS_MIGRATION: 2, + ERROR: 3 +}; + +const printState = ({state, logging}) => { + if (state === states.READY) { + logging.info('Database is in a ready state.'); + } + + if (state === states.NEEDS_INITIALISATION) { + logging.warn('Database state requires initialisation.'); + } + + if (state === states.NEEDS_MIGRATION) { + logging.warn('Database state requires migration.'); + } + + if (state === states.ERROR) { + logging.error('Database is in an error state.'); + } +}; + +class DatabaseStateManager { + constructor({knexMigratorFilePath}) { + this.knexMigrator = new KnexMigrator({ + knexMigratorFilePath + }); + } + + async getState() { + let state = states.READY; + try { + await this.knexMigrator.isDatabaseOK(); + return state; + } catch (error) { + // CASE: database has not yet been initialised + if (error.code === 'DB_NOT_INITIALISED') { + state = states.NEEDS_INITIALISATION; + return state; + } + + // CASE: there's no migration table so we can't understand + if (error.code === 'MIGRATION_TABLE_IS_MISSING') { + state = states.NEEDS_INITIALISATION; + return state; + } + + // CASE: database needs migrations + if (error.code === 'DB_NEEDS_MIGRATION') { + state = states.NEEDS_MIGRATION; + return state; + } + + // CASE: database connection errors, unknown cases + let errorToThrow = error; + if (!errors.utils.isIgnitionError(errorToThrow)) { + errorToThrow = new errors.GhostError({message: errorToThrow.message, err: errorToThrow}); + } + + throw errorToThrow; + } + } + + async makeReady({logging}) { + try { + let state = await this.getState(); + + if (logging) { + printState({state, logging}); + } + + if (state === states.READY) { + return; + } + + if (state === states.NEEDS_INITIALISATION) { + await this.knexMigrator.init(); + } + + if (state === states.NEEDS_MIGRATION) { + await this.knexMigrator.migrate(); + } + + state = await this.getState(); + + if (logging) { + printState({state, logging}); + } + } catch (error) { + let errorToThrow = error; + if (!errors.utils.isIgnitionError(error)) { + errorToThrow = new errors.GhostError({message: errorToThrow.message, err: errorToThrow}); + } + + throw errorToThrow; + } + } +} + +module.exports = DatabaseStateManager; diff --git a/core/server/ghost-server.js b/core/server/ghost-server.js index 3d03e929d6..04c58996b0 100644 --- a/core/server/ghost-server.js +++ b/core/server/ghost-server.js @@ -159,14 +159,18 @@ class GhostServer { * Stops the server, handles cleanup and exits the process = a full shutdown * Called on SIGINT or SIGTERM */ - async shutdown() { + async shutdown(code = 0) { try { logging.warn(i18n.t('notices.httpServer.ghostIsShuttingDown')); await this.stop(); - process.exit(0); + setTimeout(() => { + process.exit(code); + }, 100); } catch (error) { logging.error(error); - process.exit(1); + setTimeout(() => { + process.exit(1); + }, 100); } } diff --git a/core/server/views/maintenance.html b/core/server/views/maintenance.html new file mode 100644 index 0000000000..17a4dd3dc0 --- /dev/null +++ b/core/server/views/maintenance.html @@ -0,0 +1,86 @@ + + + + + +We'll be right back + + + +
+

We'll be right back.

+

We're busy updating our site to give you the best experience, and will be back soon.

+
+ + diff --git a/ghost.js b/ghost.js new file mode 100644 index 0000000000..d05f0b897a --- /dev/null +++ b/ghost.js @@ -0,0 +1,15 @@ +const argv = process.argv; + +const mode = argv[2] || 'new'; + +// Switch between boot modes +switch (mode) { +case 'old': +case '3': + // Old boot sequence + require('./startup'); + break; +default: + // New boot sequence + require('./core/boot')(); +} diff --git a/index.js b/index.js index d215670413..3eb69a6823 100644 --- a/index.js +++ b/index.js @@ -1,42 +1 @@ -// # Ghost Startup -// Orchestrates the startup of Ghost when run from command line. - -const startTime = Date.now(); -const debug = require('ghost-ignition').debug('boot:index'); -// Sentry must be initialised early on -const sentry = require('./core/shared/sentry'); - -debug('First requires...'); - -const ghost = require('./core'); - -debug('Required ghost'); - -const express = require('./core/shared/express'); -const logging = require('./core/shared/logging'); -const urlService = require('./core/frontend/services/url'); -// This is what listen gets called on, it needs to be a full Express App -const ghostApp = express('ghost'); - -// Use the request handler at the top level -// @TODO: decide if this should be here or in parent App - should it come after request id mw? -ghostApp.use(sentry.requestHandler); - -debug('Initialising Ghost'); - -ghost().then(function (ghostServer) { - // Mount our Ghost instance on our desired subdirectory path if it exists. - ghostApp.use(urlService.utils.getSubdir(), ghostServer.rootApp); - - debug('Starting Ghost'); - // Let Ghost handle starting our server instance. - return ghostServer.start(ghostApp) - .then(function afterStart() { - logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's'); - }); -}).catch(function (err) { - logging.error(err); - setTimeout(() => { - process.exit(1); - }, 100); -}); +require('./ghost'); diff --git a/startup.js b/startup.js new file mode 100644 index 0000000000..1cde9e5f1a --- /dev/null +++ b/startup.js @@ -0,0 +1,44 @@ +// # Ghost Startup +// Orchestrates the startup of Ghost when run from command line. + +const startTime = Date.now(); +const debug = require('ghost-ignition').debug('boot:index'); +// Sentry must be initialised early on +const sentry = require('./core/shared/sentry'); + +debug('First requires...'); + +const ghost = require('./core'); + +debug('Required ghost'); + +const express = require('./core/shared/express'); +const logging = require('./core/shared/logging'); +const urlService = require('./core/frontend/services/url'); + +logging.info('Boot Mode: 3.0'); +// This is what listen gets called on, it needs to be a full Express App +const ghostApp = express('ghost'); + +// Use the request handler at the top level +// @TODO: decide if this should be here or in parent App - should it come after request id mw? +ghostApp.use(sentry.requestHandler); + +debug('Initialising Ghost'); + +ghost().then(function (ghostServer) { + // Mount our Ghost instance on our desired subdirectory path if it exists. + ghostApp.use(urlService.utils.getSubdir(), ghostServer.rootApp); + + debug('Starting Ghost'); + // Let Ghost handle starting our server instance. + return ghostServer.start(ghostApp) + .then(function afterStart() { + logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's'); + }); +}).catch(function (err) { + logging.error(err); + setTimeout(() => { + process.exit(-1); + }, 100); +});