✨ Added support for worker threads in scheduled jobs

no issue

- When jobs are performing CPU intensive tasks they block main process'
event loop. They also can cause memory leaks or unexpected crashes
effectively crashing the parent proccess. To address these issues jobs need to be performed off of main
process. Worker Threads (https://nodejs.org/dist/latest-v12.x/docs/api/worker_threads.html)
are the best candidate for such work.
- These changes introduce an integration on top of bree
(https://github.com/breejs/bree/) which allows to run recurring
jobs in worker thereads. It falls back to child process execution for
Node v10 running without `--experimental-worker` flag.
- bree was chosen not only because it gives a polyfill for older Node
versions. It has support for some of the future use-cases Ghost is looking to
implement, like scheduled jobs.
- This changeset also includes a complete example of job running on an
interval with a possibility for graceful shutdown

ghost/job-manager/lib/assemble-bree-job.js

@@ -0,0 +1,32 @@
+const isCronExpression = require('./is-cron-expression');
+
+const assemble = (when, job, data, name) => {
+    const breeJob = {
+        name: name,
+        // NOTE: both function and path syntaxes work with 'path' parameter
+        path: job,
+        outputWorkerMetadata: true
+    };
+
+    if (data) {
+        Object.assign(breeJob, {
+            worker: {
+                workerData: data
+            }
+        });
+    }
+
+    if (isCronExpression(when)) {
+        Object.assign(breeJob, {
+            cron: when
+        });
+    } else {
+        Object.assign(breeJob, {
+            interval: when
+        });
+    }
+
+    return breeJob;
+};
+
+module.exports = assemble;

ghost/job-manager/lib/job-manager.js

@@ -1,8 +1,11 @@
+const path = require('path');
 const fastq = require('fastq');
 const later = require('@breejs/later');
+const Bree = require('bree');
 const pWaitFor = require('p-wait-for');
 const errors = require('@tryghost/errors');
 const isCronExpression = require('./is-cron-expression');
+const assembleBreeJob = require('./assemble-bree-job');
 
 const worker = async (task, callback) => {
     try {
@@ -25,7 +28,13 @@ const handler = (error, result) => {
 class JobManager {
     constructor(logging) {
         this.queue = fastq(this, worker, 1);
-        this.schedule = [];
+
+        this.bree = new Bree({
+            root: false, // set this to `false` to prevent requiring a root directory of jobs
+            hasSeconds: true, // precission is needed to avoid task ovelaps after immidiate execution
+            logger: logging
+        });
+
         this.logging = logging;
     }
 
@@ -67,10 +76,19 @@ class JobManager {
      * @param {String} when - cron or human readable schedule format
      * @param {Function|String} job - function or path to a module defining a job
      * @param {Object} [data] - data to be passed into the job
+     * @param {String} [name] - job name
      */
-    scheduleJob(when, job, data) {
+    scheduleJob(when, job, data, name) {
         let schedule;
 
+        if (!name) {
+            if (typeof job === 'string') {
+                name = path.parse(job).name;
+            } else {
+                throw new Error('Name parameter should be present if job is a function');
+            }
+        }
+
         schedule = later.parse.text(when);
 
         if (isCronExpression(when)) {
@@ -83,21 +101,16 @@ class JobManager {
 
         this.logging.info(`Scheduling job. Next run on: ${later.schedule(schedule).next()}`);
 
-        const cancelInterval = later.setInterval(() => {
-            this.logging.info(`Scheduled job added to the queue.`);
-            this.addJob(job, data);
-        }, schedule);
-
-        this.schedule.push(cancelInterval);
+        const breeJob = assembleBreeJob(when, job, data, name);
+        this.bree.add(breeJob);
+        return this.bree.start(name);
     }
 
     /**
      * @param {import('p-wait-for').Options} [options]
      */
     async shutdown(options) {
-        this.schedule.forEach((cancelHandle) => {
-            cancelHandle.clear();
-        });
+        await this.bree.stop();
 
         if (this.queue.idle()) {
             return;

ghost/job-manager/package.json

@@ -26,6 +26,7 @@
   },
   "dependencies": {
     "@breejs/later": "4.0.2",
+    "bree": "3.4.0",
     "cron-validate": "1.4.0",
     "fastq": "1.9.0",
     "p-wait-for": "3.1.0"

ghost/job-manager/test/examples/graceful-shutdown.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+
+const pWaitFor = require('p-wait-for');
+const path = require('path');
+const setTimeoutPromise = require('util').promisify(setTimeout);
+const JobManager = require('../../lib/job-manager');
+
+const jobManager = new JobManager({
+    info: console.log,
+    warn: console.log,
+    error: console.log
+});
+
+process.on('SIGINT', () => {
+    shutdown('SIGINT');
+});
+
+async function shutdown(signal) {
+    console.log(`shutting down via: ${signal}`);
+
+    await jobManager.shutdown();
+}
+
+(async () => {
+    jobManager.scheduleJob('every 10 days', path.resolve(__dirname, '../jobs/graceful.js'));
+
+    await setTimeoutPromise(100); // allow job to get scheduled
+
+    await pWaitFor(() => Object.keys(jobManager.bree.workers).length === 0);
+
+    process.exit(0);
+})();

ghost/job-manager/test/job-manager.test.js

@@ -1,6 +1,7 @@
 // Switch these lines once there are useful utils
 // const testUtils = require('./utils');
 require('./utils');
+const path = require('path');
 const sinon = require('sinon');
 const delay = require('delay');
 
@@ -58,19 +59,29 @@ describe('Job Manager', function () {
     });
 
     describe('Schedule Job', function () {
-        it('fails to run for invalid scheduling expression', function () {
+        it('fails to schedule for invalid scheduling expression', function () {
+            const jobManager = new JobManager(logging);
+
+            try {
+                jobManager.scheduleJob('invalid expression', 'jobName', {});
+            } catch (err) {
+                err.message.should.equal('Invalid schedule format');
+            }
+        });
+
+        it('fails to schedule for no job name', function () {
             const jobManager = new JobManager(logging);
 
             try {
                 jobManager.scheduleJob('invalid expression', () => {}, {});
             } catch (err) {
-                err.message.should.equal('Invalid schedule format');
+                err.message.should.equal('Name parameter should be present if job is a function');
             }
         });
     });
 
     describe('Shutdown', function () {
-        it('gracefully shuts down synchronous jobs', async function () {
+        it('gracefully shuts down a synchronous jobs', async function () {
             const jobManager = new JobManager(logging);
 
             jobManager.addJob(require('./jobs/timed-job'), 200);
@@ -81,5 +92,19 @@ describe('Job Manager', function () {
 
             should(jobManager.queue.idle()).be.true();
         });
+
+        it('gracefully shuts down an interval job', async function () {
+            const jobManager = new JobManager(logging);
+
+            jobManager.scheduleJob('every 5 seconds', path.resolve(__dirname, './jobs/graceful.js'));
+
+            await delay(1); // let the job execution kick in
+
+            should(Object.keys(jobManager.bree.workers).length).equal(1);
+
+            await jobManager.shutdown();
+
+            should(Object.keys(jobManager.bree.workers).length).equal(0);
+        });
     });
 });

ghost/job-manager/test/jobs/graceful.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+
+const setTimeoutPromise = require('util').promisify(setTimeout);
+const threads = require('bthreads');
+
+let shutdown = false;
+
+if (!threads.isMainThread) {
+    threads.parentPort.on('message', (message) => {
+        console.log(`paret message received: ${message}`);
+        if (message === 'cancel') {
+            shutdown = true;
+        }
+    });
+}
+
+(async () => {
+    console.log('started graceful job');
+
+    for (;;) {
+        await setTimeoutPromise(1000);
+        console.log('worked for 1000 ms');
+
+        if (shutdown) {
+            console.log('exiting gracefully');
+
+            await setTimeoutPromise(100); // async cleanup imitation
+
+            process.exit(0);
+        }
+    }
+})();