From 2c1417da882d7e860124d5ba38aa1d6616c083d2 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Tue, 11 Aug 2020 21:13:31 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Added=20new=20job=20manager=20packa?= =?UTF-8?q?ge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Minimal working version of a job manager - Uses fastq for handling the queue - Exposes 2 methods: addJob and shutdown --- ghost/job-manager/.eslintrc.js | 6 +++ ghost/job-manager/LICENSE | 21 ++++++++++ ghost/job-manager/README.md | 39 ++++++++++++++++++ ghost/job-manager/index.js | 1 + ghost/job-manager/lib/job-manager.js | 46 ++++++++++++++++++++++ ghost/job-manager/package.json | 30 ++++++++++++++ ghost/job-manager/test/.eslintrc.js | 6 +++ ghost/job-manager/test/hello.test.js | 10 +++++ ghost/job-manager/test/utils/assertions.js | 11 ++++++ ghost/job-manager/test/utils/index.js | 11 ++++++ ghost/job-manager/test/utils/overrides.js | 10 +++++ 11 files changed, 191 insertions(+) create mode 100644 ghost/job-manager/.eslintrc.js create mode 100644 ghost/job-manager/LICENSE create mode 100644 ghost/job-manager/README.md create mode 100644 ghost/job-manager/index.js create mode 100644 ghost/job-manager/lib/job-manager.js create mode 100644 ghost/job-manager/package.json create mode 100644 ghost/job-manager/test/.eslintrc.js create mode 100644 ghost/job-manager/test/hello.test.js create mode 100644 ghost/job-manager/test/utils/assertions.js create mode 100644 ghost/job-manager/test/utils/index.js create mode 100644 ghost/job-manager/test/utils/overrides.js diff --git a/ghost/job-manager/.eslintrc.js b/ghost/job-manager/.eslintrc.js new file mode 100644 index 0000000000..c9c1bcb522 --- /dev/null +++ b/ghost/job-manager/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ] +}; diff --git a/ghost/job-manager/LICENSE b/ghost/job-manager/LICENSE new file mode 100644 index 0000000000..a8ebdea81d --- /dev/null +++ b/ghost/job-manager/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2020 Ghost Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ghost/job-manager/README.md b/ghost/job-manager/README.md new file mode 100644 index 0000000000..84baa4b0a0 --- /dev/null +++ b/ghost/job-manager/README.md @@ -0,0 +1,39 @@ +# Job Manager + +## Install + +`npm install @tryghost/job-manager --save` + +or + +`yarn add @tryghost/job-manager` + + +## Usage + + +## Develop + +This is a mono repository, managed with [lerna](https://lernajs.io/). + +Follow the instructions for the top-level repo. +1. `git clone` this repo & `cd` into it as usual +2. Run `yarn` to install top-level dependencies. + + +## Run + +- `yarn dev` + + +## Test + +- `yarn lint` run just eslint +- `yarn test` run lint and tests + + + + +# Copyright & License + +Copyright (c) 2020 Ghost Foundation - Released under the [MIT license](LICENSE). \ No newline at end of file diff --git a/ghost/job-manager/index.js b/ghost/job-manager/index.js new file mode 100644 index 0000000000..34f8bdac0c --- /dev/null +++ b/ghost/job-manager/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/job-manager'); diff --git a/ghost/job-manager/lib/job-manager.js b/ghost/job-manager/lib/job-manager.js new file mode 100644 index 0000000000..52648d647b --- /dev/null +++ b/ghost/job-manager/lib/job-manager.js @@ -0,0 +1,46 @@ +const fastq = require('fastq'); +const pWaitFor = require('p-wait-for'); + +const worker = async (task, callback) => { + try { + let result = await task(); + callback(null, result); + } catch (error) { + callback(error); + } +}; + +const handler = (error, result) => { + if (error) { + throw error; + } + // Can potentially standardise the result here + return result; +}; + +class JobManager { + constructor(logging) { + this.queue = fastq(this, worker, 1); + this.logging = logging; + } + + addJob(job, data) { + this.queue.push(async () => { + await job(data); + }, handler); + } + + async shutdown(options) { + if (this.queue.idle()) { + return; + } + + this.logging.warn('Waiting for busy job queue'); + + await pWaitFor(() => this.queue.idle() === true, options); + + this.logging.warn('Job queue finished'); + } +} + +module.exports = JobManager; diff --git a/ghost/job-manager/package.json b/ghost/job-manager/package.json new file mode 100644 index 0000000000..5c06272f17 --- /dev/null +++ b/ghost/job-manager/package.json @@ -0,0 +1,30 @@ +{ + "name": "@tryghost/job-manager", + "version": "0.0.0", + "repository": "https://github.com/TryGhost/Ghost-Utils/tree/master/packages/job-manager", + "author": "Ghost Foundation", + "license": "MIT", + "main": "index.js", + "scripts": { + "dev": "echo \"Implement me!\"", + "test": "NODE_ENV=testing mocha './test/**/*.test.js'", + "lint": "eslint . --ext .js --cache", + "posttest": "yarn lint" + }, + "files": [ + "index.js", + "lib" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "mocha": "8.1.1", + "should": "13.2.3", + "sinon": "9.0.2" + }, + "dependencies": { + "fastq": "1.8.0", + "p-wait-for": "3.1.0" + } +} diff --git a/ghost/job-manager/test/.eslintrc.js b/ghost/job-manager/test/.eslintrc.js new file mode 100644 index 0000000000..829b601eb0 --- /dev/null +++ b/ghost/job-manager/test/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/test' + ] +}; diff --git a/ghost/job-manager/test/hello.test.js b/ghost/job-manager/test/hello.test.js new file mode 100644 index 0000000000..85d69d1e08 --- /dev/null +++ b/ghost/job-manager/test/hello.test.js @@ -0,0 +1,10 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); + +describe('Hello world', function () { + it('Runs a test', function () { + // TODO: Write me! + 'hello'.should.eql('hello'); + }); +}); diff --git a/ghost/job-manager/test/utils/assertions.js b/ghost/job-manager/test/utils/assertions.js new file mode 100644 index 0000000000..7364ee8aa1 --- /dev/null +++ b/ghost/job-manager/test/utils/assertions.js @@ -0,0 +1,11 @@ +/** + * Custom Should Assertions + * + * Add any custom assertions to this file. + */ + +// Example Assertion +// should.Assertion.add('ExampleAssertion', function () { +// this.params = {operator: 'to be a valid Example Assertion'}; +// this.obj.should.be.an.Object; +// }); diff --git a/ghost/job-manager/test/utils/index.js b/ghost/job-manager/test/utils/index.js new file mode 100644 index 0000000000..0d67d86ff8 --- /dev/null +++ b/ghost/job-manager/test/utils/index.js @@ -0,0 +1,11 @@ +/** + * Test Utilities + * + * Shared utils for writing tests + */ + +// Require overrides - these add globals for tests +require('./overrides'); + +// Require assertions - adds custom should assertions +require('./assertions'); diff --git a/ghost/job-manager/test/utils/overrides.js b/ghost/job-manager/test/utils/overrides.js new file mode 100644 index 0000000000..90203424ee --- /dev/null +++ b/ghost/job-manager/test/utils/overrides.js @@ -0,0 +1,10 @@ +// This file is required before any test is run + +// Taken from the should wiki, this is how to make should global +// Should is a global in our eslint test config +global.should = require('should').noConflict(); +should.extend(); + +// Sinon is a simple case +// Sinon is a global in our eslint test config +global.sinon = require('sinon');