Enabled and tracked retries in CI tests (#16909)

ghost/core/package.json

@@ -25,14 +25,15 @@
     "build": "yarn build:css",
     "build:css": "postcss core/frontend/public/ghost.css --no-map --use cssnano -o core/frontend/public/ghost.min.css",
     "test": "yarn test:unit",
+    "test:base": "mocha --require=./test/utils/overrides.js --exit --trace-warnings --recursive --extension=test.js --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",
     "test:single": "mocha --require=./test/utils/overrides.js --exit --trace-warnings --recursive --extension=test.js --timeout=60000",
     "test:all": "yarn test:unit && yarn test:integration && yarn test:e2e && yarn lint",
     "test:debug": "DEBUG=ghost:test* yarn test",
     "test:unit": "c8 yarn test:unit:base",
-    "test:unit:base": "mocha --require=./test/utils/overrides.js --exit --trace-warnings --recursive --extension=test.js './test/unit' --timeout=2000",
+    "test:unit:base": "yarn test:base './test/unit' --timeout=2000",
-    "test:integration": "mocha --require=./test/utils/overrides.js --exit --trace-warnings --recursive --extension=test.js  './test/integration' --timeout=10000",
+    "test:integration": "yarn test:base './test/integration' --timeout=10000",
-    "test:e2e": "mocha --require=./test/utils/overrides.js --exit --trace-warnings --recursive --extension=test.js './test/e2e-api' './test/e2e-frontend' './test/e2e-server' './test/e2e-webhooks' --timeout=15000",
+    "test:e2e": "yarn test:base ./test/e2e-* --timeout=15000",
-    "test:regression": "mocha --require=./test/utils/overrides.js --exit --trace-warnings --recursive --extension=test.js  './test/regression' --timeout=60000",
+    "test:regression": "yarn test:base './test/regression' --timeout=60000",
     "test:browser": "yarn test:browser:admin && yarn test:browser:portal",
     "test:browser:admin": "NODE_ENV=testing-browser  playwright test test/e2e-browser --project=admin",
     "test:browser:portal": "NODE_ENV=testing-browser playwright test test/e2e-browser --project=portal",
@@ -224,6 +225,7 @@
     "sqlite3": "5.1.6"
   },
   "devDependencies": {
+    "@actions/core": "1.10.0",
     "@playwright/test": "1.34.3",
     "@tryghost/express-test": "0.13.4",
     "@tryghost/webhook-mock-receiver": "0.2.6",

ghost/core/test/utils/mocha-retry-reporter.js

@@ -0,0 +1,67 @@
+const _ = require('lodash');
+const core = require('@actions/core');
+const mocha = require('mocha');
+
+// From https://github.com/findmypast-oss/mocha-json-streamier-reporter/blob/master/lib/parse-stack-trace.js
+function extractModuleLineAndColumn(stackTrace) {
+    const matches = /^\s*at Context.* \(([^\(\)]+):([0-9]+):([0-9]+)\)/gm.exec(stackTrace);
+
+    if (matches === null) {
+        return {};
+    }
+
+    return {
+        file: matches[1],
+        line: parseInt(matches[2]),
+        col: parseInt(matches[3])
+    };
+}
+
+module.exports = class RetryReporter extends mocha.reporters.Spec {
+    constructor(runner) {
+        super(runner);
+
+        if (!process.env.CI || !process.env.GITHUB_ACTIONS) {
+            return;
+        }
+
+        if (runner.suite.retries() === 0) {
+            return;
+        }
+
+        const retriedTests = [];
+        const failedTests = [];
+
+        runner
+            .on(mocha.Runner.constants.EVENT_TEST_RETRY, (test, err) => {
+                // eslint-disable-next-line no-console
+                console.log(`Retrying '${test.fullTitle()}' due to '${err.message}'`);
+                retriedTests.push({test, err});
+            })
+            .on(mocha.Runner.constants.EVENT_TEST_FAIL, (test, err) => {
+                failedTests.push({test, err});
+            })
+            .on(mocha.Runner.constants.EVENT_RUN_END, () => {
+                _.uniqWith(retriedTests
+                    .filter(({test}) => !failedTests.find(failed => failed.test === test))
+                    .map(({test, err}) => {
+                        const {file, line, col} = extractModuleLineAndColumn(err.stack);
+
+                        return {
+                            file,
+                            line,
+                            col,
+                            err: err.message,
+                            testName: test.fullTitle()
+                        };
+                    }), _.isEqual)
+                    .forEach(({file, line, col, testName, err}) => {
+                        core.warning(`Retried '${testName}' due to '${err}'`, {
+                            file,
+                            startLine: line,
+                            startColumn: col
+                        });
+                    });
+            });
+    }
+};

yarn.lock

@@ -2,6 +2,21 @@
 # yarn lockfile v1
 
 
+"@actions/core@1.10.0":
+  version "1.10.0"
+  resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.10.0.tgz#44551c3c71163949a2f06e94d9ca2157a0cfac4f"
+  integrity sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==
+  dependencies:
+    "@actions/http-client" "^2.0.1"
+    uuid "^8.3.2"
+
+"@actions/http-client@^2.0.1":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@actions/http-client/-/http-client-2.1.0.tgz#b6d8c3934727d6a50d10d19f00a711a964599a9f"
+  integrity sha512-BonhODnXr3amchh4qkmjPMUO8mFi/zLaaCeCAJZqch8iQqyDnVIkySjB38VHAC8IJ+bnlgfOqlhpyCUZHlQsqw==
+  dependencies:
+    tunnel "^0.0.6"
+
 "@adobe/css-tools@^4.0.1":
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.2.0.tgz#e1a84fca468f4b337816fcb7f0964beb620ba855"
@@ -31804,6 +31819,11 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+tunnel@^0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
+  integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"