mirror of
https://github.com/microsoft/playwright.git
synced 2024-10-27 13:50:25 +03:00
devops: migrate flakiness dashboard to the new folio reporter format (#6089)
New folio changed the JSON report, so we have to keep with the changes. The most notable changes: - there are no parameters any more. We recreate these as `testInfo.data` arguments that are saved under test result's data. - `test.runs` are renamed into `test.results` I didn't find other changes so far - let's see if this works in the cloud!
This commit is contained in:
parent
6a767d1a9c
commit
1a44f68155
@ -162,6 +162,18 @@ export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
|
|||||||
async beforeEach(testInfo: TestInfo) {
|
async beforeEach(testInfo: TestInfo) {
|
||||||
// Different screenshots per browser.
|
// Different screenshots per browser.
|
||||||
testInfo.snapshotPathSegment = this._browserName;
|
testInfo.snapshotPathSegment = this._browserName;
|
||||||
|
testInfo.data = {
|
||||||
|
browserName: this._browserName,
|
||||||
|
};
|
||||||
|
const headful = !this._browserOptions.headless;
|
||||||
|
if (headful)
|
||||||
|
testInfo.data.headful = true;
|
||||||
|
if (this._options.mode !== 'default')
|
||||||
|
testInfo.data.mode = this._options.mode;
|
||||||
|
if (this._options.video)
|
||||||
|
testInfo.data.video = true;
|
||||||
|
if (this._options.trace)
|
||||||
|
testInfo.data.trace = true;
|
||||||
return {
|
return {
|
||||||
playwright: this._playwright,
|
playwright: this._playwright,
|
||||||
browserName: this._browserName,
|
browserName: this._browserName,
|
||||||
@ -174,7 +186,7 @@ export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
|
|||||||
isWindows: os.platform() === 'win32',
|
isWindows: os.platform() === 'win32',
|
||||||
isMac: os.platform() === 'darwin',
|
isMac: os.platform() === 'darwin',
|
||||||
isLinux: os.platform() === 'linux',
|
isLinux: os.platform() === 'linux',
|
||||||
headful: !this._browserOptions.headless,
|
headful,
|
||||||
video: !!this._options.video,
|
video: !!this._options.video,
|
||||||
mode: this._options.mode,
|
mode: this._options.mode,
|
||||||
platform: os.platform() as ('win32' | 'darwin' | 'linux'),
|
platform: os.platform() as ('win32' | 'darwin' | 'linux'),
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
# Flakiness Dashboard Backend
|
# Flakiness Dashboard Backend
|
||||||
|
|
||||||
This directory contains source code for the Azure function that we use to aggregate test reports.
|
This directory contains source code for the Azure function that we use to aggregate test reports.
|
||||||
The data is consumed by https://devops.aslushnikov.com/flakiness.html
|
The data is consumed by https://devops.aslushnikov.com/flakiness2.html
|
||||||
|
@ -51,21 +51,31 @@ function compressReports(reports) {
|
|||||||
specs.set(specId, specObject);
|
specs.set(specId, specObject);
|
||||||
}
|
}
|
||||||
for (const test of spec.tests || []) {
|
for (const test of spec.tests || []) {
|
||||||
if (test.runs.length === 1 && !test.runs[0].status)
|
// It's unclear how many results we get in the new test runner - let's
|
||||||
|
// stay on the safe side and skip test without any results.
|
||||||
|
if (!test.results || !test.results.length)
|
||||||
continue;
|
continue;
|
||||||
|
// We get tests with a single result without status for sharded
|
||||||
|
// tests that are inside shard that we don't run.
|
||||||
|
if (test.results.length === 1 && !test.results[0].status)
|
||||||
|
continue;
|
||||||
|
// Folio currently reports `data` as part of test results.
|
||||||
|
// In our case, all data will be identical - so pick
|
||||||
|
// from the first result.
|
||||||
|
const testParameters = test.results[0].data;
|
||||||
// Overwrite test platform parameter with a more specific information from
|
// Overwrite test platform parameter with a more specific information from
|
||||||
// build run.
|
// build run.
|
||||||
const osName = report.metadata.osName.toUpperCase().startsWith('MINGW') ? 'Windows' : report.metadata.osName;
|
const osName = report.metadata.osName.toUpperCase().startsWith('MINGW') ? 'Windows' : report.metadata.osName;
|
||||||
const arch = report.metadata.arch && !report.metadata.arch.includes('x86') ? report.metadata.arch : '';
|
const arch = report.metadata.arch && !report.metadata.arch.includes('x86') ? report.metadata.arch : '';
|
||||||
const platform = (osName + ' ' + report.metadata.osVersion + ' ' + arch).trim();
|
const platform = (osName + ' ' + report.metadata.osVersion + ' ' + arch).trim();
|
||||||
const browserName = test.parameters.browserName || 'N/A';
|
const browserName = testParameters.browserName || 'N/A';
|
||||||
|
|
||||||
const testName = getTestName(browserName, platform, test.parameters);
|
const testName = getTestName(browserName, platform, testParameters);
|
||||||
let testObject = specObject.tests.get(testName);
|
let testObject = specObject.tests.get(testName);
|
||||||
if (!testObject) {
|
if (!testObject) {
|
||||||
testObject = {
|
testObject = {
|
||||||
parameters: {
|
parameters: {
|
||||||
...test.parameters,
|
...testParameters,
|
||||||
browserName,
|
browserName,
|
||||||
platform,
|
platform,
|
||||||
},
|
},
|
||||||
@ -78,7 +88,7 @@ function compressReports(reports) {
|
|||||||
specObject.tests.set(testName, testObject);
|
specObject.tests.set(testName, testObject);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const run of test.runs) {
|
for (const run of test.results) {
|
||||||
// Record duration of slow tests only, i.e. > 1s.
|
// Record duration of slow tests only, i.e. > 1s.
|
||||||
if (run.status === 'passed' && run.duration > 1000) {
|
if (run.status === 'passed' && run.duration > 1000) {
|
||||||
testObject.minTime = Math.min((testObject.minTime || Number.MAX_VALUE), run.duration);
|
testObject.minTime = Math.min((testObject.minTime || Number.MAX_VALUE), run.duration);
|
||||||
|
Loading…
Reference in New Issue
Block a user