devops: migrate flakiness dashboard to the new folio reporter format (#6089)

New folio changed the JSON report, so we have to keep with the changes.

The most notable changes:
- there are no parameters any more. We recreate these as `testInfo.data`
  arguments that are saved under test result's data.
- `test.runs` are renamed into `test.results`

I didn't find other changes so far - let's see if this works in the
cloud!

tests/config/browserEnv.ts

@@ -162,6 +162,18 @@ export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
   async beforeEach(testInfo: TestInfo) {
     // Different screenshots per browser.
     testInfo.snapshotPathSegment = this._browserName;
+    testInfo.data = {
+      browserName: this._browserName,
+    };
+    const headful = !this._browserOptions.headless;
+    if (headful)
+      testInfo.data.headful = true;
+    if (this._options.mode !== 'default')
+      testInfo.data.mode = this._options.mode;
+    if (this._options.video)
+      testInfo.data.video = true;
+    if (this._options.trace)
+      testInfo.data.trace = true;
     return {
       playwright: this._playwright,
       browserName: this._browserName,
@@ -174,7 +186,7 @@ export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
       isWindows: os.platform() === 'win32',
       isMac: os.platform() === 'darwin',
       isLinux: os.platform() === 'linux',
-      headful: !this._browserOptions.headless,
+      headful,
       video: !!this._options.video,
       mode: this._options.mode,
       platform: os.platform() as ('win32' | 'darwin' | 'linux'),

utils/flakiness-dashboard/README.md

@@ -1,4 +1,4 @@
 # Flakiness Dashboard Backend
 
 This directory contains source code for the Azure function that we use to aggregate test reports.
-The data is consumed by https://devops.aslushnikov.com/flakiness.html
+The data is consumed by https://devops.aslushnikov.com/flakiness2.html

utils/flakiness-dashboard/processing/dashboard_compressed_v1.js

@@ -51,21 +51,31 @@ function compressReports(reports) {
         specs.set(specId, specObject);
       }
       for (const test of spec.tests || []) {
-        if (test.runs.length === 1 && !test.runs[0].status)
+        // It's unclear how many results we get in the new test runner - let's
+        // stay on the safe side and skip test without any results.
+        if (!test.results || !test.results.length)
           continue;
+        // We get tests with a single result without status for sharded
+        // tests that are inside shard that we don't run.
+        if (test.results.length === 1 && !test.results[0].status)
+          continue;
+        // Folio currently reports `data` as part of test results.
+        // In our case, all data will be identical - so pick
+        // from the first result.
+        const testParameters = test.results[0].data;
         // Overwrite test platform parameter with a more specific information from
         // build run.
         const osName = report.metadata.osName.toUpperCase().startsWith('MINGW') ? 'Windows' : report.metadata.osName;
         const arch = report.metadata.arch && !report.metadata.arch.includes('x86') ? report.metadata.arch : '';
         const platform = (osName + ' ' + report.metadata.osVersion + ' ' + arch).trim();
-        const browserName = test.parameters.browserName || 'N/A';
+        const browserName = testParameters.browserName || 'N/A';
 
-        const testName = getTestName(browserName, platform, test.parameters);
+        const testName = getTestName(browserName, platform, testParameters);
         let testObject = specObject.tests.get(testName);
         if (!testObject) {
           testObject = {
             parameters: {
-              ...test.parameters,
+              ...testParameters,
               browserName,
               platform,
             },
@@ -78,7 +88,7 @@ function compressReports(reports) {
           specObject.tests.set(testName, testObject);
         }
 
-        for (const run of test.runs) {
+        for (const run of test.results) {
           // Record duration of slow tests only, i.e. > 1s.
           if (run.status === 'passed' && run.duration > 1000) {
             testObject.minTime = Math.min((testObject.minTime || Number.MAX_VALUE), run.duration);