test: add a tracing bot that collects a trace for most contexts (#8316)

.github/workflows/tests_secondary.yml

@@ -187,6 +187,30 @@ jobs:
         name: mode-${{ matrix.mode }}-linux-test-results
         path: test-results
 
+  tracing_linux:
+    name: "Tracing"
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, firefox, webkit]
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 12
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+    - run: npm run build
+    - run: node lib/cli/cli install --with-deps ${{ matrix.browser }} chromium
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}
+      env:
+        PWTEST_TRACE: 1
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: always()
+
   chrome_stable_linux:
     name: "Chrome Stable (Linux)"
     runs-on: ubuntu-20.04

src/client/browserContext.ts

@@ -50,7 +50,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
   };
 
   readonly tracing: Tracing;
-
+  private _closed = false;
   readonly _backgroundPages = new Set<Page>();
   readonly _serviceWorkers = new Set<Worker>();
   readonly _isChromium: boolean;
@@ -320,6 +320,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
   }
 
   _onClose() {
+    this._closed = true;
     if (this._browser)
       this._browser._contexts.delete(this);
     this._browserType?._contexts?.delete(this);

tests/chromium/js-coverage.spec.ts

@@ -16,6 +16,8 @@
 
 import { contextTest as it, expect } from '../config/browserTest';
 
+it.skip(({ trace }) => !!trace);
+
 it('should work', async function({page, server}) {
   await page.coverage.startJSCoverage();
   await page.goto(server.PREFIX + '/jscoverage/simple.html', { waitUntil: 'load' });

tests/config/baseTest.ts

@@ -33,6 +33,7 @@ type BaseOptions = {
   browserName: BrowserName;
   channel: LaunchOptions['channel'];
   video: boolean | undefined;
+  trace: boolean | undefined;
   headless: boolean | undefined;
 };
 type BaseFixtures = {
@@ -105,6 +106,7 @@ const baseFixtures: Fixtures<{}, BaseOptions & BaseFixtures> = {
   browserName: [ 'chromium' , { scope: 'worker' } ],
   channel: [ undefined, { scope: 'worker' } ],
   video: [ undefined, { scope: 'worker' } ],
+  trace: [ undefined, { scope: 'worker' } ],
   headless: [ undefined, { scope: 'worker' } ],
   platform: [ process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' } ],
   playwright: [ async ({ mode }, run, workerInfo) => {

tests/config/browserTest.ts

@@ -24,7 +24,6 @@ import { RemoteServer, RemoteServerOptions } from './remoteServer';
 import { baseTest, CommonWorkerFixtures } from './baseTest';
 
 type PlaywrightWorkerOptions = {
-  tracesDir: LaunchOptions['tracesDir'];
   executablePath: LaunchOptions['executablePath'];
   proxy: LaunchOptions['proxy'];
   args: LaunchOptions['args'];
@@ -50,7 +49,6 @@ type PlaywrightTestFixtures = {
 export type PlaywrightOptions = PlaywrightWorkerOptions & PlaywrightTestOptions;
 
 export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTestFixtures, PlaywrightWorkerOptions & PlaywrightWorkerFixtures, {}, CommonWorkerFixtures> = {
-  tracesDir: [ undefined, { scope: 'worker' } ],
   executablePath: [ undefined, { scope: 'worker' } ],
   proxy: [ undefined, { scope: 'worker' } ],
   args: [ undefined, { scope: 'worker' } ],
@@ -60,12 +58,11 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
     await run(playwright[browserName]);
   }, { scope: 'worker' } ],
 
-  browserOptions: [async ({ headless, channel, executablePath, tracesDir, proxy, args }, run) => {
+  browserOptions: [async ({ headless, channel, executablePath, proxy, args }, run) => {
     await run({
       headless,
       channel,
       executablePath,
-      tracesDir,
       proxy,
       args,
       handleSIGINT: false,
@@ -125,7 +122,7 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
       await remoteServer.close();
   },
 
-  contextOptions: async ({ video, hasTouch, browserVersion }, run, testInfo) => {
+  contextOptions: async ({ video, hasTouch }, run, testInfo) => {
     const debugName = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-');
     const contextOptions = {
       recordVideo: video ? { dir: testInfo.outputPath('') } : undefined,
@@ -135,10 +132,12 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
     await run(contextOptions);
   },
 
-  contextFactory: async ({ browser, contextOptions, video }, run, testInfo) => {
+  contextFactory: async ({ browser, contextOptions, trace }, run, testInfo) => {
     const contexts: BrowserContext[] = [];
     await run(async options => {
       const context = await browser.newContext({ ...contextOptions, ...options });
+      if (trace)
+        await context.tracing.start({ screenshots: true, snapshots: true });
       (context as any)._csi = {
         onApiCall: (name: string) => {
           return (testInfo as any)._addStep('pw:api', name);
@@ -149,6 +148,8 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
     });
     await Promise.all(contexts.map(async context => {
       const videos = context.pages().map(p => p.video()).filter(Boolean);
+      if (!(context as any)._closed && trace)
+        await context.tracing.stop({ path: testInfo.outputPath('trace.zip') });
       await context.close();
       for (const v of videos) {
         const videoPath = await v.path().catch(() => null);

tests/config/default.config.ts

@@ -42,13 +42,14 @@ const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | '
 const headed = !!process.env.HEADFUL;
 const channel = process.env.PWTEST_CHANNEL as any;
 const video = !!process.env.PWTEST_VIDEO;
+const trace = !!process.env.PWTEST_TRACE;
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: Config<CommonOptions & PlaywrightOptions> = {
   testDir,
   outputDir,
-  timeout: video || process.env.PWTRACE ? 60000 : 30000,
+  timeout: video ? 60000 : 30000,
   globalTimeout: 5400000,
   workers: process.env.CI ? 1 : undefined,
   forbidOnly: !!process.env.CI,
@@ -79,7 +80,7 @@ for (const browserName of browserNames) {
       channel,
       video,
       executablePath,
-      tracesDir: process.env.PWTRACE ? path.join(outputDir, 'trace') : undefined,
+      trace,
       coverageName: browserName,
     },
     define: { test: pageTest, fixtures: pageFixtures },
@@ -91,6 +92,7 @@ for (const browserName of browserNames) {
       channel,
       mode,
       video: !!video,
+      trace: !!trace,
     },
   });
 }

tests/config/remoteServer.ts

@@ -50,7 +50,6 @@ export class RemoteServer {
       args: browserOptions.args,
       headless: browserOptions.headless,
       channel: browserOptions.channel,
-      tracesDir: browserOptions.tracesDir,
       handleSIGINT: true,
       handleSIGTERM: true,
       handleSIGHUP: true,

tests/page/page-dialog.spec.ts

@@ -66,7 +66,7 @@ it('should dismiss the confirm prompt', async ({page}) => {
   expect(result).toBe(false);
 });
 
-it('should be able to close context with open alert', async ({page}) => {
+it('should be able to close context with open alert', async ({page, trace}) => {
   const alertPromise = page.waitForEvent('dialog');
   await page.evaluate(() => {
     setTimeout(() => alert('hello'), 0);

tests/tracing.spec.ts

@@ -18,6 +18,8 @@ import { expect, contextTest as test, browserTest } from './config/browserTest';
 import yauzl from 'yauzl';
 import jpeg from 'jpeg-js';
 
+test.skip(({ trace }) => !!trace);
+
 test('should collect trace with resources, but no js', async ({ context, page, server }, testInfo) => {
   await context.tracing.start({ screenshots: true, snapshots: true });
   await page.goto(server.PREFIX + '/frames/frame.html');