From 7818f5fa0ba3de2bd74ab7a52948879888f736bb Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 19 Aug 2021 19:09:19 -0700 Subject: [PATCH] test: add a tracing bot that collects a trace for most contexts (#8316) --- .github/workflows/tests_secondary.yml | 24 ++++++++++++++++++++++++ src/client/browserContext.ts | 3 ++- tests/chromium/js-coverage.spec.ts | 2 ++ tests/config/baseTest.ts | 2 ++ tests/config/browserTest.ts | 13 +++++++------ tests/config/default.config.ts | 6 ++++-- tests/config/remoteServer.ts | 1 - tests/page/page-dialog.spec.ts | 2 +- tests/tracing.spec.ts | 2 ++ 9 files changed, 44 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index df3ff28168..a06698a404 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -187,6 +187,30 @@ jobs: name: mode-${{ matrix.mode }}-linux-test-results path: test-results + tracing_linux: + name: "Tracing" + strategy: + fail-fast: false + matrix: + browser: [chromium, firefox, webkit] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 12 + - run: npm ci + env: + DEBUG: pw:install + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + - run: npm run build + - run: node lib/cli/cli install --with-deps ${{ matrix.browser }} chromium + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} + env: + PWTEST_TRACE: 1 + - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json + if: always() + chrome_stable_linux: name: "Chrome Stable (Linux)" runs-on: ubuntu-20.04 diff --git a/src/client/browserContext.ts b/src/client/browserContext.ts index fd3366f891..6615392227 100644 --- a/src/client/browserContext.ts +++ b/src/client/browserContext.ts @@ -50,7 +50,7 @@ export class BrowserContext extends ChannelOwner(); readonly _serviceWorkers = new Set(); readonly _isChromium: boolean; @@ -320,6 +320,7 @@ export class BrowserContext extends ChannelOwner !!trace); + it('should work', async function({page, server}) { await page.coverage.startJSCoverage(); await page.goto(server.PREFIX + '/jscoverage/simple.html', { waitUntil: 'load' }); diff --git a/tests/config/baseTest.ts b/tests/config/baseTest.ts index d63ae11688..8d1a8504a6 100644 --- a/tests/config/baseTest.ts +++ b/tests/config/baseTest.ts @@ -33,6 +33,7 @@ type BaseOptions = { browserName: BrowserName; channel: LaunchOptions['channel']; video: boolean | undefined; + trace: boolean | undefined; headless: boolean | undefined; }; type BaseFixtures = { @@ -105,6 +106,7 @@ const baseFixtures: Fixtures<{}, BaseOptions & BaseFixtures> = { browserName: [ 'chromium' , { scope: 'worker' } ], channel: [ undefined, { scope: 'worker' } ], video: [ undefined, { scope: 'worker' } ], + trace: [ undefined, { scope: 'worker' } ], headless: [ undefined, { scope: 'worker' } ], platform: [ process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' } ], playwright: [ async ({ mode }, run, workerInfo) => { diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.ts index b9eacd376c..27daf64f91 100644 --- a/tests/config/browserTest.ts +++ b/tests/config/browserTest.ts @@ -24,7 +24,6 @@ import { RemoteServer, RemoteServerOptions } from './remoteServer'; import { baseTest, CommonWorkerFixtures } from './baseTest'; type PlaywrightWorkerOptions = { - tracesDir: LaunchOptions['tracesDir']; executablePath: LaunchOptions['executablePath']; proxy: LaunchOptions['proxy']; args: LaunchOptions['args']; @@ -50,7 +49,6 @@ type PlaywrightTestFixtures = { export type PlaywrightOptions = PlaywrightWorkerOptions & PlaywrightTestOptions; export const playwrightFixtures: Fixtures = { - tracesDir: [ undefined, { scope: 'worker' } ], executablePath: [ undefined, { scope: 'worker' } ], proxy: [ undefined, { scope: 'worker' } ], args: [ undefined, { scope: 'worker' } ], @@ -60,12 +58,11 @@ export const playwrightFixtures: Fixtures { + browserOptions: [async ({ headless, channel, executablePath, proxy, args }, run) => { await run({ headless, channel, executablePath, - tracesDir, proxy, args, handleSIGINT: false, @@ -125,7 +122,7 @@ export const playwrightFixtures: Fixtures { + contextOptions: async ({ video, hasTouch }, run, testInfo) => { const debugName = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-'); const contextOptions = { recordVideo: video ? { dir: testInfo.outputPath('') } : undefined, @@ -135,10 +132,12 @@ export const playwrightFixtures: Fixtures { + contextFactory: async ({ browser, contextOptions, trace }, run, testInfo) => { const contexts: BrowserContext[] = []; await run(async options => { const context = await browser.newContext({ ...contextOptions, ...options }); + if (trace) + await context.tracing.start({ screenshots: true, snapshots: true }); (context as any)._csi = { onApiCall: (name: string) => { return (testInfo as any)._addStep('pw:api', name); @@ -149,6 +148,8 @@ export const playwrightFixtures: Fixtures { const videos = context.pages().map(p => p.video()).filter(Boolean); + if (!(context as any)._closed && trace) + await context.tracing.stop({ path: testInfo.outputPath('trace.zip') }); await context.close(); for (const v of videos) { const videoPath = await v.path().catch(() => null); diff --git a/tests/config/default.config.ts b/tests/config/default.config.ts index 7a5f95b403..918de01c64 100644 --- a/tests/config/default.config.ts +++ b/tests/config/default.config.ts @@ -42,13 +42,14 @@ const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | ' const headed = !!process.env.HEADFUL; const channel = process.env.PWTEST_CHANNEL as any; const video = !!process.env.PWTEST_VIDEO; +const trace = !!process.env.PWTEST_TRACE; const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: Config = { testDir, outputDir, - timeout: video || process.env.PWTRACE ? 60000 : 30000, + timeout: video ? 60000 : 30000, globalTimeout: 5400000, workers: process.env.CI ? 1 : undefined, forbidOnly: !!process.env.CI, @@ -79,7 +80,7 @@ for (const browserName of browserNames) { channel, video, executablePath, - tracesDir: process.env.PWTRACE ? path.join(outputDir, 'trace') : undefined, + trace, coverageName: browserName, }, define: { test: pageTest, fixtures: pageFixtures }, @@ -91,6 +92,7 @@ for (const browserName of browserNames) { channel, mode, video: !!video, + trace: !!trace, }, }); } diff --git a/tests/config/remoteServer.ts b/tests/config/remoteServer.ts index a7f02262b6..89495e55d3 100644 --- a/tests/config/remoteServer.ts +++ b/tests/config/remoteServer.ts @@ -50,7 +50,6 @@ export class RemoteServer { args: browserOptions.args, headless: browserOptions.headless, channel: browserOptions.channel, - tracesDir: browserOptions.tracesDir, handleSIGINT: true, handleSIGTERM: true, handleSIGHUP: true, diff --git a/tests/page/page-dialog.spec.ts b/tests/page/page-dialog.spec.ts index 46e39237c3..a0dc251560 100644 --- a/tests/page/page-dialog.spec.ts +++ b/tests/page/page-dialog.spec.ts @@ -66,7 +66,7 @@ it('should dismiss the confirm prompt', async ({page}) => { expect(result).toBe(false); }); -it('should be able to close context with open alert', async ({page}) => { +it('should be able to close context with open alert', async ({page, trace}) => { const alertPromise = page.waitForEvent('dialog'); await page.evaluate(() => { setTimeout(() => alert('hello'), 0); diff --git a/tests/tracing.spec.ts b/tests/tracing.spec.ts index 1601f1bf26..fb87c1eeda 100644 --- a/tests/tracing.spec.ts +++ b/tests/tracing.spec.ts @@ -18,6 +18,8 @@ import { expect, contextTest as test, browserTest } from './config/browserTest'; import yauzl from 'yauzl'; import jpeg from 'jpeg-js'; +test.skip(({ trace }) => !!trace); + test('should collect trace with resources, but no js', async ({ context, page, server }, testInfo) => { await context.tracing.start({ screenshots: true, snapshots: true }); await page.goto(server.PREFIX + '/frames/frame.html');