diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 108767961c..fa79f001cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,7 @@ env: # Force terminal colors. @see https://www.npmjs.com/package/colors FORCE_COLOR: 1 FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }} + FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" jobs: test_linux: @@ -38,17 +39,8 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json" - env: - BROWSER: ${{ matrix.browser }} - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json" - env: - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - # Checking coverage across two test suites is hard. Temporary disabled. - # - run: node test/checkCoverage.js - # env: - # BROWSER: ${{ matrix.browser }} + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json" + - run: node test/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -73,13 +65,7 @@ jobs: - run: npm ci - run: npm run build - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium - - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json - env: - BROWSER: ${{ matrix.browser }} - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json - env: - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" + - run: npm run test -- ${{ matrix.browser }} --reporter=dot,json - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -106,15 +92,8 @@ jobs: - run: npm ci - run: npm run build - run: node lib/cli/cli install-deps - - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json + - run: npm run test -- ${{ matrix.browser }} --reporter=dot,json shell: bash - env: - BROWSER: ${{ matrix.browser }} - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json - shell: bash - env: - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) shell: bash @@ -164,17 +143,10 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json" - if: ${{ always() }} - env: - BROWSER: ${{ matrix.browser }} - HEADFUL: 1 - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json" if: ${{ always() }} env: HEADFUL: 1 - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -204,15 +176,9 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json" - env: - BROWSER: "chromium" - PWMODE: "${{ matrix.mode }}" - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- chromium --reporter=dot,json" env: PWMODE: "${{ matrix.mode }}" - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -242,14 +208,9 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json -p video" - env: - BROWSER: ${{ matrix.browser }} - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json" env: PWVIDEO: 1 - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -266,7 +227,6 @@ jobs: shard: [1, 2] runs-on: macos-10.15 env: - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" PW_ANDROID_TESTS: 1 steps: - uses: actions/checkout@v2 @@ -281,7 +241,7 @@ jobs: - name: Start Android Emulator run: utils/avd_start.sh - name: Run tests - run: npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts --reporter=dot,json --shard=${{ matrix.shard }}/2 + run: npm run atest -- --reporter=dot,json --shard=${{ matrix.shard }}/2 - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -315,15 +275,9 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json" - env: - BROWSER: "chromium" - PW_CHROMIUM_CHANNEL: "chrome" - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- chromium --reporter=dot,json" env: PW_CHROMIUM_CHANNEL: "chrome" - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -347,17 +301,10 @@ jobs: - run: npm run build # This only created problems, should we move ffmpeg back into npm? - run: node lib/cli/cli install ffmpeg - - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json - shell: bash - env: - BROWSER: "chromium" - PW_CHROMIUM_CHANNEL: "chrome" - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: npm run folio -- chromium --reporter=dot,json + - run: npm run test -- chromium --reporter=dot,json shell: bash env: PW_CHROMIUM_CHANNEL: "chrome" - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -378,15 +325,9 @@ jobs: - run: npm run build # This only created problems, should we move ffmpeg back into npm? - run: node lib/cli/cli install ffmpeg - - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json - env: - BROWSER: "chromium" - PW_CHROMIUM_CHANNEL: "chrome" - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: npm run folio -- chromium --reporter=dot,json + - run: npm run test -- chromium --reporter=dot,json env: PW_CHROMIUM_CHANNEL: "chrome" - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -410,17 +351,10 @@ jobs: - run: npm run build # This only created problems, should we move ffmpeg back into npm? - run: node lib/cli/cli install ffmpeg - - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json - shell: bash - env: - BROWSER: "chromium" - PW_CHROMIUM_CHANNEL: "msedge" - FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json" - - run: npm run folio -- chromium --reporter=dot,json + - run: npm run test -- chromium --reporter=dot,json shell: bash env: PW_CHROMIUM_CHANNEL: "msedge" - FOLIO_JSON_OUTPUT_NAME: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: diff --git a/package.json b/package.json index fccda6a5e7..5d5300f460 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,11 @@ "node": ">=10.17.0" }, "scripts": { - "ctest": "cross-env BROWSER=chromium folio test/", - "ftest": "cross-env BROWSER=firefox folio test/", - "wtest": "cross-env BROWSER=webkit folio test/", - "atest": "cross-env BROWSER=chromium PW_ANDROID_TESTS=1 npx folio test/android --workers=1 --reporter=list", - "test": "folio test/", + "ctest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts chromium", + "ftest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts firefox", + "wtest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts webkit", + "atest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts", + "test": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .", "tsc": "tsc -p .", "tsc-installer": "tsc -p ./src/install/tsconfig.json", @@ -29,8 +29,7 @@ "build-android-driver": "./utils/build_android_driver.sh", "storybook": "start-storybook -p 6006 -s public", "build-storybook": "build-storybook -s public", - "build-folio": "tsc -p ./tests/folio", - "folio": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts" + "build-folio": "tsc -p ./tests/folio" }, "author": { "name": "Microsoft Corporation" diff --git a/src/server/supplements/recorder/recorderApp.ts b/src/server/supplements/recorder/recorderApp.ts index f44d463766..021ce366cf 100644 --- a/src/server/supplements/recorder/recorderApp.ts +++ b/src/server/supplements/recorder/recorderApp.ts @@ -99,15 +99,15 @@ export class RecorderApp extends EventEmitter { '--window-size=600,600', '--window-position=1280,10', ]; - if (isUnderTest()) - args.push(`--remote-debugging-port=0`); + if (process.env.PW_RECORDER_PORT) + args.push(`--remote-debugging-port=${process.env.PW_RECORDER_PORT}`); const context = await recorderPlaywright.chromium.launchPersistentContext(internalCallMetadata(), '', { channel: inspectedContext._browser.options.channel, sdkLanguage: inspectedContext._options.sdkLanguage, args, noDefaultViewport: true, headless: !!process.env.PWCLI_HEADLESS_FOR_TEST || (isUnderTest() && !inspectedContext._browser.options.headful), - useWebSocket: isUnderTest() + useWebSocket: !!process.env.PW_RECORDER_PORT }); const controller = new ProgressController(internalCallMetadata(), context._browser); await controller.run(async progress => { diff --git a/test/checkCoverage.js b/test/checkCoverage.js index 1cdaa73e36..658c118a43 100644 --- a/test/checkCoverage.js +++ b/test/checkCoverage.js @@ -17,7 +17,7 @@ const path = require('path'); const fs = require('fs'); const {installCoverageHooks} = require('./coverage'); -const browserName = process.env.BROWSER || 'chromium'; +const browserName = process.argv[2] || 'chromium'; let api = new Set(installCoverageHooks(browserName).coverage.keys()); @@ -30,8 +30,21 @@ if (browserName === 'chromium') { if (browserName !== 'chromium') { // we don't have CDPSession in non-chromium browsers + api.delete('browser.newBrowserCDPSession'); + api.delete('browser.startTracing'); + api.delete('browser.stopTracing'); + api.delete('browserContext.backgroundPages'); + api.delete('browserContext.serviceWorkers'); + api.delete('browserContext.newCDPSession'); + api.delete('browserContext.emit("backgroundpage")'); + api.delete('browserContext.emit("serviceworker")'); api.delete('cDPSession.send'); api.delete('cDPSession.detach'); + api.delete('coverage.startJSCoverage'); + api.delete('coverage.stopJSCoverage'); + api.delete('coverage.startCSSCoverage'); + api.delete('coverage.stopCSSCoverage'); + api.delete('page.pdf'); } // Some permissions tests are disabled in webkit. See permissions.jest.js diff --git a/test/utils.ts b/test/utils.ts index b03ca5aa71..a2e45f08c8 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -15,8 +15,7 @@ */ import { expect } from './fixtures'; -import type { Frame, Page, BrowserContext } from '../index'; -import { chromium } from '../index'; +import type { Frame, Page } from '../index'; export async function attachFrame(page: Page, frameId: string, url: string): Promise { const handle = await page.evaluateHandle(async ({ frameId, url }) => { @@ -59,12 +58,3 @@ export function expectedSSLError(browserName: string): string { } return expectedSSLError; } - -export async function recorderPageGetter(context: BrowserContext, toImpl: (x: any) => any) { - while (!toImpl(context).recorderAppForTest) - await new Promise(f => setTimeout(f, 100)); - const wsEndpoint = toImpl(context).recorderAppForTest.wsEndpoint; - const browser = await chromium.connectOverCDP({ wsEndpoint }); - const c = browser.contexts()[0]; - return c.pages()[0] || await c.waitForEvent('page'); -} diff --git a/test/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png b/tests/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png similarity index 100% rename from test/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png rename to tests/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png diff --git a/test/chromium/chromium.spec.ts b/tests/chromium/chromium.spec.ts similarity index 78% rename from test/chromium/chromium.spec.ts rename to tests/chromium/chromium.spec.ts index ab861e3c4e..8c115645f6 100644 --- a/test/chromium/chromium.spec.ts +++ b/tests/chromium/chromium.spec.ts @@ -14,21 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { it, expect, describe } from '../fixtures'; + +import { test as pageTest, expect } from '../config/pageTest'; +import { test as playwrightTest } from '../config/playwrightTest'; import http from 'http'; -describe('chromium', (suite, { browserName }) => { - suite.skip(browserName !== 'chromium'); -}, () => { - it('should create a worker from a service worker', async ({page, server, context}) => { +pageTest.describe('chromium', () => { + pageTest.beforeEach(async ({ browserName }) => { + pageTest.skip(browserName !== 'chromium'); + pageTest.skip(!!process.env.PW_ANDROID_TESTS); + }); + + pageTest('should create a worker from a service worker', async ({page, server}) => { const [worker] = await Promise.all([ - context.waitForEvent('serviceworker'), + page.context().waitForEvent('serviceworker'), page.goto(server.PREFIX + '/serviceworkers/empty/sw.html') ]); expect(await worker.evaluate(() => self.toString())).toBe('[object ServiceWorkerGlobalScope]'); }); - it('serviceWorkers() should return current workers', async ({page, server, context}) => { + pageTest('serviceWorkers() should return current workers', async ({page, server}) => { + const context = page.context(); const [worker1] = await Promise.all([ context.waitForEvent('serviceworker'), page.goto(server.PREFIX + '/serviceworkers/empty/sw.html') @@ -46,31 +52,17 @@ describe('chromium', (suite, { browserName }) => { expect(workers).toContain(worker2); }); - it('should not create a worker from a shared worker', async ({page, server, context}) => { + pageTest('should not create a worker from a shared worker', async ({page, server}) => { await page.goto(server.EMPTY_PAGE); let serviceWorkerCreated; - context.once('serviceworker', () => serviceWorkerCreated = true); + page.context().once('serviceworker', () => serviceWorkerCreated = true); await page.evaluate(() => { new SharedWorker('data:text/javascript,console.log("hi")'); }); expect(serviceWorkerCreated).not.toBeTruthy(); }); - it('should close service worker together with the context', async ({browser, server}) => { - const context = await browser.newContext(); - const page = await context.newPage(); - const [worker] = await Promise.all([ - context.waitForEvent('serviceworker'), - page.goto(server.PREFIX + '/serviceworkers/empty/sw.html') - ]); - const messages = []; - context.on('close', () => messages.push('context')); - worker.on('close', () => messages.push('worker')); - await context.close(); - expect(messages.join('|')).toBe('worker|context'); - }); - - it('Page.route should work with intervention headers', async ({server, page}) => { + pageTest('Page.route should work with intervention headers', async ({server, page}) => { server.setRoute('/intervention', (req, res) => res.end(`