diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 108767961c..fa79f001cb 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -14,6 +14,7 @@ env:
# Force terminal colors. @see https://www.npmjs.com/package/colors
FORCE_COLOR: 1
FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }}
+ FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
jobs:
test_linux:
@@ -38,17 +39,8 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
- env:
- BROWSER: ${{ matrix.browser }}
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
- env:
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- # Checking coverage across two test suites is hard. Temporary disabled.
- # - run: node test/checkCoverage.js
- # env:
- # BROWSER: ${{ matrix.browser }}
+ - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json"
+ - run: node test/checkCoverage.js ${{ matrix.browser }}
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@@ -73,13 +65,7 @@ jobs:
- run: npm ci
- run: npm run build
- run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
- - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
- env:
- BROWSER: ${{ matrix.browser }}
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
- env:
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
+ - run: npm run test -- ${{ matrix.browser }} --reporter=dot,json
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@@ -106,15 +92,8 @@ jobs:
- run: npm ci
- run: npm run build
- run: node lib/cli/cli install-deps
- - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
+ - run: npm run test -- ${{ matrix.browser }} --reporter=dot,json
shell: bash
- env:
- BROWSER: ${{ matrix.browser }}
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
- shell: bash
- env:
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
shell: bash
@@ -164,17 +143,10 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
- if: ${{ always() }}
- env:
- BROWSER: ${{ matrix.browser }}
- HEADFUL: 1
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
+ - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json"
if: ${{ always() }}
env:
HEADFUL: 1
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@@ -204,15 +176,9 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
- env:
- BROWSER: "chromium"
- PWMODE: "${{ matrix.mode }}"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
+ - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- chromium --reporter=dot,json"
env:
PWMODE: "${{ matrix.mode }}"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@@ -242,14 +208,9 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json -p video"
- env:
- BROWSER: ${{ matrix.browser }}
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
+ - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json"
env:
PWVIDEO: 1
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@@ -266,7 +227,6 @@ jobs:
shard: [1, 2]
runs-on: macos-10.15
env:
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
PW_ANDROID_TESTS: 1
steps:
- uses: actions/checkout@v2
@@ -281,7 +241,7 @@ jobs:
- name: Start Android Emulator
run: utils/avd_start.sh
- name: Run tests
- run: npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts --reporter=dot,json --shard=${{ matrix.shard }}/2
+ run: npm run atest -- --reporter=dot,json --shard=${{ matrix.shard }}/2
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@@ -315,15 +275,9 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
- env:
- BROWSER: "chromium"
- PW_CHROMIUM_CHANNEL: "chrome"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
+ - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- chromium --reporter=dot,json"
env:
PW_CHROMIUM_CHANNEL: "chrome"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- uses: actions/upload-artifact@v1
if: ${{ always() }}
with:
@@ -347,17 +301,10 @@ jobs:
- run: npm run build
# This only created problems, should we move ffmpeg back into npm?
- run: node lib/cli/cli install ffmpeg
- - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
- shell: bash
- env:
- BROWSER: "chromium"
- PW_CHROMIUM_CHANNEL: "chrome"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: npm run folio -- chromium --reporter=dot,json
+ - run: npm run test -- chromium --reporter=dot,json
shell: bash
env:
PW_CHROMIUM_CHANNEL: "chrome"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- uses: actions/upload-artifact@v1
if: ${{ always() }}
with:
@@ -378,15 +325,9 @@ jobs:
- run: npm run build
# This only created problems, should we move ffmpeg back into npm?
- run: node lib/cli/cli install ffmpeg
- - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
- env:
- BROWSER: "chromium"
- PW_CHROMIUM_CHANNEL: "chrome"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: npm run folio -- chromium --reporter=dot,json
+ - run: npm run test -- chromium --reporter=dot,json
env:
PW_CHROMIUM_CHANNEL: "chrome"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- uses: actions/upload-artifact@v1
if: ${{ always() }}
with:
@@ -410,17 +351,10 @@ jobs:
- run: npm run build
# This only created problems, should we move ffmpeg back into npm?
- run: node lib/cli/cli install ffmpeg
- - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
- shell: bash
- env:
- BROWSER: "chromium"
- PW_CHROMIUM_CHANNEL: "msedge"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
- - run: npm run folio -- chromium --reporter=dot,json
+ - run: npm run test -- chromium --reporter=dot,json
shell: bash
env:
PW_CHROMIUM_CHANNEL: "msedge"
- FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
- uses: actions/upload-artifact@v1
if: ${{ always() }}
with:
diff --git a/package.json b/package.json
index fccda6a5e7..5d5300f460 100644
--- a/package.json
+++ b/package.json
@@ -9,11 +9,11 @@
"node": ">=10.17.0"
},
"scripts": {
- "ctest": "cross-env BROWSER=chromium folio test/",
- "ftest": "cross-env BROWSER=firefox folio test/",
- "wtest": "cross-env BROWSER=webkit folio test/",
- "atest": "cross-env BROWSER=chromium PW_ANDROID_TESTS=1 npx folio test/android --workers=1 --reporter=list",
- "test": "folio test/",
+ "ctest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts chromium",
+ "ftest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts firefox",
+ "wtest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts webkit",
+ "atest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts",
+ "test": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts",
"eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
"tsc": "tsc -p .",
"tsc-installer": "tsc -p ./src/install/tsconfig.json",
@@ -29,8 +29,7 @@
"build-android-driver": "./utils/build_android_driver.sh",
"storybook": "start-storybook -p 6006 -s public",
"build-storybook": "build-storybook -s public",
- "build-folio": "tsc -p ./tests/folio",
- "folio": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts"
+ "build-folio": "tsc -p ./tests/folio"
},
"author": {
"name": "Microsoft Corporation"
diff --git a/src/server/supplements/recorder/recorderApp.ts b/src/server/supplements/recorder/recorderApp.ts
index f44d463766..021ce366cf 100644
--- a/src/server/supplements/recorder/recorderApp.ts
+++ b/src/server/supplements/recorder/recorderApp.ts
@@ -99,15 +99,15 @@ export class RecorderApp extends EventEmitter {
'--window-size=600,600',
'--window-position=1280,10',
];
- if (isUnderTest())
- args.push(`--remote-debugging-port=0`);
+ if (process.env.PW_RECORDER_PORT)
+ args.push(`--remote-debugging-port=${process.env.PW_RECORDER_PORT}`);
const context = await recorderPlaywright.chromium.launchPersistentContext(internalCallMetadata(), '', {
channel: inspectedContext._browser.options.channel,
sdkLanguage: inspectedContext._options.sdkLanguage,
args,
noDefaultViewport: true,
headless: !!process.env.PWCLI_HEADLESS_FOR_TEST || (isUnderTest() && !inspectedContext._browser.options.headful),
- useWebSocket: isUnderTest()
+ useWebSocket: !!process.env.PW_RECORDER_PORT
});
const controller = new ProgressController(internalCallMetadata(), context._browser);
await controller.run(async progress => {
diff --git a/test/checkCoverage.js b/test/checkCoverage.js
index 1cdaa73e36..658c118a43 100644
--- a/test/checkCoverage.js
+++ b/test/checkCoverage.js
@@ -17,7 +17,7 @@ const path = require('path');
const fs = require('fs');
const {installCoverageHooks} = require('./coverage');
-const browserName = process.env.BROWSER || 'chromium';
+const browserName = process.argv[2] || 'chromium';
let api = new Set(installCoverageHooks(browserName).coverage.keys());
@@ -30,8 +30,21 @@ if (browserName === 'chromium') {
if (browserName !== 'chromium') {
// we don't have CDPSession in non-chromium browsers
+ api.delete('browser.newBrowserCDPSession');
+ api.delete('browser.startTracing');
+ api.delete('browser.stopTracing');
+ api.delete('browserContext.backgroundPages');
+ api.delete('browserContext.serviceWorkers');
+ api.delete('browserContext.newCDPSession');
+ api.delete('browserContext.emit("backgroundpage")');
+ api.delete('browserContext.emit("serviceworker")');
api.delete('cDPSession.send');
api.delete('cDPSession.detach');
+ api.delete('coverage.startJSCoverage');
+ api.delete('coverage.stopJSCoverage');
+ api.delete('coverage.startCSSCoverage');
+ api.delete('coverage.stopCSSCoverage');
+ api.delete('page.pdf');
}
// Some permissions tests are disabled in webkit. See permissions.jest.js
diff --git a/test/utils.ts b/test/utils.ts
index b03ca5aa71..a2e45f08c8 100644
--- a/test/utils.ts
+++ b/test/utils.ts
@@ -15,8 +15,7 @@
*/
import { expect } from './fixtures';
-import type { Frame, Page, BrowserContext } from '../index';
-import { chromium } from '../index';
+import type { Frame, Page } from '../index';
export async function attachFrame(page: Page, frameId: string, url: string): Promise {
const handle = await page.evaluateHandle(async ({ frameId, url }) => {
@@ -59,12 +58,3 @@ export function expectedSSLError(browserName: string): string {
}
return expectedSSLError;
}
-
-export async function recorderPageGetter(context: BrowserContext, toImpl: (x: any) => any) {
- while (!toImpl(context).recorderAppForTest)
- await new Promise(f => setTimeout(f, 100));
- const wsEndpoint = toImpl(context).recorderAppForTest.wsEndpoint;
- const browser = await chromium.connectOverCDP({ wsEndpoint });
- const c = browser.contexts()[0];
- return c.pages()[0] || await c.waitForEvent('page');
-}
diff --git a/test/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png b/tests/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png
similarity index 100%
rename from test/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png
rename to tests/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png
diff --git a/test/chromium/chromium.spec.ts b/tests/chromium/chromium.spec.ts
similarity index 78%
rename from test/chromium/chromium.spec.ts
rename to tests/chromium/chromium.spec.ts
index ab861e3c4e..8c115645f6 100644
--- a/test/chromium/chromium.spec.ts
+++ b/tests/chromium/chromium.spec.ts
@@ -14,21 +14,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { it, expect, describe } from '../fixtures';
+
+import { test as pageTest, expect } from '../config/pageTest';
+import { test as playwrightTest } from '../config/playwrightTest';
import http from 'http';
-describe('chromium', (suite, { browserName }) => {
- suite.skip(browserName !== 'chromium');
-}, () => {
- it('should create a worker from a service worker', async ({page, server, context}) => {
+pageTest.describe('chromium', () => {
+ pageTest.beforeEach(async ({ browserName }) => {
+ pageTest.skip(browserName !== 'chromium');
+ pageTest.skip(!!process.env.PW_ANDROID_TESTS);
+ });
+
+ pageTest('should create a worker from a service worker', async ({page, server}) => {
const [worker] = await Promise.all([
- context.waitForEvent('serviceworker'),
+ page.context().waitForEvent('serviceworker'),
page.goto(server.PREFIX + '/serviceworkers/empty/sw.html')
]);
expect(await worker.evaluate(() => self.toString())).toBe('[object ServiceWorkerGlobalScope]');
});
- it('serviceWorkers() should return current workers', async ({page, server, context}) => {
+ pageTest('serviceWorkers() should return current workers', async ({page, server}) => {
+ const context = page.context();
const [worker1] = await Promise.all([
context.waitForEvent('serviceworker'),
page.goto(server.PREFIX + '/serviceworkers/empty/sw.html')
@@ -46,31 +52,17 @@ describe('chromium', (suite, { browserName }) => {
expect(workers).toContain(worker2);
});
- it('should not create a worker from a shared worker', async ({page, server, context}) => {
+ pageTest('should not create a worker from a shared worker', async ({page, server}) => {
await page.goto(server.EMPTY_PAGE);
let serviceWorkerCreated;
- context.once('serviceworker', () => serviceWorkerCreated = true);
+ page.context().once('serviceworker', () => serviceWorkerCreated = true);
await page.evaluate(() => {
new SharedWorker('data:text/javascript,console.log("hi")');
});
expect(serviceWorkerCreated).not.toBeTruthy();
});
- it('should close service worker together with the context', async ({browser, server}) => {
- const context = await browser.newContext();
- const page = await context.newPage();
- const [worker] = await Promise.all([
- context.waitForEvent('serviceworker'),
- page.goto(server.PREFIX + '/serviceworkers/empty/sw.html')
- ]);
- const messages = [];
- context.on('close', () => messages.push('context'));
- worker.on('close', () => messages.push('worker'));
- await context.close();
- expect(messages.join('|')).toBe('worker|context');
- });
-
- it('Page.route should work with intervention headers', async ({server, page}) => {
+ pageTest('Page.route should work with intervention headers', async ({server, page}) => {
server.setRoute('/intervention', (req, res) => res.end(`