mirror of
https://github.com/microsoft/playwright.git
synced 2024-09-11 20:37:54 +03:00
feat(testrunner): allow unexpected passes (#3665)
This commit is contained in:
parent
5c0f93301d
commit
6ffdd4dfa1
9
.github/workflows/tests.yml
vendored
9
.github/workflows/tests.yml
vendored
@ -40,8 +40,6 @@ jobs:
|
||||
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json && npm run coverage"
|
||||
env:
|
||||
BROWSER: ${{ matrix.browser }}
|
||||
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
|
||||
DEBUG_FILE: "test-results/debug.log"
|
||||
PWRUNNER_JSON_REPORT: "test-results/report.json"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: always()
|
||||
@ -67,8 +65,6 @@ jobs:
|
||||
- run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json
|
||||
env:
|
||||
BROWSER: ${{ matrix.browser }}
|
||||
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
|
||||
DEBUG_FILE: "test-results/debug.log"
|
||||
PWRUNNER_JSON_REPORT: "test-results/report.json"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: ${{ always() }}
|
||||
@ -98,8 +94,6 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
BROWSER: ${{ matrix.browser }}
|
||||
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
|
||||
DEBUG_FILE: "test-results/debug.log"
|
||||
PWRUNNER_JSON_REPORT: "test-results/report.json"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: ${{ always() }}
|
||||
@ -152,7 +146,6 @@ jobs:
|
||||
env:
|
||||
BROWSER: ${{ matrix.browser }}
|
||||
HEADLESS: "false"
|
||||
DEBUG_FILE: "test-results/debug.log"
|
||||
PWRUNNER_JSON_REPORT: "test-results/report.json"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: ${{ always() }}
|
||||
@ -184,8 +177,6 @@ jobs:
|
||||
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json"
|
||||
env:
|
||||
BROWSER: ${{ matrix.browser }}
|
||||
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
|
||||
DEBUG_FILE: "test-results/debug.log"
|
||||
PWWIRE: true
|
||||
PWRUNNER_JSON_REPORT: "test-results/report.json"
|
||||
- uses: actions/upload-artifact@v1
|
||||
|
@ -32,6 +32,7 @@ declare global {
|
||||
|
||||
type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void> | void) => void) & {
|
||||
fail(condition: boolean): ItFunction<STATE>;
|
||||
fixme(condition: boolean): ItFunction<STATE>;
|
||||
flaky(condition: boolean): ItFunction<STATE>;
|
||||
skip(condition: boolean): ItFunction<STATE>;
|
||||
slow(): ItFunction<STATE>;
|
||||
|
@ -30,10 +30,9 @@ const stackUtils = new StackUtils();
|
||||
|
||||
export class BaseReporter implements Reporter {
|
||||
skipped: Test[] = [];
|
||||
passed: Test[] = [];
|
||||
asExpected: Test[] = [];
|
||||
unexpected = new Set<Test>();
|
||||
flaky: Test[] = [];
|
||||
failed: Test[] = [];
|
||||
timedOut: Test[] = [];
|
||||
duration = 0;
|
||||
startTime: number;
|
||||
config: RunnerConfig;
|
||||
@ -66,28 +65,24 @@ export class BaseReporter implements Reporter {
|
||||
}
|
||||
|
||||
onTestEnd(test: Test, result: TestResult) {
|
||||
switch (result.status) {
|
||||
case 'skipped': {
|
||||
this.skipped.push(test);
|
||||
return;
|
||||
}
|
||||
case 'passed':
|
||||
if (test.results.length === 1)
|
||||
this.passed.push(test);
|
||||
else
|
||||
this.flaky.push(test);
|
||||
return;
|
||||
case 'failed':
|
||||
// Fall through.
|
||||
case 'timedOut': {
|
||||
if (test.results.length === this.config.retries + 1) {
|
||||
if (result.status === 'timedOut')
|
||||
this.timedOut.push(test);
|
||||
else
|
||||
this.failed.push(test);
|
||||
}
|
||||
return;
|
||||
if (result.status === 'skipped') {
|
||||
this.skipped.push(test);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.status === result.expectedStatus) {
|
||||
if (test.results.length === 1) {
|
||||
// as expected from the first attempt
|
||||
this.asExpected.push(test);
|
||||
} else {
|
||||
// as expected after unexpected -> flaky.
|
||||
this.flaky.push(test);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (result.status === 'passed' || result.status === 'timedOut' || test.results.length === this.config.retries + 1) {
|
||||
// We made as many retries as we could, still failing.
|
||||
this.unexpected.add(test);
|
||||
}
|
||||
}
|
||||
|
||||
@ -98,15 +93,16 @@ export class BaseReporter implements Reporter {
|
||||
epilogue() {
|
||||
console.log('');
|
||||
|
||||
console.log(colors.green(` ${this.passed.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
|
||||
console.log(colors.green(` ${this.asExpected.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
|
||||
|
||||
if (this.skipped.length)
|
||||
console.log(colors.yellow(` ${this.skipped.length} skipped`));
|
||||
|
||||
if (this.failed.length) {
|
||||
console.log(colors.red(` ${this.failed.length} failed`));
|
||||
const filteredUnexpected = [...this.unexpected].filter(t => !t._hasResultWithStatus('timedOut'));
|
||||
if (filteredUnexpected.length) {
|
||||
console.log(colors.red(` ${filteredUnexpected.length} failed`));
|
||||
console.log('');
|
||||
this._printFailures(this.failed);
|
||||
this._printFailures(filteredUnexpected);
|
||||
}
|
||||
|
||||
if (this.flaky.length) {
|
||||
@ -115,11 +111,13 @@ export class BaseReporter implements Reporter {
|
||||
this._printFailures(this.flaky);
|
||||
}
|
||||
|
||||
if (this.timedOut.length) {
|
||||
console.log(colors.red(` ${this.timedOut.length} timed out`));
|
||||
const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
|
||||
if (timedOut.length) {
|
||||
console.log(colors.red(` ${timedOut.length} timed out`));
|
||||
console.log('');
|
||||
this._printFailures(this.timedOut);
|
||||
this._printFailures(timedOut);
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
private _printFailures(failures: Test[]) {
|
||||
@ -131,7 +129,8 @@ export class BaseReporter implements Reporter {
|
||||
formatFailure(test: Test, index?: number): string {
|
||||
const tokens: string[] = [];
|
||||
const relativePath = path.relative(process.cwd(), test.file);
|
||||
const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}`;
|
||||
const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
|
||||
const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}${passedUnexpectedlySuffix}`;
|
||||
tokens.push(colors.bold(colors.red(header)));
|
||||
for (const result of test.results) {
|
||||
if (result.status === 'passed')
|
||||
|
@ -23,8 +23,8 @@ class DotReporter extends BaseReporter {
|
||||
super.onTestEnd(test, result);
|
||||
switch (result.status) {
|
||||
case 'skipped': process.stdout.write(colors.yellow('∘')); break;
|
||||
case 'passed': process.stdout.write(colors.green('·')); break;
|
||||
case 'failed': process.stdout.write(colors.red(test.results.length > 1 ? '' + test.results.length : 'F')); break;
|
||||
case 'passed': process.stdout.write(result.status === result.expectedStatus ? colors.green('·') : colors.red('P')); break;
|
||||
case 'failed': process.stdout.write(result.status === result.expectedStatus ? colors.green('f') : colors.red('F')); break;
|
||||
case 'timedOut': process.stdout.write(colors.red('T')); break;
|
||||
}
|
||||
}
|
||||
|
@ -35,12 +35,14 @@ class ListReporter extends BaseReporter {
|
||||
onTestEnd(test: Test, result: TestResult) {
|
||||
super.onTestEnd(test, result);
|
||||
let text = '';
|
||||
switch (result.status) {
|
||||
case 'skipped': text = colors.green(' - ') + colors.cyan(test.fullTitle()); break;
|
||||
case 'passed': text = '\u001b[2K\u001b[0G' + colors.green(' ✓ ') + colors.gray(test.fullTitle()); break;
|
||||
case 'failed':
|
||||
// fall through
|
||||
case 'timedOut': text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle()); break;
|
||||
if (result.status === 'skipped') {
|
||||
text = colors.green(' - ') + colors.cyan(test.fullTitle());
|
||||
} else {
|
||||
const statusMark = result.status === 'passed' ? colors.green(' ✓ ') : colors.red(' x ');
|
||||
if (result.status === result.expectedStatus)
|
||||
text = '\u001b[2K\u001b[0G' + statusMark + colors.gray(test.fullTitle());
|
||||
else
|
||||
text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle());
|
||||
}
|
||||
process.stdout.write(text + '\n');
|
||||
}
|
||||
|
@ -149,14 +149,15 @@ class PytestReporter extends BaseReporter {
|
||||
}
|
||||
|
||||
const status = [];
|
||||
if (this.passed.length)
|
||||
status.push(colors.green(`${this.passed.length} passed`));
|
||||
if (this.asExpected.length)
|
||||
status.push(colors.green(`${this.asExpected.length} as expected`));
|
||||
if (this.skipped.length)
|
||||
status.push(colors.yellow(`${this.skipped.length} skipped`));
|
||||
if (this.failed.length)
|
||||
status.push(colors.red(`${this.failed.length} failed`));
|
||||
if (this.timedOut.length)
|
||||
status.push(colors.red(`${this.timedOut.length} timed out`));
|
||||
const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
|
||||
if (this.unexpected.size - timedOut.length)
|
||||
status.push(colors.red(`${this.unexpected.size - timedOut.length} unexpected failures`));
|
||||
if (timedOut.length)
|
||||
status.push(colors.red(`${timedOut.length} timed out`));
|
||||
status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`));
|
||||
|
||||
for (let i = lines.length; i < this._visibleRows; ++i)
|
||||
|
@ -103,7 +103,11 @@ export class Runner {
|
||||
let doneCallback;
|
||||
const result = new Promise(f => doneCallback = f);
|
||||
worker.once('done', params => {
|
||||
if (!params.failedTestId && !params.fatalError) {
|
||||
// We won't file remaining if:
|
||||
// - there are no remaining
|
||||
// - we are here not because something failed
|
||||
// - no unrecoverable worker error
|
||||
if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
|
||||
this._workerAvailable(worker);
|
||||
doneCallback();
|
||||
return;
|
||||
@ -112,8 +116,8 @@ export class Runner {
|
||||
// When worker encounters error, we will restart it.
|
||||
this._restartWorker(worker);
|
||||
|
||||
// In case of fatal error without test id, we are done with the entry.
|
||||
if (params.fatalError && !params.failedTestId) {
|
||||
// In case of fatal error, we are done with the entry.
|
||||
if (params.fatalError) {
|
||||
// Report all the tests are failing with this error.
|
||||
for (const id of entry.ids) {
|
||||
const { test, result } = this._testById.get(id);
|
||||
@ -127,13 +131,16 @@ export class Runner {
|
||||
}
|
||||
|
||||
const remaining = params.remaining;
|
||||
if (this._config.retries) {
|
||||
|
||||
// Only retry expected failures, not passes and only if the test failed.
|
||||
if (this._config.retries && params.failedTestId) {
|
||||
const pair = this._testById.get(params.failedTestId);
|
||||
if (pair.test.results.length < this._config.retries + 1) {
|
||||
if (pair.result.expectedStatus === 'passed' && pair.test.results.length < this._config.retries + 1) {
|
||||
pair.result = pair.test._appendResult();
|
||||
remaining.unshift(pair.test._id);
|
||||
}
|
||||
}
|
||||
|
||||
if (remaining.length)
|
||||
this._queue.unshift({ ...entry, ids: remaining });
|
||||
|
||||
@ -169,6 +176,7 @@ export class Runner {
|
||||
const { test } = this._testById.get(params.id);
|
||||
test._skipped = params.skipped;
|
||||
test._flaky = params.flaky;
|
||||
test._expectedStatus = params.expectedStatus;
|
||||
this._reporter.onTestBegin(test);
|
||||
});
|
||||
worker.on('testEnd', params => {
|
||||
|
@ -53,7 +53,7 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
|
||||
const suites = [suite];
|
||||
suite.file = file;
|
||||
|
||||
const it = specBuilder(['skip', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
|
||||
const it = specBuilder(['skip', 'fixme', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
|
||||
const suite = suites[0];
|
||||
const test = new Test(title, fn);
|
||||
test.file = file;
|
||||
@ -65,8 +65,10 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
|
||||
test.only = true;
|
||||
if (!only && specs.skip && specs.skip[0])
|
||||
test._skipped = true;
|
||||
if (!only && specs.fail && specs.fail[0])
|
||||
if (!only && specs.fixme && specs.fixme[0])
|
||||
test._skipped = true;
|
||||
if (specs.fail && specs.fail[0])
|
||||
test._expectedStatus = 'failed';
|
||||
if (specs.flaky && specs.flaky[0])
|
||||
test._flaky = true;
|
||||
suite._addTest(test);
|
||||
@ -81,7 +83,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
|
||||
if (only)
|
||||
child.only = true;
|
||||
if (!only && specs.skip && specs.skip[0])
|
||||
child.skipped = true;
|
||||
child._skipped = true;
|
||||
if (!only && specs.fixme && specs.fixme[0])
|
||||
child._skipped = true;
|
||||
suites.unshift(child);
|
||||
fn();
|
||||
suites.shift();
|
||||
|
@ -16,6 +16,8 @@
|
||||
|
||||
export type Configuration = { name: string, value: string }[];
|
||||
|
||||
type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
|
||||
|
||||
export class Test {
|
||||
suite: Suite;
|
||||
title: string;
|
||||
@ -27,10 +29,13 @@ export class Test {
|
||||
results: TestResult[] = [];
|
||||
|
||||
_id: string;
|
||||
// Skipped & flaky are resolved based on options in worker only
|
||||
// We will compute them there and send to the runner (front-end)
|
||||
_skipped = false;
|
||||
_flaky = false;
|
||||
_overriddenFn: Function;
|
||||
_startTime: number;
|
||||
_expectedStatus: TestStatus = 'passed';
|
||||
|
||||
constructor(title: string, fn: Function) {
|
||||
this.title = title;
|
||||
@ -48,7 +53,7 @@ export class Test {
|
||||
_appendResult(): TestResult {
|
||||
const result: TestResult = {
|
||||
duration: 0,
|
||||
status: 'none',
|
||||
expectedStatus: 'passed',
|
||||
stdout: [],
|
||||
stderr: [],
|
||||
data: {}
|
||||
@ -58,17 +63,21 @@ export class Test {
|
||||
}
|
||||
|
||||
_ok(): boolean {
|
||||
if (this._skipped)
|
||||
if (this._skipped || this.suite._isSkipped())
|
||||
return true;
|
||||
const hasFailedResults = !!this.results.find(r => r.status !== 'passed' && r.status !== 'skipped');
|
||||
const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus);
|
||||
if (!hasFailedResults)
|
||||
return true;
|
||||
if (!this._flaky)
|
||||
return false;
|
||||
const hasPassedResults = !!this.results.find(r => r.status === 'passed');
|
||||
const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus);
|
||||
return hasPassedResults;
|
||||
}
|
||||
|
||||
_hasResultWithStatus(status: TestStatus): boolean {
|
||||
return !!this.results.find(r => r.status === status);
|
||||
}
|
||||
|
||||
_clone(): Test {
|
||||
const test = new Test(this.title, this.fn);
|
||||
test.suite = this.suite;
|
||||
@ -83,7 +92,8 @@ export class Test {
|
||||
|
||||
export type TestResult = {
|
||||
duration: number;
|
||||
status: 'none' | 'passed' | 'failed' | 'timedOut' | 'skipped';
|
||||
status?: TestStatus;
|
||||
expectedStatus: TestStatus;
|
||||
error?: any;
|
||||
stdout: (string | Buffer)[];
|
||||
stderr: (string | Buffer)[];
|
||||
@ -96,9 +106,12 @@ export class Suite {
|
||||
suites: Suite[] = [];
|
||||
tests: Test[] = [];
|
||||
only = false;
|
||||
skipped = false;
|
||||
file: string;
|
||||
configuration: Configuration;
|
||||
|
||||
// Skipped & flaky are resolved based on options in worker only
|
||||
// We will compute them there and send to the runner (front-end)
|
||||
_skipped = false;
|
||||
_configurationString: string;
|
||||
|
||||
_hooks: { type: string, fn: Function } [] = [];
|
||||
@ -124,7 +137,7 @@ export class Suite {
|
||||
}
|
||||
|
||||
_isSkipped(): boolean {
|
||||
return this.skipped || (this.parent && this.parent._isSkipped());
|
||||
return this._skipped || (this.parent && this.parent._isSkipped());
|
||||
}
|
||||
|
||||
_addTest(test: Test) {
|
||||
@ -163,7 +176,7 @@ export class Suite {
|
||||
const suite = new Suite(this.title);
|
||||
suite.only = this.only;
|
||||
suite.file = this.file;
|
||||
suite.skipped = this.skipped;
|
||||
suite._skipped = this._skipped;
|
||||
return suite;
|
||||
}
|
||||
|
||||
|
@ -46,7 +46,6 @@ export class TestRunner extends EventEmitter {
|
||||
private _ids: Set<string>;
|
||||
private _remaining: Set<string>;
|
||||
private _trialRun: any;
|
||||
private _configuredFile: any;
|
||||
private _parsedGeneratorConfiguration: any = {};
|
||||
private _config: RunnerConfig;
|
||||
private _timeout: number;
|
||||
@ -55,6 +54,7 @@ export class TestRunner extends EventEmitter {
|
||||
private _stdErrBuffer: (string | Buffer)[] = [];
|
||||
private _testResult: TestResult | null = null;
|
||||
private _suite: Suite;
|
||||
private _loaded = false;
|
||||
|
||||
constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) {
|
||||
super();
|
||||
@ -66,7 +66,6 @@ export class TestRunner extends EventEmitter {
|
||||
this._trialRun = config.trialRun;
|
||||
this._timeout = config.timeout;
|
||||
this._config = config;
|
||||
this._configuredFile = entry.file + `::[${entry.configurationString}]`;
|
||||
for (const {name, value} of entry.configuration)
|
||||
this._parsedGeneratorConfiguration[name] = value;
|
||||
this._parsedGeneratorConfiguration['parallelIndex'] = workerId;
|
||||
@ -77,14 +76,16 @@ export class TestRunner extends EventEmitter {
|
||||
this._trialRun = true;
|
||||
}
|
||||
|
||||
fatalError(error: Error | any) {
|
||||
this._fatalError = serializeError(error);
|
||||
unhandledError(error: Error | any) {
|
||||
if (this._testResult) {
|
||||
this._testResult.error = this._fatalError;
|
||||
this._testResult.error = serializeError(error);
|
||||
this.emit('testEnd', {
|
||||
id: this._testId,
|
||||
result: this._testResult
|
||||
});
|
||||
} else if (!this._loaded) {
|
||||
// No current test - fatal error.
|
||||
this._fatalError = serializeError(error);
|
||||
}
|
||||
this._reportDone();
|
||||
}
|
||||
@ -114,6 +115,7 @@ export class TestRunner extends EventEmitter {
|
||||
require(this._suite.file);
|
||||
revertBabelRequire();
|
||||
this._suite._renumber();
|
||||
this._loaded = true;
|
||||
|
||||
rerunRegistrations(this._suite.file, 'test');
|
||||
await this._runSuite(this._suite);
|
||||
@ -132,7 +134,6 @@ export class TestRunner extends EventEmitter {
|
||||
await this._runSuite(entry);
|
||||
else
|
||||
await this._runTest(entry);
|
||||
|
||||
}
|
||||
try {
|
||||
await this._runHooks(suite, 'afterAll', 'after');
|
||||
@ -151,6 +152,9 @@ export class TestRunner extends EventEmitter {
|
||||
|
||||
const id = test._id;
|
||||
this._testId = id;
|
||||
// We only know resolved skipped/flaky value in the worker,
|
||||
// send it to the runner.
|
||||
test._skipped = test._skipped || test.suite._isSkipped();
|
||||
this.emit('testBegin', {
|
||||
id,
|
||||
skipped: test._skipped,
|
||||
@ -160,13 +164,14 @@ export class TestRunner extends EventEmitter {
|
||||
const result: TestResult = {
|
||||
duration: 0,
|
||||
status: 'passed',
|
||||
expectedStatus: test._expectedStatus,
|
||||
stdout: [],
|
||||
stderr: [],
|
||||
data: {}
|
||||
};
|
||||
this._testResult = result;
|
||||
|
||||
if (test._skipped || test.suite._isSkipped()) {
|
||||
if (test._skipped) {
|
||||
result.status = 'skipped';
|
||||
this.emit('testEnd', { id, result });
|
||||
return;
|
||||
@ -181,7 +186,7 @@ export class TestRunner extends EventEmitter {
|
||||
await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo);
|
||||
await this._runHooks(test.suite, 'afterEach', 'after', testInfo);
|
||||
} else {
|
||||
result.status = 'passed';
|
||||
result.status = result.expectedStatus;
|
||||
}
|
||||
} catch (error) {
|
||||
// Error in the test fixture teardown.
|
||||
|
@ -49,12 +49,12 @@ let testRunner: TestRunner;
|
||||
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (testRunner)
|
||||
testRunner.fatalError(reason);
|
||||
testRunner.unhandledError(reason);
|
||||
});
|
||||
|
||||
process.on('uncaughtException', error => {
|
||||
if (testRunner)
|
||||
testRunner.fatalError(error);
|
||||
testRunner.unhandledError(error);
|
||||
});
|
||||
|
||||
process.on('message', async message => {
|
||||
|
24
test-runner/test/assets/expected-failure.js
Normal file
24
test-runner/test/assets/expected-failure.js
Normal file
@ -0,0 +1,24 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
require('../../lib');
|
||||
|
||||
it.fail(true)('fails',() => {
|
||||
expect(1 + 1).toBe(3);
|
||||
});
|
||||
|
||||
it('non-empty remaining',() => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
22
test-runner/test/assets/nested-skip.js
Normal file
22
test-runner/test/assets/nested-skip.js
Normal file
@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
require('../../');
|
||||
|
||||
describe.skip(true)('skipped', () => {
|
||||
it('succeeds',() => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
});
|
20
test-runner/test/assets/one-timeout.js
Normal file
20
test-runner/test/assets/one-timeout.js
Normal file
@ -0,0 +1,20 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
require('../../');
|
||||
|
||||
it('timeout', async () => {
|
||||
await new Promise(f => setTimeout(f, 10000));
|
||||
});
|
20
test-runner/test/assets/unexpected-pass.js
Normal file
20
test-runner/test/assets/unexpected-pass.js
Normal file
@ -0,0 +1,20 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
require('../../');
|
||||
|
||||
it.fail(true)('succeeds',() => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
@ -14,6 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import colors from 'colors/safe';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@ -30,6 +31,14 @@ it('should fail', async () => {
|
||||
expect(result.failed).toBe(1);
|
||||
});
|
||||
|
||||
it('should timeout', async () => {
|
||||
const { exitCode, passed, failed, timedOut } = await runTest('one-timeout.js', { timeout: 100 });
|
||||
expect(exitCode).toBe(1);
|
||||
expect(passed).toBe(0);
|
||||
expect(failed).toBe(0);
|
||||
expect(timedOut).toBe(1);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
const result = await runTest('one-success.js');
|
||||
expect(result.exitCode).toBe(0);
|
||||
@ -85,6 +94,15 @@ it('should retry failures', async () => {
|
||||
expect(result.flaky).toBe(1);
|
||||
});
|
||||
|
||||
it('should retry timeout', async () => {
|
||||
const { exitCode, passed, failed, timedOut, output } = await runTest('one-timeout.js', { timeout: 100, retries: 2 });
|
||||
expect(exitCode).toBe(1);
|
||||
expect(passed).toBe(0);
|
||||
expect(failed).toBe(0);
|
||||
expect(timedOut).toBe(1);
|
||||
expect(output.split('\n')[0]).toBe(colors.red('T').repeat(3));
|
||||
});
|
||||
|
||||
it('should repeat each', async () => {
|
||||
const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 });
|
||||
expect(exitCode).toBe(0);
|
||||
@ -106,6 +124,44 @@ it('should allow flaky', async () => {
|
||||
expect(result.flaky).toBe(1);
|
||||
});
|
||||
|
||||
it('should fail on unexpected pass', async () => {
|
||||
const { exitCode, failed, output } = await runTest('unexpected-pass.js');
|
||||
expect(exitCode).toBe(1);
|
||||
expect(failed).toBe(1);
|
||||
expect(output).toContain('passed unexpectedly');
|
||||
});
|
||||
|
||||
it('should fail on unexpected pass with retries', async () => {
|
||||
const { exitCode, failed, output } = await runTest('unexpected-pass.js', { retries: 1 });
|
||||
expect(exitCode).toBe(1);
|
||||
expect(failed).toBe(1);
|
||||
expect(output).toContain('passed unexpectedly');
|
||||
});
|
||||
|
||||
it('should not retry unexpected pass', async () => {
|
||||
const { exitCode, passed, failed, output } = await runTest('unexpected-pass.js', { retries: 2 });
|
||||
expect(exitCode).toBe(1);
|
||||
expect(passed).toBe(0);
|
||||
expect(failed).toBe(1);
|
||||
expect(output.split('\n')[0]).toBe(colors.red('P'));
|
||||
});
|
||||
|
||||
it('should not retry expected failure', async () => {
|
||||
const { exitCode, passed, failed, output } = await runTest('expected-failure.js', { retries: 2 });
|
||||
expect(exitCode).toBe(0);
|
||||
expect(passed).toBe(2);
|
||||
expect(failed).toBe(0);
|
||||
expect(output.split('\n')[0]).toBe(colors.green('f') + colors.green('·'));
|
||||
});
|
||||
|
||||
it('should respect nested skip', async () => {
|
||||
const { exitCode, passed, failed, skipped } = await runTest('nested-skip.js');
|
||||
expect(exitCode).toBe(0);
|
||||
expect(passed).toBe(0);
|
||||
expect(failed).toBe(0);
|
||||
expect(skipped).toBe(1);
|
||||
});
|
||||
|
||||
async function runTest(filePath: string, params: any = {}) {
|
||||
const outputDir = path.join(__dirname, 'test-results');
|
||||
const reportFile = path.join(outputDir, 'results.json');
|
||||
@ -125,14 +181,20 @@ async function runTest(filePath: string, params: any = {}) {
|
||||
});
|
||||
const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
|
||||
const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
|
||||
const timedOut = (/(\d+) timed out/.exec(output.toString()) || [])[1];
|
||||
const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1];
|
||||
const skipped = (/(\d+) skipped/.exec(output.toString()) || [])[1];
|
||||
const report = JSON.parse(fs.readFileSync(reportFile).toString());
|
||||
let outputStr = output.toString();
|
||||
outputStr = outputStr.substring(1, outputStr.length - 1);
|
||||
return {
|
||||
exitCode: status,
|
||||
output: output.toString(),
|
||||
output: outputStr,
|
||||
passed: parseInt(passed, 10),
|
||||
failed: parseInt(failed || '0', 10),
|
||||
timedOut: parseInt(timedOut || '0', 10),
|
||||
flaky: parseInt(flaky || '0', 10),
|
||||
skipped: parseInt(skipped || '0', 10),
|
||||
report
|
||||
};
|
||||
}
|
||||
|
BIN
test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png
Normal file
BIN
test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 3.9 KiB |
BIN
test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png
Normal file
BIN
test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 3.4 KiB |
@ -48,7 +48,7 @@ it('should work with correct credentials', async({browser, server}) => {
|
||||
await context.close();
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM && !options.HEADLESS)('should fail with wrong credentials', async({browser, server}) => {
|
||||
it('should fail with wrong credentials', async({browser, server}) => {
|
||||
server.setAuth('/empty.html', 'user', 'pass');
|
||||
const context = await browser.newContext({
|
||||
httpCredentials: { username: 'foo', password: 'bar' }
|
||||
|
@ -137,7 +137,7 @@ it('should be isolated between contexts', async({browser, server}) => {
|
||||
]);
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should not change default locale in another context', async({browser, server}) => {
|
||||
it('should not change default locale in another context', async({browser, server}) => {
|
||||
async function getContextLocale(context) {
|
||||
const page = await context.newPage();
|
||||
return await page.evaluate(() => (new Intl.NumberFormat()).resolvedOptions().locale);
|
||||
|
@ -156,7 +156,7 @@ it('should fire page lifecycle events', async function({browser, server}) {
|
||||
await context.close();
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => {
|
||||
it.fixme(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => {
|
||||
// WebKit: Shift+Click does not open a new window.
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
@ -170,7 +170,7 @@ it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, serve
|
||||
await context.close();
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => {
|
||||
it.fixme(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => {
|
||||
// Firefox: reports an opener in this case.
|
||||
// WebKit: Ctrl+Click does not open a new tab.
|
||||
const context = await browser.newContext();
|
||||
|
@ -69,7 +69,7 @@ it('should work for multiple pages sharing same process', async({browser, server
|
||||
await context.close();
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should not change default timezone in another context', async({browser, server}) => {
|
||||
it('should not change default timezone in another context', async({browser, server}) => {
|
||||
async function getContextTimezone(context) {
|
||||
const page = await context.newPage();
|
||||
return await page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone);
|
||||
|
@ -81,7 +81,7 @@ it.skip(options.WIRE).slow()('disconnected event should be emitted when browser
|
||||
expect(disconnected2).toBe(1);
|
||||
});
|
||||
|
||||
it.skip(options.WIRE).fail(options.CHROMIUM && WIN).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => {
|
||||
it.skip(options.WIRE).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => {
|
||||
const __testHookBeforeCreateBrowser = () => { throw new Error('Dummy') };
|
||||
const error = await browserType.connect({ wsEndpoint: remoteServer.wsEndpoint(), __testHookBeforeCreateBrowser } as any).catch(e => e);
|
||||
expect(error.message).toContain('Dummy');
|
||||
|
@ -43,7 +43,7 @@ it.skip(options.FIREFOX)('should throw if page argument is passed', async({brows
|
||||
expect(waitError.message).toContain('can not specify page');
|
||||
});
|
||||
|
||||
it.fail(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => {
|
||||
it.fixme(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => {
|
||||
// I'm getting ENCONRESET on this one.
|
||||
const options = Object.assign({}, defaultBrowserOptions, {executablePath: path.join(__dirname, 'assets', 'dummy_bad_browser_executable.js')});
|
||||
let waitError = null;
|
||||
|
@ -48,7 +48,7 @@ it('should respect CSP', async({page, server}) => {
|
||||
expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS');
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
|
||||
it.fixme(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
|
||||
// TODO: the test passes on Windows locally but fails on GitHub Action bot,
|
||||
// apparently due to a Media Pack issue in the Windows Server.
|
||||
// Also the test is very flaky on Linux WebKit.
|
||||
|
@ -65,7 +65,7 @@ it.skip(!options.CHROMIUM)('should handle remote -> local -> remote transitions'
|
||||
expect(await countOOPIFs(browser)).toBe(1);
|
||||
});
|
||||
|
||||
it.fail(true)('should get the proper viewport', async({browser, page, server}) => {
|
||||
it.fixme(options.CHROMIUM).skip(!options.CHROMIUM)('should get the proper viewport', async({browser, page, server}) => {
|
||||
expect(page.viewportSize()).toEqual({width: 1280, height: 720});
|
||||
await page.goto(server.PREFIX + '/dynamic-oopif.html');
|
||||
expect(page.frames().length).toBe(2);
|
||||
|
@ -39,7 +39,7 @@ it.fail(true)('should report that selector does not match anymore', async ({page
|
||||
expect(error.message).toContain('element does not match the selector anymore');
|
||||
});
|
||||
|
||||
it.fail(true)('should not retarget the handle when element is recycled', async ({page, server}) => {
|
||||
it.fixme(true)('should not retarget the handle when element is recycled', async ({page, server}) => {
|
||||
await page.goto(server.PREFIX + '/react.html');
|
||||
await page.evaluate(() => {
|
||||
renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] ));
|
||||
@ -66,7 +66,7 @@ it('should timeout when click opens alert', async({page, server}) => {
|
||||
await dialog.dismiss();
|
||||
});
|
||||
|
||||
it.fail(true)('should retarget when element is recycled during hit testing', async ({page, server}) => {
|
||||
it.fixme(true)('should retarget when element is recycled during hit testing', async ({page, server}) => {
|
||||
await page.goto(server.PREFIX + '/react.html');
|
||||
await page.evaluate(() => {
|
||||
renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2' })] ));
|
||||
@ -81,7 +81,7 @@ it.fail(true)('should retarget when element is recycled during hit testing', asy
|
||||
expect(await page.evaluate('window.button2')).toBe(undefined);
|
||||
});
|
||||
|
||||
it.fail(true)('should retarget when element is recycled before enabled check', async ({page, server}) => {
|
||||
it.fixme(true)('should retarget when element is recycled before enabled check', async ({page, server}) => {
|
||||
await page.goto(server.PREFIX + '/react.html');
|
||||
await page.evaluate(() => {
|
||||
renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] ));
|
||||
|
@ -322,7 +322,7 @@ it('should click the button inside an iframe', async({page, server}) => {
|
||||
expect(await frame.evaluate(() => window['result'])).toBe('Clicked');
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => {
|
||||
it.fixme(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => {
|
||||
// @see https://github.com/GoogleChrome/puppeteer/issues/4110
|
||||
// @see https://bugs.chromium.org/p/chromium/issues/detail?id=986390
|
||||
// @see https://chromium-review.googlesource.com/c/chromium/src/+/1742784
|
||||
|
@ -24,12 +24,12 @@
|
||||
function traceAPICoverage(apiCoverage, api, events) {
|
||||
const uninstalls = [];
|
||||
for (const [name, classType] of Object.entries(api)) {
|
||||
// console.log('trace', name);
|
||||
const className = name.substring(0, 1).toLowerCase() + name.substring(1);
|
||||
for (const methodName of Reflect.ownKeys(classType.prototype)) {
|
||||
const method = Reflect.get(classType.prototype, methodName);
|
||||
if (methodName === 'constructor' || typeof methodName !== 'string' || methodName.startsWith('_') || typeof method !== 'function')
|
||||
continue;
|
||||
|
||||
apiCoverage.set(`${className}.${methodName}`, false);
|
||||
const override = function(...args) {
|
||||
apiCoverage.set(`${className}.${methodName}`, true);
|
||||
|
@ -75,7 +75,7 @@ it('should support extraHTTPHeaders option', async ({server, launchPersistent})
|
||||
expect(request.headers['foo']).toBe('bar');
|
||||
});
|
||||
|
||||
it('should accept userDataDir', async ({launchPersistent, tmpDir}) => {
|
||||
it.flaky(options.CHROMIUM)('should accept userDataDir', async ({launchPersistent, tmpDir}) => {
|
||||
const {page, context} = await launchPersistent();
|
||||
// Note: we need an open page to make sure its functional.
|
||||
expect(fs.readdirSync(tmpDir).length).toBeGreaterThan(0);
|
||||
@ -111,7 +111,7 @@ it.slow()('should restore state from userDataDir', async({browserType, defaultBr
|
||||
await removeUserDataDir(userDataDir2);
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM && (WIN || MAC)).slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions, server, launchPersistent}) => {
|
||||
it.slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions, server, launchPersistent}) => {
|
||||
const userDataDir = await makeUserDataDir();
|
||||
const browserContext = await browserType.launchPersistentContext(userDataDir, defaultBrowserOptions);
|
||||
const page = await browserContext.newPage();
|
||||
|
@ -62,7 +62,7 @@ it('should dismiss the confirm prompt', async({page}) => {
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT)('should be able to close context with open alert', async({browser}) => {
|
||||
it.fixme(options.WEBKIT && MAC)('should be able to close context with open alert', async({browser}) => {
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
const alertPromise = page.waitForEvent('dialog');
|
||||
|
@ -250,7 +250,7 @@ it(`should report download path within page.on('download', …) handler for Blob
|
||||
expect(fs.readFileSync(path).toString()).toBe('Hello world');
|
||||
await page.close();
|
||||
})
|
||||
it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => {
|
||||
it.fixme(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => {
|
||||
// Firefox does not download on alt-click by default.
|
||||
// Our WebKit embedder does not download on alt-click, although Safari does.
|
||||
server.setRoute('/download', (req, res) => {
|
||||
@ -271,7 +271,7 @@ it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads',
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => {
|
||||
it.fixme(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => {
|
||||
// TODO: - the test fails in headful Chromium as the popup page gets closed along
|
||||
// with the session before download completed event arrives.
|
||||
// - WebKit doesn't close the popup page
|
||||
|
@ -17,6 +17,7 @@
|
||||
|
||||
import './playwright.fixtures';
|
||||
import utils from './utils';
|
||||
import { options } from './playwright.fixtures';
|
||||
|
||||
it('should work', async ({ page, server }) => {
|
||||
await page.goto(server.EMPTY_PAGE);
|
||||
@ -34,7 +35,7 @@ it('should work for cross-process iframes', async ({ page, server }) => {
|
||||
expect(await elementHandle.ownerFrame()).toBe(frame);
|
||||
});
|
||||
|
||||
it('should work for document', async ({ page, server }) => {
|
||||
it.flaky(WIN && options.WEBKIT)('should work for document', async ({ page, server }) => {
|
||||
await page.goto(server.EMPTY_PAGE);
|
||||
await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
|
||||
const frame = page.frames()[1];
|
||||
|
@ -351,7 +351,7 @@ it.skip(ffheadful || options.WIRE)('should restore viewport after element screen
|
||||
await context.close();
|
||||
});
|
||||
|
||||
it.skip(ffheadful)('should wait for element to stop moving', async({page, server, golden}) => {
|
||||
it.skip(ffheadful).flaky(options.WEBKIT && !options.HEADLESS && LINUX)('should wait for element to stop moving', async ({ page, server, golden }) => {
|
||||
await page.setViewportSize({ width: 500, height: 500 });
|
||||
await page.goto(server.PREFIX + '/grid.html');
|
||||
const elementHandle = await page.$('.box:nth-of-type(3)');
|
||||
|
@ -16,6 +16,7 @@
|
||||
*/
|
||||
|
||||
import './playwright.fixtures';
|
||||
import { options } from './playwright.fixtures';
|
||||
|
||||
async function giveItAChanceToResolve(page) {
|
||||
for (let i = 0; i < 5; i++)
|
||||
|
@ -147,7 +147,7 @@ it.fail(options.CHROMIUM || options.FIREFOX)('should work in iframes that failed
|
||||
expect(await page.frames()[1].$('div')).toBeTruthy();
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => {
|
||||
it.fixme(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => {
|
||||
// Chromium does not report isolated world for the iframe.
|
||||
await page.goto(server.EMPTY_PAGE);
|
||||
await page.evaluate(() => {
|
||||
|
@ -171,7 +171,7 @@ it('should report different frame instance when frame re-attaches', async({page,
|
||||
expect(frame1).not.toBe(frame2);
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => {
|
||||
it.fixme(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => {
|
||||
server.setRoute('/x-frame-options-deny.html', async (req, res) => {
|
||||
res.setHeader('Content-Type', 'text/html');
|
||||
res.setHeader('X-Frame-Options', 'DENY');
|
||||
|
@ -128,7 +128,7 @@ it('should(not) block third party cookies', async({browserType, defaultBrowserOp
|
||||
await browser.close();
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) {
|
||||
it.fixme(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) {
|
||||
// Our WebKit embedder does not respect window features.
|
||||
const browser = await browserType.launch({...defaultBrowserOptions, headless: false });
|
||||
const context = await browser.newContext({ viewport: null });
|
||||
|
@ -27,7 +27,7 @@ function dimensions() {
|
||||
};
|
||||
}
|
||||
|
||||
it.fail(options.FIREFOX && WIN)('should click the document', async({page, server}) => {
|
||||
it.flaky(options.FIREFOX && WIN)('should click the document', async({page, server}) => {
|
||||
// Occasionally times out on options.FIREFOX on Windows: https://github.com/microsoft/playwright/pull/1911/checks?check_run_id=607149016
|
||||
await page.evaluate(() => {
|
||||
window["clickPromise"] = new Promise(resolve => {
|
||||
|
@ -88,7 +88,7 @@ it('should work with content', async({page, server}) => {
|
||||
expect(await page.evaluate(() => window['__injected'])).toBe(35);
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should throw when added with content to the CSP page', async({page, server}) => {
|
||||
it('should throw when added with content to the CSP page', async({page, server}) => {
|
||||
// Firefox fires onload for blocked script before it issues the CSP console error.
|
||||
await page.goto(server.PREFIX + '/csp.html');
|
||||
let error = null;
|
||||
|
@ -114,7 +114,7 @@ it('should work in cross-process iframe', async({browser, server}) => {
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should change the actual colors in css', async({page}) => {
|
||||
it('should change the actual colors in css', async({page}) => {
|
||||
await page.setContent(`
|
||||
<style>
|
||||
@media (prefers-color-scheme: dark) {
|
||||
|
@ -415,7 +415,7 @@ it('should not throw an error when evaluation does a navigation', async ({ page,
|
||||
expect(result).toEqual([42]);
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT)('should not throw an error when evaluation does a synchronous navigation and returns an object', async ({ page, server }) => {
|
||||
it.fixme(options.WEBKIT)('should not throw an error when evaluation does a synchronous navigation and returns an object', async ({ page, server }) => {
|
||||
// It is imporant to be on about:blank for sync reload.
|
||||
const result = await page.evaluate(() => {
|
||||
window.location.reload();
|
||||
@ -433,7 +433,7 @@ it('should not throw an error when evaluation does a synchronous navigation and
|
||||
expect(result).toBe(undefined);
|
||||
});
|
||||
|
||||
it.fail(options.WIRE)('should transfer 100Mb of data from page to node.js', async ({ page }) => {
|
||||
it('should transfer 100Mb of data from page to node.js', async ({ page }) => {
|
||||
// This is too slow with wire.
|
||||
const a = await page.evaluate(() => Array(100 * 1024 * 1024 + 1).join('a'));
|
||||
expect(a.length).toBe(100 * 1024 * 1024);
|
||||
@ -453,7 +453,7 @@ it('should work even when JSON is set to null', async ({ page }) => {
|
||||
expect(result).toEqual({ abc: 123 });
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should await promise from popup', async ({ page, server }) => {
|
||||
it('should await promise from popup', async ({ page, server }) => {
|
||||
// Something is wrong about the way Firefox waits for the chained promise
|
||||
await page.goto(server.EMPTY_PAGE);
|
||||
const result = await page.evaluate(() => {
|
||||
|
@ -17,8 +17,6 @@
|
||||
|
||||
import { options } from './playwright.fixtures';
|
||||
|
||||
const CRASH_FAIL = (options.FIREFOX && WIN) || options.WIRE;
|
||||
// Firefox Win: it just doesn't crash sometimes.
|
||||
function crash(pageImpl, browserName) {
|
||||
if (browserName === 'chromium')
|
||||
pageImpl.mainFrame().goto('chrome://crash').catch(e => {});
|
||||
@ -28,13 +26,13 @@ function crash(pageImpl, browserName) {
|
||||
pageImpl._delegate._session.send('Page.crash', {}).catch(e => {});
|
||||
}
|
||||
|
||||
it.fail(CRASH_FAIL)('should emit crash event when page crashes', async({page, browserName, toImpl}) => {
|
||||
it.fail(options.WIRE)('should emit crash event when page crashes', async({page, browserName, toImpl}) => {
|
||||
await page.setContent(`<div>This page should crash</div>`);
|
||||
crash(toImpl(page), browserName);
|
||||
await new Promise(f => page.on('crash', f));
|
||||
});
|
||||
|
||||
it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page, browserName, toImpl}) => {
|
||||
it.fail(options.WIRE)('should throw on any action after page crashes', async({page, browserName, toImpl}) => {
|
||||
await page.setContent(`<div>This page should crash</div>`);
|
||||
crash(toImpl(page), browserName);
|
||||
await page.waitForEvent('crash');
|
||||
@ -43,7 +41,7 @@ it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page
|
||||
expect(err.message).toContain('crash');
|
||||
});
|
||||
|
||||
it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page, browserName, toImpl}) => {
|
||||
it.fail(options.WIRE)('should cancel waitForEvent when page crashes', async({page, browserName, toImpl}) => {
|
||||
await page.setContent(`<div>This page should crash</div>`);
|
||||
const promise = page.waitForEvent('response').catch(e => e);
|
||||
crash(toImpl(page), browserName);
|
||||
@ -51,7 +49,7 @@ it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page,
|
||||
expect(error.message).toContain('Page crashed');
|
||||
});
|
||||
|
||||
it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, browserName, toImpl, server}) => {
|
||||
it.fixme(options.WIRE)('should cancel navigation when page crashes', async({page, browserName, toImpl, server}) => {
|
||||
await page.setContent(`<div>This page should crash</div>`);
|
||||
server.setRoute('/one-style.css', () => {});
|
||||
const promise = page.goto(server.PREFIX + '/one-style.html').catch(e => e);
|
||||
@ -61,7 +59,7 @@ it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, b
|
||||
expect(error.message).toContain('Navigation failed because page crashed');
|
||||
});
|
||||
|
||||
it.fail(CRASH_FAIL)('should be able to close context when page crashes', async({page, browserName, toImpl}) => {
|
||||
it.fixme(options.WIRE)('should be able to close context when page crashes', async({page, browserName, toImpl}) => {
|
||||
await page.setContent(`<div>This page should crash</div>`);
|
||||
crash(toImpl(page), browserName);
|
||||
await page.waitForEvent('crash');
|
||||
|
@ -214,7 +214,7 @@ it.skip(ffheadful)('should work for translateZ', async({page, server, golden}) =
|
||||
expect(screenshot).toMatchImage(golden('screenshot-translateZ.png'));
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX || options.WEBKIT)('should work for webgl', async({page, server, golden}) => {
|
||||
it.fixme(options.FIREFOX).flaky(options.WEBKIT && LINUX)('should work for webgl', async({page, server, golden}) => {
|
||||
await page.setViewportSize({width: 640, height: 480});
|
||||
await page.goto(server.PREFIX + '/screenshots/webgl.html');
|
||||
const screenshot = await page.screenshot();
|
||||
|
@ -141,7 +141,7 @@ it('should work with DOM history.back()/history.forward()', async({page, server}
|
||||
expect(page.url()).toBe(server.PREFIX + '/second.html');
|
||||
});
|
||||
|
||||
it.fail(options.FIREFOX)('should work when subframe issues window.stop()', async({page, server}) => {
|
||||
it('should work when subframe issues window.stop()', async({page, server}) => {
|
||||
server.setRoute('/frames/style.css', (req, res) => {});
|
||||
const navigationPromise = page.goto(server.PREFIX + '/frames/one-frame.html');
|
||||
const frame = await new Promise<Frame>(f => page.once('frameattached', f));
|
||||
|
@ -98,7 +98,7 @@ describe.skip(options.WEBKIT)('permissions', () => {
|
||||
expect(await getPermission(page, 'geolocation')).toBe('prompt');
|
||||
});
|
||||
|
||||
it.fail(options.WEBKIT || options.FIREFOX || (options.CHROMIUM && !options.HEADLESS))('should trigger permission onchange', async({page, server, context}) => {
|
||||
it.fail(options.WEBKIT || (options.CHROMIUM && !options.HEADLESS))('should trigger permission onchange', async({page, server, context}) => {
|
||||
//TODO: flaky
|
||||
// - Linux: https://github.com/microsoft/playwright/pull/1790/checks?check_run_id=587327883
|
||||
// - Win: https://ci.appveyor.com/project/aslushnikov/playwright/builds/32402536
|
||||
|
@ -173,7 +173,7 @@ class VideoPlayer {
|
||||
}
|
||||
}
|
||||
|
||||
it.fail(options.CHROMIUM)('should capture static page', async({page, tmpDir, videoPlayer, toImpl}) => {
|
||||
it.fixme(options.CHROMIUM)('should capture static page', async({page, tmpDir, videoPlayer, toImpl}) => {
|
||||
if (!toImpl)
|
||||
return;
|
||||
const videoFile = path.join(tmpDir, 'v.webm');
|
||||
@ -199,7 +199,7 @@ it.fail(options.CHROMIUM)('should capture static page', async({page, tmpDir, vid
|
||||
expectAll(pixels, almostRed);
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM)('should capture navigation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
|
||||
it.fixme(options.CHROMIUM).flaky(options.WEBKIT)('should capture navigation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
|
||||
if (!toImpl)
|
||||
return;
|
||||
const videoFile = path.join(tmpDir, 'v.webm');
|
||||
@ -233,7 +233,7 @@ it.fail(options.CHROMIUM)('should capture navigation', async({page, tmpDir, serv
|
||||
});
|
||||
|
||||
// Accelerated compositing is disabled in WebKit on Windows.
|
||||
it.fail(options.CHROMIUM || (options.WEBKIT && WIN))('should capture css transformation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
|
||||
it.fixme(options.CHROMIUM || (options.WEBKIT && WIN)).flaky(options.WEBKIT && LINUX)('should capture css transformation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
|
||||
if (!toImpl)
|
||||
return;
|
||||
const videoFile = path.join(tmpDir, 'v.webm');
|
||||
@ -258,7 +258,7 @@ it.fail(options.CHROMIUM || (options.WEBKIT && WIN))('should capture css transfo
|
||||
}
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM)('should fire start/stop events when page created/closed', async({browser, tmpDir, server, toImpl}) => {
|
||||
it.slow().fixme(options.CHROMIUM)('should fire start/stop events when page created/closed', async({browser, tmpDir, server, toImpl}) => {
|
||||
if (!toImpl)
|
||||
return;
|
||||
// Use server side of the context. All the code below also uses server side APIs.
|
||||
@ -280,7 +280,7 @@ it.fail(options.CHROMIUM)('should fire start/stop events when page created/close
|
||||
await context.close();
|
||||
});
|
||||
|
||||
it.fail(options.CHROMIUM)('should fire start event for popups', async({browser, tmpDir, server, toImpl}) => {
|
||||
it.fixme(options.CHROMIUM)('should fire start event for popups', async({browser, tmpDir, server, toImpl}) => {
|
||||
if (!toImpl)
|
||||
return;
|
||||
// Use server side of the context. All the code below also uses server side APIs.
|
||||
|
Loading…
Reference in New Issue
Block a user