playwright/tests/playwright-test/loader.spec.ts
Dmitry Gozman b86926e691
fix(test runner): update timeout error messages (#15117)
Some of the current timeout error messages are confusing, because they do not suggest that the issue is most likely a slow test. This PR updates timeout messages as follows:
- Test timeout of 30000ms exceeded.
- Test timeout of 30000ms exceeded while setting up "browser".
- Test timeout of 30000ms exceeded while tearing down "context".
- Test timeout of 30000ms exceeded while setting up "playwright configuration".
- Test timeout of 30000ms exceeded while running "beforeEach" hook.
- Test timeout of 30000ms exceeded while running "afterEach" hook.
- "beforeAll" hook timeout of 30000ms exceeded.
- "afterAll" hook timeout of 30000ms exceeded.
- Worker teardown timeout of 30000ms exceeded.
- "skip" modifier timeout of 30000ms exceeded.
- Fixture "myCustomFixture" timeout of 5000ms exceeded.
2022-06-30 17:05:08 -07:00

415 lines
13 KiB
TypeScript

/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect, stripAnsi } from './playwright-test-fixtures';
import path from 'path';
test('should return the location of a syntax error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'error.spec.js': `
const x = {
foo: 'bar';
};
`
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.output).toContain('error.spec.js:6');
});
test('should print an improper error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'error.spec.js': `
throw 123;
`
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.output).toContain('123');
});
test('should print a null error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'error.spec.js': `
throw null;
`
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.output).toContain('null');
});
test('should return the location of a syntax error in typescript', async ({ runInlineTest }) => {
const result = await runInlineTest({
'error.spec.ts': `
const x = {
foo: 'bar';
};
`
}, {}, {
FORCE_COLOR: '0'
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.output).toContain('error.spec.ts');
expect(result.output).toContain(`'bar';`);
});
test('should allow export default form the config file', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
export default { timeout: 1000 };
`,
'a.test.ts': `
const { test } = pwt;
test('fails', async ({}, testInfo) => {
await new Promise(f => setTimeout(f, 2000));
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
test('should validate configuration object', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
export default { timeout: '1000' };
`,
'a.test.ts': `
const { test } = pwt;
test('works', () => {});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(0);
expect(result.output).toContain('playwright.config.ts: config.timeout must be a non-negative number');
});
test('should match tests well', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test('works', () => {});
`,
'hello.spec.ts': `
const { test } = pwt;
test('works', () => {});
`,
'test.ts': `
const { test } = pwt;
test('works', () => {});
`,
'spec.ts': `
const { test } = pwt;
test('works', () => {});
`,
'strange.....spec.ts': `
const { test } = pwt;
test('works', () => {});
`,
'badspec.ts': `
const { test } = pwt;
test('bad', () => { throw new Error('badspec.ts')});
`,
'specspec.ts': `
const { test } = pwt;
test('bad', () => { throw new Error('specspec.ts')});
`,
'a.testtest.ts': `
const { test } = pwt;
test('bad', () => { throw new Error('a.testtest.ts')});
`,
'b.testspec.ts': `
const { test } = pwt;
test('bad', () => { throw new Error('b.testspec.ts')});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(5);
});
test('should load an mjs file', async ({ runInlineTest }) => {
const { exitCode, passed } = await runInlineTest({
'a.spec.mjs': `
const { test } = pwt;
test('succeeds', () => {
expect(1 + 1).toBe(2);
});
`
});
expect(passed).toBe(1);
expect(exitCode).toBe(0);
});
test('should throw a nice error if a js file uses import', async ({ runInlineTest }) => {
const { exitCode, output } = await runInlineTest({
'a.spec.js': `
import fs from 'fs';
const { test } = folio;
test('succeeds', () => {
expect(1 + 1).toBe(2);
});
`
});
expect(exitCode).toBe(1);
expect(output).toContain('a.spec.js');
expect(output).toContain('Cannot use import statement outside a module');
});
test('should load esm when package.json has type module', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.js': `
//@no-header
import * as fs from 'fs';
export default { projects: [{name: 'foo'}] };
`,
'package.json': JSON.stringify({ type: 'module' }),
'a.esm.test.js': `
const { test } = pwt;
test('check project name', ({}, testInfo) => {
expect(testInfo.project.name).toBe('foo');
});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('should load esm config files', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.mjs': `
//@no-header
import * as fs from 'fs';
export default { projects: [{name: 'foo'}] };
`,
'a.test.ts': `
const { test } = pwt;
test('check project name', ({}, testInfo) => {
expect(testInfo.project.name).toBe('foo');
});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('should fail to load ts from esm when package.json has type module', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.js': `
//@no-header
import * as fs from 'fs';
export default { projects: [{name: 'foo'}] };
`,
'package.json': JSON.stringify({ type: 'module' }),
'a.test.js': `
import { foo } from './b.ts';
const { test } = pwt;
test('check project name', ({}, testInfo) => {
expect(testInfo.project.name).toBe('foo');
});
`,
'b.ts': `
export const foo: string = 'foo';
`
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Unknown file extension ".ts"');
});
test('should filter stack trace for simple expect', async ({ runInlineTest }) => {
const result = await runInlineTest({
'expect-test.spec.ts': `
const { test } = pwt;
test('should work', () => {
test.expect(1+1).toEqual(3);
});
`
});
expect(result.exitCode).toBe(1);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-test`);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-core`);
expect(stripAnsi(result.output)).not.toContain('internal');
});
test('should filter stack trace for web-first assertions', async ({ runInlineTest }) => {
const result = await runInlineTest({
'expect-test.spec.ts': `
const { test } = pwt;
test('should work', async ({page}) => {
await expect(page.locator('x-foo'), 'x-foo must be visible').toBeVisible({timeout: 1});
});
`
});
expect(result.exitCode).toBe(1);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-test`);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-core`);
expect(stripAnsi(result.output)).not.toContain('internal');
});
test('should filter out event emitter from stack traces', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'expect-test.spec.ts': `
const { test } = pwt;
const EventEmitter = require('events');
test('should work', async ({}) => {
const emitter = new EventEmitter();
emitter.on('event', function handle() { expect(1).toBe(2); });
emitter.emit('event');
});
`
});
expect(result.exitCode).toBe(1);
const outputWithoutGoodStackFrames = stripAnsi(result.output).split('\n').filter(line => !line.includes(testInfo.outputPath())).join('\n');
expect(outputWithoutGoodStackFrames).not.toContain('EventEmitter.emit');
});
test('should filter out syntax error stack traces', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'expect-test.spec.ts': `
const { test } = pwt;
test('should work', ({}) => {
// syntax error: cannot have await in non-async function
await Proimse.resolve();
});
`
});
expect(result.exitCode).toBe(1);
expect(stripAnsi(result.output)).not.toContain('babel');
expect(stripAnsi(result.output)).not.toContain(' at ');
});
test('should filter stack trace for raw errors', async ({ runInlineTest }) => {
const result = await runInlineTest({
'expect-test.spec.ts': `
const { test } = pwt;
test('should work', async ({}) => {
throw new Error('foobar!');
});
`
});
expect(result.exitCode).toBe(1);
expect(stripAnsi(result.output)).toContain('foobar!');
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-test`);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-core`);
expect(stripAnsi(result.output)).not.toContain('internal');
});
test('should not filter out POM', async ({ runInlineTest }) => {
const result = await runInlineTest({
'helper.ts': `
export function foo() {
throw new Error('foo');
}
`,
'expect-test.spec.ts': `
const { test } = pwt;
const { foo } = require('./helper');
test('should work', ({}) => {
foo();
});
`
});
expect(result.exitCode).toBe(1);
expect(stripAnsi(result.output)).toContain('foo');
expect(stripAnsi(result.output)).toContain('helper.ts');
expect(stripAnsi(result.output)).toContain('expect-test.spec.ts');
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-test`);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-core`);
expect(stripAnsi(result.output)).not.toContain('internal');
});
test('should filter stack even without default Error.prepareStackTrace', async ({ runInlineTest }) => {
const result = await runInlineTest({
'expect-test.spec.ts': `
const { test } = pwt;
test('should work', ({}) => {
Error.prepareStackTrace = undefined;
throw new Error('foobar');
});
`
});
expect(result.exitCode).toBe(1);
expect(stripAnsi(result.output)).toContain('foobar');
expect(stripAnsi(result.output)).toContain('expect-test.spec.ts');
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-test`);
expect(stripAnsi(result.output)).not.toContain(path.sep + `playwright-core`);
expect(stripAnsi(result.output)).not.toContain('internal');
const stackLines = stripAnsi(result.output).split('\n').filter(line => line.includes(' at '));
expect(stackLines.length).toBe(1);
});
test('should work with cross-imports - 1', async ({ runInlineTest }) => {
const result = await runInlineTest({
'test1.spec.ts': `
const { test } = pwt;
test('test 1', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-1');
});
`,
'test2.spec.ts': `
import * as _ from './test1.spec';
const { test } = pwt;
test('test 2', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-2');
});
`
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.failed).toBe(0);
expect(result.output).toContain('TEST-1');
expect(result.output).toContain('TEST-2');
});
test('should work with cross-imports - 2', async ({ runInlineTest }) => {
const result = await runInlineTest({
'test1.spec.ts': `
const { test } = pwt;
import * as _ from './test2.spec';
test('test 1', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-1');
});
`,
'test2.spec.ts': `
const { test } = pwt;
test('test 2', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-2');
});
`
}, { workers: 2, reporter: 'list' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.failed).toBe(0);
expect(result.output).toContain('TEST-1');
expect(result.output).toContain('TEST-2');
});