playwright/tests/playwright-test/deps.spec.ts
2023-07-17 16:48:52 -07:00

646 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from './playwright-test-fixtures';
test('should run projects with dependencies', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A' },
{ name: 'B', dependencies: ['A'] },
{ name: 'C', dependencies: ['A'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['--project=B', '--project=C'] });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual(['A', 'B', 'C']);
});
test('should inherit env changes from dependencies', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A', testMatch: '**/a.spec.ts' },
{ name: 'B', testMatch: '**/b.spec.ts', teardown: 'E' },
{ name: 'C', testMatch: '**/c.spec.ts', dependencies: ['A'] },
{ name: 'D', testMatch: '**/d.spec.ts', dependencies: ['B'] },
{ name: 'E', testMatch: '**/e.spec.ts' },
] };
`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', async ({}, testInfo) => {
process.env.SET_IN_A = 'valuea';
delete process.env.SET_OUTSIDE;
console.log('\\n%%A');
});
`,
'b.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', async ({}, testInfo) => {
process.env.SET_IN_B = 'valueb';
console.log('\\n%%B');
});
`,
'c.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', async ({}, testInfo) => {
console.log('\\n%%C-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE);
});
`,
'd.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', async ({}, testInfo) => {
console.log('\\n%%D-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE);
});
`,
'e.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', async ({}, testInfo) => {
console.log('\\n%%E-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE);
});
`,
}, {}, { SET_OUTSIDE: 'outside' });
expect(result.passed).toBe(5);
expect(result.failed).toBe(0);
expect(result.skipped).toBe(0);
expect(result.outputLines.sort()).toEqual(['A', 'B', 'C-valuea-undefined-undefined', 'D-undefined-valueb-outside', 'E-undefined-valueb-outside']);
});
test('should not run projects with dependencies when --no-deps is passed', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A' },
{ name: 'B', dependencies: ['A'] },
{ name: 'C', dependencies: ['A'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['--no-deps', '--project=B', '--project=C'] });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['B', 'C']);
});
test('should not run project if dependency failed', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A' },
{ name: 'B', dependencies: ['A'] },
{ name: 'C', dependencies: ['B'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
if (testInfo.project.name === 'B')
throw new Error('Failed project B');
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(1);
expect(result.output).toContain('Failed project B');
expect(result.outputLines).toEqual(['A', 'B']);
});
test('should not run project if dependency failed (2)', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A1' },
{ name: 'A2', dependencies: ['A1'] },
{ name: 'A3', dependencies: ['A2'] },
{ name: 'B1' },
{ name: 'B2', dependencies: ['B1'] },
{ name: 'B3', dependencies: ['B2'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
if (testInfo.project.name === 'B1')
throw new Error('Failed project B1');
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.outputLines.sort()).toEqual(['A1', 'A2', 'A3', 'B1']);
});
test('should filter by project list, but run deps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A' },
{ name: 'B' },
{ name: 'C', dependencies: ['A'] },
{ name: 'D' },
] };
`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`
}, { project: ['C', 'D'] });
expect(result.passed).toBe(3);
expect(result.failed).toBe(0);
expect(result.skipped).toBe(0);
expect(result.outputLines.sort()).toEqual(['A', 'C', 'D']);
});
test('should not filter dependency by file name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A' },
{ name: 'B', dependencies: ['A'] },
] };
`,
'one.spec.ts': `
import { test, expect } from '@playwright/test';
test('fails', () => { expect(1).toBe(2); });
`,
'two.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', () => { });
`,
}, undefined, undefined, { additionalArgs: ['two.spec.ts'] });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('1) [A] one.spec.ts:3:11 fails');
});
test('should filter dependency by only', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'browser', dependencies: ['setup'] },
] };
`,
'setup.ts': `
import { test, expect } from '@playwright/test';
test('passes', () => {
console.log('\\n%% setup in ' + test.info().project.name);
});
test.only('passes 2', () => {
console.log('\\n%% setup 2 in ' + test.info().project.name);
});
`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', () => {
console.log('\\n%% test in ' + test.info().project.name);
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.outputLines).toEqual(['setup 2 in setup']);
});
test('should filter dependency by only when running explicitly', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'browser', dependencies: ['setup'] },
] };
`,
'setup.ts': `
import { test, expect } from '@playwright/test';
test('passes', () => {
console.log('\\n%% setup in ' + test.info().project.name);
});
test.only('passes 2', () => {
console.log('\\n%% setup 2 in ' + test.info().project.name);
});
`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', () => {
console.log('\\n%% test in ' + test.info().project.name);
});
`,
}, { project: ['setup'] });
expect(result.exitCode).toBe(0);
expect(result.outputLines).toEqual(['setup 2 in setup']);
});
test('should not filter dependency by only 3', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.*.ts/ },
{ name: 'browser', dependencies: ['setup'] },
] };
`,
'setup-1.ts': `
import { test, expect } from '@playwright/test';
test('setup 1', () => {
console.log('\\n%% setup in ' + test.info().project.name);
});
`,
'setup-2.ts': `
import { test, expect } from '@playwright/test';
test('setup 2', () => {
console.log('\\n%% setup 2 in ' + test.info().project.name);
});
`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', () => {
console.log('\\n%% test in ' + test.info().project.name);
});
`,
}, undefined, undefined, { additionalArgs: ['setup-2.ts'] });
expect(result.exitCode).toBe(0);
expect(result.outputLines).toEqual(['setup 2 in setup']);
});
test('should report skipped dependent tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'browser', dependencies: ['setup'] },
] };
`,
'setup.ts': `
import { test, expect } from '@playwright/test';
test('setup', () => {
expect(1).toBe(2);
});
`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', () => {});
`,
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.skipped).toBe(1);
expect(result.results.length).toBe(2);
});
test('should report circular dependencies', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A', dependencies: ['B'] },
{ name: 'B', dependencies: ['A'] },
] };
`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('pass', () => {});
`,
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Circular dependency detected between projects.');
});
test('should run dependency in each shard', async ({ runInlineTest }) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'chromium', dependencies: ['setup'] },
],
};
`,
'setup.ts': `
import { test, expect } from '@playwright/test';
test('setup', async ({}) => {
console.log('\\n%%setup');
});
`,
'test1.spec.ts': `
import { test, expect } from '@playwright/test';
test('test1', async ({}) => {
console.log('\\n%%test1');
});
`,
'test2.spec.ts': `
import { test, expect } from '@playwright/test';
test('test2', async ({}) => {
console.log('\\n%%test2');
});
`,
};
{
const result = await runInlineTest(files, { workers: 1, shard: '1/2' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['setup', 'test1']);
}
{
const result = await runInlineTest(files, { workers: 1, shard: '2/2' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['setup', 'test2']);
}
});
test('should run setup project with zero tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'setup', testMatch: /not-matching/ },
{ name: 'real', dependencies: ['setup'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual(['real']);
});
test('should run setup project with zero tests recursively', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', testMatch: /a.spec/ },
{ name: 'B', testMatch: /not-matching/, dependencies: ['A'] },
{ name: 'C', testMatch: /c.spec/, dependencies: ['B'] },
],
};`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
'c.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1, project: 'C' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['A', 'C']);
});
test('should run project with teardown', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', teardown: 'B' },
{ name: 'B' },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['--project=A'] });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['A', 'B']);
});
test('should run teardown after depedents', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', teardown: 'E' },
{ name: 'B', dependencies: ['A'] },
{ name: 'C', dependencies: ['B'], teardown: 'D' },
{ name: 'D' },
{ name: 'E' },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['--project=C'] });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(5);
expect(result.outputLines).toEqual(['A', 'B', 'C', 'D', 'E']);
});
test('should run teardown after failure', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', teardown: 'D' },
{ name: 'B', dependencies: ['A'] },
{ name: 'C', dependencies: ['B'] },
{ name: 'D' },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
if (testInfo.project.name === 'A')
throw new Error('ouch');
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['--project=C'] });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(2);
expect(result.outputLines).toEqual(['A', 'D']);
});
test('should complain about teardown being a dependency', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', teardown: 'B' },
{ name: 'B' },
{ name: 'C', dependencies: ['B'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', () => {});
`,
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`Project C must not depend on a teardown project B`);
});
test('should complain about teardown having a dependency', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', teardown: 'B' },
{ name: 'B', dependencies: ['C'] },
{ name: 'C' },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', () => {});
`,
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`Teardown project B must not have dependencies`);
});
test('should support the same teardown used multiple times', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A', teardown: 'D' },
{ name: 'B', teardown: 'D' },
{ name: 'D' },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual(['A', 'B', 'D']);
});
test('should only apply --repeat-each to top-level', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A' },
{ name: 'B', dependencies: ['A'] },
{ name: 'C', dependencies: ['A'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { 'workers': 1, 'repeat-each': 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(5);
expect(result.outputLines).toEqual(['A', 'B', 'B', 'C', 'C']);
});
test('should run teardown when all projects are top-level at run point', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'setup', teardown: 'teardown' },
{ name: 'teardown' },
{ name: 'project', dependencies: ['setup'] },
],
};`,
'test.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['test.spec.ts'] });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual(['setup', 'project', 'teardown']);
});
test('should not run deps for projects filtered with grep', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'setupA', teardown: 'teardownA', testMatch: '**/hook.spec.ts' },
{ name: 'teardownA', testMatch: '**/hook.spec.ts' },
{ name: 'projectA', dependencies: ['setupA'], testMatch: '**/a.spec.ts' },
{ name: 'setupB', teardown: 'teardownB', testMatch: '**/hook.spec.ts' },
{ name: 'teardownB', testMatch: '**/hook.spec.ts' },
{ name: 'projectB', dependencies: ['setupB'], testMatch: '**/b.spec.ts' },
],
};`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
'hook.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
'b.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 }, undefined, { additionalArgs: ['--grep=b.spec.ts'] });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual(['setupB', 'projectB', 'teardownB']);
});