microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-12-11 12:33:45 +03:00
Code Issues Projects Releases Wiki Activity

chore: make sure to call task's teardown if it has ever started (#24317)

Browse Source 
This way things like WebServerPlugin can cleanup after themselves even
if they failed to start or were interrupted mid-way.
This commit is contained in:
Dmitry Gozman 2023-07-20 17:16:22 -07:00 committed by GitHub
parent 59d5198d17
commit 767addec8c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 180 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/playwright-ct-core/src/vitePlugin.ts
View File

@ -173,7 +173,8 @@ export function createPlugin(
}, },
end: async () => { end: async () => {
await new Promise(f => stoppableServer.stop(f)); if (stoppableServer)
await new Promise(f => stoppableServer.stop(f));
}, },
}; };
} }

15
packages/playwright-test/src/runner/taskRunner.ts
View File

@ -21,8 +21,8 @@ import { SigIntWatcher } from './sigIntWatcher';
import { serializeError } from '../util'; import { serializeError } from '../util';
import type { ReporterV2 } from '../reporters/reporterV2'; import type { ReporterV2 } from '../reporters/reporterV2';
type TaskTeardown = () => Promise<any> | undefined; type TaskPhase<Context> = (context: Context, errors: TestError[], softErrors: TestError[]) => Promise<void> | void;
export type Task<Context> = (context: Context, errors: TestError[], softErrors: TestError[]) => Promise<TaskTeardown | void> | undefined; export type Task<Context> = { setup?: TaskPhase<Context>, teardown?: TaskPhase<Context> };
export class TaskRunner<Context> { export class TaskRunner<Context> {
private _tasks: { name: string, task: Task<Context> }[] = []; private _tasks: { name: string, task: Task<Context> }[] = [];
@ -50,7 +50,7 @@ export class TaskRunner<Context> {
async runDeferCleanup(context: Context, deadline: number, cancelPromise = new ManualPromise<void>()): Promise<{ status: FullResult['status'], cleanup: () => Promise<FullResult['status']> }> { async runDeferCleanup(context: Context, deadline: number, cancelPromise = new ManualPromise<void>()): Promise<{ status: FullResult['status'], cleanup: () => Promise<FullResult['status']> }> {
const sigintWatcher = new SigIntWatcher(); const sigintWatcher = new SigIntWatcher();
const timeoutWatcher = new TimeoutWatcher(deadline); const timeoutWatcher = new TimeoutWatcher(deadline);
const teardownRunner = new TaskRunner(this._reporter, this._globalTimeoutForError); const teardownRunner = new TaskRunner<Context>(this._reporter, this._globalTimeoutForError);
teardownRunner._isTearDown = true; teardownRunner._isTearDown = true;
let currentTaskName: string | undefined; let currentTaskName: string | undefined;
@ -64,9 +64,8 @@ export class TaskRunner<Context> {
const errors: TestError[] = []; const errors: TestError[] = [];
const softErrors: TestError[] = []; const softErrors: TestError[] = [];
try { try {
const teardown = await task(context, errors, softErrors); teardownRunner._tasks.unshift({ name: `teardown for ${name}`, task: { setup: task.teardown } });
if (teardown) await task.setup?.(context, errors, softErrors);
teardownRunner._tasks.unshift({ name: `teardown for ${name}`, task: teardown });
} catch (e) { } catch (e) {
debug('pw:test:task')(`error in "${name}": `, e); debug('pw:test:task')(`error in "${name}": `, e);
errors.push(serializeError(e)); errors.push(serializeError(e));
@ -106,12 +105,14 @@ export class TaskRunner<Context> {
status = 'failed'; status = 'failed';
} }
cancelPromise?.resolve(); cancelPromise?.resolve();
// Note that upon hitting deadline, we "run cleanup", but it exits immediately
// because of the same deadline. Essentially, we're not perfomring any cleanup.
const cleanup = () => teardownRunner.runDeferCleanup(context, deadline).then(r => r.status); const cleanup = () => teardownRunner.runDeferCleanup(context, deadline).then(r => r.status);
return { status, cleanup }; return { status, cleanup };
} }
} }
export class TimeoutWatcher { class TimeoutWatcher {
private _timedOut = false; private _timedOut = false;
readonly promise = new ManualPromise(); readonly promise = new ManualPromise();
private _timer: NodeJS.Timeout | undefined; private _timer: NodeJS.Timeout | undefined;

301
packages/playwright-test/src/runner/tasks.ts
View File

@ -93,7 +93,6 @@ function addRunTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal
taskRunner.addTask('report begin', createReportBeginTask()); taskRunner.addTask('report begin', createReportBeginTask());
for (const plugin of config.plugins) for (const plugin of config.plugins)
taskRunner.addTask('plugin begin', createPluginBeginTask(plugin)); taskRunner.addTask('plugin begin', createPluginBeginTask(plugin));
taskRunner.addTask('start workers', createWorkersTask());
taskRunner.addTask('test suite', createRunTestsTask()); taskRunner.addTask('test suite', createRunTestsTask());
return taskRunner; return taskRunner;
} }
@ -106,189 +105,209 @@ export function createTaskRunnerForList(config: FullConfigInternal, reporter: Re
} }
function createReportBeginTask(): Task<TestRun> { function createReportBeginTask(): Task<TestRun> {
return async ({ config, reporter, rootSuite }) => { let montonicStartTime = 0;
const montonicStartTime = monotonicTime(); return {
reporter.onBegin(rootSuite!); setup: async ({ config, reporter, rootSuite }) => {
return async () => { montonicStartTime = monotonicTime();
reporter.onBegin(rootSuite!);
},
teardown: async ({ config }) => {
config.config.metadata.totalTime = monotonicTime() - montonicStartTime; config.config.metadata.totalTime = monotonicTime() - montonicStartTime;
}; },
}; };
} }
function createPluginSetupTask(plugin: TestRunnerPluginRegistration): Task<TestRun> { function createPluginSetupTask(plugin: TestRunnerPluginRegistration): Task<TestRun> {
return async ({ config, reporter }) => { return {
if (typeof plugin.factory === 'function') setup: async ({ config, reporter }) => {
plugin.instance = await plugin.factory(); if (typeof plugin.factory === 'function')
else plugin.instance = await plugin.factory();
plugin.instance = plugin.factory; else
await plugin.instance?.setup?.(config.config, config.configDir, reporter); plugin.instance = plugin.factory;
return () => plugin.instance?.teardown?.(); await plugin.instance?.setup?.(config.config, config.configDir, reporter);
},
teardown: async () => {
await plugin.instance?.teardown?.();
},
}; };
} }
function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task<TestRun> { function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task<TestRun> {
return async ({ rootSuite }) => { return {
await plugin.instance?.begin?.(rootSuite!); setup: async ({ rootSuite }) => {
return () => plugin.instance?.end?.(); await plugin.instance?.begin?.(rootSuite!);
},
teardown: async () => {
await plugin.instance?.end?.();
},
}; };
} }
function createGlobalSetupTask(): Task<TestRun> { function createGlobalSetupTask(): Task<TestRun> {
return async ({ config }) => { let globalSetupResult: any;
const setupHook = config.config.globalSetup ? await loadGlobalHook(config, config.config.globalSetup) : undefined; let globalSetupFinished = false;
const teardownHook = config.config.globalTeardown ? await loadGlobalHook(config, config.config.globalTeardown) : undefined; let teardownHook: any;
const globalSetupResult = setupHook ? await setupHook(config.config) : undefined; return {
return async () => { setup: async ({ config }) => {
const setupHook = config.config.globalSetup ? await loadGlobalHook(config, config.config.globalSetup) : undefined;
teardownHook = config.config.globalTeardown ? await loadGlobalHook(config, config.config.globalTeardown) : undefined;
globalSetupResult = setupHook ? await setupHook(config.config) : undefined;
globalSetupFinished = true;
},
teardown: async ({ config }) => {
if (typeof globalSetupResult === 'function') if (typeof globalSetupResult === 'function')
await globalSetupResult(); await globalSetupResult();
await teardownHook?.(config.config); if (globalSetupFinished)
}; await teardownHook?.(config.config);
},
}; };
} }
function createRemoveOutputDirsTask(): Task<TestRun> { function createRemoveOutputDirsTask(): Task<TestRun> {
return async ({ config }) => { return {
if (process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS) setup: async ({ config }) => {
return; if (process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS)
const outputDirs = new Set<string>(); return;
for (const p of config.projects) { const outputDirs = new Set<string>();
if (!config.cliProjectFilter || config.cliProjectFilter.includes(p.project.name)) for (const p of config.projects) {
outputDirs.add(p.project.outputDir); if (!config.cliProjectFilter || config.cliProjectFilter.includes(p.project.name))
} outputDirs.add(p.project.outputDir);
await Promise.all(Array.from(outputDirs).map(outputDir => rimraf(outputDir).catch(async (error: any) => {
if ((error as any).code === 'EBUSY') {
// We failed to remove folder, might be due to the whole folder being mounted inside a container:
// https://github.com/microsoft/playwright/issues/12106
// Do a best-effort to remove all files inside of it instead.
const entries = await readDirAsync(outputDir).catch(e => []);
await Promise.all(entries.map(entry => rimraf(path.join(outputDir, entry))));
} else {
throw error;
} }
})));
await Promise.all(Array.from(outputDirs).map(outputDir => rimraf(outputDir).catch(async (error: any) => {
if ((error as any).code === 'EBUSY') {
// We failed to remove folder, might be due to the whole folder being mounted inside a container:
// https://github.com/microsoft/playwright/issues/12106
// Do a best-effort to remove all files inside of it instead.
const entries = await readDirAsync(outputDir).catch(e => []);
await Promise.all(entries.map(entry => rimraf(path.join(outputDir, entry))));
} else {
throw error;
}
})));
},
}; };
} }
function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunTestsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> { function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunTestsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> {
return async (testRun, errors, softErrors) => { return {
await collectProjectsAndTestFiles(testRun, !!options.doNotRunTestsOutsideProjectFilter, options.additionalFileMatcher); setup: async (testRun, errors, softErrors) => {
await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors); await collectProjectsAndTestFiles(testRun, !!options.doNotRunTestsOutsideProjectFilter, options.additionalFileMatcher);
testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly); await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
// Fail when no tests. testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard) // Fail when no tests.
throw new Error(`No tests found`); if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard)
throw new Error(`No tests found`);
},
}; };
} }
function createPhasesTask(): Task<TestRun> { function createPhasesTask(): Task<TestRun> {
return async testRun => { return {
let maxConcurrentTestGroups = 0; setup: async testRun => {
let maxConcurrentTestGroups = 0;
const processed = new Set<FullProjectInternal>(); const processed = new Set<FullProjectInternal>();
const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._fullProject!, suite])); const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._fullProject!, suite]));
const allProjects = [...projectToSuite.keys()]; const allProjects = [...projectToSuite.keys()];
const teardownToSetups = buildTeardownToSetupsMap(allProjects); const teardownToSetups = buildTeardownToSetupsMap(allProjects);
const teardownToSetupsDependents = new Map<FullProjectInternal, FullProjectInternal[]>(); const teardownToSetupsDependents = new Map<FullProjectInternal, FullProjectInternal[]>();
for (const [teardown, setups] of teardownToSetups) { for (const [teardown, setups] of teardownToSetups) {
const closure = buildDependentProjects(setups, allProjects); const closure = buildDependentProjects(setups, allProjects);
closure.delete(teardown); closure.delete(teardown);
teardownToSetupsDependents.set(teardown, [...closure]); teardownToSetupsDependents.set(teardown, [...closure]);
}
for (let i = 0; i < projectToSuite.size; i++) {
// Find all projects that have all their dependencies processed by previous phases.
const phaseProjects: FullProjectInternal[] = [];
for (const project of projectToSuite.keys()) {
if (processed.has(project))
continue;
const projectsThatShouldFinishFirst = [...project.deps, ...(teardownToSetupsDependents.get(project) || [])];
if (projectsThatShouldFinishFirst.find(p => !processed.has(p)))
continue;
phaseProjects.push(project);
} }
// Create a new phase. for (let i = 0; i < projectToSuite.size; i++) {
for (const project of phaseProjects) // Find all projects that have all their dependencies processed by previous phases.
processed.add(project); const phaseProjects: FullProjectInternal[] = [];
if (phaseProjects.length) { for (const project of projectToSuite.keys()) {
let testGroupsInPhase = 0; if (processed.has(project))
const phase: Phase = { dispatcher: new Dispatcher(testRun.config, testRun.reporter), projects: [] }; continue;
testRun.phases.push(phase); const projectsThatShouldFinishFirst = [...project.deps, ...(teardownToSetupsDependents.get(project) || [])];
for (const project of phaseProjects) { if (projectsThatShouldFinishFirst.find(p => !processed.has(p)))
const projectSuite = projectToSuite.get(project)!; continue;
const testGroups = createTestGroups(projectSuite, testRun.config.config.workers); phaseProjects.push(project);
phase.projects.push({ project, projectSuite, testGroups }); }
testGroupsInPhase += testGroups.length;
// Create a new phase.
for (const project of phaseProjects)
processed.add(project);
if (phaseProjects.length) {
let testGroupsInPhase = 0;
const phase: Phase = { dispatcher: new Dispatcher(testRun.config, testRun.reporter), projects: [] };
testRun.phases.push(phase);
for (const project of phaseProjects) {
const projectSuite = projectToSuite.get(project)!;
const testGroups = createTestGroups(projectSuite, testRun.config.config.workers);
phase.projects.push({ project, projectSuite, testGroups });
testGroupsInPhase += testGroups.length;
}
debug('pw:test:task')(`created phase #${testRun.phases.length} with ${phase.projects.map(p => p.project.project.name).sort()} projects, ${testGroupsInPhase} testGroups`);
maxConcurrentTestGroups = Math.max(maxConcurrentTestGroups, testGroupsInPhase);
} }
debug('pw:test:task')(`created phase #${testRun.phases.length} with ${phase.projects.map(p => p.project.project.name).sort()} projects, ${testGroupsInPhase} testGroups`);
maxConcurrentTestGroups = Math.max(maxConcurrentTestGroups, testGroupsInPhase);
} }
}
testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups); testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups);
}; },
}
function createWorkersTask(): Task<TestRun> {
return async ({ phases }) => {
return async () => {
for (const { dispatcher } of phases.reverse())
await dispatcher.stop();
};
}; };
} }
function createRunTestsTask(): Task<TestRun> { function createRunTestsTask(): Task<TestRun> {
return async testRun => { return {
const { phases } = testRun; setup: async ({ phases }) => {
const successfulProjects = new Set<FullProjectInternal>(); const successfulProjects = new Set<FullProjectInternal>();
const extraEnvByProjectId: EnvByProjectId = new Map(); const extraEnvByProjectId: EnvByProjectId = new Map();
const teardownToSetups = buildTeardownToSetupsMap(phases.map(phase => phase.projects.map(p => p.project)).flat()); const teardownToSetups = buildTeardownToSetupsMap(phases.map(phase => phase.projects.map(p => p.project)).flat());
for (const { dispatcher, projects } of phases) { for (const { dispatcher, projects } of phases) {
// Each phase contains dispatcher and a set of test groups. // Each phase contains dispatcher and a set of test groups.
// We don't want to run the test groups beloning to the projects // We don't want to run the test groups beloning to the projects
// that depend on the projects that failed previously. // that depend on the projects that failed previously.
const phaseTestGroups: TestGroup[] = []; const phaseTestGroups: TestGroup[] = [];
for (const { project, testGroups } of projects) { for (const { project, testGroups } of projects) {
// Inherit extra enviroment variables from dependencies. // Inherit extra enviroment variables from dependencies.
let extraEnv: Record<string, string | undefined> = {}; let extraEnv: Record<string, string | undefined> = {};
for (const dep of project.deps) for (const dep of project.deps)
extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(dep.id) }; extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(dep.id) };
for (const setup of teardownToSetups.get(project) || []) for (const setup of teardownToSetups.get(project) || [])
extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(setup.id) }; extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(setup.id) };
extraEnvByProjectId.set(project.id, extraEnv); extraEnvByProjectId.set(project.id, extraEnv);
const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p)); const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p));
if (!hasFailedDeps) { if (!hasFailedDeps) {
phaseTestGroups.push(...testGroups); phaseTestGroups.push(...testGroups);
} else { } else {
for (const testGroup of testGroups) { for (const testGroup of testGroups) {
for (const test of testGroup.tests) for (const test of testGroup.tests)
test._appendTestResult().status = 'skipped'; test._appendTestResult().status = 'skipped';
}
}
}
if (phaseTestGroups.length) {
await dispatcher!.run(phaseTestGroups, extraEnvByProjectId);
await dispatcher.stop();
for (const [projectId, envProduced] of dispatcher.producedEnvByProjectId()) {
const extraEnv = extraEnvByProjectId.get(projectId) || {};
extraEnvByProjectId.set(projectId, { ...extraEnv, ...envProduced });
}
}
// If the worker broke, fail everything, we have no way of knowing which
// projects failed.
if (!dispatcher.hasWorkerErrors()) {
for (const { project, projectSuite } of projects) {
const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p));
if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok()))
successfulProjects.add(project);
} }
} }
} }
},
if (phaseTestGroups.length) { teardown: async ({ phases }) => {
await dispatcher!.run(phaseTestGroups, extraEnvByProjectId); for (const { dispatcher } of phases.reverse())
await dispatcher.stop(); await dispatcher.stop();
for (const [projectId, envProduced] of dispatcher.producedEnvByProjectId()) { },
const extraEnv = extraEnvByProjectId.get(projectId) || {};
extraEnvByProjectId.set(projectId, { ...extraEnv, ...envProduced });
}
}
// If the worker broke, fail everything, we have no way of knowing which
// projects failed.
if (!dispatcher.hasWorkerErrors()) {
for (const { project, projectSuite } of projects) {
const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p));
if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok()))
successfulProjects.add(project);
}
}
}
}; };
} }

14
tests/playwright-test/runner.spec.ts
View File

@ -399,16 +399,19 @@ test('sigint should stop plugins', async ({ runInlineTest }) => {
`, `,
'a.spec.js': ` 'a.spec.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('test', async () => { }); test('test', async () => {
console.log('testing!');
});
`, `,
}, { 'workers': 1 }, {}, { sendSIGINTAfter: 1 }); }, { 'workers': 1 }, {}, { sendSIGINTAfter: 1 });
expect(result.exitCode).toBe(130); expect(result.exitCode).toBe(130);
expect(result.passed).toBe(0); expect(result.passed).toBe(0);
const output = result.output; const output = result.output;
expect(output).toContain('Plugin1 setup'); expect(output).toContain('Plugin1 setup');
expect(output).not.toContain('Plugin1 teardown'); expect(output).toContain('Plugin1 teardown');
expect(output).not.toContain('Plugin2 setup'); expect(output).not.toContain('Plugin2 setup');
expect(output).not.toContain('Plugin2 teardown'); expect(output).not.toContain('Plugin2 teardown');
expect(output).not.toContain('testing!');
}); });
test('sigint should stop plugins 2', async ({ runInlineTest }) => { test('sigint should stop plugins 2', async ({ runInlineTest }) => {
@ -440,7 +443,9 @@ test('sigint should stop plugins 2', async ({ runInlineTest }) => {
`, `,
'a.spec.js': ` 'a.spec.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('test', async () => { }); test('test', async () => {
console.log('testing!');
});
`, `,
}, { 'workers': 1 }, {}, { sendSIGINTAfter: 1 }); }, { 'workers': 1 }, {}, { sendSIGINTAfter: 1 });
expect(result.exitCode).toBe(130); expect(result.exitCode).toBe(130);
@ -449,7 +454,8 @@ test('sigint should stop plugins 2', async ({ runInlineTest }) => {
expect(output).toContain('Plugin1 setup'); expect(output).toContain('Plugin1 setup');
expect(output).toContain('Plugin2 setup'); expect(output).toContain('Plugin2 setup');
expect(output).toContain('Plugin1 teardown'); expect(output).toContain('Plugin1 teardown');
expect(output).not.toContain('Plugin2 teardown'); expect(output).toContain('Plugin2 teardown');
expect(output).not.toContain('testing!');
}); });
test('should not crash with duplicate titles and .only', async ({ runInlineTest }) => { test('should not crash with duplicate titles and .only', async ({ runInlineTest }) => {