chore: allow opt-into the legacy global setup mode (#11888)

This commit is contained in:
Pavel Feldman 2022-02-07 10:41:56 -08:00 committed by GitHub
parent c3c99a5f66
commit 9116adc684
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 316 additions and 278 deletions

View File

@ -140,6 +140,14 @@ export class Runner {
async runAllTests(options: RunOptions = {}): Promise<FullResult> {
this._reporter = await this._createReporter(!!options.listOnly);
const config = this._loader.fullConfig();
let legacyGlobalTearDown: (() => Promise<void>) | undefined;
if (process.env.PW_TEST_LEGACY_GLOBAL_SETUP_MODE) {
legacyGlobalTearDown = await this._performGlobalSetup(config);
if (!legacyGlobalTearDown)
return { status: 'failed' };
}
const result = await raceAgainstTimeout(() => this._run(!!options.listOnly, options.filePatternFilter || [], options.projectFilter), config.globalTimeout);
let fullResult: FullResult;
if (result.timedOut) {
@ -149,6 +157,7 @@ export class Runner {
fullResult = result.result;
}
await this._reporter.onEnd?.(fullResult);
await legacyGlobalTearDown?.();
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
@ -351,21 +360,19 @@ export class Runner {
if (list)
return { status: 'passed' };
// 13. Declare global setup to tear down in finally.
const internalGlobalTeardowns: (() => Promise<void>)[] = [];
let webServer: WebServer | undefined;
let globalSetupResult: any;
// 13. Run Global setup.
let globalTearDown: (() => Promise<void>) | undefined;
if (!process.env.PW_TEST_LEGACY_GLOBAL_SETUP_MODE) {
globalTearDown = await this._performGlobalSetup(config);
if (!globalTearDown)
return { status: 'failed' };
}
const result: FullResult = { status: 'passed' };
// 14. Run tests.
try {
// 14. Perform global setup.
for (const internalGlobalSetup of this._internalGlobalSetups)
internalGlobalTeardowns.push(await internalGlobalSetup());
webServer = config.webServer ? await WebServer.create(config.webServer) : undefined;
if (config.globalSetup)
globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig());
const sigintWatcher = new SigIntWatcher();
let hasWorkerErrors = false;
@ -389,35 +396,59 @@ export class Runner {
this._reporter.onError?.(serializeError(e));
return { status: 'failed' };
} finally {
await this._runAndAssignError(async () => {
if (globalSetupResult && typeof globalSetupResult === 'function')
await globalSetupResult(this._loader.fullConfig());
}, result);
await this._runAndAssignError(async () => {
if (config.globalTeardown)
await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig());
}, result);
await this._runAndAssignError(async () => {
await webServer?.kill();
}, result);
await this._runAndAssignError(async () => {
for (const internalGlobalTeardown of internalGlobalTeardowns)
await internalGlobalTeardown();
}, result);
await globalTearDown?.();
}
return result;
}
private async _runAndAssignError(callback: () => Promise<void>, result: FullResult) {
private async _performGlobalSetup(config: FullConfig): Promise<(() => Promise<void>) | undefined> {
const result: FullResult = { status: 'passed' };
const internalGlobalTeardowns: (() => Promise<void>)[] = [];
let globalSetupResult: any;
let webServer: WebServer | undefined;
const tearDown = async () => {
await this._runAndReportError(async () => {
if (globalSetupResult && typeof globalSetupResult === 'function')
await globalSetupResult(this._loader.fullConfig());
}, result);
await this._runAndReportError(async () => {
if (config.globalTeardown)
await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig());
}, result);
await this._runAndReportError(async () => {
await webServer?.kill();
}, result);
await this._runAndReportError(async () => {
for (const internalGlobalTeardown of internalGlobalTeardowns)
await internalGlobalTeardown();
}, result);
};
await this._runAndReportError(async () => {
for (const internalGlobalSetup of this._internalGlobalSetups)
internalGlobalTeardowns.push(await internalGlobalSetup());
webServer = config.webServer ? await WebServer.create(config.webServer) : undefined;
if (config.globalSetup)
globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig());
}, result);
if (result.status !== 'passed') {
tearDown();
return;
}
return tearDown;
}
private async _runAndReportError(callback: () => Promise<void>, result: FullResult) {
try {
await callback();
} catch (e) {
if (result.status === 'passed')
result.status = 'failed';
result.status = 'failed';
this._reporter.onError?.(serializeError(e));
}
}

View File

@ -16,252 +16,259 @@
import { test, expect } from './playwright-test-fixtures';
test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => {
const { results, output } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup',
globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
};
`,
'globalSetup.ts': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 100));
global.value = 42;
process.env.FOO = String(global.value);
};
`,
'globalTeardown.ts': `
module.exports = async () => {
console.log('teardown=' + global.value);
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}, testInfo) => {
expect(process.env.FOO).toBe('42');
});
`,
});
expect(results[0].status).toBe('passed');
expect(output).toContain('teardown=42');
});
for (const mode of ['legacy', 'default']) {
test.describe(`${mode} mode`, () => {
const env = { PW_TEST_LEGACY_GLOBAL_SETUP_MODE: mode === 'legacy' ? '1' : undefined };
test('globalTeardown runs after failures', async ({ runInlineTest }) => {
const { results, output } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: 'globalSetup.ts',
globalTeardown: './globalTeardown.ts',
};
`,
'globalSetup.ts': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 100));
global.value = 42;
process.env.FOO = String(global.value);
};
`,
'globalTeardown.ts': `
module.exports = async () => {
console.log('teardown=' + global.value);
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}, testInfo) => {
expect(process.env.FOO).toBe('43');
});
`,
});
expect(results[0].status).toBe('failed');
expect(output).toContain('teardown=42');
});
test('globalTeardown does not run when globalSetup times out', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
globalTeardown: 'globalTeardown.ts',
globalTimeout: 1000,
};
`,
'globalSetup.ts': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 10000));
};
`,
'globalTeardown.ts': `
module.exports = async () => {
console.log('teardown=');
};
`,
'a.test.js': `
const { test } = pwt;
test('should not run', async ({}, testInfo) => {
});
`,
});
// We did not run tests, so we should only have 1 skipped test.
expect(result.skipped).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.exitCode).toBe(1);
expect(result.output).not.toContain('teardown=');
});
test('globalSetup should work with sync function', async ({ runInlineTest }) => {
const { passed } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
};
`,
'globalSetup.ts': `
module.exports = () => {
process.env.FOO = JSON.stringify({ foo: 'bar' });
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}) => {
const value = JSON.parse(process.env.FOO);
expect(value).toEqual({ foo: 'bar' });
});
`,
});
expect(passed).toBe(1);
});
test('globalSetup should throw when passed non-function', async ({ runInlineTest }) => {
const { output } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
};
`,
'globalSetup.ts': `
module.exports = 42;
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}) => {
});
`,
});
expect(output).toContain(`globalSetup.ts: globalSetup file must export a single function.`);
});
test('globalSetup should work with default export and run the returned fn', async ({ runInlineTest }) => {
const { output, exitCode, passed } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
};
`,
'globalSetup.ts': `
function setup() {
let x = 42;
console.log('\\n%%setup: ' + x);
return async () => {
await x;
console.log('\\n%%teardown: ' + x);
};
}
export default setup;
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}) => {
});
`,
});
expect(passed).toBe(1);
expect(exitCode).toBe(0);
expect(output).toContain(`%%setup: 42`);
expect(output).toContain(`%%teardown: 42`);
});
test('globalSetup should allow requiring a package from node_modules', async ({ runInlineTest }) => {
const { results } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: 'my-global-setup'
};
`,
'node_modules/my-global-setup/index.js': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 100));
global.value = 42;
process.env.FOO = String(global.value);
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}, testInfo) => {
expect(process.env.FOO).toBe('42');
});
`,
});
expect(results[0].status).toBe('passed');
});
const authFiles = {
'playwright.config.ts': `
const config: pwt.PlaywrightTestConfig = {
globalSetup: require.resolve('./auth'),
use: {
baseURL: 'https://www.example.com',
storageState: 'state.json',
},
};
export default config;
`,
'auth.ts': `
async function globalSetup(config: pwt.FullConfig) {
const { baseURL, storageState } = config.projects[0].use;
const browser = await pwt.chromium.launch();
const page = await browser.newPage();
await page.route('**/*', route => {
route.fulfill({ body: '<html></html>' }).catch(() => {});
});
await page.goto(baseURL!);
await page.evaluate(() => {
localStorage['name'] = 'value';
});
await page.context().storageState({ path: storageState as string });
await browser.close();
};
export default globalSetup;
`,
'a.test.ts': `
const { test } = pwt;
test('should have storage state', async ({ page }) => {
await page.route('**/*', route => {
route.fulfill({ body: '<html></html>' }).catch(() => {});
});
await page.goto('/');
const value = await page.evaluate(() => localStorage['name']);
expect(value).toBe('value');
test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => {
const { results, output } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup',
globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
};
`,
'globalSetup.ts': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 100));
global.value = 42;
process.env.FOO = String(global.value);
};
`,
'globalTeardown.ts': `
module.exports = async () => {
console.log('teardown=' + global.value);
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}, testInfo) => {
expect(process.env.FOO).toBe('42');
});
`,
}, undefined, env);
expect(results[0].status).toBe('passed');
expect(output).toContain('teardown=42');
});
`,
};
test('globalSetup should work for auth', async ({ runInlineTest }) => {
const result = await runInlineTest(authFiles);
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('globalTeardown runs after failures', async ({ runInlineTest }) => {
const { results, output } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: 'globalSetup.ts',
globalTeardown: './globalTeardown.ts',
};
`,
'globalSetup.ts': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 100));
global.value = 42;
process.env.FOO = String(global.value);
};
`,
'globalTeardown.ts': `
module.exports = async () => {
console.log('teardown=' + global.value);
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}, testInfo) => {
expect(process.env.FOO).toBe('43');
});
`,
}, undefined, env);
expect(results[0].status).toBe('failed');
expect(output).toContain('teardown=42');
});
test('globalSetup auth should compile', async ({ runTSC }) => {
const result = await runTSC(authFiles);
expect(result.exitCode).toBe(0);
});
test('globalTeardown does not run when globalSetup times out', async ({ runInlineTest }) => {
test.skip(!!env.PW_TEST_LEGACY_GLOBAL_SETUP_MODE);
const result = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
globalTeardown: 'globalTeardown.ts',
globalTimeout: 1000,
};
`,
'globalSetup.ts': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 10000));
};
`,
'globalTeardown.ts': `
module.exports = async () => {
console.log('teardown=');
};
`,
'a.test.js': `
const { test } = pwt;
test('should not run', async ({}, testInfo) => {
});
`,
}, undefined, env);
// We did not run tests, so we should only have 1 skipped test.
expect(result.skipped).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.exitCode).toBe(1);
expect(result.output).not.toContain('teardown=');
});
test('globalSetup should work with sync function', async ({ runInlineTest }) => {
const { passed } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
};
`,
'globalSetup.ts': `
module.exports = () => {
process.env.FOO = JSON.stringify({ foo: 'bar' });
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}) => {
const value = JSON.parse(process.env.FOO);
expect(value).toEqual({ foo: 'bar' });
});
`,
}, undefined, env);
expect(passed).toBe(1);
});
test('globalSetup should throw when passed non-function', async ({ runInlineTest }) => {
const { output } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
};
`,
'globalSetup.ts': `
module.exports = 42;
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}) => {
});
`,
}, undefined, env);
expect(output).toContain(`globalSetup.ts: globalSetup file must export a single function.`);
});
test('globalSetup should work with default export and run the returned fn', async ({ runInlineTest }) => {
const { output, exitCode, passed } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: './globalSetup.ts',
};
`,
'globalSetup.ts': `
function setup() {
let x = 42;
console.log('\\n%%setup: ' + x);
return async () => {
await x;
console.log('\\n%%teardown: ' + x);
};
}
export default setup;
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}) => {
});
`,
}, undefined, env);
expect(passed).toBe(1);
expect(exitCode).toBe(0);
expect(output).toContain(`%%setup: 42`);
expect(output).toContain(`%%teardown: 42`);
});
test('globalSetup should allow requiring a package from node_modules', async ({ runInlineTest }) => {
const { results } = await runInlineTest({
'playwright.config.ts': `
import * as path from 'path';
module.exports = {
globalSetup: 'my-global-setup'
};
`,
'node_modules/my-global-setup/index.js': `
module.exports = async () => {
await new Promise(f => setTimeout(f, 100));
global.value = 42;
process.env.FOO = String(global.value);
};
`,
'a.test.js': `
const { test } = pwt;
test('should work', async ({}, testInfo) => {
expect(process.env.FOO).toBe('42');
});
`,
}, undefined, env);
expect(results[0].status).toBe('passed');
});
const authFiles = {
'playwright.config.ts': `
const config: pwt.PlaywrightTestConfig = {
globalSetup: require.resolve('./auth'),
use: {
baseURL: 'https://www.example.com',
storageState: 'state.json',
},
};
export default config;
`,
'auth.ts': `
async function globalSetup(config: pwt.FullConfig) {
const { baseURL, storageState } = config.projects[0].use;
const browser = await pwt.chromium.launch();
const page = await browser.newPage();
await page.route('**/*', route => {
route.fulfill({ body: '<html></html>' }).catch(() => {});
});
await page.goto(baseURL!);
await page.evaluate(() => {
localStorage['name'] = 'value';
});
await page.context().storageState({ path: storageState as string });
await browser.close();
};
export default globalSetup;
`,
'a.test.ts': `
const { test } = pwt;
test('should have storage state', async ({ page }) => {
await page.route('**/*', route => {
route.fulfill({ body: '<html></html>' }).catch(() => {});
});
await page.goto('/');
const value = await page.evaluate(() => localStorage['name']);
expect(value).toBe('value');
});
`,
};
test('globalSetup should work for auth', async ({ runInlineTest }) => {
const result = await runInlineTest(authFiles, undefined, env);
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('globalSetup auth should compile', async ({ runTSC }) => {
const result = await runTSC(authFiles);
expect(result.exitCode).toBe(0);
});
});
}