feat(test-runner): add webServer (#7368)

This commit is contained in:
Max Schmitt 2021-07-07 20:19:42 +02:00 committed by GitHub
parent ee0497c725
commit 98bcf26656
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 442 additions and 21 deletions

View File

@ -42,6 +42,7 @@ These options would be typically different between local development and CI oper
- `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold.
- `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information.
- `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run.
- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automaticially detect the port when it got printed to the stdout.
- `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests.
Note that each [test project](#projects) can provide its own test suite options, for example two projects can run different tests by providing different `testDir`s. However, test run options are shared between all projects.
@ -200,6 +201,61 @@ export const test = base.extend<{ saveLogs: void }>({
});
```
## Launching a development web server during the tests
To launch a web server during the tests, use the `webServer` option in the [configuration file](#configuration-object).
Playwright Test does automatically detect if a localhost URL like `http://localhost:3000` gets printed to the stdout.
The port from the printed URL gets then used to check when its accepting requests and passed over to Playwright as a
[`param: baseURL`] when creating the context [`method: Browser.newContext`]. You can also manually specify a `port` or additional environment variables, see [here](#configuration-object).
```js js-flavor=ts
// playwright.config.ts
import { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
webServer: {
command: 'npm run start',
timeout: 120 * 1000,
},
};
export default config;
```
```js js-flavor=ts
// test.spec.ts
import { test } = from '@playwright/test';
test('test', async ({ page }) => {
// This will result in e.g. http://localhost:3000/foo when your dev-server prints a http://localhost:3000 address
await page.goto('/foo');
});
```
```js js-flavor=js
// @ts-check
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
webServer: {
command: 'npm run start',
timeout: 120 * 1000,
},
};
mode.exports = config;
```
```js js-flavor=js
// test.spec.js
const { test } = require('@playwright/test');
test('test', async ({ page }) => {
// This will result in e.g. http://localhost:3000/foo when your dev-server prints a http://localhost:3000 address
await page.goto('/foo');
});
```
## Global setup and teardown
To set something up once before running all tests, use `globalSetup` option in the [configuration file](#configuration-object).

View File

@ -161,6 +161,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots,
- `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`.
- `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files.
- `timeout`: Time in milliseconds given to each test.
- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automaticially detect the port when it got printed to the stdout.
- `workers`: The maximum number of concurrent worker processes to use for parallelizing tests.
You can specify these options in the configuration file.

View File

@ -182,7 +182,7 @@ export abstract class BrowserType extends SdkObject {
let transport: ConnectionTransport | undefined = undefined;
let browserProcess: BrowserProcess | undefined = undefined;
const { launchedProcess, gracefullyClose, kill } = await launchProcess({
executablePath: executable,
command: executable,
args: browserArguments,
env: this._amendEnvironment(env, userDataDir, executable, browserArguments),
handleSIGINT,

View File

@ -92,7 +92,7 @@ export class VideoRecorder {
const progress = this._progress;
const { launchedProcess, gracefullyClose } = await launchProcess({
executablePath: this._ffmpegPath,
command: this._ffmpegPath,
args,
stdio: 'stdin',
log: (message: string) => progress.log(message),

View File

@ -127,7 +127,7 @@ export class Electron extends SdkObject {
const browserLogsCollector = new RecentLogsCollector();
const { launchedProcess, gracefullyClose, kill } = await launchProcess({
executablePath: options.executablePath || require('electron/index.js'),
command: options.executablePath || require('electron/index.js'),
args: electronArguments,
env: options.env ? envArrayToObject(options.env) : process.env,
log: (message: string) => {

View File

@ -24,9 +24,10 @@ import { isUnderTest, removeFolders } from '../utils/utils';
export type Env = {[key: string]: string | number | boolean | undefined};
export type LaunchProcessOptions = {
executablePath: string,
args: string[],
command: string,
args?: string[],
env?: Env,
shell?: boolean,
handleSIGINT?: boolean,
handleSIGTERM?: boolean,
@ -62,20 +63,18 @@ if (maxListeners !== 0)
export async function launchProcess(options: LaunchProcessOptions): Promise<LaunchResult> {
const stdio: ('ignore' | 'pipe')[] = options.stdio === 'pipe' ? ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] : ['pipe', 'pipe', 'pipe'];
options.log(`<launching> ${options.executablePath} ${options.args.join(' ')}`);
const spawnedProcess = childProcess.spawn(
options.executablePath,
options.args,
{
// On non-windows platforms, `detached: true` makes child process a leader of a new
// process group, making it possible to kill child process tree with `.kill(-pid)` command.
// @see https://nodejs.org/api/child_process.html#child_process_options_detached
detached: process.platform !== 'win32',
env: (options.env as {[key: string]: string}),
cwd: options.cwd,
stdio,
}
);
options.log(`<launching> ${options.command} ${options.args ? options.args.join(' ') : ''}`);
const spawnOptions: childProcess.SpawnOptions = {
// On non-windows platforms, `detached: true` makes child process a leader of a new
// process group, making it possible to kill child process tree with `.kill(-pid)` command.
// @see https://nodejs.org/api/child_process.html#child_process_options_detached
detached: process.platform !== 'win32',
env: (options.env as {[key: string]: string}),
cwd: options.cwd,
shell: options.shell,
stdio,
};
const spawnedProcess = childProcess.spawn(options.command, options.args, spawnOptions);
const cleanup = async () => {
options.log(`[pid=${spawnedProcess.pid || 'N/A'}] starting temporary directories cleanup`);

View File

@ -74,9 +74,12 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
timezoneId: undefined,
userAgent: undefined,
viewport: undefined,
baseURL: async ({ }, use) => {
await use(process.env.PLAYWRIGHT_TEST_BASE_URL);
},
contextOptions: {},
context: async ({ browser, screenshot, trace, video, acceptDownloads, bypassCSP, colorScheme, deviceScaleFactor, extraHTTPHeaders, hasTouch, geolocation, httpCredentials, ignoreHTTPSErrors, isMobile, javaScriptEnabled, locale, offline, permissions, proxy, storageState, viewport, timezoneId, userAgent, contextOptions }, use, testInfo) => {
context: async ({ browser, screenshot, trace, video, acceptDownloads, bypassCSP, colorScheme, deviceScaleFactor, extraHTTPHeaders, hasTouch, geolocation, httpCredentials, ignoreHTTPSErrors, isMobile, javaScriptEnabled, locale, offline, permissions, proxy, storageState, viewport, timezoneId, userAgent, baseURL, contextOptions }, use, testInfo) => {
testInfo.snapshotSuffix = process.platform;
if (process.env.PWDEBUG)
testInfo.setTimeout(0);
@ -139,6 +142,8 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
options.userAgent = userAgent;
if (viewport !== undefined)
options.viewport = viewport;
if (baseURL !== undefined)
options.baseURL = baseURL;
const context = await browser.newContext(options);
context.setDefaultTimeout(0);

View File

@ -103,6 +103,7 @@ export class Loader {
this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard);
this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots);
this._fullConfig.workers = takeFirst(this._configOverrides.workers, this._config.workers, baseFullConfig.workers);
this._fullConfig.webServer = takeFirst(this._configOverrides.webServer, this._config.webServer, baseFullConfig.webServer);
for (const project of projects)
this._addProject(project, this._fullConfig.rootDir);
@ -429,4 +430,5 @@ const baseFullConfig: FullConfig = {
shard: null,
updateSnapshots: 'missing',
workers: 1,
webServer: null,
};

View File

@ -35,6 +35,7 @@ import EmptyReporter from './reporters/empty';
import { ProjectImpl } from './project';
import { Minimatch } from 'minimatch';
import { Config } from './types';
import { WebServer } from './webServer';
const removeFolderAsync = promisify(rimraf);
const readDirAsync = promisify(fs.readdir);
@ -169,6 +170,7 @@ export class Runner {
let globalSetupResult: any;
if (config.globalSetup)
globalSetupResult = await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup')(this._loader.fullConfig());
const webServer: WebServer|null = config.webServer ? await WebServer.create(config.webServer) : null;
try {
for (const file of allTestFiles)
await this._loader.loadTestFile(file);
@ -261,6 +263,7 @@ export class Runner {
await this._reporter.onEnd({ status: failed ? 'failed' : 'passed' });
return { status: failed ? 'failed' : 'passed' };
} finally {
await webServer?.kill();
if (globalSetupResult && typeof globalSetupResult === 'function')
await globalSetupResult(this._loader.fullConfig());
if (config.globalTeardown)

145
src/test/webServer.ts Normal file
View File

@ -0,0 +1,145 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable no-console */
import net from 'net';
import os from 'os';
import stream from 'stream';
import { monotonicTime, raceAgainstDeadline } from './util';
import { WebServerConfig } from '../../types/test';
import { launchProcess } from '../server/processLauncher';
const DEFAULT_ENVIRONMENT_VARIABLES = {
'BROWSER': 'none', // Disable that create-react-app will open the page in the browser
};
const newProcessLogPrefixer = () => new stream.Transform({
transform(this: stream.Transform, chunk: Buffer, encoding: string, callback: stream.TransformCallback) {
this.push(chunk.toString().split(os.EOL).map((line: string): string => line ? `[WebServer] ${line}` : line).join(os.EOL));
callback();
},
});
export class WebServer {
private _killProcess?: () => Promise<void>;
private _processExitedPromise!: Promise<any>;
constructor(private readonly config: WebServerConfig) { }
public static async create(config: WebServerConfig): Promise<WebServer> {
const webServer = new WebServer(config);
if (config.port)
await webServer._verifyFreePort(config.port);
try {
const port = await webServer._startWebServer();
await webServer._waitForAvailability(port);
const baseURL = `http://localhost:${port}`;
process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL;
console.log(`Using WebServer at '${baseURL}'.`);
return webServer;
} catch (error) {
await webServer.kill();
throw error;
}
}
private async _verifyFreePort(port: number) {
const cancellationToken = { canceled: false };
const portIsUsed = await Promise.race([
new Promise(resolve => setTimeout(() => resolve(false), 100)),
waitForSocket(port, 100, cancellationToken),
]);
cancellationToken.canceled = true;
if (portIsUsed)
throw new Error(`Port ${port} is used, make sure that nothing is running on the port`);
}
private async _startWebServer(): Promise<number> {
let collectPortResolve = (port: number) => { };
const collectPortPromise = new Promise<number>(resolve => collectPortResolve = resolve);
function collectPort(data: Buffer) {
const regExp = /http:\/\/localhost:(\d+)/.exec(data.toString());
if (regExp)
collectPortResolve(parseInt(regExp[1], 10));
}
let processExitedReject = (error: Error) => { };
this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject);
console.log(`Starting WebServer with '${this.config.command}'...`);
const { launchedProcess, kill } = await launchProcess({
command: this.config.command,
env: {
...DEFAULT_ENVIRONMENT_VARIABLES,
...process.env,
...this.config.env,
},
cwd: this.config.cwd,
stdio: 'stdin',
shell: true,
attemptToGracefullyClose: async () => {},
log: () => {},
onExit: code => processExitedReject(new Error(`WebServer was not able to start. Exit code: ${code}`)),
tempDirectories: [],
});
this._killProcess = kill;
launchedProcess.stderr.pipe(newProcessLogPrefixer()).pipe(process.stderr);
launchedProcess.stdout.on('data', () => {});
if (this.config.port)
return this.config.port;
launchedProcess.stdout.on('data', collectPort);
const detectedPort = await Promise.race([
this._processExitedPromise,
collectPortPromise,
]);
return detectedPort;
}
private async _waitForAvailability(port: number) {
const launchTimeout = this.config.timeout || 60 * 1000;
const cancellationToken = { canceled: false };
const { timedOut } = (await Promise.race([
raceAgainstDeadline(waitForSocket(port, 100, cancellationToken), launchTimeout + monotonicTime()),
this._processExitedPromise,
]));
cancellationToken.canceled = true;
if (timedOut)
throw new Error(`Timed out waiting ${launchTimeout}ms for WebServer"`);
}
public async kill() {
await this._killProcess?.();
}
}
async function waitForSocket(port: number, delay: number, cancellationToken: { canceled: boolean }) {
while (!cancellationToken.canceled) {
const connected = await new Promise(resolve => {
const conn = net
.connect(port)
.on('error', () => {
resolve(false);
})
.on('connect', () => {
conn.end();
resolve(true);
});
});
if (connected)
return;
await new Promise(x => setTimeout(x, delay));
}
}

View File

@ -0,0 +1,10 @@
const { TestServer } = require('../../../utils/testserver/');
// delay creating the server to test waiting for it
setTimeout(() => {
TestServer.create(__dirname, process.argv[2] || 3000).then(server => {
console.log(`Listening on http://localhost:${server.PORT}`);
server.setRoute('/hello', (message, response) => {
response.end('hello');
});
});
}, 750);

View File

@ -0,0 +1,13 @@
const { TestServer } = require('../../../utils/testserver/');
// delay creating the server to test waiting for it
setTimeout(() => {
TestServer.create(__dirname, process.argv[2] || 3000).then(server => {
console.log('listening on port', server.PORT);
server.setRoute('/hello', (message, response) => {
response.end('hello');
});
server.setRoute('/env-FOO', (message, response) => {
response.end(process.env.FOO);
});
});
}, 750);

View File

@ -0,0 +1,151 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import http from 'http';
import path from 'path';
import { test, expect } from './playwright-test-fixtures';
test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
const port = workerIndex + 10500;
const result = await runInlineTest({
'test.spec.ts': `
const { test } = pwt;
test('connect to the server via the baseURL', async ({baseURL, page}) => {
await page.goto('/hello');
await page.waitForURL('/hello');
expect(page.url()).toBe('http://localhost:${port}/hello');
expect(await page.textContent('body')).toBe('hello');
});
`,
'playwright.config.ts': `
module.exports = {
webServer: {
command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
port: ${port},
}
};
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
});
test('should create a server with environment variables', async ({ runInlineTest }, { workerIndex }) => {
const port = workerIndex + 10500;
const result = await runInlineTest({
'test.spec.ts': `
const { test } = pwt;
test('connect to the server', async ({baseURL, page}) => {
expect(baseURL).toBe('http://localhost:${port}');
await page.goto(baseURL + '/env-FOO');
expect(await page.textContent('body')).toBe('BAR');
});
`,
'playwright.config.ts': `
module.exports = {
webServer: {
command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
port: ${port},
env: {
'FOO': 'BAR',
}
}
};
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
});
test('should time out waiting for a server', async ({ runInlineTest }, { workerIndex }) => {
const port = workerIndex + 10500;
const result = await runInlineTest({
'test.spec.ts': `
const { test } = pwt;
test('connect to the server', async ({baseURL, page}) => {
expect(baseURL).toBe('http://localhost:${port}');
await page.goto(baseURL + '/hello');
expect(await page.textContent('body')).toBe('hello');
});
`,
'playwright.config.ts': `
module.exports = {
webServer: {
command: 'node ${JSON.stringify(JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js')))} ${port}',
port: ${port},
timeout: 100,
}
};
`,
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`Timed out waiting 100ms for WebServer`);
});
test('should be able to detect the port from the process stdout', async ({ runInlineTest }, { workerIndex }) => {
const port = workerIndex + 10500;
const result = await runInlineTest({
'test.spec.ts': `
const { test } = pwt;
test('connect to the server', async ({baseURL, page}) => {
expect(baseURL).toBe('http://localhost:${port}');
await page.goto(baseURL + '/hello');
expect(await page.textContent('body')).toBe('hello');
});
`,
'playwright.config.ts': `
module.exports = {
webServer: {
command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server-with-stdout.js'))} ${port}',
}
};
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
});
test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => {
const port = workerIndex + 10500;
const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
res.end('<html><body>hello</body></html>');
});
await new Promise(resolve => server.listen(port, resolve));
const result = await runInlineTest({
'test.spec.ts': `
const { test } = pwt;
test('connect to the server', async ({baseURL, page}) => {
expect(baseURL).toBe('http://localhost:${port}');
await page.goto(baseURL + '/hello');
expect(await page.textContent('body')).toBe('hello');
});
`,
'playwright.config.ts': `
module.exports = {
use: {
baseURL: 'http://localhost:${port}',
}
};
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
server.close();
});

36
types/test.d.ts vendored
View File

@ -118,6 +118,30 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends ProjectBase {
export type FullProject<TestArgs = {}, WorkerArgs = {}> = Required<Project<TestArgs, WorkerArgs>>;
export type WebServerConfig = {
/**
* Shell command to start the webserver. For example `npm run start`.
*/
command: string,
/**
* The port that your server is expected to appear on. If not specified, it does get automatically collected via the
* command output when a localhost URL gets printed.
*/
port?: number,
/**
* WebServer environment variables, process.env by default
*/
env?: Record<string, string>,
/**
* Current working directory of the spawned process. Default is process.cwd().
*/
cwd?: string,
/**
* How long to wait for the server to start up in milliseconds. Defaults to 60000.
*/
timeout?: number,
};
/**
* Testing configuration.
*/
@ -206,6 +230,11 @@ interface ConfigBase {
*/
updateSnapshots?: UpdateSnapshots;
/**
* Launch a web server before running tests.
*/
webServer?: WebServerConfig;
/**
* The maximum number of concurrent worker processes to use for parallelizing tests.
*/
@ -239,6 +268,7 @@ export interface FullConfig {
shard: Shard;
updateSnapshots: UpdateSnapshots;
workers: number;
webServer: WebServerConfig | null;
}
export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
@ -1147,6 +1177,12 @@ export type PlaywrightTestOptions = {
*/
viewport: ViewportSize | null | undefined;
/**
* BaseURL used for all the contexts in the test. Takes priority over `contextOptions`.
* @see BrowserContextOptions
*/
baseURL: string | undefined;
/**
* Options used to create the context. Other options above (e.g. `viewport`) take priority.
* @see BrowserContextOptions

View File

@ -161,7 +161,7 @@ DEPS['src/utils/'] = ['src/common/', 'src/protocol/'];
DEPS['src/server/trace/common/'] = ['src/server/snapshot/', ...DEPS['src/server/']];
DEPS['src/server/trace/recorder/'] = ['src/server/trace/common/', ...DEPS['src/server/trace/common/']];
DEPS['src/server/trace/viewer/'] = ['src/server/trace/common/', 'src/server/trace/recorder/', 'src/server/chromium/', ...DEPS['src/server/trace/common/']];
DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts'];
DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts', 'src/server/processLauncher.ts'];
checkDeps().catch(e => {
console.error(e && e.stack ? e.stack : e);