From 98bcf2665667410ac7ab7d707afa854db70f63be Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Wed, 7 Jul 2021 20:19:42 +0200 Subject: [PATCH] feat(test-runner): add webServer (#7368) --- docs/src/test-advanced.md | 56 +++++++ docs/src/test-configuration.md | 1 + src/server/browserType.ts | 2 +- src/server/chromium/videoRecorder.ts | 2 +- src/server/electron/electron.ts | 2 +- src/server/processLauncher.ts | 31 ++-- src/test/index.ts | 7 +- src/test/loader.ts | 2 + src/test/runner.ts | 3 + src/test/webServer.ts | 145 +++++++++++++++++ .../assets/simple-server-with-stdout.js | 10 ++ tests/playwright-test/assets/simple-server.js | 13 ++ tests/playwright-test/web-server.spec.ts | 151 ++++++++++++++++++ types/test.d.ts | 36 +++++ utils/check_deps.js | 2 +- 15 files changed, 442 insertions(+), 21 deletions(-) create mode 100644 src/test/webServer.ts create mode 100644 tests/playwright-test/assets/simple-server-with-stdout.js create mode 100644 tests/playwright-test/assets/simple-server.js create mode 100644 tests/playwright-test/web-server.spec.ts diff --git a/docs/src/test-advanced.md b/docs/src/test-advanced.md index 8d536ec6fd..af47a26375 100644 --- a/docs/src/test-advanced.md +++ b/docs/src/test-advanced.md @@ -42,6 +42,7 @@ These options would be typically different between local development and CI oper - `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold. - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information. - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run. +- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automaticially detect the port when it got printed to the stdout. - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests. Note that each [test project](#projects) can provide its own test suite options, for example two projects can run different tests by providing different `testDir`s. However, test run options are shared between all projects. @@ -200,6 +201,61 @@ export const test = base.extend<{ saveLogs: void }>({ }); ``` +## Launching a development web server during the tests + +To launch a web server during the tests, use the `webServer` option in the [configuration file](#configuration-object). + +Playwright Test does automatically detect if a localhost URL like `http://localhost:3000` gets printed to the stdout. +The port from the printed URL gets then used to check when its accepting requests and passed over to Playwright as a +[`param: baseURL`] when creating the context [`method: Browser.newContext`]. You can also manually specify a `port` or additional environment variables, see [here](#configuration-object). + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + webServer: { + command: 'npm run start', + timeout: 120 * 1000, + }, +}; + +export default config; +``` + +```js js-flavor=ts +// test.spec.ts +import { test } = from '@playwright/test'; + +test('test', async ({ page }) => { + // This will result in e.g. http://localhost:3000/foo when your dev-server prints a http://localhost:3000 address + await page.goto('/foo'); +}); +``` + +```js js-flavor=js +// @ts-check +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + webServer: { + command: 'npm run start', + timeout: 120 * 1000, + }, +}; + +mode.exports = config; +``` + +```js js-flavor=js +// test.spec.js +const { test } = require('@playwright/test'); + +test('test', async ({ page }) => { + // This will result in e.g. http://localhost:3000/foo when your dev-server prints a http://localhost:3000 address + await page.goto('/foo'); +}); +``` + ## Global setup and teardown To set something up once before running all tests, use `globalSetup` option in the [configuration file](#configuration-object). diff --git a/docs/src/test-configuration.md b/docs/src/test-configuration.md index bc3f254607..e330c0153d 100644 --- a/docs/src/test-configuration.md +++ b/docs/src/test-configuration.md @@ -161,6 +161,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots, - `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`. - `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files. - `timeout`: Time in milliseconds given to each test. +- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automaticially detect the port when it got printed to the stdout. - `workers`: The maximum number of concurrent worker processes to use for parallelizing tests. You can specify these options in the configuration file. diff --git a/src/server/browserType.ts b/src/server/browserType.ts index d489d5ecfb..54e673b7ca 100644 --- a/src/server/browserType.ts +++ b/src/server/browserType.ts @@ -182,7 +182,7 @@ export abstract class BrowserType extends SdkObject { let transport: ConnectionTransport | undefined = undefined; let browserProcess: BrowserProcess | undefined = undefined; const { launchedProcess, gracefullyClose, kill } = await launchProcess({ - executablePath: executable, + command: executable, args: browserArguments, env: this._amendEnvironment(env, userDataDir, executable, browserArguments), handleSIGINT, diff --git a/src/server/chromium/videoRecorder.ts b/src/server/chromium/videoRecorder.ts index 301a7418fb..485c7accbd 100644 --- a/src/server/chromium/videoRecorder.ts +++ b/src/server/chromium/videoRecorder.ts @@ -92,7 +92,7 @@ export class VideoRecorder { const progress = this._progress; const { launchedProcess, gracefullyClose } = await launchProcess({ - executablePath: this._ffmpegPath, + command: this._ffmpegPath, args, stdio: 'stdin', log: (message: string) => progress.log(message), diff --git a/src/server/electron/electron.ts b/src/server/electron/electron.ts index a1535e97c2..09f8b3c894 100644 --- a/src/server/electron/electron.ts +++ b/src/server/electron/electron.ts @@ -127,7 +127,7 @@ export class Electron extends SdkObject { const browserLogsCollector = new RecentLogsCollector(); const { launchedProcess, gracefullyClose, kill } = await launchProcess({ - executablePath: options.executablePath || require('electron/index.js'), + command: options.executablePath || require('electron/index.js'), args: electronArguments, env: options.env ? envArrayToObject(options.env) : process.env, log: (message: string) => { diff --git a/src/server/processLauncher.ts b/src/server/processLauncher.ts index aaf6eb37ce..0540d34db2 100644 --- a/src/server/processLauncher.ts +++ b/src/server/processLauncher.ts @@ -24,9 +24,10 @@ import { isUnderTest, removeFolders } from '../utils/utils'; export type Env = {[key: string]: string | number | boolean | undefined}; export type LaunchProcessOptions = { - executablePath: string, - args: string[], + command: string, + args?: string[], env?: Env, + shell?: boolean, handleSIGINT?: boolean, handleSIGTERM?: boolean, @@ -62,20 +63,18 @@ if (maxListeners !== 0) export async function launchProcess(options: LaunchProcessOptions): Promise { const stdio: ('ignore' | 'pipe')[] = options.stdio === 'pipe' ? ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] : ['pipe', 'pipe', 'pipe']; - options.log(` ${options.executablePath} ${options.args.join(' ')}`); - const spawnedProcess = childProcess.spawn( - options.executablePath, - options.args, - { - // On non-windows platforms, `detached: true` makes child process a leader of a new - // process group, making it possible to kill child process tree with `.kill(-pid)` command. - // @see https://nodejs.org/api/child_process.html#child_process_options_detached - detached: process.platform !== 'win32', - env: (options.env as {[key: string]: string}), - cwd: options.cwd, - stdio, - } - ); + options.log(` ${options.command} ${options.args ? options.args.join(' ') : ''}`); + const spawnOptions: childProcess.SpawnOptions = { + // On non-windows platforms, `detached: true` makes child process a leader of a new + // process group, making it possible to kill child process tree with `.kill(-pid)` command. + // @see https://nodejs.org/api/child_process.html#child_process_options_detached + detached: process.platform !== 'win32', + env: (options.env as {[key: string]: string}), + cwd: options.cwd, + shell: options.shell, + stdio, + }; + const spawnedProcess = childProcess.spawn(options.command, options.args, spawnOptions); const cleanup = async () => { options.log(`[pid=${spawnedProcess.pid || 'N/A'}] starting temporary directories cleanup`); diff --git a/src/test/index.ts b/src/test/index.ts index c53f99cc4b..d348008c47 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -74,9 +74,12 @@ export const test = _baseTest.extend { + await use(process.env.PLAYWRIGHT_TEST_BASE_URL); + }, contextOptions: {}, - context: async ({ browser, screenshot, trace, video, acceptDownloads, bypassCSP, colorScheme, deviceScaleFactor, extraHTTPHeaders, hasTouch, geolocation, httpCredentials, ignoreHTTPSErrors, isMobile, javaScriptEnabled, locale, offline, permissions, proxy, storageState, viewport, timezoneId, userAgent, contextOptions }, use, testInfo) => { + context: async ({ browser, screenshot, trace, video, acceptDownloads, bypassCSP, colorScheme, deviceScaleFactor, extraHTTPHeaders, hasTouch, geolocation, httpCredentials, ignoreHTTPSErrors, isMobile, javaScriptEnabled, locale, offline, permissions, proxy, storageState, viewport, timezoneId, userAgent, baseURL, contextOptions }, use, testInfo) => { testInfo.snapshotSuffix = process.platform; if (process.env.PWDEBUG) testInfo.setTimeout(0); @@ -139,6 +142,8 @@ export const test = _baseTest.extend new stream.Transform({ + transform(this: stream.Transform, chunk: Buffer, encoding: string, callback: stream.TransformCallback) { + this.push(chunk.toString().split(os.EOL).map((line: string): string => line ? `[WebServer] ${line}` : line).join(os.EOL)); + callback(); + }, +}); + +export class WebServer { + private _killProcess?: () => Promise; + private _processExitedPromise!: Promise; + constructor(private readonly config: WebServerConfig) { } + + public static async create(config: WebServerConfig): Promise { + const webServer = new WebServer(config); + if (config.port) + await webServer._verifyFreePort(config.port); + try { + const port = await webServer._startWebServer(); + await webServer._waitForAvailability(port); + const baseURL = `http://localhost:${port}`; + process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL; + console.log(`Using WebServer at '${baseURL}'.`); + return webServer; + } catch (error) { + await webServer.kill(); + throw error; + } + } + + private async _verifyFreePort(port: number) { + const cancellationToken = { canceled: false }; + const portIsUsed = await Promise.race([ + new Promise(resolve => setTimeout(() => resolve(false), 100)), + waitForSocket(port, 100, cancellationToken), + ]); + cancellationToken.canceled = true; + if (portIsUsed) + throw new Error(`Port ${port} is used, make sure that nothing is running on the port`); + } + + private async _startWebServer(): Promise { + let collectPortResolve = (port: number) => { }; + const collectPortPromise = new Promise(resolve => collectPortResolve = resolve); + function collectPort(data: Buffer) { + const regExp = /http:\/\/localhost:(\d+)/.exec(data.toString()); + if (regExp) + collectPortResolve(parseInt(regExp[1], 10)); + } + + let processExitedReject = (error: Error) => { }; + this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject); + + console.log(`Starting WebServer with '${this.config.command}'...`); + const { launchedProcess, kill } = await launchProcess({ + command: this.config.command, + env: { + ...DEFAULT_ENVIRONMENT_VARIABLES, + ...process.env, + ...this.config.env, + }, + cwd: this.config.cwd, + stdio: 'stdin', + shell: true, + attemptToGracefullyClose: async () => {}, + log: () => {}, + onExit: code => processExitedReject(new Error(`WebServer was not able to start. Exit code: ${code}`)), + tempDirectories: [], + }); + this._killProcess = kill; + + launchedProcess.stderr.pipe(newProcessLogPrefixer()).pipe(process.stderr); + launchedProcess.stdout.on('data', () => {}); + + if (this.config.port) + return this.config.port; + launchedProcess.stdout.on('data', collectPort); + const detectedPort = await Promise.race([ + this._processExitedPromise, + collectPortPromise, + ]); + return detectedPort; + } + + private async _waitForAvailability(port: number) { + const launchTimeout = this.config.timeout || 60 * 1000; + const cancellationToken = { canceled: false }; + const { timedOut } = (await Promise.race([ + raceAgainstDeadline(waitForSocket(port, 100, cancellationToken), launchTimeout + monotonicTime()), + this._processExitedPromise, + ])); + cancellationToken.canceled = true; + if (timedOut) + throw new Error(`Timed out waiting ${launchTimeout}ms for WebServer"`); + } + public async kill() { + await this._killProcess?.(); + } +} + +async function waitForSocket(port: number, delay: number, cancellationToken: { canceled: boolean }) { + while (!cancellationToken.canceled) { + const connected = await new Promise(resolve => { + const conn = net + .connect(port) + .on('error', () => { + resolve(false); + }) + .on('connect', () => { + conn.end(); + resolve(true); + }); + }); + if (connected) + return; + await new Promise(x => setTimeout(x, delay)); + } +} diff --git a/tests/playwright-test/assets/simple-server-with-stdout.js b/tests/playwright-test/assets/simple-server-with-stdout.js new file mode 100644 index 0000000000..84650d5b2b --- /dev/null +++ b/tests/playwright-test/assets/simple-server-with-stdout.js @@ -0,0 +1,10 @@ +const { TestServer } = require('../../../utils/testserver/'); +// delay creating the server to test waiting for it +setTimeout(() => { + TestServer.create(__dirname, process.argv[2] || 3000).then(server => { + console.log(`Listening on http://localhost:${server.PORT}`); + server.setRoute('/hello', (message, response) => { + response.end('hello'); + }); + }); +}, 750); diff --git a/tests/playwright-test/assets/simple-server.js b/tests/playwright-test/assets/simple-server.js new file mode 100644 index 0000000000..ccf7d7484f --- /dev/null +++ b/tests/playwright-test/assets/simple-server.js @@ -0,0 +1,13 @@ +const { TestServer } = require('../../../utils/testserver/'); +// delay creating the server to test waiting for it +setTimeout(() => { + TestServer.create(__dirname, process.argv[2] || 3000).then(server => { + console.log('listening on port', server.PORT); + server.setRoute('/hello', (message, response) => { + response.end('hello'); + }); + server.setRoute('/env-FOO', (message, response) => { + response.end(process.env.FOO); + }); + }); +}, 750); diff --git a/tests/playwright-test/web-server.spec.ts b/tests/playwright-test/web-server.spec.ts new file mode 100644 index 0000000000..8ef2fb4355 --- /dev/null +++ b/tests/playwright-test/web-server.spec.ts @@ -0,0 +1,151 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import http from 'http'; +import path from 'path'; +import { test, expect } from './playwright-test-fixtures'; + +test('should create a server', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server via the baseURL', async ({baseURL, page}) => { + await page.goto('/hello'); + await page.waitForURL('/hello'); + expect(page.url()).toBe('http://localhost:${port}/hello'); + expect(await page.textContent('body')).toBe('hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + webServer: { + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', + port: ${port}, + } + }; + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); +}); + +test('should create a server with environment variables', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server', async ({baseURL, page}) => { + expect(baseURL).toBe('http://localhost:${port}'); + await page.goto(baseURL + '/env-FOO'); + expect(await page.textContent('body')).toBe('BAR'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + webServer: { + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', + port: ${port}, + env: { + 'FOO': 'BAR', + } + } + }; + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); +}); + +test('should time out waiting for a server', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server', async ({baseURL, page}) => { + expect(baseURL).toBe('http://localhost:${port}'); + await page.goto(baseURL + '/hello'); + expect(await page.textContent('body')).toBe('hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + webServer: { + command: 'node ${JSON.stringify(JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js')))} ${port}', + port: ${port}, + timeout: 100, + } + }; + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Timed out waiting 100ms for WebServer`); +}); + +test('should be able to detect the port from the process stdout', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server', async ({baseURL, page}) => { + expect(baseURL).toBe('http://localhost:${port}'); + await page.goto(baseURL + '/hello'); + expect(await page.textContent('body')).toBe('hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + webServer: { + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server-with-stdout.js'))} ${port}', + } + }; + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); +}); + +test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { + res.end('hello'); + }); + await new Promise(resolve => server.listen(port, resolve)); + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server', async ({baseURL, page}) => { + expect(baseURL).toBe('http://localhost:${port}'); + await page.goto(baseURL + '/hello'); + expect(await page.textContent('body')).toBe('hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + use: { + baseURL: 'http://localhost:${port}', + } + }; + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); + server.close(); +}); diff --git a/types/test.d.ts b/types/test.d.ts index ef0adf352c..d739c83083 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -118,6 +118,30 @@ export interface Project extends ProjectBase { export type FullProject = Required>; +export type WebServerConfig = { + /** + * Shell command to start the webserver. For example `npm run start`. + */ + command: string, + /** + * The port that your server is expected to appear on. If not specified, it does get automatically collected via the + * command output when a localhost URL gets printed. + */ + port?: number, + /** + * WebServer environment variables, process.env by default + */ + env?: Record, + /** + * Current working directory of the spawned process. Default is process.cwd(). + */ + cwd?: string, + /** + * How long to wait for the server to start up in milliseconds. Defaults to 60000. + */ + timeout?: number, +}; + /** * Testing configuration. */ @@ -206,6 +230,11 @@ interface ConfigBase { */ updateSnapshots?: UpdateSnapshots; + /** + * Launch a web server before running tests. + */ + webServer?: WebServerConfig; + /** * The maximum number of concurrent worker processes to use for parallelizing tests. */ @@ -239,6 +268,7 @@ export interface FullConfig { shard: Shard; updateSnapshots: UpdateSnapshots; workers: number; + webServer: WebServerConfig | null; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; @@ -1147,6 +1177,12 @@ export type PlaywrightTestOptions = { */ viewport: ViewportSize | null | undefined; + /** + * BaseURL used for all the contexts in the test. Takes priority over `contextOptions`. + * @see BrowserContextOptions + */ + baseURL: string | undefined; + /** * Options used to create the context. Other options above (e.g. `viewport`) take priority. * @see BrowserContextOptions diff --git a/utils/check_deps.js b/utils/check_deps.js index bfb6146778..3262a49e34 100644 --- a/utils/check_deps.js +++ b/utils/check_deps.js @@ -161,7 +161,7 @@ DEPS['src/utils/'] = ['src/common/', 'src/protocol/']; DEPS['src/server/trace/common/'] = ['src/server/snapshot/', ...DEPS['src/server/']]; DEPS['src/server/trace/recorder/'] = ['src/server/trace/common/', ...DEPS['src/server/trace/common/']]; DEPS['src/server/trace/viewer/'] = ['src/server/trace/common/', 'src/server/trace/recorder/', 'src/server/chromium/', ...DEPS['src/server/trace/common/']]; -DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts']; +DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts', 'src/server/processLauncher.ts']; checkDeps().catch(e => { console.error(e && e.stack ? e.stack : e);