feat: introduce experimental general-purpose grid (#8941)

This patch adds a general-purpose grid framework to parallelize
Playwright across multiple agents.

This patch adds two CLI commands to manage grid:

- `npx playwright experimental-grid-server` - to launch grid
- `npx playwrigth experimental-grid-agent` - to launch agent in a host
  environment.

Grid server accepts an `--agent-factory` argument. A simple
`factory.js` might look like this:

```js
const child_process = require('child_process');

module.exports = {
  name: 'My Simple Factory',
  capacity: Infinity, // How many workers launch per agent
  timeout: 10_000, // 10 seconds timeout to create agent
  launch: ({agentId, gridURL, playwrightVersion}) => child_process.spawn(`npx`, [
    'playwright'
    'experimental-grid-agent',
    '--grid-url', gridURL,
    '--agent-id', agentId,
  ], {
    cwd: __dirname,
    shell: true,
    stdio: 'inherit',
  }),
};
```

With this `factory.js`, grid server could be launched like this:

```bash
npx playwright experimental-grid-server --factory=./factory.js
```

Once launched, it could be used with Playwright Test using env variable:

```bash
PW_GRID=http://localhost:3000 npx playwright test
```

src/cli/cli.ts

@@ -32,6 +32,8 @@ import { BrowserType } from '../client/browserType';
 import { BrowserContextOptions, LaunchOptions } from '../client/types';
 import { spawn } from 'child_process';
 import { registry, Executable } from '../utils/registry';
+import { launchGridAgent } from '../grid/gridAgent';
+import { launchGridServer } from '../grid/gridServer';
 
 const packageJSON = require('../../package.json');
 
@@ -203,6 +205,23 @@ commandWithOpenOptions('pdf <url> <filename>', 'save page as pdf',
   console.log('  $ pdf https://example.com example.pdf');
 });
 
+program
+    .command('experimental-grid-server', { hidden: true })
+    .option('--port <port>', 'grid port; defaults to 3333')
+    .option('--agent-factory <factory>', 'path to grid agent factory or npm package')
+    .option('--auth-token <authToken>', 'optional authentication token')
+    .action(function(options) {
+      launchGridServer(options.agentFactory, options.port || 3333, options.authToken);
+    });
+
+program
+    .command('experimental-grid-agent', { hidden: true })
+    .requiredOption('--agent-id <agentId>', 'agent ID')
+    .requiredOption('--grid-url <gridURL>', 'grid URL')
+    .action(function(options) {
+      launchGridAgent(options.agentId, options.gridUrl);
+    });
+
 program
     .command('show-trace [trace]')
     .option('-b, --browser <browserType>', 'browser to use, one of cr, chromium, ff, firefox, wk, webkit', 'chromium')

src/grid/gridAgent.ts

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import debug from 'debug';
+import WebSocket from 'ws';
+import { fork } from 'child_process';
+import { getPlaywrightVersion } from '../utils/utils';
+
+export function launchGridAgent(agentId: string, gridURL: string) {
+  const log = debug(`[agent ${agentId}]`);
+  log('created');
+  const params = new URLSearchParams();
+  params.set('pwVersion', getPlaywrightVersion(true /* majorMinorOnly */));
+  params.set('agentId', agentId);
+  const ws = new WebSocket(gridURL + `/registerAgent?` + params.toString());
+  ws.on('message', (workerId: string) => {
+    log('Worker requested ' + workerId);
+    fork(require.resolve('./gridWorker.js'), [gridURL, agentId, workerId], { detached: true });
+  });
+  ws.on('close', () => process.exit(0));
+}

src/grid/gridClient.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import WebSocket from 'ws';
+import { Connection } from '../client/connection';
+import { Playwright } from '../client/playwright';
+import { getPlaywrightVersion } from '../utils/utils';
+
+export class GridClient {
+  private _ws: WebSocket;
+  private _playwright: Playwright;
+
+  static async connect(gridURL: string) {
+    const params = new URLSearchParams();
+    params.set('pwVersion', getPlaywrightVersion(true /* majorMinorOnly */));
+    const ws = new WebSocket(`${gridURL}/claimWorker?` + params.toString());
+    const errorText = await Promise.race([
+      new Promise(f => ws.once('message', () => f(undefined))),
+      new Promise(f => ws.once('close', (code, reason) => f(reason))),
+    ]);
+    if (errorText)
+      throw errorText;
+    const connection = new Connection();
+    connection.onmessage = (message: Object) => ws.send(JSON.stringify(message));
+    ws.on('message', message => connection.dispatch(JSON.parse(message.toString())));
+    ws.on('close', (code, reason) => connection.didDisconnect(reason));
+    const playwright = await connection.initializePlaywright();
+    playwright._enablePortForwarding();
+    return new GridClient(ws, playwright);
+  }
+
+  constructor(ws: WebSocket, playwright: Playwright) {
+    this._ws = ws;
+    this._playwright = playwright;
+  }
+
+  playwright(): Playwright {
+    return this._playwright;
+  }
+
+  close() {
+    this._ws.close();
+  }
+}
+
+

src/grid/gridServer.ts

@@ -0,0 +1,377 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import debug from 'debug';
+import path from 'path';
+import assert from 'assert';
+import { EventEmitter } from 'events';
+import { URL } from 'url';
+import WebSocket, { Server as WebSocketServer } from 'ws';
+import { HttpServer } from '../utils/httpServer';
+import { createGuid, getPlaywrightVersion } from '../utils/utils';
+
+export type GridAgentLaunchOptions = {
+  agentId: string,
+  gridURL: string,
+  playwrightVersion: string,
+};
+
+export type GridFactory = {
+  name?: string,
+  capacity?: number,
+  timeout?: number,
+  launch: (launchOptions: GridAgentLaunchOptions) => Promise<void>,
+};
+
+type ErrorCode = {
+  code: number,
+  reason: string,
+};
+
+const WSErrors = {
+  NO_ERROR: { code: 1000, reason: '' },
+  AUTH_FAILED: { code: 1008, reason: 'Grid authentication failed' },
+  AGENT_CREATION_FAILED: { code: 1013, reason: 'Grid agent creationg failed' },
+  AGENT_NOT_FOUND: { code: 1013, reason: 'Grid agent registration failed - agent with given ID not found' },
+  AGENT_NOT_CONNECTED: { code: 1013, reason: 'Grid worker registration failed - agent has unsupported status' },
+  AGENT_CREATION_TIMED_OUT: { code: 1013, reason: 'Grid agent creationg timed out' },
+  AGENT_RETIRED: { code: 1000, reason: 'Grid agent was retired' },
+  CLIENT_SOCKET_ERROR: { code: 1011, reason: 'Grid client socket error' },
+  WORKER_SOCKET_ERROR: { code: 1011, reason: 'Grid worker socket error' },
+  CLIENT_PLAYWRIGHT_VERSION_MISMATCH: { code: 1013, reason: 'Grid Playwright and grid client versions are different' },
+  AGENT_PLAYWRIGHT_VERSION_MISMATCH: { code: 1013, reason: 'Grid Playwright and grid agent versions are different' },
+  GRID_SHUTDOWN: { code: 1000, reason: 'Grid was shutdown' },
+  AGENT_MANUALLY_STOPPED: { code: 1000, reason: 'Grid agent was manually stopped' },
+};
+
+class GridWorker extends EventEmitter {
+  readonly workerId = createGuid();
+  private _workerSocket: WebSocket | undefined;
+  private _clientSocket: WebSocket;
+  private _log: debug.Debugger;
+
+  constructor(clientSocket: WebSocket) {
+    super();
+    this._log = debug(`[worker ${this.workerId}]`);
+    this._clientSocket = clientSocket;
+    clientSocket.on('close', (code: number, reason: string) => this.closeWorker(WSErrors.NO_ERROR));
+    clientSocket.on('error', (error: Error) => this.closeWorker(WSErrors.CLIENT_SOCKET_ERROR));
+  }
+
+  workerConnected(workerSocket: WebSocket) {
+    this._log('connected');
+    this._workerSocket = workerSocket;
+    workerSocket.on('close', (code: number, reason: string) => this.closeWorker(WSErrors.NO_ERROR));
+    workerSocket.on('error', (error: Error) => this.closeWorker(WSErrors.WORKER_SOCKET_ERROR));
+    this._clientSocket.on('message', data => workerSocket!.send(data));
+    workerSocket.on('message', data => this._clientSocket!.send(data));
+    this._clientSocket.send('run');
+  }
+
+  closeWorker(errorCode: ErrorCode) {
+    this._log('close');
+    this._workerSocket?.close(errorCode.code, errorCode.reason);
+    this._clientSocket.close(errorCode.code, errorCode.reason);
+    this.emit('close');
+  }
+
+  debugInfo() {
+    return { worker: !!this._workerSocket, client: !!this._clientSocket };
+  }
+}
+
+type AgentStatus = 'none' | 'created' | 'connected' | 'retiring';
+
+class GridAgent extends EventEmitter {
+  private _capacity: number;
+  readonly agentId = createGuid();
+  private _ws: WebSocket | undefined;
+  readonly _workers = new Map<string, GridWorker>();
+  private _status: AgentStatus = 'none';
+  private _workersWaitingForAgentConnected: GridWorker[] = [];
+  private _retireTimeout: NodeJS.Timeout | undefined;
+  private _log: debug.Debugger;
+  private _agentCreationTimeout: NodeJS.Timeout;
+
+  constructor(capacity = Infinity, creationTimeout = 5 * 60_000) {
+    super();
+    this._capacity = capacity;
+    this._log = debug(`[agent ${this.agentId}]`);
+    this.setStatus('created');
+    this._agentCreationTimeout = setTimeout(() => {
+      this.closeAgent(WSErrors.AGENT_CREATION_TIMED_OUT);
+    }, creationTimeout);
+  }
+
+  public status(): AgentStatus {
+    return this._status;
+  }
+
+  setStatus(status: AgentStatus) {
+    this._log(`status ${this._status} => ${status}`);
+    this._status = status;
+  }
+
+  agentConnected(ws: WebSocket) {
+    clearTimeout(this._agentCreationTimeout);
+    this.setStatus('connected');
+    this._ws = ws;
+    for (const worker of this._workersWaitingForAgentConnected) {
+      this._log(`send worker id: ${worker.workerId}`);
+      ws.send(worker.workerId);
+    }
+    this._workersWaitingForAgentConnected = [];
+  }
+
+  canCreateWorker() {
+    return this._workers.size < this._capacity;
+  }
+
+  async createWorker(clientSocket: WebSocket) {
+    if (this._retireTimeout)
+      clearTimeout(this._retireTimeout);
+    if (this._ws)
+      this.setStatus('connected');
+    const worker = new GridWorker(clientSocket);
+    this._log(`create worker: ${worker.workerId}`);
+    this._workers.set(worker.workerId, worker);
+    worker.on('close', () => {
+      this._workers.delete(worker.workerId);
+      if (!this._workers.size) {
+        this.setStatus('retiring');
+        if (this._retireTimeout)
+          clearTimeout(this._retireTimeout);
+        this._retireTimeout = setTimeout(() => this.closeAgent(WSErrors.AGENT_RETIRED), 30000);
+      }
+    });
+    if (this._ws) {
+      this._log(`send worker id: ${worker.workerId}`);
+      this._ws.send(worker.workerId);
+    } else {
+      this._workersWaitingForAgentConnected.push(worker);
+    }
+  }
+
+  workerConnected(workerId: string, ws: WebSocket) {
+    this._log(`worker connected: ${workerId}`);
+    const worker = this._workers.get(workerId)!;
+    worker.workerConnected(ws);
+  }
+
+  closeAgent(errorCode: ErrorCode) {
+    for (const worker of this._workersWaitingForAgentConnected)
+      worker.closeWorker(errorCode);
+    for (const worker of this._workers.values())
+      worker.closeWorker(errorCode);
+    this._log('close');
+    this._ws?.close(errorCode.code, errorCode.reason);
+    this.emit('close');
+  }
+}
+
+class GridServer {
+  private _server: HttpServer;
+  private _wsServer: WebSocketServer;
+  private _agents = new Map<string, GridAgent>();
+  private _log: debug.Debugger;
+  private _authToken: string;
+  private _factory: GridFactory;
+  private _pwVersion: string;
+
+  constructor(factory: GridFactory, authToken: string = '') {
+    this._log = debug(`[grid]`);
+    this._authToken = authToken || '';
+    this._server = new HttpServer();
+    this._factory = factory;
+    this._pwVersion = getPlaywrightVersion(true /* majorMinorOnly */);
+
+    this._server.routePath(this._securePath('/'), (request, response) => {
+      response.statusCode = 200;
+      response.setHeader('Content-Type', 'text/html');
+      response.end(this._state());
+      return true;
+    });
+
+    this._server.routePath(this._securePath('/stopAll'), (request, response) => {
+      for (const agent of this._agents.values())
+        agent.closeAgent(WSErrors.AGENT_MANUALLY_STOPPED);
+      response.statusCode = 302;
+      response.setHeader('Location', this._securePath('/'));
+      response.end();
+      return true;
+    });
+
+    this._wsServer = this._server.createWebSocketServer();
+
+    this._wsServer.shouldHandle = request => {
+      this._log(request.url);
+      if (request.url!.startsWith(this._securePath('/claimWorker'))) {
+        // shouldHandle claims it accepts promise, except it doesn't.
+        return true;
+      }
+
+      if (request.url!.startsWith('/registerAgent') || request.url!.startsWith('/registerWorker')) {
+        const params = new URL('http://localhost/' + request.url).searchParams;
+        const agentId = params.get('agentId');
+        return !!agentId && this._agents.has(agentId);
+      }
+
+      return false;
+    };
+
+    this._wsServer.on('connection', async (ws, request) => {
+      if (request.url?.startsWith(this._securePath('/claimWorker'))) {
+        const params = new URL('http://localhost/' + request.url).searchParams;
+        if (params.get('pwVersion') !== this._pwVersion) {
+          ws.close(WSErrors.CLIENT_PLAYWRIGHT_VERSION_MISMATCH.code, WSErrors.CLIENT_PLAYWRIGHT_VERSION_MISMATCH.reason);
+          return;
+        }
+        const agent = [...this._agents.values()].find(w => w.canCreateWorker()) || this._createAgent();
+        if (!agent) {
+          ws.close(WSErrors.AGENT_CREATION_FAILED.code, WSErrors.AGENT_CREATION_FAILED.reason);
+          return;
+        }
+
+        agent.createWorker(ws);
+        return;
+      }
+
+      if (request.url?.startsWith('/registerAgent')) {
+        const params = new URL('http://localhost/' + request.url).searchParams;
+        if (params.get('pwVersion') !== this._pwVersion) {
+          ws.close(WSErrors.AGENT_PLAYWRIGHT_VERSION_MISMATCH.code, WSErrors.AGENT_PLAYWRIGHT_VERSION_MISMATCH.reason);
+          return;
+        }
+        const agentId = params.get('agentId')!;
+        const agent = this._agents.get(agentId);
+        if (!agent) {
+          ws.close(WSErrors.AGENT_NOT_FOUND.code, WSErrors.AGENT_NOT_FOUND.reason);
+          return;
+        }
+
+        agent.agentConnected(ws);
+        return;
+      }
+
+      if (request.url?.startsWith('/registerWorker')) {
+        const params = new URL('http://localhost/' + request.url).searchParams;
+        const agentId = params.get('agentId')!;
+        const workerId = params.get('workerId')!;
+        const agent = this._agents.get(agentId);
+        if (!agent)
+          ws.close(WSErrors.AGENT_NOT_FOUND.code, WSErrors.AGENT_NOT_FOUND.reason);
+        else if (agent.status() !== 'connected')
+          ws.close(WSErrors.AGENT_NOT_CONNECTED.code, WSErrors.AGENT_NOT_CONNECTED.reason);
+        else
+          agent.workerConnected(workerId, ws);
+        return;
+      }
+    });
+  }
+
+  private _createAgent(): GridAgent {
+    const agent = new GridAgent(this._factory.capacity, this._factory.timeout);
+    this._agents.set(agent.agentId, agent);
+    agent.on('close', () => {
+      this._agents.delete(agent.agentId);
+    });
+    Promise.resolve()
+        .then(() => this._factory.launch({
+          agentId: agent.agentId,
+          gridURL: this._server.urlPrefix(),
+          playwrightVersion: getPlaywrightVersion(),
+        })).then(() => {
+          this._log('created');
+        }).catch(e => {
+          this._log('failed to launch agent ' + agent.agentId);
+          console.error(e);
+          agent.closeAgent(WSErrors.AGENT_CREATION_FAILED);
+        });
+    return agent;
+  }
+
+  _securePath(suffix: string): string {
+    return this._authToken ? '/' + this._authToken + suffix : suffix;
+  }
+
+  private _state(): string {
+    return `
+        <section style="display: flex; flex-direction: row">
+          <div style="display: flex; flex-direction: column; align-items: end; margin-right: 1ex;">
+            <span>Grid Playwright Version:</span>
+            <span>Agent Factory:</span>
+            <span>Agents:</span>
+          </div>
+          <div style="display: flex; flex-direction: column">
+            <span>${this._pwVersion}</span>
+            <span>${this._factory.name}</span>
+            <span>${this._agents.size} <a href="./stopAll">(Stop All)</a></span>
+          </div>
+        </section>
+        <hr/>
+        <ul>
+          ${[...this._agents].map(([agentId, agent]) => `
+            <li>
+              <div>Agent <code>${mangle(agentId)}</code>: ${agent.status()}</div>
+              <div>Workers: ${agent._workers.size}</div>
+              <ul>
+                ${[...agent._workers].map(([workerId, worker]) => `
+                  <li>worker <code>${mangle(workerId)}</code> - ${JSON.stringify(worker.debugInfo())}</li>
+                `)}
+              </ul>
+            </li>
+          `)}
+        </ul>
+    `;
+  }
+
+  async start(port: number) {
+    await this._server.start(port);
+  }
+
+  urlPrefix(): string {
+    return this._server.urlPrefix() + this._securePath('/');
+  }
+
+  async stop() {
+    for (const agent of this._agents.values())
+      agent.closeAgent(WSErrors.GRID_SHUTDOWN);
+    assert(this._agents.size === 0);
+    await this._server.stop();
+  }
+}
+
+function mangle(sessionId: string) {
+  return sessionId.replace(/\w{28}/, 'x'.repeat(28));
+}
+
+export async function launchGridServer(factoryPathOrPackageName: string, port: number, authToken: string|undefined): Promise<void> {
+  if (!factoryPathOrPackageName)
+    factoryPathOrPackageName = './simpleGridFactory';
+  let factory;
+  try {
+    factory = require(path.resolve(factoryPathOrPackageName));
+  } catch (e) {
+    factory = require(factoryPathOrPackageName);
+  }
+  if (!factory || !factory.launch || typeof factory.launch !== 'function')
+    throw new Error('factory does not export `launch` method');
+  factory.name = factory.name || factoryPathOrPackageName;
+  const gridServer = new GridServer(factory as GridFactory, authToken);
+  await gridServer.start(port);
+  /* eslint-disable no-console */
+  console.log('Grid server is running at ' + gridServer.urlPrefix());
+}

src/grid/gridWorker.ts

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import WebSocket from 'ws';
+import debug from 'debug';
+import { DispatcherConnection, Root } from '../dispatchers/dispatcher';
+import { PlaywrightDispatcher } from '../dispatchers/playwrightDispatcher';
+import { createPlaywright } from '../server/playwright';
+import { gracefullyCloseAll } from '../utils/processLauncher';
+
+function launchGridWorker(gridURL: string, agentId: string, workerId: string) {
+  const log = debug(`[worker ${workerId}]`);
+  log('created');
+  const ws = new WebSocket(gridURL + `/registerWorker?agentId=${agentId}&workerId=${workerId}`);
+  const dispatcherConnection = new DispatcherConnection();
+  dispatcherConnection.onmessage = message => ws.send(JSON.stringify(message));
+  ws.once('open', () => {
+    new Root(dispatcherConnection, async rootScope => {
+      const playwright = createPlaywright('javascript');
+      const dispatcher = new PlaywrightDispatcher(rootScope, playwright);
+      dispatcher.enableSocksProxy();
+      return dispatcher;
+    });
+  });
+  ws.on('message', message => dispatcherConnection.dispatch(JSON.parse(message.toString())));
+  ws.on('close', async () => {
+    // Drop any messages during shutdown on the floor.
+    dispatcherConnection.onmessage = () => {};
+    setTimeout(() => process.exit(0), 30000);
+    // Meanwhile, try to gracefully close all browsers.
+    await gracefullyCloseAll();
+    process.exit(0);
+  });
+}
+
+launchGridWorker(process.argv[2], process.argv[3], process.argv[4]);

src/grid/simpleGridFactory.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import child_process from 'child_process';
+import { GridAgentLaunchOptions } from './gridServer';
+import path from 'path';
+
+export const name = 'Agents co-located with grid';
+export const capacity = Infinity;
+export const timeout = 10_000;
+export function launch({ agentId, gridURL }: GridAgentLaunchOptions) {
+  child_process.spawn(process.argv[0], [
+    path.join(__dirname, '..', 'cli', 'cli.js'),
+    'experimental-grid-agent',
+    '--grid-url', gridURL,
+    '--agent-id', agentId,
+  ], {
+    cwd: __dirname,
+    shell: true,
+    stdio: 'inherit',
+  });
+}
+

src/test/index.ts

@@ -20,6 +20,7 @@ import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext, Browse
 import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, TestInfo } from '../../types/test';
 import { rootTestType } from './testType';
 import { createGuid, removeFolders } from '../utils/utils';
+import { GridClient } from '../grid/gridClient';
 export { expect } from './expect';
 export const _baseTest: TestType<{}, {}> = rootTestType.test;
 
@@ -35,7 +36,15 @@ type WorkerAndFileFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
 export const test = _baseTest.extend<TestFixtures, WorkerAndFileFixtures>({
   defaultBrowserType: [ 'chromium', { scope: 'worker' } ],
   browserName: [ ({ defaultBrowserType }, use) => use(defaultBrowserType), { scope: 'worker' } ],
-  playwright: [ require('../inprocess'), { scope: 'worker' } ],
+  playwright: [async ({}, use, workerInfo) => {
+    if (process.env.PW_GRID) {
+      const gridClient = await GridClient.connect(process.env.PW_GRID);
+      await use(gridClient.playwright() as any);
+      await gridClient.close();
+    } else {
+      await use(require('../inprocess'));
+    }
+  }, { scope: 'worker' } ],
   headless: [ undefined, { scope: 'worker' } ],
   channel: [ undefined, { scope: 'worker' } ],
   launchOptions: [ {}, { scope: 'worker' } ],

src/utils/httpServer.ts

@@ -17,17 +17,24 @@
 import * as http from 'http';
 import fs from 'fs';
 import path from 'path';
+import { Server as WebSocketServer } from 'ws';
 import * as mime from 'mime';
 
 export type ServerRouteHandler = (request: http.IncomingMessage, response: http.ServerResponse) => boolean;
 
 export class HttpServer {
-  private _server: http.Server | undefined;
+  private _server: http.Server;
   private _urlPrefix: string;
+  private _port: number = 0;
   private _routes: { prefix?: string, exact?: string, handler: ServerRouteHandler }[] = [];
   private _activeSockets = new Set<import('net').Socket>();
   constructor() {
     this._urlPrefix = '';
+    this._server = http.createServer(this._onRequest.bind(this));
+  }
+
+  createWebSocketServer(): WebSocketServer {
+    return new WebSocketServer({ server: this._server });
   }
 
   routePrefix(prefix: string, handler: ServerRouteHandler) {
@@ -38,9 +45,12 @@ export class HttpServer {
     this._routes.push({ exact: path, handler });
   }
 
+  port(): number {
+    return this._port;
+  }
+
   async start(port?: number): Promise<string> {
-    console.assert(!this._server, 'server already started');
-    this._server = http.createServer(this._onRequest.bind(this));
+    console.assert(!this._urlPrefix, 'server already started');
     this._server.on('connection', socket => {
       this._activeSockets.add(socket);
       socket.once('close', () => this._activeSockets.delete(socket));
@@ -48,7 +58,12 @@ export class HttpServer {
     this._server.listen(port);
     await new Promise(cb => this._server!.once('listening', cb));
     const address = this._server.address();
-    this._urlPrefix = typeof address === 'string' ? address : `http://127.0.0.1:${address.port}`;
+    if (typeof address === 'string') {
+      this._urlPrefix = address;
+    } else {
+      this._port = address.port;
+      this._urlPrefix = `http://127.0.0.1:${address.port}`;
+    }
     return this._urlPrefix;
   }
 
@@ -58,7 +73,7 @@ export class HttpServer {
     await new Promise(cb => this._server!.close(cb));
   }
 
-  urlPrefix() {
+  urlPrefix(): string {
     return this._urlPrefix;
   }
 

src/utils/utils.ts

@@ -351,8 +351,12 @@ export function canAccessFile(file: string) {
 }
 
 export function getUserAgent() {
+  return `Playwright/${getPlaywrightVersion()} (${os.arch()}/${os.platform()}/${os.release()})`;
+}
+
+export function getPlaywrightVersion(majorMinorOnly = false) {
   const packageJson = require('./../../package.json');
-  return `Playwright/${packageJson.version} (${os.arch()}/${os.platform()}/${os.release()})`;
+  return majorMinorOnly ? packageJson.version.split('.').slice(0, 2).join('.') : packageJson.version;
 }
 
 export function constructURLBasedOnBaseURL(baseURL: string | undefined, givenURL: string): string {

utils/check_deps.js

@@ -181,7 +181,7 @@ DEPS['src/web/traceViewer/ui/'] = ['src/common/', 'src/protocol/', 'src/web/trac
 DEPS['src/remote/'] = ['src/client/', 'src/debug/', 'src/dispatchers/', 'src/server/', 'src/server/supplements/', 'src/server/electron/', 'src/server/trace/', 'src/utils/**'];
 
 // CLI should only use client-side features.
-DEPS['src/cli/'] = ['src/cli/**', 'src/client/**', 'src/generated/', 'src/server/injected/', 'src/debug/injected/', 'src/server/trace/**', 'src/utils/**'];
+DEPS['src/cli/'] = ['src/cli/**', 'src/client/**', 'src/generated/', 'src/server/injected/', 'src/debug/injected/', 'src/server/trace/**', 'src/utils/**', 'src/grid/**'];
 
 DEPS['src/server/supplements/recorder/recorderApp.ts'] = ['src/common/', 'src/utils/', 'src/server/', 'src/server/chromium/'];
 DEPS['src/server/supplements/recorderSupplement.ts'] = ['src/server/snapshot/', ...DEPS['src/server/']];
@@ -191,11 +191,16 @@ DEPS['src/utils/'] = ['src/common/', 'src/protocol/'];
 DEPS['src/server/trace/common/'] = ['src/server/snapshot/', ...DEPS['src/server/']];
 DEPS['src/server/trace/recorder/'] = ['src/server/trace/common/', ...DEPS['src/server/trace/common/']];
 DEPS['src/server/trace/viewer/'] = ['src/server/trace/common/', 'src/server/trace/recorder/', 'src/server/chromium/', ...DEPS['src/server/trace/common/']];
+
+// Playwright Test
 DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts', 'src/utils/**'];
+DEPS['src/test/index.ts'] = [... DEPS['src/test/'], 'src/grid/gridClient.ts' ];
 
 // HTML report
 DEPS['src/web/htmlReport/'] = ['src/test/**', 'src/web/'];
 
+// Grid
+DEPS['src/grid/'] = ['src/utils/**', 'src/dispatchers/**', 'src/server/', 'src/client/'];
 
 checkDeps().catch(e => {
   console.error(e && e.stack ? e.stack : e);