feat: introduce Dockerfile.remote image (#20691)

When this image is launched, it exposes a single endpoint
that can be used to connect to and to launch browsers.
This commit is contained in:
Andrey Lushnikov 2023-02-07 10:50:44 -08:00 committed by GitHub
parent 8f53bf7b41
commit b67cef2c4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 373 additions and 22 deletions

View File

@ -180,3 +180,31 @@ jobs:
if: always()
shell: bash
smoke_test_docker_integration:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 14
- run: npm i -g npm@8
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps
- run: |
./utils/docker/build.sh --amd64 remote playwright:localbuild-remote
docker run -p 5400:5400 --name pw-remote --rm -d playwright:localbuild-remote
while ! curl localhost:5400; do sleep 0.2; done
DOCKER_REMOTE=1 xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --grep '@smoke'
docker kill pw-remote
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
- uses: actions/upload-artifact@v3
if: always()
with:
name: docker-remote-test-results
path: test-results

View File

@ -14,6 +14,8 @@
lib/**/injected/
# Include all binaries that we ship with the package.
!bin/*
# Include all shell files in the lib/containers/*
!lib/containers/*.sh
# Include FFMPEG
!third_party/ffmpeg/*
# Include generated types and entrypoint.

View File

@ -38,6 +38,7 @@ import type { GridFactory } from '../grid/gridServer';
import { GridServer } from '../grid/gridServer';
import type { Executable } from '../server';
import { registry, writeDockerVersion } from '../server';
import { addContainerCLI } from '../containers/';
const packageJSON = require('../../package.json');
@ -311,6 +312,8 @@ Examples:
$ show-trace https://example.com/trace.zip`);
addContainerCLI(program);
if (!process.env.PW_LANG_NAME) {
let playwrightTestPackagePath = null;
const resolvePwTestPaths = [__dirname, process.cwd()];
@ -581,13 +584,15 @@ async function openPage(context: BrowserContext, url: string | undefined): Promi
async function open(options: Options, url: string | undefined, language: string) {
const { context, launchOptions, contextOptions } = await launchContext(options, !!process.env.PWTEST_CLI_HEADLESS, process.env.PWTEST_CLI_EXECUTABLE_PATH);
await context._enableRecorder({
language,
launchOptions,
contextOptions,
device: options.device,
saveStorage: options.saveStorage,
});
if (!process.env.PW_DISABLE_RECORDER) {
await context._enableRecorder({
language,
launchOptions,
contextOptions,
device: options.device,
saveStorage: options.saveStorage,
});
}
await openPage(context, url);
if (process.env.PWTEST_CLI_EXIT)
await Promise.all(context.pages().map(p => p.close()));

View File

@ -0,0 +1,6 @@
[*]
../utils/
../utilsBundle.ts
../cli/
../remote/
../third_party/

View File

@ -0,0 +1,40 @@
#!/bin/bash
set -e
trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
SCREEN_WIDTH=1360
SCREEN_HEIGHT=1020
SCREEN_DEPTH=24
SCREEN_DPI=96
GEOMETRY="$SCREEN_WIDTH""x""$SCREEN_HEIGHT""x""$SCREEN_DEPTH"
# Launch x11
nohup /usr/bin/xvfb-run --server-num=$DISPLAY_NUM \
--listen-tcp \
--server-args="-screen 0 "$GEOMETRY" -fbdir /var/tmp -dpi "$SCREEN_DPI" -listen tcp -noreset -ac +extension RANDR" \
/usr/bin/fluxbox -display "$DISPLAY" >/dev/null 2>&1 &
# Launch x11vnc
nohup x11vnc -noprimary -nosetprimary -forever -shared -rfbport 5900 -rfbportv6 5900 -display "$DISPLAY" >/dev/null 2>&1 &
# Launch novnc
nohup /opt/bin/noVNC/utils/novnc_proxy --listen 7900 --vnc localhost:5900 >/dev/null 2>&1 &
# Wait for x11 display to start
for i in $(seq 1 500); do
if xdpyinfo -display $DISPLAY >/dev/null 2>&1; then
break
fi
sleep 0.1
done
# Make sure to re-start container agent if something goes wrong.
# The approach taken from: https://stackoverflow.com/a/697064/314883
until npx playwright container start-agent --novnc-endpoint="http://127.0.0.1:7900" --port 5400; do
echo "Server crashed with exit code $?. Respawning.." >&2
sleep 1
done

View File

@ -0,0 +1,85 @@
export NOVNC_REF='1.3.0'
export WEBSOCKIFY_REF='0.10.0'
export DEBIAN_FRONTEND=noninteractive
# Install FluxBox, VNC & noVNC
mkdir -p /opt/bin && chmod +x /dev/shm \
&& apt-get update && apt-get install -y unzip fluxbox x11vnc \
&& curl -L -o noVNC.zip "https://github.com/novnc/noVNC/archive/v${NOVNC_REF}.zip" \
&& unzip -x noVNC.zip \
&& rm -rf noVNC-${NOVNC_REF}/{docs,tests} \
&& mv noVNC-${NOVNC_REF} /opt/bin/noVNC \
&& cp /opt/bin/noVNC/vnc.html /opt/bin/noVNC/index.html \
&& rm noVNC.zip \
&& curl -L -o websockify.zip "https://github.com/novnc/websockify/archive/v${WEBSOCKIFY_REF}.zip" \
&& unzip -x websockify.zip \
&& rm websockify.zip \
&& rm -rf websockify-${WEBSOCKIFY_REF}/{docs,tests} \
&& mv websockify-${WEBSOCKIFY_REF} /opt/bin/noVNC/utils/websockify
# Patch noVNC
cat <<'EOF' > /opt/bin/noVNC/clip.patch
diff --git a/app/ui.js b/app/ui.js
index cb6a9fd..dbe42e0 100644
--- a/app/ui.js
+++ b/app/ui.js
@@ -951,6 +951,7 @@ const UI = {
clipboardReceive(e) {
Log.Debug(">> UI.clipboardReceive: " + e.detail.text.substr(0, 40) + "...");
document.getElementById('noVNC_clipboard_text').value = e.detail.text;
+ navigator.clipboard.writeText(e.detail.text).catch(() => {});
Log.Debug("<< UI.clipboardReceive");
},
diff --git a/core/rfb.js b/core/rfb.js
index ea3bf58..fad57bc 100644
--- a/core/rfb.js
+++ b/core/rfb.js
@@ -176,6 +176,7 @@ export default class RFB extends EventTargetMixin {
handleMouse: this._handleMouse.bind(this),
handleWheel: this._handleWheel.bind(this),
handleGesture: this._handleGesture.bind(this),
+ handleFocus: () => navigator.clipboard.readText().then(this.clipboardPasteFrom.bind(this)).catch(() => {})
};
// main setup
@@ -515,6 +516,7 @@ export default class RFB extends EventTargetMixin {
this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture);
this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture);
this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture);
+ window.addEventListener('focus', this._eventHandlers.handleFocus);
Log.Debug("<< RFB.connect");
}
@@ -522,6 +524,7 @@ export default class RFB extends EventTargetMixin {
_disconnect() {
Log.Debug(">> RFB.disconnect");
this._cursor.detach();
+ window.removeEventListener('focus', this._eventHandlers.handleFocus);
this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture);
this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture);
this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture);
EOF
cd /opt/bin/noVNC
git apply clip.patch
# Configure FluxBox menus
mkdir /root/.fluxbox
cat <<'EOF' > /root/.fluxbox/menu
[begin] (fluxbox)
[submenu] (Browsers) {}
[exec] (Chromium) { cd /ms-playwright-agent && PW_DISABLE_RECORDER=1 npx playwright open --browser chromium } <>
[exec] (Firefox) { cd /ms-playwright-agent && PW_DISABLE_RECORDER=1 npx playwright open --browser firefox } <>
[exec] (WebKit) { cd /ms-playwright-agent && PW_DISABLE_RECORDER=1 npx playwright open --browser webkit } <>
[end]
[include] (/etc/X11/fluxbox/fluxbox-menu)
[end]
EOF
cat <<'EOF' > /root/.fluxbox/lastwallpaper
$center $full|/ms-playwright-agent/node_modules/playwright-core/lib/server/chromium/appIcon.png||:99
$center $full|/ms-playwright-agent/node_modules/playwright-core/lib/server/chromium/appIcon.png||:99.0
EOF

View File

@ -0,0 +1,105 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the 'License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable no-console */
import path from 'path';
import { spawnAsync } from '../utils/spawnAsync';
import { gracefullyCloseAll } from '../utils/processLauncher';
import { createGuid } from '../utils';
import type { Command } from '../utilsBundle';
import { debug } from '../utilsBundle';
import type { AddressInfo } from 'net';
import http from 'http';
import { PlaywrightServer } from '../remote/playwrightServer';
const { ProxyServer } = require('../third_party/http_proxy.js');
const debugLog = debug('pw:container');
export function addContainerCLI(program: Command) {
const ctrCommand = program.command('container', { hidden: true })
.description(`Manage container integration (EXPERIMENTAL)`);
ctrCommand.command('install-services', { hidden: true })
.description('install services required to run container agent')
.action(async function() {
const { code } = await spawnAsync('bash', [path.join(__dirname, 'container_install_deps.sh')], { stdio: 'inherit' });
if (code !== 0)
throw new Error('Failed to install server dependencies!');
});
ctrCommand.command('entrypoint', { hidden: true })
.description('launch all services and container agent')
.action(async function() {
await spawnAsync('bash', [path.join(__dirname, 'container_entrypoint.sh')], { stdio: 'inherit' });
});
ctrCommand.command('start-agent', { hidden: true })
.description('start container agent')
.option('--port <number>', 'port number')
.option('--novnc-endpoint <url>', 'novnc server endpoint')
.action(async function(options) {
launchContainerAgent(+(options.port ?? '0'), options.novncEndpoint);
});
}
async function launchContainerAgent(port: number, novncEndpoint: string) {
const novncWSPath = createGuid();
const server = new PlaywrightServer({
path: '/' + createGuid(),
maxConnections: Infinity,
});
await server.listen(undefined);
const serverEndpoint = server.address();
process.on('exit', () => server.close().catch(console.error));
process.stdin.on('close', () => selfDestruct());
const vncProxy = new ProxyServer(novncEndpoint, debugLog);
const serverProxy = new ProxyServer(serverEndpoint, debugLog);
const httpServer = http.createServer((request, response) => {
if (request.url === '/' && request.method === 'GET') {
response.writeHead(307, {
Location: `/screen/?resize=scale&autoconnect=1&path=${novncWSPath}`,
}).end();
} else if (request.url?.startsWith('/screen')) {
request.url = request.url.substring('/screen'.length);
vncProxy.web(request, response);
} else {
serverProxy.web(request, response);
}
});
httpServer.on('error', error => debugLog(error));
httpServer.on('upgrade', (request, socket, head) => {
if (request.url === '/' + novncWSPath)
vncProxy.ws(request, socket, head);
else
serverProxy.ws(request, socket, head);
});
httpServer.listen(port, '0.0.0.0', () => {
const { port } = httpServer.address() as AddressInfo;
console.log(`Playwright Container running on http://localhost:${port}`);
});
}
function selfDestruct() {
// Force exit after 30 seconds.
setTimeout(() => process.exit(0), 30000);
// Meanwhile, try to gracefully close all browsers.
gracefullyCloseAll().then(() => {
process.exit(0);
});
}

View File

@ -49,6 +49,7 @@ export class PlaywrightServer {
private _preLaunchedPlaywright: Playwright | undefined;
private _wsServer: WebSocketServer | undefined;
private _options: ServerOptions;
private _address: string = '';
constructor(options: ServerOptions) {
this._options = options;
@ -58,6 +59,10 @@ export class PlaywrightServer {
this._preLaunchedPlaywright = options.preLaunchedAndroidDevice._android._playwrightOptions.rootSdkObject as Playwright;
}
address(): string {
return this._address;
}
async listen(port: number = 0): Promise<string> {
const server = http.createServer((request: http.IncomingMessage, response: http.ServerResponse) => {
if (request.method === 'GET' && request.url === '/json') {
@ -74,11 +79,12 @@ export class PlaywrightServer {
const wsEndpoint = await new Promise<string>((resolve, reject) => {
server.listen(port, () => {
const address = server.address();
if (!address) {
if (!address || typeof address === 'string') {
reject(new Error('Could not bind server socket'));
return;
}
const wsEndpoint = typeof address === 'string' ? `${address}${this._options.path}` : `ws://127.0.0.1:${address.port}${this._options.path}`;
this._address = `http://127.0.0.1:${address.port}`;
const wsEndpoint = `ws://127.0.0.1:${address.port}${this._options.path}`;
resolve(wsEndpoint);
}).on('error', reject);
});

View File

@ -56,7 +56,7 @@ test('androidDevice.launchBrowser should throw for bad proxy server value', asyn
});
test('androidDevice.launchBrowser should pass proxy config', async ({ androidDevice, server, mode, loopback }) => {
test.skip(mode === 'docker', 'proxy is not supported for remote connection');
test.skip(mode === 'docker_remote', 'proxy is not supported for remote connection');
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});

View File

@ -24,7 +24,7 @@ export type PlatformWorkerFixtures = {
};
export const platformTest = test.extend<{}, PlatformWorkerFixtures>({
platform: [process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }],
platform: [process.env.PWTEST_MODE === 'docker_remote' ? 'linux' : process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }],
isWindows: [process.platform === 'win32', { scope: 'worker' }],
isMac: [process.platform === 'darwin', { scope: 'worker' }],
isLinux: [process.platform === 'linux', { scope: 'worker' }],

View File

@ -17,7 +17,7 @@
import { start } from '../../packages/playwright-core/lib/outofprocess';
import type { Playwright } from '../../packages/playwright-core/lib/client/playwright';
export type TestModeName = 'default' | 'driver' | 'service' | 'service2' | 'docker';
export type TestModeName = 'default' | 'driver' | 'service' | 'service2' | 'docker_remote';
interface TestMode {
setup(): Promise<Playwright>;

View File

@ -37,7 +37,7 @@ export const testModeTest = test.extend<TestModeTestFixtures, TestModeWorkerOpti
playwright: [async ({ mode }, run) => {
const testMode = {
default: new DefaultTestMode(),
docker: new DefaultTestMode(),
docker_remote: new DefaultTestMode(),
service: new DefaultTestMode(),
driver: new DriverTestMode(),
service2: new DefaultTestMode(),

View File

@ -66,7 +66,7 @@ it('should respect CSP @smoke', async ({ page, server }) => {
});
it('should play video @smoke', async ({ page, asset, browserName, platform, mode }) => {
it.skip(mode === 'docker', 'local paths do not work with remote setup');
it.skip(mode === 'docker_remote', 'local paths do not work with remote setup');
// TODO: the test passes on Windows locally but fails on GitHub Action bot,
// apparently due to a Media Pack issue in the Windows Server.
// Also the test is very flaky on Linux WebKit.
@ -85,7 +85,7 @@ it('should play video @smoke', async ({ page, asset, browserName, platform, mode
});
it('should play webm video @smoke', async ({ page, asset, browserName, platform, mode }) => {
it.skip(mode === 'docker', 'local paths do not work with remote setup');
it.skip(mode === 'docker_remote', 'local paths do not work with remote setup');
it.fixme(browserName === 'webkit' && platform === 'darwin' && parseInt(os.release(), 10) === 20, 'Does not work on BigSur');
it.fixme(browserName === 'webkit' && platform === 'win32');

View File

@ -51,7 +51,7 @@ it.describe('download event', () => {
});
it('should report download when navigation turns into download @smoke', async ({ browser, server, browserName, mode }) => {
it.skip(mode === 'docker', 'local paths do not work remote connection');
it.skip(mode === 'docker_remote', 'local paths do not work remote connection');
const page = await browser.newPage();
const [download, responseOrError] = await Promise.all([
page.waitForEvent('download'),

View File

@ -17,7 +17,7 @@
import { playwrightTest as it, expect } from '../config/browserTest';
it('should log @smoke', async ({ browserType, mode }) => {
it.skip(mode === 'docker', 'logger is not plumbed into the remote connection');
it.skip(mode === 'docker_remote', 'logger is not plumbed into the remote connection');
const log = [];
const browser = await browserType.launch({ logger: {

View File

@ -34,7 +34,7 @@ const getExecutablePath = (browserName: BrowserName) => {
return process.env.WKPATH;
};
const mode: TestModeName = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service' | 'service2');
const mode: TestModeName = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service' | 'service2' | 'docker_remote');
const headed = process.argv.includes('--headed');
const channel = process.env.PWTEST_CHANNEL as any;
const video = !!process.env.PWTEST_VIDEO;
@ -135,6 +135,10 @@ for (const browserName of browserNames) {
executablePath,
devtools
},
connectOptions: mode === 'docker_remote' ? {
wsEndpoint: 'http://localhost:5400',
_exposeNetwork: '*',
} as any : undefined,
trace: trace ? 'on' : undefined,
coverageName: browserName,
},

View File

@ -29,7 +29,7 @@ it('should throw for bad server value', async ({ browserType }) => {
});
it('should use proxy @smoke', async ({ browserType, server, mode }) => {
it.skip(mode === 'docker', 'proxy is not supported for remote connection');
it.skip(mode === 'docker_remote', 'proxy is not supported for remote connection');
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});

View File

@ -318,6 +318,13 @@ copyFiles.push({
ignored: ['**/.eslintrc.js', '**/webpack*.config.js', '**/injected/**/*']
});
// Copy all shell files if we happen to use any.
copyFiles.push({
files: 'packages/playwright-core/src/**/*.sh',
from: 'packages/playwright-core/src',
to: 'packages/playwright-core/lib',
});
// Sometimes we require JSON files that babel ignores.
// For example, deviceDescriptorsSource.json
copyFiles.push({

View File

@ -0,0 +1,51 @@
FROM ubuntu:focal
ARG DEBIAN_FRONTEND=noninteractive
ARG TZ=America/Los_Angeles
ARG DOCKER_IMAGE_NAME_TEMPLATE="mcr.microsoft.com/playwright:v%version%-vrt"
# === INSTALL Node.js ===
RUN apt-get update && \
# Install Node 18
apt-get install -y curl wget gpg && \
curl -sL https://deb.nodesource.com/setup_18.x | bash - && \
apt-get install -y nodejs && \
# Feature-parity with node.js base images.
apt-get install -y --no-install-recommends git openssh-client && \
npm install -g yarn && \
# clean apt cache
rm -rf /var/lib/apt/lists/* && \
# Create the pwuser
adduser pwuser
# === BAKE BROWSERS INTO IMAGE ===
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
# 1. Add tip-of-tree Playwright package to install its browsers.
# The package should be built beforehand from tip-of-tree Playwright.
COPY ./playwright-core.tar.gz /tmp/playwright-core.tar.gz
# 2. Bake in Playwright Agent.
# Playwright Agent is used to bake in browsers and browser dependencies,
# and run docker server later on.
# Browsers will be downloaded in `/ms-playwright`.
# Note: make sure to set 777 to the registry so that any user can access
# registry.
RUN mkdir /ms-playwright && \
mkdir /ms-playwright-agent && \
cd /ms-playwright-agent && npm init -y && \
npm i /tmp/playwright-core.tar.gz && \
npx playwright mark-docker-image "${DOCKER_IMAGE_NAME_TEMPLATE}" && \
npx playwright install --with-deps && \
npx playwright container install-services && \
rm /tmp/playwright-core.tar.gz && \
chmod -R 777 /ms-playwright && \
rm -rf /var/lib/apt/lists/*
WORKDIR /ms-playwright-agent
ENV DISPLAY_NUM=99
ENV DISPLAY=:99
EXPOSE 5400
ENTRYPOINT npx playwright container entrypoint

View File

@ -3,7 +3,7 @@ set -e
set +x
if [[ ($1 == '--help') || ($1 == '-h') || ($1 == '') || ($2 == '') ]]; then
echo "usage: $(basename $0) {--arm64,--amd64} {focal,jammy} playwright:localbuild-focal"
echo "usage: $(basename $0) {--arm64,--amd64} {focal,jammy,remote} playwright:localbuild-focal"
echo
echo "Build Playwright docker image and tag it as 'playwright:localbuild-focal'."
echo "Once image is built, you can run it with"

View File

@ -53,6 +53,11 @@ if [[ "$RELEASE_CHANNEL" == "stable" ]]; then
JAMMY_TAGS+=("jammy")
fi
REMOTE_TAGS=(
"next-remote"
"v${PW_VERSION}-remote"
)
tag_and_push() {
local source="$1"
local target="$2"
@ -68,8 +73,10 @@ publish_docker_images_with_arch_suffix() {
TAGS=("${FOCAL_TAGS[@]}")
elif [[ "$FLAVOR" == "jammy" ]]; then
TAGS=("${JAMMY_TAGS[@]}")
elif [[ "$FLAVOR" == "remote" ]]; then
TAGS=("${REMOTE_TAGS[@]}")
else
echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal' or 'jammy'"
echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal', 'jammy' or 'remote'"
exit 1
fi
local ARCH="$2"
@ -94,8 +101,10 @@ publish_docker_manifest () {
TAGS=("${FOCAL_TAGS[@]}")
elif [[ "$FLAVOR" == "jammy" ]]; then
TAGS=("${JAMMY_TAGS[@]}")
elif [[ "$FLAVOR" == "remote" ]]; then
TAGS=("${REMOTE_TAGS[@]}")
else
echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal' or 'jammy'"
echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal', 'jammy' or 'remote'"
exit 1
fi
@ -122,3 +131,6 @@ publish_docker_images_with_arch_suffix jammy amd64
publish_docker_images_with_arch_suffix jammy arm64
publish_docker_manifest jammy amd64 arm64
publish_docker_images_with_arch_suffix remote amd64
publish_docker_images_with_arch_suffix remote arm64
publish_docker_manifest remote amd64 arm64