feat(tracing): pack sources to trace on the driver side (#10815)

This commit is contained in:
Yury Semikhatsky 2021-12-09 17:21:17 -08:00 committed by GitHub
parent a52c6219a7
commit 976af162b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 266 additions and 101 deletions

View File

@ -24,6 +24,7 @@ import { isSafeCloseError, kBrowserClosedError } from '../utils/errors';
import * as api from '../../types/types';
import { CDPSession } from './cdpSession';
import type { BrowserType } from './browserType';
import { LocalUtils } from './localUtils';
export class Browser extends ChannelOwner<channels.BrowserChannel> implements api.Browser {
readonly _contexts = new Set<BrowserContext>();
@ -32,6 +33,7 @@ export class Browser extends ChannelOwner<channels.BrowserChannel> implements ap
_shouldCloseConnectionOnClose = false;
private _browserType!: BrowserType;
readonly _name: string;
_localUtils!: LocalUtils;
static from(browser: channels.BrowserChannel): Browser {
return (browser as any)._object;
@ -62,6 +64,7 @@ export class Browser extends ChannelOwner<channels.BrowserChannel> implements ap
this._contexts.add(context);
context._logger = options.logger || this._logger;
context._setBrowserType(this._browserType);
context._localUtils = this._localUtils;
await this._browserType._onDidCreateContext?.(context);
return context;
}

View File

@ -38,12 +38,14 @@ import type { BrowserType } from './browserType';
import { Artifact } from './artifact';
import { APIRequestContext } from './fetch';
import { createInstrumentation } from './clientInstrumentation';
import { LocalUtils } from './localUtils';
export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel> implements api.BrowserContext {
_pages = new Set<Page>();
private _routes: network.RouteHandler[] = [];
readonly _browser: Browser | null = null;
private _browserType: BrowserType | undefined;
_localUtils!: LocalUtils;
readonly _bindings = new Map<string, (source: structs.BindingSource, ...args: any[]) => any>();
_timeoutSettings = new TimeoutSettings();
_ownerPage: Page | undefined;

View File

@ -80,6 +80,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
const browser = Browser.from((await this._channel.launch(launchOptions)).browser);
browser._logger = logger;
browser._setBrowserType(this);
browser._localUtils = this._playwright._utils;
return browser;
}
@ -108,6 +109,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
context._options = contextParams;
context._logger = logger;
context._setBrowserType(this);
context._localUtils = this._playwright._utils;
await this._onDidCreateContext?.(context);
return context;
}
@ -172,6 +174,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
browser._logger = logger;
browser._shouldCloseConnectionOnClose = true;
browser._setBrowserType((playwright as any)[browser._name]);
browser._localUtils = this._playwright._utils;
browser.on(Events.Browser.Disconnected, closePipe);
fulfill(browser);
} catch (e) {
@ -216,6 +219,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
browser._contexts.add(BrowserContext.from(result.defaultContext));
browser._logger = logger;
browser._setBrowserType(this);
browser._localUtils = this._playwright._utils;
return browser;
}
}

View File

@ -40,6 +40,7 @@ import { Artifact } from './artifact';
import { EventEmitter } from 'events';
import { JsonPipe } from './jsonPipe';
import { APIRequestContext } from './fetch';
import { LocalUtils } from './localUtils';
class Root extends ChannelOwner<channels.RootChannel> {
constructor(connection: Connection) {
@ -229,6 +230,9 @@ export class Connection extends EventEmitter {
case 'JsonPipe':
result = new JsonPipe(parent, type, guid, initializer);
break;
case 'LocalUtils':
result = new LocalUtils(parent, type, guid, initializer);
break;
case 'Page':
result = new Page(parent, type, guid, initializer);
break;

View File

@ -0,0 +1,32 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as channels from '../protocol/channels';
import { ChannelOwner } from './channelOwner';
export class LocalUtils extends ChannelOwner<channels.LocalUtilsChannel> {
static from(channel: channels.LocalUtilsChannel): LocalUtils {
return (channel as any)._object;
}
constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.LocalUtilsInitializer) {
super(parent, type, guid, initializer);
}
async zip(zipFile: string, entries: channels.NameValue[]): Promise<void> {
await this._channel.zip({ zipFile, entries });
}
}

View File

@ -25,6 +25,7 @@ import { BrowserType } from './browserType';
import { ChannelOwner } from './channelOwner';
import { Electron } from './electron';
import { APIRequest } from './fetch';
import { LocalUtils } from './localUtils';
import { Selectors, SelectorsOwner } from './selectors';
import { Size } from './types';
const dnsLookupAsync = util.promisify(dns.lookup);
@ -49,6 +50,7 @@ export class Playwright extends ChannelOwner<channels.PlaywrightChannel> {
selectors: Selectors;
readonly request: APIRequest;
readonly errors: { TimeoutError: typeof TimeoutError };
_utils: LocalUtils;
private _sockets = new Map<string, net.Socket>();
private _redirectPortForTest: number | undefined;
@ -68,6 +70,7 @@ export class Playwright extends ChannelOwner<channels.PlaywrightChannel> {
this.devices[name] = descriptor;
this.selectors = new Selectors();
this.errors = { TimeoutError };
this._utils = LocalUtils.from(initializer.utils);
const selectorsOwner = SelectorsOwner.from(initializer.selectors);
this.selectors._addChannel(selectorsOwner);

View File

@ -16,17 +16,11 @@
import * as api from '../../types/types';
import * as channels from '../protocol/channels';
import { ParsedStackTrace } from '../utils/stackTrace';
import { calculateSha1 } from '../utils/utils';
import { Artifact } from './artifact';
import { BrowserContext } from './browserContext';
import fs from 'fs';
import path from 'path';
import yauzl from 'yauzl';
import yazl from 'yazl';
import { assert, calculateSha1 } from '../utils/utils';
import { ManualPromise } from '../utils/async';
import EventEmitter from 'events';
import { ClientInstrumentationListener } from './clientInstrumentation';
import { ParsedStackTrace } from '../utils/stackTrace';
export class Tracing implements api.Tracing {
private _context: BrowserContext;
@ -72,80 +66,26 @@ export class Tracing implements api.Tracing {
const sources = this._sources;
this._sources = new Set();
this._context._instrumentation!.removeListener(this._instrumentationListener);
const skipCompress = !this._context._connection.isRemote();
const result = await channel.tracingStopChunk({ save: !!filePath, skipCompress });
const isLocal = !this._context._connection.isRemote();
const result = await channel.tracingStopChunk({ save: !!filePath, skipCompress: isLocal });
if (!filePath) {
// Not interested in artifacts.
return;
}
// If we don't have anything locally and we run against remote Playwright, compress on remote side.
if (!skipCompress && !sources) {
const sourceEntries: channels.NameValue[] = [];
for (const value of sources)
sourceEntries.push({ name: 'resources/src@' + calculateSha1(value) + '.txt', value });
if (!isLocal) {
// We run against remote Playwright, compress on remote side.
const artifact = Artifact.from(result.artifact!);
await artifact.saveAs(filePath);
await artifact.delete();
return;
}
// We either have sources to append or we were running locally, compress on client side
const promise = new ManualPromise<void>();
const zipFile = new yazl.ZipFile();
(zipFile as any as EventEmitter).on('error', error => promise.reject(error));
// Add sources.
if (sources) {
for (const source of sources) {
try {
if (fs.statSync(source).isFile())
zipFile.addFile(source, 'resources/src@' + calculateSha1(source) + '.txt');
} catch (e) {
}
}
}
await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
if (skipCompress) {
// Local scenario, compress the entries.
for (const entry of result.entries!)
zipFile.addFile(entry.value, entry.name);
zipFile.end(undefined, () => {
zipFile.outputStream.pipe(fs.createWriteStream(filePath)).on('close', () => promise.resolve());
});
return promise;
}
// Remote scenario, repack.
const artifact = Artifact.from(result.artifact!);
const tmpPath = filePath! + '.tmp';
await artifact.saveAs(tmpPath);
await artifact.delete();
yauzl.open(tmpPath!, (err, inZipFile) => {
if (err) {
promise.reject(err);
return;
}
assert(inZipFile);
let pendingEntries = inZipFile.entryCount;
inZipFile.on('entry', entry => {
inZipFile.openReadStream(entry, (err, readStream) => {
if (err) {
promise.reject(err);
return;
}
zipFile.addReadStream(readStream!, entry.fileName);
if (--pendingEntries === 0) {
zipFile.end(undefined, () => {
zipFile.outputStream.pipe(fs.createWriteStream(filePath)).on('close', () => {
fs.promises.unlink(tmpPath).then(() => {
promise.resolve();
});
});
});
}
});
});
});
return promise;
if (isLocal || sourceEntries)
await this._context._localUtils.zip(filePath, sourceEntries.concat(result.entries));
}
}

View File

@ -0,0 +1,88 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the 'License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import EventEmitter from 'events';
import fs from 'fs';
import path from 'path';
import yauzl from 'yauzl';
import yazl from 'yazl';
import * as channels from '../protocol/channels';
import { ManualPromise } from '../utils/async';
import { assert, createGuid } from '../utils/utils';
import { Dispatcher, DispatcherScope } from './dispatcher';
export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel> implements channels.LocalUtilsChannel {
_type_LocalUtils: boolean;
constructor(scope: DispatcherScope) {
super(scope, { guid: 'localUtils@' + createGuid() }, 'LocalUtils', {});
this._type_LocalUtils = true;
}
async zip(params: channels.LocalUtilsZipParams, metadata?: channels.Metadata): Promise<void> {
const promise = new ManualPromise<void>();
const zipFile = new yazl.ZipFile();
(zipFile as any as EventEmitter).on('error', error => promise.reject(error));
for (const entry of params.entries) {
try {
if (fs.statSync(entry.value).isFile())
zipFile.addFile(entry.value, entry.name);
} catch (e) {
}
}
if (!fs.existsSync(params.zipFile)) {
// New file, just compress the entries.
await fs.promises.mkdir(path.dirname(params.zipFile), { recursive: true });
zipFile.end(undefined, () => {
zipFile.outputStream.pipe(fs.createWriteStream(params.zipFile)).on('close', () => promise.resolve());
});
return promise;
}
// File already exists. Repack and add new entries.
const tempFile = params.zipFile + '.tmp';
await fs.promises.rename(params.zipFile, tempFile);
yauzl.open(tempFile, (err, inZipFile) => {
if (err) {
promise.reject(err);
return;
}
assert(inZipFile);
let pendingEntries = inZipFile.entryCount;
inZipFile.on('entry', entry => {
inZipFile.openReadStream(entry, (err, readStream) => {
if (err) {
promise.reject(err);
return;
}
zipFile.addReadStream(readStream!, entry.fileName);
if (--pendingEntries === 0) {
zipFile.end(undefined, () => {
zipFile.outputStream.pipe(fs.createWriteStream(params.zipFile)).on('close', () => {
fs.promises.unlink(tempFile).then(() => {
promise.resolve();
});
});
});
}
});
});
});
return promise;
}
}

View File

@ -26,6 +26,7 @@ import { AndroidDispatcher } from './androidDispatcher';
import { BrowserTypeDispatcher } from './browserTypeDispatcher';
import { Dispatcher, DispatcherScope } from './dispatcher';
import { ElectronDispatcher } from './electronDispatcher';
import { LocalUtilsDispatcher } from './localUtilsDispatcher';
import { APIRequestContextDispatcher } from './networkDispatchers';
import { SelectorsDispatcher } from './selectorsDispatcher';
@ -43,6 +44,7 @@ export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.Playwr
webkit: new BrowserTypeDispatcher(scope, playwright.webkit),
android: new AndroidDispatcher(scope, playwright.android),
electron: new ElectronDispatcher(scope, playwright.electron),
utils: new LocalUtilsDispatcher(scope),
deviceDescriptors,
selectors: customSelectors || new SelectorsDispatcher(scope, playwright.selectors),
preLaunchedBrowser,

View File

@ -51,6 +51,7 @@ export type InitializerTraits<T> =
T extends SelectorsChannel ? SelectorsInitializer :
T extends PlaywrightChannel ? PlaywrightInitializer :
T extends RootChannel ? RootInitializer :
T extends LocalUtilsChannel ? LocalUtilsInitializer :
T extends APIRequestContextChannel ? APIRequestContextInitializer :
object;
@ -84,6 +85,7 @@ export type EventsTraits<T> =
T extends SelectorsChannel ? SelectorsEvents :
T extends PlaywrightChannel ? PlaywrightEvents :
T extends RootChannel ? RootEvents :
T extends LocalUtilsChannel ? LocalUtilsEvents :
T extends APIRequestContextChannel ? APIRequestContextEvents :
undefined;
@ -117,6 +119,7 @@ export type EventTargetTraits<T> =
T extends SelectorsChannel ? SelectorsEventTarget :
T extends PlaywrightChannel ? PlaywrightEventTarget :
T extends RootChannel ? RootEventTarget :
T extends LocalUtilsChannel ? LocalUtilsEventTarget :
T extends APIRequestContextChannel ? APIRequestContextEventTarget :
undefined;
@ -346,6 +349,26 @@ export type APIResponse = {
};
export type LifecycleEvent = 'load' | 'domcontentloaded' | 'networkidle' | 'commit';
// ----------- LocalUtils -----------
export type LocalUtilsInitializer = {};
export interface LocalUtilsEventTarget {
}
export interface LocalUtilsChannel extends LocalUtilsEventTarget, Channel {
_type_LocalUtils: boolean;
zip(params: LocalUtilsZipParams, metadata?: Metadata): Promise<LocalUtilsZipResult>;
}
export type LocalUtilsZipParams = {
zipFile: string,
entries: NameValue[],
};
export type LocalUtilsZipOptions = {
};
export type LocalUtilsZipResult = void;
export interface LocalUtilsEvents {
}
// ----------- Root -----------
export type RootInitializer = {};
export interface RootEventTarget {
@ -374,6 +397,7 @@ export type PlaywrightInitializer = {
webkit: BrowserTypeChannel,
android: AndroidChannel,
electron: ElectronChannel,
utils: LocalUtilsChannel,
deviceDescriptors: {
name: string,
descriptor: {

View File

@ -415,6 +415,18 @@ ContextOptions:
path: string
strictSelectors: boolean?
LocalUtils:
type: interface
commands:
zip:
parameters:
zipFile: string
entries:
type: array
items: NameValue
Root:
type: interface
@ -435,6 +447,7 @@ Playwright:
webkit: BrowserType
android: Android
electron: Electron
utils: LocalUtils
deviceDescriptors:
type: array
items:

View File

@ -189,6 +189,10 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
headers: tArray(tType('NameValue')),
});
scheme.LifecycleEvent = tEnum(['load', 'domcontentloaded', 'networkidle', 'commit']);
scheme.LocalUtilsZipParams = tObject({
zipFile: tString,
entries: tArray(tType('NameValue')),
});
scheme.RootInitializeParams = tObject({
sdkLanguage: tString,
});

View File

@ -15,12 +15,12 @@
* limitations under the License.
*/
import { playwrightTest as test, expect } from './config/browserTest';
import fs from 'fs';
import * as path from 'path';
import { getUserAgent } from 'playwright-core/lib/utils/utils';
import WebSocket from 'ws';
import { suppressCertificateWarning } from './config/utils';
import { expect, playwrightTest as test } from './config/browserTest';
import { parseTrace, suppressCertificateWarning } from './config/utils';
test.slow(true, 'All connect tests are slow');
@ -531,3 +531,26 @@ test('should save har', async ({ browserType, startRemoteServer, server }, testI
expect(entry.pageref).toBe(log.pages[0].id);
expect(entry.request.url).toBe(server.EMPTY_PAGE);
});
test('should record trace with sources', async ({ browserType, startRemoteServer, server }, testInfo) => {
const remoteServer = await startRemoteServer();
const browser = await browserType.connect(remoteServer.wsEndpoint());
const context = await browser.newContext();
const page = await context.newPage();
await context.tracing.start({ sources: true });
await page.goto(server.EMPTY_PAGE);
await page.setContent('<button>Click</button>');
await page.click('"Click"');
await context.tracing.stop({ path: testInfo.outputPath('trace1.zip') });
await context.close();
await browser.close();
const { resources } = await parseTrace(testInfo.outputPath('trace1.zip'));
const sourceNames = Array.from(resources.keys()).filter(k => k.endsWith('.txt'));
expect(sourceNames.length).toBe(1);
const sourceFile = resources.get(sourceNames[0]);
const thisFile = await fs.promises.readFile(__filename);
expect(sourceFile).toEqual(thisFile);
});

View File

@ -38,6 +38,7 @@ it('should scope context handles', async ({ browserType, server }) => {
{ _guid: 'browser', objects: [] }
] },
{ _guid: 'electron', objects: [] },
{ _guid: 'localUtils', objects: [] },
{ _guid: 'Playwright', objects: [] },
{ _guid: 'selectors', objects: [] },
]
@ -65,6 +66,7 @@ it('should scope context handles', async ({ browserType, server }) => {
] },
] },
{ _guid: 'electron', objects: [] },
{ _guid: 'localUtils', objects: [] },
{ _guid: 'Playwright', objects: [] },
{ _guid: 'selectors', objects: [] },
]
@ -89,6 +91,7 @@ it('should scope CDPSession handles', async ({ browserType, browserName }) => {
{ _guid: 'browser', objects: [] }
] },
{ _guid: 'electron', objects: [] },
{ _guid: 'localUtils', objects: [] },
{ _guid: 'Playwright', objects: [] },
{ _guid: 'selectors', objects: [] },
]
@ -108,6 +111,7 @@ it('should scope CDPSession handles', async ({ browserType, browserName }) => {
] },
] },
{ _guid: 'electron', objects: [] },
{ _guid: 'localUtils', objects: [] },
{ _guid: 'Playwright', objects: [] },
{ _guid: 'selectors', objects: [] },
]
@ -128,6 +132,7 @@ it('should scope browser handles', async ({ browserType }) => {
{ _guid: 'browser-type', objects: [] },
{ _guid: 'browser-type', objects: [] },
{ _guid: 'electron', objects: [] },
{ _guid: 'localUtils', objects: [] },
{ _guid: 'Playwright', objects: [] },
{ _guid: 'selectors', objects: [] },
]
@ -152,6 +157,7 @@ it('should scope browser handles', async ({ browserType }) => {
]
},
{ _guid: 'electron', objects: [] },
{ _guid: 'localUtils', objects: [] },
{ _guid: 'Playwright', objects: [] },
{ _guid: 'selectors', objects: [] },
]

View File

@ -16,6 +16,7 @@
import { expect } from '@playwright/test';
import type { Frame, Page } from 'playwright-core';
import { ZipFileSystem } from '../../packages/playwright-core/lib/utils/vfs';
export async function attachFrame(page: Page, frameId: string, url: string): Promise<Frame> {
const handle = await page.evaluateHandle(async ({ frameId, url }) => {
@ -87,4 +88,26 @@ export function suppressCertificateWarning() {
}
return originalEmitWarning.call(process, warning, ...args);
};
}
}
export async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
const zipFS = new ZipFileSystem(file);
const resources = new Map<string, Buffer>();
for (const entry of await zipFS.entries())
resources.set(entry, await zipFS.read(entry));
zipFS.close();
const events = [];
for (const line of resources.get('trace.trace').toString().split('\n')) {
if (line)
events.push(JSON.parse(line));
}
for (const line of resources.get('trace.network').toString().split('\n')) {
if (line)
events.push(JSON.parse(line));
}
return {
events,
resources,
};
}

View File

@ -14,10 +14,11 @@
* limitations under the License.
*/
import { expect, contextTest as test, browserTest } from './config/browserTest';
import { ZipFileSystem } from '../packages/playwright-core/lib/utils/vfs';
import fs from 'fs';
import jpeg from 'jpeg-js';
import path from 'path';
import { browserTest, contextTest as test, expect } from './config/browserTest';
import { parseTrace } from './config/utils';
test.skip(({ trace }) => trace === 'on');
@ -131,6 +132,21 @@ test('should collect two traces', async ({ context, page, server }, testInfo) =>
}
});
test('should collect sources', async ({ context, page, server }, testInfo) => {
await context.tracing.start({ sources: true });
await page.goto(server.EMPTY_PAGE);
await page.setContent('<button>Click</button>');
await page.click('"Click"');
await context.tracing.stop({ path: testInfo.outputPath('trace1.zip') });
const { resources } = await parseTrace(testInfo.outputPath('trace1.zip'));
const sourceNames = Array.from(resources.keys()).filter(k => k.endsWith('.txt'));
expect(sourceNames.length).toBe(1);
const sourceFile = resources.get(sourceNames[0]);
const thisFile = await fs.promises.readFile(__filename);
expect(sourceFile).toEqual(thisFile);
});
test('should not stall on dialogs', async ({ page, context, server }) => {
await context.tracing.start({ screenshots: true, snapshots: true });
await page.goto(server.EMPTY_PAGE);
@ -342,28 +358,6 @@ test('should hide internal stack frames in expect', async ({ context, page }, te
expect(relativeStack(action)).toEqual(['tracing.spec.ts']);
});
async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
const zipFS = new ZipFileSystem(file);
const resources = new Map<string, Buffer>();
for (const entry of await zipFS.entries())
resources.set(entry, await zipFS.read(entry));
zipFS.close();
const events = [];
for (const line of resources.get('trace.trace').toString().split('\n')) {
if (line)
events.push(JSON.parse(line));
}
for (const line of resources.get('trace.network').toString().split('\n')) {
if (line)
events.push(JSON.parse(line));
}
return {
events,
resources,
};
}
function expectRed(pixels: Buffer, offset: number) {
const r = pixels.readUInt8(offset);
const g = pixels.readUInt8(offset + 1);

View File

@ -94,7 +94,7 @@ async function innerCheckDeps(root, checkDepsFile) {
}
const importPath = path.resolve(path.dirname(fileName), importName) + '.ts';
if (checkDepsFile && !allowImport(fileName, importPath))
errors.push(`Disallowed import from ${path.relative(root, fileName)} to ${path.relative(root, importPath)}`);
errors.push(`Disallowed import ${path.relative(root, importPath)} in ${path.relative(root, fileName)}`);
if (checkDepsFile && !allowExternalImport(fileName, importPath, importName))
errors.push(`Disallowed external dependency ${importName} from ${path.relative(root, fileName)}`);
}