microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-12-13 17:14:02 +03:00
Code Issues Projects Releases Wiki Activity

feat(tracing): trace context APIRequest calls (#10684)

Browse Source
This commit is contained in:
Yury Semikhatsky 2021-12-02 15:53:47 -08:00 committed by GitHub
parent 98e2f40bb0
commit 8afd0b7d6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 236 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/playwright-core/src/server/browserContext.ts
View File

@ -78,11 +78,12 @@ export abstract class BrowserContext extends SdkObject {
// Create instrumentation per context.
this.instrumentation = createInstrumentation();
this.fetchRequest = new BrowserContextAPIRequestContext(this);
if (this._options.recordHar)
this._harRecorder = new HarRecorder(this, { ...this._options.recordHar, path: path.join(this._browser.options.artifactsDir, `${createGuid()}.har`) });
this.tracing = new Tracing(this);
this.fetchRequest = new BrowserContextAPIRequestContext(this);
}
isPersistentContext(): boolean {

64
packages/playwright-core/src/server/fetch.ts
View File

@ -44,9 +44,31 @@ type FetchRequestOptions = {
baseURL?: string;
};
export type APIRequestEvent = {
url: URL,
method: string,
headers: { [name: string]: string },
cookies: types.NameValueList,
postData?: Buffer
};
export type APIRequestFinishedEvent = {
requestEvent: APIRequestEvent,
httpVersion: string;
headers: http.IncomingHttpHeaders;
cookies: types.NetworkCookie[];
rawHeaders: string[];
statusCode: number;
statusMessage: string;
body?: Buffer;
};
export abstract class APIRequestContext extends SdkObject {
static Events = {
Dispose: 'dispose',
Request: 'request',
RequestFinished: 'requestfinished',
};
readonly fetchResponses: Map<string, Buffer> = new Map();
@ -166,7 +188,9 @@ export abstract class APIRequestContext extends SdkObject {
return { ...fetchResponse, fetchUid };
}
private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) {
private _parseSetCookieHeader(responseUrl: string, setCookie: string[] | undefined): types.NetworkCookie[] {
if (!setCookie)
return [];
const url = new URL(responseUrl);
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
@ -188,8 +212,7 @@ export abstract class APIRequestContext extends SdkObject {
cookie.path = defaultPath;
cookies.push(cookie);
}
if (cookies.length)
await this._addCookies(cookies);
return cookies;
}
private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
@ -204,15 +227,43 @@ export abstract class APIRequestContext extends SdkObject {
private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.APIResponse>{
await this._updateRequestCookieHeader(url, options);
const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => {
const [name, value] = p.split('=').map(v => v.trim());
return { name, value };
}) || [];
const requestEvent: APIRequestEvent = {
url,
method: options.method!,
headers: options.headers as { [name: string]: string },
cookies: requestCookies,
postData
};
this.emit(APIRequestContext.Events.Request, requestEvent);
return new Promise<types.APIResponse>((fulfill, reject) => {
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
= (url.protocol === 'https:' ? https : http).request;
const request = requestConstructor(url, options, async response => {
const notifyRequestFinished = (body?: Buffer) => {
const requestFinishedEvent: APIRequestFinishedEvent = {
requestEvent,
statusCode: -1,
statusMessage: '',
...response,
cookies,
body
};
this.emit(APIRequestContext.Events.RequestFinished, requestFinishedEvent);
};
progress.log(`← ${response.statusCode} ${response.statusMessage}`);
for (const [name, value] of Object.entries(response.headers))
progress.log(` ${name}: ${value}`);
if (response.headers['set-cookie'])
await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']);
const cookies = this._parseSetCookieHeader(response.url || url.toString(), response.headers['set-cookie']) ;
if (cookies.length)
await this._addCookies(cookies);
if (redirectStatus.includes(response.statusCode!)) {
if (!options.maxRedirects) {
reject(new Error('Max redirect count exceeded'));
@ -251,6 +302,7 @@ export abstract class APIRequestContext extends SdkObject {
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
if (response.headers.location) {
const locationURL = new URL(response.headers.location, url);
notifyRequestFinished();
fulfill(this._sendRequest(progress, locationURL, redirectOptions, postData));
request.destroy();
return;
@ -263,6 +315,7 @@ export abstract class APIRequestContext extends SdkObject {
const { username, password } = credentials;
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
options.headers!['authorization'] = `Basic ${encoded}`;
notifyRequestFinished();
fulfill(this._sendRequest(progress, url, options, postData));
request.destroy();
return;
@ -294,6 +347,7 @@ export abstract class APIRequestContext extends SdkObject {
body.on('data', chunk => chunks.push(chunk));
body.on('end', () => {
const body = Buffer.concat(chunks);
notifyRequestFinished(body);
fulfill({
url: response.url || url.toString(),
status: response.statusCode || 0,

183
packages/playwright-core/src/server/supplements/har/harTracer.ts
View File

@ -15,6 +15,7 @@
*/
import { BrowserContext } from '../../browserContext';
import { APIRequestContext, APIRequestEvent, APIRequestFinishedEvent } from '../../fetch';
import { helper } from '../../helper';
import * as network from '../../network';
import { Page } from '../../page';
@ -64,10 +65,12 @@ export class HarTracer {
eventsHelper.addEventListener(this._context, BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)),
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFinished, ({ request, response }) => this._onRequestFinished(request, response).catch(() => {})),
eventsHelper.addEventListener(this._context, BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response)),
eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.Request, (event: APIRequestEvent) => this._onAPIRequest(event)),
eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.RequestFinished, (event: APIRequestFinishedEvent) => this._onAPIRequestFinished(event)),
];
}
private _entryForRequest(request: network.Request): har.Entry | undefined {
private _entryForRequest(request: network.Request | APIRequestEvent): har.Entry | undefined {
return (request as any)[this._entrySymbol];
}
@ -132,6 +135,49 @@ export class HarTracer {
this._barrierPromises.add(race);
}
private _onAPIRequest(event: APIRequestEvent) {
const harEntry = createHarEntry(event.method, event.url, '', '');
harEntry.request.cookies = event.cookies;
harEntry.request.headers = Object.entries(event.headers).map(([name, value]) => ({ name, value }));
harEntry.request.postData = postDataForBuffer(event.postData || null, event.headers['content-type'], this._options.content);
harEntry.request.bodySize = event.postData?.length || 0;
(event as any)[this._entrySymbol] = harEntry;
if (this._started)
this._delegate.onEntryStarted(harEntry);
}
private _onAPIRequestFinished(event: APIRequestFinishedEvent): void {
const harEntry = this._entryForRequest(event.requestEvent);
if (!harEntry)
return;
harEntry.response.status = event.statusCode;
harEntry.response.statusText = event.statusMessage;
harEntry.response.httpVersion = event.httpVersion;
harEntry.response.redirectURL = event.headers.location || '';
for (let i = 0; i < event.rawHeaders.length; i += 2) {
harEntry.response.headers.push({
name: event.rawHeaders[i],
value: event.rawHeaders[i + 1]
});
}
harEntry.response.cookies = event.cookies.map(c => {
return {
...c,
expires: c.expires === -1 ? undefined : new Date(c.expires)
};
});
const content = harEntry.response.content;
const contentType = event.headers['content-type'];
if (contentType)
content.mimeType = contentType;
this._storeResponseContent(event.body, content);
if (this._started)
this._delegate.onEntryFinished(harEntry);
}
private _onRequest(request: network.Request) {
const page = request.frame()._page;
const url = network.parsedURL(request.url());
@ -139,49 +185,10 @@ export class HarTracer {
return;
const pageEntry = this._ensurePageEntry(page);
const harEntry: har.Entry = {
pageref: pageEntry.id,
_requestref: request.guid,
_frameref: request.frame().guid,
_monotonicTime: monotonicTime(),
startedDateTime: new Date(),
time: -1,
request: {
method: request.method(),
url: request.url(),
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
postData: postDataForHar(request, this._options.content),
headersSize: -1,
bodySize: request.bodySize(),
},
response: {
status: -1,
statusText: '',
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
content: {
size: -1,
mimeType: request.headerValue('content-type') || 'x-unknown',
},
headersSize: -1,
bodySize: -1,
redirectURL: '',
_transferSize: -1
},
cache: {
beforeRequest: null,
afterRequest: null,
},
timings: {
send: -1,
wait: -1,
receive: -1
},
};
const harEntry = createHarEntry(request.method(), url, request.guid, request.frame().guid);
harEntry.pageref = pageEntry.id;
harEntry.request.postData = postDataForRequest(request, this._options.content);
harEntry.request.bodySize = request.bodySize();
if (request.redirectedFrom()) {
const fromEntry = this._entryForRequest(request.redirectedFrom()!);
if (fromEntry)
@ -232,18 +239,8 @@ export class HarTracer {
}
const content = harEntry.response.content;
content.size = buffer.length;
compressionCalculationBarrier.setDecodedBodySize(buffer.length);
if (buffer && buffer.length > 0) {
if (this._options.content === 'embedded') {
content.text = buffer.toString('base64');
content.encoding = 'base64';
} else if (this._options.content === 'sha1') {
content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
if (this._started)
this._delegate.onContentBlob(content._sha1, buffer);
}
}
this._storeResponseContent(buffer, content);
}).catch(() => {
compressionCalculationBarrier.setDecodedBodySize(0);
}).then(() => {
@ -267,6 +264,22 @@ export class HarTracer {
}));
}
private _storeResponseContent(buffer: Buffer | undefined, content: har.Content) {
if (!buffer) {
content.size = 0;
return;
}
content.size = buffer.length;
if (this._options.content === 'embedded') {
content.text = buffer.toString('base64');
content.encoding = 'base64';
} else if (this._options.content === 'sha1') {
content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
if (this._started)
this._delegate.onContentBlob(content._sha1, buffer);
}
}
private _onResponse(response: network.Response) {
const page = response.frame()._page;
const pageEntry = this._ensurePageEntry(page);
@ -275,7 +288,7 @@ export class HarTracer {
return;
const request = response.request();
harEntry.request.postData = postDataForHar(request, this._options.content);
harEntry.request.postData = postDataForRequest(request, this._options.content);
harEntry.response = {
status: response.status(),
@ -372,12 +385,66 @@ export class HarTracer {
}
}
function postDataForHar(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
function createHarEntry(method: string, url: URL, requestref: string, frameref: string): har.Entry {
const harEntry: har.Entry = {
_requestref: requestref,
_frameref: frameref,
_monotonicTime: monotonicTime(),
startedDateTime: new Date(),
time: -1,
request: {
method: method,
url: url.toString(),
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
headersSize: -1,
bodySize: 0,
},
response: {
status: -1,
statusText: '',
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
content: {
size: -1,
mimeType: 'x-unknown',
},
headersSize: -1,
bodySize: -1,
redirectURL: '',
_transferSize: -1
},
cache: {
beforeRequest: null,
afterRequest: null,
},
timings: {
send: -1,
wait: -1,
receive: -1
},
};
return harEntry;
}
function postDataForRequest(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
const postData = request.postDataBuffer();
if (!postData)
return;
const contentType = request.headerValue('content-type') || 'application/octet-stream';
const contentType = request.headerValue('content-type');
return postDataForBuffer(postData, contentType, content);
}
function postDataForBuffer(postData: Buffer | null, contentType: string | undefined, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
if (!postData)
return;
contentType ??= 'application/octet-stream';
const result: har.PostData = {
mimeType: contentType,
text: '',

37
tests/har.spec.ts
View File

@ -634,3 +634,40 @@ it('should include _requestref for redirects', async ({ contextFactory, server }
expect(entryEmptyPage.request.url).toBe(server.EMPTY_PAGE);
expect(entryEmptyPage._requestref).toBe(requests.get(entryEmptyPage.request.url));
});
it('should include API request', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
const url = server.PREFIX + '/simple.json';
const response = await page.request.post(url, {
headers: { cookie: 'a=b; c=d' },
data: { foo: 'bar' }
});
const responseBody = await response.body();
const log = await getLog();
expect(log.entries.length).toBe(1);
const entry = log.entries[0];
expect(entry.request.url).toBe(url);
expect(entry.request.method).toBe('POST');
expect(entry.request.httpVersion).toBe('HTTP/1.1');
expect(entry.request.cookies).toEqual([
{
'name': 'a',
'value': 'b'
},
{
'name': 'c',
'value': 'd'
}
]);
expect(entry.request.headers.length).toBeGreaterThan(1);
expect(entry.request.headers.find(h => h.name.toLowerCase() === 'user-agent')).toBeTruthy();
expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toBe('application/json');
expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-length')?.value).toBe('13');
expect(entry.request.bodySize).toBe(13);
expect(entry.response.status).toBe(200);
expect(entry.response.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toContain('application/json');
expect(entry.response.content.size).toBe(15);
expect(entry.response.content.text).toBe(responseBody.toString('base64'));
});

13
tests/tracing.spec.ts
View File

@ -85,6 +85,19 @@ test('should exclude internal pages', async ({ browserName, context, page, serve
expect(pageIds.size).toBe(1);
});
test('should include context API requests', async ({ browserName, context, page, server }, testInfo) => {
await context.tracing.start({ snapshots: true });
await page.request.post(server.PREFIX + '/simple.json', { data: { foo: 'bar' } });
await context.tracing.stop({ path: testInfo.outputPath('trace.zip') });
const { events } = await parseTrace(testInfo.outputPath('trace.zip'));
const postEvent = events.find(e => e.metadata?.apiName === 'apiRequestContext.post');
expect(postEvent).toBeTruthy();
const harEntry = events.find(e => e.type === 'resource-snapshot');
expect(harEntry).toBeTruthy();
expect(harEntry.snapshot.request.url).toBe(server.PREFIX + '/simple.json');
expect(harEntry.snapshot.response.status).toBe(200);
});
test('should collect two traces', async ({ context, page, server }, testInfo) => {
await context.tracing.start({ screenshots: true, snapshots: true });
await page.goto(server.EMPTY_PAGE);