microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-01-05 19:04:43 +03:00
Code Issues Projects Releases Wiki Activity

fix(har): record request overrides to har (#17027)

Browse Source
This commit is contained in:
Yury Semikhatsky 2022-09-04 10:52:20 -07:00 committed by GitHub
parent c58bfd0552
commit 01d83f1d5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 169 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/playwright-core/src/server/har/harRecorder.ts
View File

@ -45,6 +45,7 @@ export class HarRecorder {
content,
slimMode: options.mode === 'minimal',
includeTraceInfo: false,
recordRequestOverrides: true,
waitForContentOnStop: true,
skipScripts: false,
urlFilter: urlFilterRe ?? options.urlGlob,

30
packages/playwright-core/src/server/har/harTracer.ts
View File

@ -30,7 +30,7 @@ import { ManualPromise } from '../../utils/manualPromise';
import { getPlaywrightVersion } from '../../common/userAgent';
import { urlMatches } from '../../common/netUtils';
import { Frame } from '../frames';
import type { LifecycleEvent } from '../types';
import type { HeadersArray, LifecycleEvent } from '../types';
import { isTextualMimeType } from '../../utils/mimeType';
const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
@ -45,6 +45,7 @@ type HarTracerOptions = {
content: 'omit' | 'attach' | 'embed';
skipScripts: boolean;
includeTraceInfo: boolean;
recordRequestOverrides: boolean;
waitForContentOnStop: boolean;
urlFilter?: string | RegExp;
slimMode?: boolean;
@ -248,6 +249,7 @@ export class HarTracer {
const harEntry = createHarEntry(request.method(), url, request.frame()?.guid, this._options);
if (pageEntry)
harEntry.pageref = pageEntry.id;
this._recordRequestHeadersAndCookies(harEntry, request.headers());
harEntry.request.postData = this._postDataForRequest(request, this._options.content);
if (!this._options.omitSizes)
harEntry.request.bodySize = request.bodySize();
@ -261,6 +263,24 @@ export class HarTracer {
this._delegate.onEntryStarted(harEntry);
}
private _recordRequestHeadersAndCookies(harEntry: har.Entry, headers: HeadersArray) {
if (!this._options.omitCookies) {
harEntry.request.cookies = [];
for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie'))
harEntry.request.cookies.push(...header.value.split(';').map(parseCookie));
}
harEntry.request.headers = headers;
}
private _recordRequestOverrides(harEntry: har.Entry, request: network.Request) {
if (!request._hasOverrides() || !this._options.recordRequestOverrides)
return;
harEntry.request.method = request.method();
harEntry.request.url = request.url();
harEntry.request.postData = this._postDataForRequest(request, this._options.content);
this._recordRequestHeadersAndCookies(harEntry, request.headers());
}
private async _onRequestFinished(request: network.Request, response: network.Response | null) {
if (!response)
return;
@ -330,6 +350,7 @@ export class HarTracer {
if (request._failureText !== null)
harEntry.response._failureText = request._failureText;
this._recordRequestOverrides(harEntry, request);
if (this._started)
this._delegate.onEntryFinished(harEntry);
}
@ -423,12 +444,9 @@ export class HarTracer {
harEntry._securityDetails = details;
}));
}
this._recordRequestOverrides(harEntry, request);
this._addBarrier(page || request.serviceWorker(), request.rawRequestHeaders().then(headers => {
if (!this._options.omitCookies) {
for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie'))
harEntry.request.cookies.push(...header.value.split(';').map(parseCookie));
}
harEntry.request.headers = headers;
this._recordRequestHeadersAndCookies(harEntry, headers);
}));
this._addBarrier(page || request.serviceWorker(), response.rawResponseHeaders().then(headers => {
if (!this._options.omitCookies) {

60
packages/playwright-core/src/server/network.ts
View File

@ -22,8 +22,9 @@ import type * as channels from '../protocol/channels';
import { assert } from '../utils';
import { ManualPromise } from '../utils/manualPromise';
import { SdkObject } from './instrumentation';
import type { NameValue } from '../common/types';
import type { HeadersArray, NameValue } from '../common/types';
import { APIRequestContext } from './fetch';
import type { NormalizedContinueOverrides } from './types';
export function filterCookies(cookies: channels.NetworkCookie[], urls: string[]): channels.NetworkCookie[] {
const parsedURLs = urls.map(s => new URL(s));
@ -97,17 +98,18 @@ export class Request extends SdkObject {
private _resourceType: string;
private _method: string;
private _postData: Buffer | null;
readonly _headers: types.HeadersArray;
readonly _headers: HeadersArray;
private _headersMap = new Map<string, string>();
readonly _frame: frames.Frame | null = null;
readonly _serviceWorker: pages.Worker | null = null;
readonly _context: contexts.BrowserContext;
private _rawRequestHeadersPromise = new ManualPromise<types.HeadersArray>();
private _rawRequestHeadersPromise = new ManualPromise<HeadersArray>();
private _waitForResponsePromise = new ManualPromise<Response | null>();
_responseEndTiming = -1;
private _overrides: NormalizedContinueOverrides | undefined;
constructor(context: contexts.BrowserContext, frame: frames.Frame | null, serviceWorker: pages.Worker | null, redirectedFrom: Request | null, documentId: string | undefined,
url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) {
url: string, resourceType: string, method: string, postData: Buffer | null, headers: HeadersArray) {
super(frame || context, 'request');
assert(!url.startsWith('data:'), 'Data urls should not fire requests');
this._context = context;
@ -122,8 +124,7 @@ export class Request extends SdkObject {
this._method = method;
this._postData = postData;
this._headers = headers;
for (const { name, value } of this._headers)
this._headersMap.set(name.toLowerCase(), value);
this._updateHeadersMap();
this._isFavicon = url.endsWith('/favicon.ico') || !!redirectedFrom?._isFavicon;
}
@ -132,8 +133,22 @@ export class Request extends SdkObject {
this._waitForResponsePromise.resolve(null);
}
_setOverrides(overrides: types.NormalizedContinueOverrides) {
this._overrides = overrides;
this._updateHeadersMap();
}
private _updateHeadersMap() {
for (const { name, value } of this.headers())
this._headersMap.set(name.toLowerCase(), value);
}
_hasOverrides() {
return !!this._overrides;
}
url(): string {
return this._url;
return this._overrides?.url || this._url;
}
resourceType(): string {
@ -141,15 +156,15 @@ export class Request extends SdkObject {
}
method(): string {
return this._method;
return this._overrides?.method || this._method;
}
postDataBuffer(): Buffer | null {
return this._postData;
return this._overrides?.postData || this._postData;
}
headers(): types.HeadersArray {
return this._headers;
headers(): HeadersArray {
return this._overrides?.headers || this._headers;
}
headerValue(name: string): string | undefined {
@ -157,13 +172,13 @@ export class Request extends SdkObject {
}
// "null" means no raw headers available - we'll use provisional headers as raw headers.
setRawRequestHeaders(headers: types.HeadersArray | null) {
setRawRequestHeaders(headers: HeadersArray | null) {
if (!this._rawRequestHeadersPromise.isDone())
this._rawRequestHeadersPromise.resolve(headers || this._headers);
}
async rawRequestHeaders(): Promise<NameValue[]> {
return this._rawRequestHeadersPromise;
async rawRequestHeaders(): Promise<HeadersArray> {
return this._overrides?.headers || this._rawRequestHeadersPromise;
}
response(): PromiseLike<Response | null> {
@ -303,6 +318,7 @@ export class Route extends SdkObject {
if (oldUrl.protocol !== newUrl.protocol)
throw new Error('New URL must have same protocol as overridden URL');
}
this._request._setOverrides(overrides);
await this._delegate.continue(this._request, overrides);
this._endHandling();
}
@ -360,20 +376,20 @@ export class Response extends SdkObject {
private _status: number;
private _statusText: string;
private _url: string;
private _headers: types.HeadersArray;
private _headers: HeadersArray;
private _headersMap = new Map<string, string>();
private _getResponseBodyCallback: GetResponseBodyCallback;
private _timing: ResourceTiming;
private _serverAddrPromise = new ManualPromise<RemoteAddr | undefined>();
private _securityDetailsPromise = new ManualPromise<SecurityDetails | undefined>();
private _rawResponseHeadersPromise = new ManualPromise<types.HeadersArray>();
private _rawResponseHeadersPromise = new ManualPromise<HeadersArray>();
private _httpVersion: string | undefined;
private _fromServiceWorker: boolean;
private _encodedBodySizePromise = new ManualPromise<number | null>();
private _transferSizePromise = new ManualPromise<number | null>();
private _responseHeadersSizePromise = new ManualPromise<number | null>();
constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, fromServiceWorker: boolean, httpVersion?: string) {
constructor(request: Request, status: number, statusText: string, headers: HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, fromServiceWorker: boolean, httpVersion?: string) {
super(request.frame() || request._context, 'response');
this._request = request;
this._timing = timing;
@ -418,7 +434,7 @@ export class Response extends SdkObject {
return this._statusText;
}
headers(): types.HeadersArray {
headers(): HeadersArray {
return this._headers;
}
@ -431,7 +447,7 @@ export class Response extends SdkObject {
}
// "null" means no raw headers available - we'll use provisional headers as raw headers.
setRawResponseHeaders(headers: types.HeadersArray | null) {
setRawResponseHeaders(headers: HeadersArray | null) {
if (!this._rawResponseHeadersPromise.isDone())
this._rawResponseHeadersPromise.resolve(headers || this._headers);
}
@ -658,11 +674,11 @@ export const STATUS_TEXTS: { [status: string]: string } = {
'511': 'Network Authentication Required',
};
export function singleHeader(name: string, value: string): types.HeadersArray {
export function singleHeader(name: string, value: string): HeadersArray {
return [{ name, value }];
}
export function mergeHeaders(headers: (types.HeadersArray | undefined | null)[]): types.HeadersArray {
export function mergeHeaders(headers: (HeadersArray | undefined | null)[]): HeadersArray {
const lowerCaseToValue = new Map<string, string>();
const lowerCaseToOriginalCase = new Map<string, string>();
for (const h of headers) {
@ -674,7 +690,7 @@ export function mergeHeaders(headers: (types.HeadersArray | undefined | null)[])
lowerCaseToValue.set(lower, value);
}
}
const result: types.HeadersArray = [];
const result: HeadersArray = [];
for (const [lower, value] of lowerCaseToValue)
result.push({ name: lowerCaseToOriginalCase.get(lower)!, value });
return result;

1
packages/playwright-core/src/server/trace/recorder/tracing.ts
View File

@ -87,6 +87,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
this._harTracer = new HarTracer(context, null, this, {
content: 'attach',
includeTraceInfo: true,
recordRequestOverrides: false,
waitForContentOnStop: false,
skipScripts: true,
});

2
packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts
View File

@ -34,7 +34,7 @@ export class InMemorySnapshotter extends BaseSnapshotStorage implements Snapshot
constructor(context: BrowserContext) {
super();
this._snapshotter = new Snapshotter(context, this);
this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, waitForContentOnStop: false, skipScripts: true });
this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, recordRequestOverrides: false, waitForContentOnStop: false, skipScripts: true });
}
async initialize(): Promise<void> {

6
packages/playwright-core/src/server/types.ts
View File

@ -15,8 +15,8 @@
* limitations under the License.
*/
import type { Size, Point, TimeoutOptions } from '../common/types';
export type { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types';
import type { Size, Point, TimeoutOptions, HeadersArray } from '../common/types';
export type { Size, Point, Rect, Quad, URLMatch, TimeoutOptions, HeadersArray } from '../common/types';
import type * as channels from '../protocol/channels';
export type StrictOptions = {
@ -129,8 +129,6 @@ export type MouseMultiClickOptions = PointerActionOptions & {
export type World = 'main' | 'utility';
export type HeadersArray = { name: string, value: string }[];
export type GotoOptions = NavigateOptions & {
referer?: string,
};

73
tests/library/har.spec.ts
View File

@ -256,6 +256,32 @@ it('should include secure set-cookies', async ({ contextFactory, httpsServer },
expect(cookies[0]).toEqual({ name: 'name1', value: 'value1', secure: true });
});
it('should record request overrides', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
page.route('**/foo', route => {
route.fallback({
url: server.EMPTY_PAGE,
method: 'POST',
headers: {
...route.request().headers(),
'content-type': 'text/plain',
'cookie': 'foo=bar',
'custom': 'value'
},
postData: 'Hi!'
});
});
await page.goto(server.PREFIX + '/foo');
const log = await getLog();
const request = log.entries[0].request;
expect(request.url).toBe(server.EMPTY_PAGE);
expect(request.method).toBe('POST');
expect(request.headers).toContainEqual({ name: 'custom', value: 'value' });
expect(request.cookies).toContainEqual({ name: 'foo', value: 'bar' });
expect(request.postData).toEqual({ 'mimeType': 'text/plain', 'params': [], 'text': 'Hi!' });
});
it('should include content @smoke', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.PREFIX + '/har.html');
@ -409,7 +435,7 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
server.setRoute('/one-style.css', (req, res) => {
res.setHeader('Content-Type', 'text/css');
res.connection.destroy();
res.socket.destroy();
});
const failedRequests = [];
page.on('requestfailed', request => failedRequests.push(request));
@ -419,6 +445,49 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac
expect(log.entries[1].response._transferSize).toBe(-1);
});
it('should record failed request headers', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
server.setRoute('/har.html', (req, res) => {
res.socket.destroy();
});
await page.goto(server.PREFIX + '/har.html').catch(() => {});
const log = await getLog();
expect(log.entries[0].response._failureText).toBeTruthy();
const request = log.entries[0].request;
expect(request.url.endsWith('/har.html')).toBe(true);
expect(request.method).toBe('GET');
expect(request.headers).toContainEqual(expect.objectContaining({ name: 'User-Agent' }));
});
it('should record failed request overrides', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
server.setRoute('/empty.html', (req, res) => {
res.socket.destroy();
});
await page.route('**/foo', route => {
route.fallback({
url: server.EMPTY_PAGE,
method: 'POST',
headers: {
...route.request().headers(),
'content-type': 'text/plain',
'cookie': 'foo=bar',
'custom': 'value'
},
postData: 'Hi!'
});
});
await page.goto(server.PREFIX + '/foo').catch(() => {});
const log = await getLog();
expect(log.entries[0].response._failureText).toBeTruthy();
const request = log.entries[0].request;
expect(request.url).toBe(server.EMPTY_PAGE);
expect(request.method).toBe('POST');
expect(request.headers).toContainEqual({ name: 'custom', value: 'value' });
expect(request.cookies).toContainEqual({ name: 'foo', value: 'bar' });
expect(request.postData).toEqual({ 'mimeType': 'text/plain', 'params': [], 'text': 'Hi!' });
});
it('should report the correct request body size', async ({ contextFactory, server }, testInfo) => {
server.setRoute('/api', (req, res) => res.end());
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
@ -556,7 +625,7 @@ it('should have connection details for redirects', async ({ contextFactory, serv
it('should have connection details for failed requests', async ({ contextFactory, server, browserName, platform, mode }, testInfo) => {
server.setRoute('/one-style.css', (_, res) => {
res.setHeader('Content-Type', 'text/css');
res.connection.destroy();
res.socket.destroy();
});
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.PREFIX + '/one-style.html');

21
tests/library/trace-viewer.spec.ts
View File

@ -686,3 +686,24 @@ test('should include requestUrl in route.abort', async ({ page, runAndTrace, ser
await expect(callLine.locator('text=requestUrl')).toContainText('http://test.com');
});
test('should serve overridden request', async ({ page, runAndTrace, server }) => {
server.setRoute('/custom.css', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/css',
});
res.end(`body { background: red }`);
});
await page.route('**/one-style.css', route => {
route.continue({
url: server.PREFIX + '/custom.css'
});
});
const traceViewer = await runAndTrace(async () => {
await page.goto(server.PREFIX + '/one-style.html');
});
// Render snapshot, check expectations.
const snapshotFrame = await traceViewer.snapshotFrame('page.goto');
const color = await snapshotFrame.locator('body').evaluate(body => getComputedStyle(body).backgroundColor);
expect(color).toBe('rgb(255, 0, 0)');
});

12
tests/page/page-request-continue.spec.ts
View File

@ -87,17 +87,15 @@ it('should amend method', async ({ page, server }) => {
});
it('should override request url', async ({ page, server }) => {
const request = server.waitForRequest('/global-var.html');
const serverRequest = server.waitForRequest('/global-var.html');
await page.route('**/foo', route => {
route.continue({ url: server.PREFIX + '/global-var.html' });
});
const [response] = await Promise.all([
page.waitForEvent('response'),
page.goto(server.PREFIX + '/foo'),
]);
expect(response.url()).toBe(server.PREFIX + '/foo');
const response = await page.goto(server.PREFIX + '/foo');
expect(response.request().url()).toBe(server.PREFIX + '/global-var.html');
expect(response.url()).toBe(server.PREFIX + '/global-var.html');
expect(await page.evaluate(() => window['globalVar'])).toBe(123);
expect((await request).method).toBe('GET');
expect((await serverRequest).method).toBe('GET');
});
it('should not allow changing protocol when overriding url', async ({ page, server }) => {

12
tests/page/page-request-fallback.spec.ts
View File

@ -199,7 +199,7 @@ it('should amend method', async ({ page, server }) => {
});
it('should override request url', async ({ page, server }) => {
const request = server.waitForRequest('/global-var.html');
const serverRequest = server.waitForRequest('/global-var.html');
let url: string;
await page.route('**/global-var.html', route => {
@ -209,14 +209,12 @@ it('should override request url', async ({ page, server }) => {
await page.route('**/foo', route => route.fallback({ url: server.PREFIX + '/global-var.html' }));
const [response] = await Promise.all([
page.waitForEvent('response'),
page.goto(server.PREFIX + '/foo'),
]);
const response = await page.goto(server.PREFIX + '/foo');
expect(url).toBe(server.PREFIX + '/global-var.html');
expect(response.url()).toBe(server.PREFIX + '/foo');
expect(response.request().url()).toBe(server.PREFIX + '/global-var.html');
expect(response.url()).toBe(server.PREFIX + '/global-var.html');
expect(await page.evaluate(() => window['globalVar'])).toBe(123);
expect((await request).method).toBe('GET');
expect((await serverRequest).method).toBe('GET');
});
it.describe('post data', () => {