diff --git a/packages/playwright-core/src/client/browser.ts b/packages/playwright-core/src/client/browser.ts index db9825d0b8..8aed8f395b 100644 --- a/packages/playwright-core/src/client/browser.ts +++ b/packages/playwright-core/src/client/browser.ts @@ -61,7 +61,7 @@ export class Browser extends ChannelOwner implements ap async newContext(options: BrowserContextOptions = {}): Promise { options = { ...this._browserType._defaultContextOptions, ...options }; - const harRouter = options.har ? await HarRouter.create(options.har) : null; + const harRouter = options.har ? await HarRouter.create(this._connection.localUtils(), options.har) : null; const contextOptions = await prepareBrowserContextParams(options); const context = BrowserContext.from((await this._channel.newContext(contextOptions)).context); context._options = contextOptions; diff --git a/packages/playwright-core/src/client/browserType.ts b/packages/playwright-core/src/client/browserType.ts index acc745734e..213c66e0df 100644 --- a/packages/playwright-core/src/client/browserType.ts +++ b/packages/playwright-core/src/client/browserType.ts @@ -95,7 +95,7 @@ export class BrowserType extends ChannelOwner imple const logger = options.logger || this._defaultLaunchOptions?.logger; assert(!(options as any).port, 'Cannot specify a port without launching as a server.'); options = { ...this._defaultLaunchOptions, ...this._defaultContextOptions, ...options }; - const harRouter = options.har ? await HarRouter.create(options.har) : null; + const harRouter = options.har ? await HarRouter.create(this._connection.localUtils(), options.har) : null; const contextParams = await prepareBrowserContextParams(options); const persistentParams: channels.BrowserTypeLaunchPersistentContextParams = { ...contextParams, diff --git a/packages/playwright-core/src/client/electron.ts b/packages/playwright-core/src/client/electron.ts index 2475fdb464..f579a94220 100644 --- a/packages/playwright-core/src/client/electron.ts +++ b/packages/playwright-core/src/client/electron.ts @@ -53,7 +53,7 @@ export class Electron extends ChannelOwner implements ...await prepareBrowserContextParams(options), env: envObjectToArray(options.env ? options.env : process.env), }; - const harRouter = options.har ? await HarRouter.create(options.har) : null; + const harRouter = options.har ? await HarRouter.create(this._connection.localUtils(), options.har) : null; const app = ElectronApplication.from((await this._channel.launch(params)).electronApplication); app._context._options = params; harRouter?.addRoute(app._context); diff --git a/packages/playwright-core/src/client/harRouter.ts b/packages/playwright-core/src/client/harRouter.ts index c178b48aca..ce15a7e222 100644 --- a/packages/playwright-core/src/client/harRouter.ts +++ b/packages/playwright-core/src/client/harRouter.ts @@ -14,13 +14,10 @@ * limitations under the License. */ -import fs from 'fs'; -import type { HAREntry, HARFile } from '../../types/types'; import { debugLogger } from '../common/debugLogger'; -import { rewriteErrorMessage } from '../utils/stackTrace'; -import { ZipFile } from '../utils/zipFile'; import type { BrowserContext } from './browserContext'; import { Events } from './events'; +import type { LocalUtils } from './localUtils'; import type { Route } from './network'; import type { BrowserContextOptions } from './types'; @@ -28,76 +25,55 @@ type HarOptions = NonNullable; export class HarRouter { private _pattern: string | RegExp; - private _harFile: HARFile; - private _zipFile: ZipFile | null; private _options: HarOptions | undefined; + private _localUtils: LocalUtils; + private _harId: string; - static async create(options: HarOptions): Promise { - if (options.path.endsWith('.zip')) { - const zipFile = new ZipFile(options.path); - const har = await zipFile.read('har.har'); - const harFile = JSON.parse(har.toString()) as HARFile; - return new HarRouter(harFile, zipFile, options); - } - const harFile = JSON.parse(await fs.promises.readFile(options.path, 'utf-8')) as HARFile; - return new HarRouter(harFile, null, options); + static async create(localUtils: LocalUtils, options: HarOptions): Promise { + const { harId } = await localUtils._channel.harOpen({ file: options.path }); + return new HarRouter(localUtils, harId, options); } - constructor(harFile: HARFile, zipFile: ZipFile | null, options?: HarOptions) { - this._harFile = harFile; - this._zipFile = zipFile; + constructor(localUtils: LocalUtils, harId: string, options?: HarOptions) { + this._localUtils = localUtils; + this._harId = harId; this._pattern = options?.urlFilter ?? /.*/; this._options = options; } private async _handle(route: Route) { - let entry; - try { - entry = harFindResponse(this._harFile, { - url: route.request().url(), - method: route.request().method() - }); - } catch (e) { - rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`); - debugLogger.log('api', e); - } + const response = await this._localUtils._channel.harLookup({ + harId: this._harId, + url: route.request().url(), + method: route.request().method(), + isNavigationRequest: route.request().isNavigationRequest() + }); - if (entry) { - // If navigation is being redirected, restart it with the final url to ensure the document's url changes. - if (entry.request.url !== route.request().url() && route.request().isNavigationRequest()) { - debugLogger.log('api', `redirecting HAR navigation: ${route.request().url()} => ${entry.request.url}`); - await route._abort(undefined, entry.request.url); - return; - } - debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`); - const response = entry.response; - const sha1 = (response.content as any)._sha1; - - if (this._zipFile && sha1) { - const body = await this._zipFile.read(sha1).catch(() => { - debugLogger.log('api', `payload ${sha1} for request ${route.request().url()} is not found in archive`); - return null; - }); - if (body) { - await route.fulfill({ - status: response.status, - headers: Object.fromEntries(response.headers.map(h => [h.name, h.value])), - body - }); - return; - } - } - - await route.fulfill({ response }); + if (response.action === 'redirect') { + debugLogger.log('api', `HAR: ${route.request().url()} redirected to ${response.redirectURL}`); + await route._abort(undefined, response.redirectURL); return; } + if (response.action === 'fulfill') { + await route.fulfill({ + status: response.status, + headers: Object.fromEntries(response.headers!.map(h => [h.name, h.value])), + body: response.base64Encoded ? Buffer.from(response.body!, 'base64') : response.body + }); + return; + } + + if (response.action === 'error') + debugLogger.log('api', response.message!); + // Report the error, but fall through to the default handler. + if (this._options?.fallback === 'continue') { await route.fallback(); return; } - debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`); + debugLogger.log('api', `HAR: ${route.request().method()} ${route.request().url()} aborted - no such entry in HAR file`); await route.abort(); } @@ -107,37 +83,6 @@ export class HarRouter { } dispose() { - this._zipFile?.close(); - } -} - -const redirectStatus = [301, 302, 303, 307, 308]; - -function harFindResponse(har: HARFile, params: { url: string, method: string }): HAREntry | undefined { - const harLog = har.log; - const visited = new Set(); - let url = params.url; - let method = params.method; - while (true) { - const entry = harLog.entries.find(entry => entry.request.url === url && entry.request.method === method); - if (!entry) - return; - if (visited.has(entry)) - throw new Error(`Found redirect cycle for ${params.url}`); - visited.add(entry); - - const locationHeader = entry.response.headers.find(h => h.name.toLowerCase() === 'location'); - if (redirectStatus.includes(entry.response.status) && locationHeader) { - const locationURL = new URL(locationHeader.value, url); - url = locationURL.toString(); - if ((entry.response.status === 301 || entry.response.status === 302) && method === 'POST' || - entry.response.status === 303 && !['GET', 'HEAD'].includes(method)) { - // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch) - method = 'GET'; - } - continue; - } - - return entry; + this._localUtils._channel.harClose({ harId: this._harId }).catch(() => {}); } } diff --git a/packages/playwright-core/src/protocol/channels.ts b/packages/playwright-core/src/protocol/channels.ts index fd754ae932..dfe5583de2 100644 --- a/packages/playwright-core/src/protocol/channels.ts +++ b/packages/playwright-core/src/protocol/channels.ts @@ -378,6 +378,9 @@ export interface LocalUtilsEventTarget { export interface LocalUtilsChannel extends LocalUtilsEventTarget, Channel { _type_LocalUtils: boolean; zip(params: LocalUtilsZipParams, metadata?: Metadata): Promise; + harOpen(params: LocalUtilsHarOpenParams, metadata?: Metadata): Promise; + harLookup(params: LocalUtilsHarLookupParams, metadata?: Metadata): Promise; + harClose(params: LocalUtilsHarCloseParams, metadata?: Metadata): Promise; } export type LocalUtilsZipParams = { zipFile: string, @@ -387,6 +390,40 @@ export type LocalUtilsZipOptions = { }; export type LocalUtilsZipResult = void; +export type LocalUtilsHarOpenParams = { + file: string, +}; +export type LocalUtilsHarOpenOptions = { + +}; +export type LocalUtilsHarOpenResult = { + harId: string, +}; +export type LocalUtilsHarLookupParams = { + harId: string, + url: string, + method: string, + isNavigationRequest: boolean, +}; +export type LocalUtilsHarLookupOptions = { + +}; +export type LocalUtilsHarLookupResult = { + action: 'error' | 'redirect' | 'fulfill' | 'noentry', + message?: string, + redirectURL?: string, + status?: number, + headers?: NameValue[], + body?: string, + base64Encoded?: boolean, +}; +export type LocalUtilsHarCloseParams = { + harId: string, +}; +export type LocalUtilsHarCloseOptions = { + +}; +export type LocalUtilsHarCloseResult = void; export interface LocalUtilsEvents { } diff --git a/packages/playwright-core/src/protocol/protocol.yml b/packages/playwright-core/src/protocol/protocol.yml index 33295daacb..b32728de6a 100644 --- a/packages/playwright-core/src/protocol/protocol.yml +++ b/packages/playwright-core/src/protocol/protocol.yml @@ -478,6 +478,39 @@ LocalUtils: type: array items: NameValue + harOpen: + parameters: + file: string + returns: + harId: string + + harLookup: + parameters: + harId: string + url: string + method: string + isNavigationRequest: boolean + returns: + action: + type: enum + literals: + - error + - redirect + - fulfill + - noentry + message: string? + redirectURL: string? + status: number? + headers: + type: array? + items: NameValue + body: string? + base64Encoded: boolean? + + harClose: + parameters: + harId: string + Root: type: interface diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 7e93725379..f6a16794a7 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -205,6 +205,18 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { zipFile: tString, entries: tArray(tType('NameValue')), }); + scheme.LocalUtilsHarOpenParams = tObject({ + file: tString, + }); + scheme.LocalUtilsHarLookupParams = tObject({ + harId: tString, + url: tString, + method: tString, + isNavigationRequest: tBoolean, + }); + scheme.LocalUtilsHarCloseParams = tObject({ + harId: tString, + }); scheme.RootInitializeParams = tObject({ sdkLanguage: tString, }); diff --git a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts index 9dce248f50..9f5f60a56e 100644 --- a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts @@ -23,9 +23,13 @@ import { assert, createGuid } from '../../utils'; import type { DispatcherScope } from './dispatcher'; import { Dispatcher } from './dispatcher'; import { yazl, yauzl } from '../../zipBundle'; +import { ZipFile } from '../../utils/zipFile'; +import type { HAREntry, HARFile } from '../../../types/types'; +import type { HeadersArray } from '../types'; export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel> implements channels.LocalUtilsChannel { _type_LocalUtils: boolean; + private _harBakends = new Map(); constructor(scope: DispatcherScope) { super(scope, { guid: 'localUtils@' + createGuid() }, 'LocalUtils', {}); @@ -86,4 +90,128 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels. }); return promise; } + + async harOpen(params: channels.LocalUtilsHarOpenParams, metadata?: channels.Metadata): Promise { + let harBackend: HarBackend; + if (params.file.endsWith('.zip')) { + const zipFile = new ZipFile(params.file); + const har = await zipFile.read('har.har'); + const harFile = JSON.parse(har.toString()) as HARFile; + harBackend = new HarBackend(harFile, zipFile); + } else { + const harFile = JSON.parse(await fs.promises.readFile(params.file, 'utf-8')) as HARFile; + harBackend = new HarBackend(harFile, null); + } + this._harBakends.set(harBackend.id, harBackend); + return { harId: harBackend.id }; + } + + async harLookup(params: channels.LocalUtilsHarLookupParams, metadata?: channels.Metadata): Promise { + const harBackend = this._harBakends.get(params.harId); + if (!harBackend) + return { action: 'error', message: `Internal error: har was not opened` }; + return await harBackend.lookup(params.url, params.method, params.isNavigationRequest); + } + + async harClose(params: channels.LocalUtilsHarCloseParams, metadata?: channels.Metadata): Promise { + const harBackend = this._harBakends.get(params.harId); + if (harBackend) { + this._harBakends.delete(harBackend.id); + harBackend.dispose(); + } + } +} + +const redirectStatus = [301, 302, 303, 307, 308]; + +class HarBackend { + readonly id = createGuid(); + private _harFile: HARFile; + private _zipFile: ZipFile | null; + + constructor(harFile: HARFile, zipFile: ZipFile | null) { + this._harFile = harFile; + this._zipFile = zipFile; + } + + async lookup(url: string, method: string, isNavigationRequest: boolean): Promise<{ + action: 'error' | 'redirect' | 'fulfill' | 'noentry', + message?: string, + redirectURL?: string, + status?: number, + headers?: HeadersArray, + body?: string, + base64Encoded?: boolean }> { + let entry; + try { + entry = this._harFindResponse(url, method); + } catch (e) { + return { action: 'error', message: 'HAR error: ' + e.message }; + } + + if (!entry) + return { action: 'noentry' }; + + // If navigation is being redirected, restart it with the final url to ensure the document's url changes. + if (entry.request.url !== url && isNavigationRequest) + return { action: 'redirect', redirectURL: entry.request.url }; + + const response = entry.response; + const sha1 = (response.content as any)._sha1; + let body: string | undefined; + let base64Encoded = false; + + if (this._zipFile && sha1) { + const buffer = await this._zipFile.read(sha1).catch(() => { + return { action: 'error', message: `Malformed HAR: payload ${sha1} for request ${url} is not found in archive` }; + }); + + if (buffer) { + body = buffer.toString('base64'); + base64Encoded = true; + } + } else { + body = response.content.text; + base64Encoded = response.content.encoding === 'base64'; + } + + return { + action: 'fulfill', + status: response.status, + headers: response.headers, + body, + base64Encoded + }; + } + + private _harFindResponse(url: string, method: string): HAREntry | undefined { + const harLog = this._harFile.log; + const visited = new Set(); + while (true) { + const entry = harLog.entries.find(entry => entry.request.url === url && entry.request.method === method); + if (!entry) + return; + if (visited.has(entry)) + throw new Error(`Found redirect cycle for ${url}`); + visited.add(entry); + + const locationHeader = entry.response.headers.find(h => h.name.toLowerCase() === 'location'); + if (redirectStatus.includes(entry.response.status) && locationHeader) { + const locationURL = new URL(locationHeader.value, url); + url = locationURL.toString(); + if ((entry.response.status === 301 || entry.response.status === 302) && method === 'POST' || + entry.response.status === 303 && !['GET', 'HEAD'].includes(method)) { + // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch) + method = 'GET'; + } + continue; + } + + return entry; + } + } + + dispose() { + this._zipFile?.close(); + } } diff --git a/tests/library/browsercontext-har.spec.ts b/tests/library/browsercontext-har.spec.ts index 2110f29b49..fb0e26eee6 100644 --- a/tests/library/browsercontext-har.spec.ts +++ b/tests/library/browsercontext-har.spec.ts @@ -116,7 +116,7 @@ it('should change document URL after redirected navigation', async ({ contextFac page.goto('https://theverge.com/') ]); await expect(page).toHaveURL('https://www.theverge.com/'); - await expect(response.request().url()).toBe('https://www.theverge.com/'); + expect(response.request().url()).toBe('https://www.theverge.com/'); expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/'); }); @@ -131,7 +131,7 @@ it('should goBack to redirected navigation', async ({ contextFactory, isAndroid, await expect(page).toHaveURL(server.EMPTY_PAGE); const response = await page.goBack(); await expect(page).toHaveURL('https://www.theverge.com/'); - await expect(response.request().url()).toBe('https://www.theverge.com/'); + expect(response.request().url()).toBe('https://www.theverge.com/'); expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/'); }); @@ -149,7 +149,7 @@ it('should goForward to redirected navigation', async ({ contextFactory, isAndro await expect(page).toHaveURL(server.EMPTY_PAGE); const response = await page.goForward(); await expect(page).toHaveURL('https://www.theverge.com/'); - await expect(response.request().url()).toBe('https://www.theverge.com/'); + expect(response.request().url()).toBe('https://www.theverge.com/'); expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/'); }); @@ -163,6 +163,6 @@ it('should reload redirected navigation', async ({ contextFactory, isAndroid, as await expect(page).toHaveURL('https://www.theverge.com/'); const response = await page.reload(); await expect(page).toHaveURL('https://www.theverge.com/'); - await expect(response.request().url()).toBe('https://www.theverge.com/'); + expect(response.request().url()).toBe('https://www.theverge.com/'); expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/'); });