chore: move har router into local utils (#14967)

packages/playwright-core/src/client/browser.ts

@@ -61,7 +61,7 @@ export class Browser extends ChannelOwner<channels.BrowserChannel> implements ap
 
   async newContext(options: BrowserContextOptions = {}): Promise<BrowserContext> {
     options = { ...this._browserType._defaultContextOptions, ...options };
-    const harRouter = options.har ? await HarRouter.create(options.har) : null;
+    const harRouter = options.har ? await HarRouter.create(this._connection.localUtils(), options.har) : null;
     const contextOptions = await prepareBrowserContextParams(options);
     const context = BrowserContext.from((await this._channel.newContext(contextOptions)).context);
     context._options = contextOptions;

packages/playwright-core/src/client/browserType.ts

@@ -95,7 +95,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
     const logger = options.logger || this._defaultLaunchOptions?.logger;
     assert(!(options as any).port, 'Cannot specify a port without launching as a server.');
     options = { ...this._defaultLaunchOptions, ...this._defaultContextOptions, ...options };
-    const harRouter = options.har ? await HarRouter.create(options.har) : null;
+    const harRouter = options.har ? await HarRouter.create(this._connection.localUtils(), options.har) : null;
     const contextParams = await prepareBrowserContextParams(options);
     const persistentParams: channels.BrowserTypeLaunchPersistentContextParams = {
       ...contextParams,

packages/playwright-core/src/client/electron.ts

@@ -53,7 +53,7 @@ export class Electron extends ChannelOwner<channels.ElectronChannel> implements
       ...await prepareBrowserContextParams(options),
       env: envObjectToArray(options.env ? options.env : process.env),
     };
-    const harRouter = options.har ? await HarRouter.create(options.har) : null;
+    const harRouter = options.har ? await HarRouter.create(this._connection.localUtils(), options.har) : null;
     const app = ElectronApplication.from((await this._channel.launch(params)).electronApplication);
     app._context._options = params;
     harRouter?.addRoute(app._context);

packages/playwright-core/src/client/harRouter.ts

@@ -14,13 +14,10 @@
  * limitations under the License.
  */
 
-import fs from 'fs';
-import type { HAREntry, HARFile } from '../../types/types';
 import { debugLogger } from '../common/debugLogger';
-import { rewriteErrorMessage } from '../utils/stackTrace';
-import { ZipFile } from '../utils/zipFile';
 import type { BrowserContext } from './browserContext';
 import { Events } from './events';
+import type { LocalUtils } from './localUtils';
 import type { Route } from './network';
 import type { BrowserContextOptions } from './types';
 
@@ -28,76 +25,55 @@ type HarOptions = NonNullable<BrowserContextOptions['har']>;
 
 export class HarRouter {
   private _pattern: string | RegExp;
-  private _harFile: HARFile;
-  private _zipFile: ZipFile | null;
   private _options: HarOptions | undefined;
+  private _localUtils: LocalUtils;
+  private _harId: string;
 
-  static async create(options: HarOptions): Promise<HarRouter> {
-    if (options.path.endsWith('.zip')) {
-      const zipFile = new ZipFile(options.path);
-      const har = await zipFile.read('har.har');
-      const harFile = JSON.parse(har.toString()) as HARFile;
-      return new HarRouter(harFile, zipFile, options);
-    }
-    const harFile = JSON.parse(await fs.promises.readFile(options.path, 'utf-8')) as HARFile;
-    return new HarRouter(harFile, null, options);
+  static async create(localUtils: LocalUtils, options: HarOptions): Promise<HarRouter> {
+    const { harId } = await localUtils._channel.harOpen({ file: options.path });
+    return new HarRouter(localUtils, harId, options);
   }
 
-  constructor(harFile: HARFile, zipFile: ZipFile | null, options?: HarOptions) {
-    this._harFile = harFile;
-    this._zipFile = zipFile;
+  constructor(localUtils: LocalUtils, harId: string, options?: HarOptions) {
+    this._localUtils = localUtils;
+    this._harId = harId;
     this._pattern = options?.urlFilter ?? /.*/;
     this._options = options;
   }
 
   private async _handle(route: Route) {
-    let entry;
-    try {
-      entry = harFindResponse(this._harFile, {
-        url: route.request().url(),
-        method: route.request().method()
-      });
-    } catch (e) {
-      rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`);
-      debugLogger.log('api', e);
-    }
+    const response = await this._localUtils._channel.harLookup({
+      harId: this._harId,
+      url: route.request().url(),
+      method: route.request().method(),
+      isNavigationRequest: route.request().isNavigationRequest()
+    });
 
-    if (entry) {
-      // If navigation is being redirected, restart it with the final url to ensure the document's url changes.
-      if (entry.request.url !== route.request().url() && route.request().isNavigationRequest()) {
-        debugLogger.log('api', `redirecting HAR navigation: ${route.request().url()} => ${entry.request.url}`);
-        await route._abort(undefined, entry.request.url);
-        return;
-      }
-      debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`);
-      const response = entry.response;
-      const sha1 = (response.content as any)._sha1;
-
-      if (this._zipFile && sha1) {
-        const body = await this._zipFile.read(sha1).catch(() => {
-          debugLogger.log('api', `payload ${sha1} for request ${route.request().url()} is not found in archive`);
-          return null;
-        });
-        if (body) {
-          await route.fulfill({
-            status: response.status,
-            headers: Object.fromEntries(response.headers.map(h => [h.name, h.value])),
-            body
-          });
-          return;
-        }
-      }
-
-      await route.fulfill({ response });
+    if (response.action === 'redirect') {
+      debugLogger.log('api', `HAR: ${route.request().url()} redirected to ${response.redirectURL}`);
+      await route._abort(undefined, response.redirectURL);
       return;
     }
 
+    if (response.action === 'fulfill') {
+      await route.fulfill({
+        status: response.status,
+        headers: Object.fromEntries(response.headers!.map(h => [h.name, h.value])),
+        body: response.base64Encoded ? Buffer.from(response.body!, 'base64') : response.body
+      });
+      return;
+    }
+
+    if (response.action === 'error')
+      debugLogger.log('api', response.message!);
+    // Report the error, but fall through to the default handler.
+
     if (this._options?.fallback === 'continue') {
       await route.fallback();
       return;
     }
 
-    debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`);
+    debugLogger.log('api', `HAR: ${route.request().method()} ${route.request().url()} aborted - no such entry in HAR file`);
     await route.abort();
   }
 
@@ -107,37 +83,6 @@ export class HarRouter {
   }
 
   dispose() {
-    this._zipFile?.close();
-  }
-}
-
-const redirectStatus = [301, 302, 303, 307, 308];
-
-function harFindResponse(har: HARFile, params: { url: string, method: string }): HAREntry | undefined {
-  const harLog = har.log;
-  const visited = new Set<HAREntry>();
-  let url = params.url;
-  let method = params.method;
-  while (true) {
-    const entry = harLog.entries.find(entry => entry.request.url === url && entry.request.method === method);
-    if (!entry)
-      return;
-    if (visited.has(entry))
-      throw new Error(`Found redirect cycle for ${params.url}`);
-    visited.add(entry);
-
-    const locationHeader = entry.response.headers.find(h => h.name.toLowerCase() === 'location');
-    if (redirectStatus.includes(entry.response.status) && locationHeader) {
-      const locationURL = new URL(locationHeader.value, url);
-      url = locationURL.toString();
-      if ((entry.response.status === 301 || entry.response.status === 302) && method === 'POST' ||
-        entry.response.status === 303 && !['GET', 'HEAD'].includes(method)) {
-        // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch)
-        method = 'GET';
-      }
-      continue;
-    }
-
-    return entry;
+    this._localUtils._channel.harClose({ harId: this._harId }).catch(() => {});
   }
 }

packages/playwright-core/src/protocol/channels.ts

@@ -378,6 +378,9 @@ export interface LocalUtilsEventTarget {
 export interface LocalUtilsChannel extends LocalUtilsEventTarget, Channel {
   _type_LocalUtils: boolean;
   zip(params: LocalUtilsZipParams, metadata?: Metadata): Promise<LocalUtilsZipResult>;
+  harOpen(params: LocalUtilsHarOpenParams, metadata?: Metadata): Promise<LocalUtilsHarOpenResult>;
+  harLookup(params: LocalUtilsHarLookupParams, metadata?: Metadata): Promise<LocalUtilsHarLookupResult>;
+  harClose(params: LocalUtilsHarCloseParams, metadata?: Metadata): Promise<LocalUtilsHarCloseResult>;
 }
 export type LocalUtilsZipParams = {
   zipFile: string,
@@ -387,6 +390,40 @@ export type LocalUtilsZipOptions = {
 
 };
 export type LocalUtilsZipResult = void;
+export type LocalUtilsHarOpenParams = {
+  file: string,
+};
+export type LocalUtilsHarOpenOptions = {
+
+};
+export type LocalUtilsHarOpenResult = {
+  harId: string,
+};
+export type LocalUtilsHarLookupParams = {
+  harId: string,
+  url: string,
+  method: string,
+  isNavigationRequest: boolean,
+};
+export type LocalUtilsHarLookupOptions = {
+
+};
+export type LocalUtilsHarLookupResult = {
+  action: 'error' | 'redirect' | 'fulfill' | 'noentry',
+  message?: string,
+  redirectURL?: string,
+  status?: number,
+  headers?: NameValue[],
+  body?: string,
+  base64Encoded?: boolean,
+};
+export type LocalUtilsHarCloseParams = {
+  harId: string,
+};
+export type LocalUtilsHarCloseOptions = {
+
+};
+export type LocalUtilsHarCloseResult = void;
 
 export interface LocalUtilsEvents {
 }

packages/playwright-core/src/protocol/protocol.yml

@@ -478,6 +478,39 @@ LocalUtils:
           type: array
           items: NameValue
 
+    harOpen:
+      parameters:
+        file: string
+      returns:
+        harId: string
+
+    harLookup:
+      parameters:
+        harId: string
+        url: string
+        method: string
+        isNavigationRequest: boolean
+      returns:
+        action:
+          type: enum
+          literals:
+          - error
+          - redirect
+          - fulfill
+          - noentry
+        message: string?
+        redirectURL: string?
+        status: number?
+        headers:
+          type: array?
+          items: NameValue
+        body: string?
+        base64Encoded: boolean?
+
+    harClose:
+      parameters:
+        harId: string
+
 Root:
   type: interface
 

packages/playwright-core/src/protocol/validator.ts

@@ -205,6 +205,18 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     zipFile: tString,
     entries: tArray(tType('NameValue')),
   });
+  scheme.LocalUtilsHarOpenParams = tObject({
+    file: tString,
+  });
+  scheme.LocalUtilsHarLookupParams = tObject({
+    harId: tString,
+    url: tString,
+    method: tString,
+    isNavigationRequest: tBoolean,
+  });
+  scheme.LocalUtilsHarCloseParams = tObject({
+    harId: tString,
+  });
   scheme.RootInitializeParams = tObject({
     sdkLanguage: tString,
   });

packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts

@@ -23,9 +23,13 @@ import { assert, createGuid } from '../../utils';
 import type { DispatcherScope } from './dispatcher';
 import { Dispatcher } from './dispatcher';
 import { yazl, yauzl } from '../../zipBundle';
+import { ZipFile } from '../../utils/zipFile';
+import type { HAREntry, HARFile } from '../../../types/types';
+import type { HeadersArray } from '../types';
 
 export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel> implements channels.LocalUtilsChannel {
   _type_LocalUtils: boolean;
+  private _harBakends = new Map<string, HarBackend>();
 
   constructor(scope: DispatcherScope) {
     super(scope, { guid: 'localUtils@' + createGuid() }, 'LocalUtils', {});
@@ -86,4 +90,128 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.
     });
     return promise;
   }
+
+  async harOpen(params: channels.LocalUtilsHarOpenParams, metadata?: channels.Metadata): Promise<channels.LocalUtilsHarOpenResult> {
+    let harBackend: HarBackend;
+    if (params.file.endsWith('.zip')) {
+      const zipFile = new ZipFile(params.file);
+      const har = await zipFile.read('har.har');
+      const harFile = JSON.parse(har.toString()) as HARFile;
+      harBackend = new HarBackend(harFile, zipFile);
+    } else {
+      const harFile = JSON.parse(await fs.promises.readFile(params.file, 'utf-8')) as HARFile;
+      harBackend = new HarBackend(harFile, null);
+    }
+    this._harBakends.set(harBackend.id, harBackend);
+    return { harId: harBackend.id };
+  }
+
+  async harLookup(params: channels.LocalUtilsHarLookupParams, metadata?: channels.Metadata): Promise<channels.LocalUtilsHarLookupResult> {
+    const harBackend = this._harBakends.get(params.harId);
+    if (!harBackend)
+      return { action: 'error', message: `Internal error: har was not opened` };
+    return await harBackend.lookup(params.url, params.method, params.isNavigationRequest);
+  }
+
+  async harClose(params: channels.LocalUtilsHarCloseParams, metadata?: channels.Metadata): Promise<void> {
+    const harBackend = this._harBakends.get(params.harId);
+    if (harBackend) {
+      this._harBakends.delete(harBackend.id);
+      harBackend.dispose();
+    }
+  }
+}
+
+const redirectStatus = [301, 302, 303, 307, 308];
+
+class HarBackend {
+  readonly id = createGuid();
+  private _harFile: HARFile;
+  private _zipFile: ZipFile | null;
+
+  constructor(harFile: HARFile, zipFile: ZipFile | null) {
+    this._harFile = harFile;
+    this._zipFile = zipFile;
+  }
+
+  async lookup(url: string, method: string, isNavigationRequest: boolean): Promise<{
+      action: 'error' | 'redirect' | 'fulfill' | 'noentry',
+      message?: string,
+      redirectURL?: string,
+      status?: number,
+      headers?: HeadersArray,
+      body?: string,
+      base64Encoded?: boolean }> {
+    let entry;
+    try {
+      entry = this._harFindResponse(url, method);
+    } catch (e) {
+      return { action: 'error', message: 'HAR error: ' + e.message };
+    }
+
+    if (!entry)
+      return { action: 'noentry' };
+
+    // If navigation is being redirected, restart it with the final url to ensure the document's url changes.
+    if (entry.request.url !== url && isNavigationRequest)
+      return { action: 'redirect', redirectURL: entry.request.url };
+
+    const response = entry.response;
+    const sha1 = (response.content as any)._sha1;
+    let body: string | undefined;
+    let base64Encoded = false;
+
+    if (this._zipFile && sha1) {
+      const buffer = await this._zipFile.read(sha1).catch(() => {
+        return { action: 'error', message: `Malformed HAR: payload ${sha1} for request ${url} is not found in archive` };
+      });
+
+      if (buffer) {
+        body = buffer.toString('base64');
+        base64Encoded = true;
+      }
+    } else {
+      body = response.content.text;
+      base64Encoded = response.content.encoding === 'base64';
+    }
+
+    return {
+      action: 'fulfill',
+      status: response.status,
+      headers: response.headers,
+      body,
+      base64Encoded
+    };
+  }
+
+  private _harFindResponse(url: string, method: string): HAREntry | undefined {
+    const harLog = this._harFile.log;
+    const visited = new Set<HAREntry>();
+    while (true) {
+      const entry = harLog.entries.find(entry => entry.request.url === url && entry.request.method === method);
+      if (!entry)
+        return;
+      if (visited.has(entry))
+        throw new Error(`Found redirect cycle for ${url}`);
+      visited.add(entry);
+
+      const locationHeader = entry.response.headers.find(h => h.name.toLowerCase() === 'location');
+      if (redirectStatus.includes(entry.response.status) && locationHeader) {
+        const locationURL = new URL(locationHeader.value, url);
+        url = locationURL.toString();
+        if ((entry.response.status === 301 || entry.response.status === 302) && method === 'POST' ||
+          entry.response.status === 303 && !['GET', 'HEAD'].includes(method)) {
+          // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch)
+          method = 'GET';
+        }
+        continue;
+      }
+
+      return entry;
+    }
+  }
+
+  dispose() {
+    this._zipFile?.close();
+  }
 }

tests/library/browsercontext-har.spec.ts

@@ -116,7 +116,7 @@ it('should change document URL after redirected navigation', async ({ contextFac
     page.goto('https://theverge.com/')
   ]);
   await expect(page).toHaveURL('https://www.theverge.com/');
-  await expect(response.request().url()).toBe('https://www.theverge.com/');
+  expect(response.request().url()).toBe('https://www.theverge.com/');
   expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/');
 });
 
@@ -131,7 +131,7 @@ it('should goBack to redirected navigation', async ({ contextFactory, isAndroid,
   await expect(page).toHaveURL(server.EMPTY_PAGE);
   const response = await page.goBack();
   await expect(page).toHaveURL('https://www.theverge.com/');
-  await expect(response.request().url()).toBe('https://www.theverge.com/');
+  expect(response.request().url()).toBe('https://www.theverge.com/');
   expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/');
 });
 
@@ -149,7 +149,7 @@ it('should goForward to redirected navigation', async ({ contextFactory, isAndro
   await expect(page).toHaveURL(server.EMPTY_PAGE);
   const response = await page.goForward();
   await expect(page).toHaveURL('https://www.theverge.com/');
-  await expect(response.request().url()).toBe('https://www.theverge.com/');
+  expect(response.request().url()).toBe('https://www.theverge.com/');
   expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/');
 });
 
@@ -163,6 +163,6 @@ it('should reload redirected navigation', async ({ contextFactory, isAndroid, as
   await expect(page).toHaveURL('https://www.theverge.com/');
   const response = await page.reload();
   await expect(page).toHaveURL('https://www.theverge.com/');
-  await expect(response.request().url()).toBe('https://www.theverge.com/');
+  expect(response.request().url()).toBe('https://www.theverge.com/');
   expect(await page.evaluate(() => location.href)).toBe('https://www.theverge.com/');
 });