feat(rawheaders): introduce initial plumbing (#8638)

docs/src/api/class-headers.md

@@ -0,0 +1,30 @@
+# class: Headers
+
+HTTP request and response raw headers collection.
+
+## method: Headers.get
+- returns: <[string|null]>
+Returns header value for the given name.
+
+### param: Headers.get.name
+- `name` <[string]>
+Header name, case-insensitive.
+
+## method: Headers.getAll
+- returns: <[Array]<[string]>>
+
+Returns all header values for the given header name.
+
+### param: Headers.getAll.name
+- `name` <[string]>
+Header name, case-insensitive.
+
+## method: Headers.headerNames
+- returns: <[Array]<[string]>>
+
+Returns all header names in this headers collection.
+
+## method: Headers.headers
+- returns: <[Array]<{ name: string, value: string }>>
+
+Returns all raw headers.

docs/src/api/class-request.md

@@ -54,7 +54,7 @@ Returns the [Frame] that initiated this request.
 ## method: Request.headers
 - returns: <[Object]<[string], [string]>>
 
-An object with HTTP headers associated with the request. All header names are lower-case.
+**DEPRECATED**  Use [`method: Request.rawHeaders`] instead.
 
 ## method: Request.isNavigationRequest
 - returns: <[boolean]>
@@ -85,6 +85,11 @@ Returns parsed request's body for `form-urlencoded` and JSON as a fallback if an
 When the response is `application/x-www-form-urlencoded` then a key/value object of the values will be returned.
 Otherwise it will be parsed as JSON.
 
+## async method: Request.rawHeaders
+- returns: <[Headers]>
+
+An object with the raw request HTTP headers associated with the request. All headers are as seen in the network stack.
+
 ## method: Request.redirectedFrom
 - returns: <[null]|[Request]>
 

docs/src/api/class-response.md

@@ -20,7 +20,7 @@ Returns the [Frame] that initiated this response.
 ## method: Response.headers
 - returns: <[Object]<[string], [string]>>
 
-Returns the object with HTTP headers associated with the response. All header names are lower-case.
+**DEPRECATED**  Use [`method: Response.rawHeaders`] instead.
 
 ## async method: Response.json
 * langs: js, python
@@ -43,6 +43,11 @@ This method will throw if the response body is not parsable via `JSON.parse`.
 
 Contains a boolean stating whether the response was successful (status in the range 200-299) or not.
 
+## async method: Response.rawHeaders
+- returns: <[Headers]>
+
+An object with the raw response HTTP headers associated with the request. All headers are as seen in the network stack.
+
 ## method: Response.request
 - returns: <[Request]>
 

src/client/api.ts

@@ -41,3 +41,4 @@ export { Video } from './video';
 export { Worker } from './worker';
 export { CDPSession } from './cdpSession';
 export { Playwright } from './playwright';
+export { RawHeaders as Headers } from './network';

src/client/browserContext.ts

@@ -28,7 +28,7 @@ import { Events } from './events';
 import { TimeoutSettings } from '../utils/timeoutSettings';
 import { Waiter } from './waiter';
 import { URLMatch, Headers, WaitForEventOptions, BrowserContextOptions, StorageState, LaunchOptions } from './types';
-import { isUnderTest, headersObjectToArray, mkdirIfNeeded, isString, headersArrayToObject } from '../utils/utils';
+import { isUnderTest, headersObjectToArray, mkdirIfNeeded, isString } from '../utils/utils';
 import { isSafeCloseError } from '../utils/errors';
 import * as api from '../../types/types';
 import * as structs from '../../types/structs';
@@ -125,15 +125,13 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
   }
 
   private _onRequestFinished(params: channels.BrowserContextRequestFinishedEvent) {
-    const { requestSizes, responseEndTiming, responseHeaders } = params;
+    const { requestSizes, responseEndTiming } = params;
     const request = network.Request.from(params.request);
     const response = network.Response.fromNullable(params.response);
     const page = Page.fromNullable(params.page);
     if (request._timing)
       request._timing.responseEnd = responseEndTiming;
     request._sizes = requestSizes;
-    if (response && responseHeaders)
-      response._headers = headersArrayToObject(responseHeaders, true /* lowerCase */);
     this.emit(Events.BrowserContext.RequestFinished, request);
     if (page)
       page.emit(Events.Page.RequestFinished, request);

src/client/network.ts

@@ -18,7 +18,7 @@ import { URLSearchParams } from 'url';
 import * as channels from '../protocol/channels';
 import { ChannelOwner } from './channelOwner';
 import { Frame } from './frame';
-import { Headers, RemoteAddr, SecurityDetails, WaitForEventOptions } from './types';
+import { Headers, HeadersArray, RemoteAddr, SecurityDetails, WaitForEventOptions } from './types';
 import fs from 'fs';
 import * as mime from 'mime';
 import { isString, headersObjectToArray, headersArrayToObject } from '../utils/utils';
@@ -29,6 +29,7 @@ import { Waiter } from './waiter';
 import * as api from '../../types/types';
 import { URLMatch } from '../common/types';
 import { urlMatches } from './clientHelper';
+import { MultiMap } from '../utils/multimap';
 
 export type NetworkCookie = {
   name: string,
@@ -58,6 +59,7 @@ export class Request extends ChannelOwner<channels.RequestChannel, channels.Requ
   private _redirectedTo: Request | null = null;
   _failureText: string | null = null;
   _headers: Headers;
+  private _rawHeadersPromise: Promise<RawHeaders> | undefined;
   private _postData: Buffer | null;
   _timing: ResourceTiming;
   _sizes: RequestSizes = { requestBodySize: 0, requestHeadersSize: 0, responseBodySize: 0, responseHeadersSize: 0, responseTransferSize: 0 };
@@ -131,10 +133,26 @@ export class Request extends ChannelOwner<channels.RequestChannel, channels.Requ
     }
   }
 
+  /**
+   * @deprecated
+   */
   headers(): Headers {
     return { ...this._headers };
   }
 
+  async rawHeaders(): Promise<RawHeaders> {
+    if (this._rawHeadersPromise)
+      return this._rawHeadersPromise;
+    this._rawHeadersPromise = this.response().then(response => {
+      if (!response)
+        return new RawHeaders([]);
+      return response._wrapApiCall(async (channel: channels.ResponseChannel) => {
+        return new RawHeaders((await channel.rawRequestHeaders()).headers);
+      });
+    });
+    return this._rawHeadersPromise;
+  }
+
   async response(): Promise<Response | null> {
     return this._wrapApiCall(async (channel: channels.RequestChannel) => {
       return Response.fromNullable((await channel.response()).response);
@@ -183,11 +201,13 @@ export class InterceptedResponse implements api.Response {
   private readonly _initializer: channels.InterceptedResponse;
   private readonly _request: Request;
   private readonly _headers: Headers;
+  private readonly _rawHeaders: RawHeaders;
 
   constructor(route: Route, initializer: channels.InterceptedResponse) {
     this._route = route;
     this._initializer = initializer;
     this._headers = headersArrayToObject(initializer.headers, true /* lowerCase */);
+    this._rawHeaders = new RawHeaders(initializer.headers);
     this._request = Request.from(initializer.request);
   }
 
@@ -230,6 +250,10 @@ export class InterceptedResponse implements api.Response {
     return { ...this._headers };
   }
 
+  async rawHeaders(): Promise<RawHeaders> {
+    return this._rawHeaders;
+  }
+
   async body(): Promise<Buffer> {
     return this._route._responseBody();
   }
@@ -386,6 +410,7 @@ export class Response extends ChannelOwner<channels.ResponseChannel, channels.Re
   _headers: Headers;
   private _request: Request;
   readonly _finishedPromise = new ManualPromise<void>();
+  private _rawHeadersPromise: Promise<RawHeaders> | undefined;
 
   static from(response: channels.ResponseChannel): Response {
     return (response as any)._object;
@@ -399,7 +424,6 @@ export class Response extends ChannelOwner<channels.ResponseChannel, channels.Re
     super(parent, type, guid, initializer);
     this._headers = headersArrayToObject(initializer.headers, true /* lowerCase */);
     this._request = Request.from(this._initializer.request);
-    this._request._headers = headersArrayToObject(initializer.requestHeaders, true /* lowerCase */);
     Object.assign(this._request._timing, this._initializer.timing);
   }
 
@@ -419,10 +443,22 @@ export class Response extends ChannelOwner<channels.ResponseChannel, channels.Re
     return this._initializer.statusText;
   }
 
+  /**
+   * @deprecated
+   */
   headers(): Headers {
     return { ...this._headers };
   }
 
+  async rawHeaders(): Promise<RawHeaders> {
+    if (this._rawHeadersPromise)
+      return this._rawHeadersPromise;
+    this._rawHeadersPromise = this._wrapApiCall(async (channel: channels.ResponseChannel) => {
+      return new RawHeaders((await channel.rawResponseHeaders()).headers);
+    });
+    return this._rawHeadersPromise;
+  }
+
   async finished(): Promise<null> {
     return this._finishedPromise.then(() => null);
   }
@@ -600,3 +636,30 @@ export class RouteHandler {
     this.handledCount++;
   }
 }
+
+export class RawHeaders implements api.Headers {
+  private _headersArray: HeadersArray;
+  private _headersMap = new MultiMap<string, string>();
+
+  constructor(headers: HeadersArray) {
+    this._headersArray = headers;
+    for (const header of headers)
+      this._headersMap.set(header.name.toLowerCase(), header.value);
+  }
+
+  get(name: string): string | null {
+    return this.getAll(name)[0] || null;
+  }
+
+  getAll(name: string): string[] {
+    return [...this._headersMap.get(name.toLowerCase())];
+  }
+
+  headerNames(): string[] {
+    return [...new Set(this._headersArray.map(h => h.name))];
+  }
+
+  headers(): HeadersArray {
+    return this._headersArray;
+  }
+}

src/client/types.ts

@@ -16,7 +16,7 @@
  */
 
 import * as channels from '../protocol/channels';
-import type { Size } from '../common/types';
+import type { NameValue, Size } from '../common/types';
 import type { ParsedStackTrace } from '../utils/stackTrace';
 export { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types';
 
@@ -32,6 +32,7 @@ export interface ClientSideInstrumentation {
 
 export type StrictOptions = { strict?: boolean };
 export type Headers = { [key: string]: string };
+export type HeadersArray = NameValue[];
 export type Env = { [key: string]: string | number | boolean | undefined };
 
 export type WaitForEventOptions = Function | { predicate?: Function, timeout?: number };

src/common/types.ts

@@ -20,3 +20,4 @@ export type Rect = Size & Point;
 export type Quad = [ Point, Point, Point, Point ];
 export type URLMatch = string | RegExp | ((url: URL) => boolean);
 export type TimeoutOptions = { timeout?: number };
+export type NameValue = { name: string, value: string };

src/dispatchers/browserContextDispatcher.ts

@@ -86,7 +86,6 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
     context.on(BrowserContext.Events.RequestFinished, ({ request, response}: { request: Request, response: Response | null }) => this._dispatchEvent('requestFinished', {
       request: RequestDispatcher.from(scope, request),
       response: ResponseDispatcher.fromNullable(scope, response),
-      responseHeaders: response?.headers(),
       responseEndTiming: request._responseEndTiming,
       requestSizes: request.sizes(),
       page: PageDispatcher.fromNullable(this._scope, request.frame()._page.initializedOrUndefined()),

src/dispatchers/networkDispatchers.ts

@@ -67,7 +67,6 @@ export class ResponseDispatcher extends Dispatcher<Response, channels.ResponseIn
       url: response.url(),
       status: response.status(),
       statusText: response.statusText(),
-      requestHeaders: response.request().headers(),
       headers: response.headers(),
       timing: response.timing()
     });
@@ -84,6 +83,14 @@ export class ResponseDispatcher extends Dispatcher<Response, channels.ResponseIn
   async serverAddr(): Promise<channels.ResponseServerAddrResult> {
     return { value: await this._object.serverAddr() || undefined };
   }
+
+  async rawRequestHeaders(params?: channels.ResponseRawRequestHeadersParams, metadata?: channels.Metadata): Promise<channels.ResponseRawRequestHeadersResult> {
+    return { headers: await this._object.rawRequestHeaders() };
+  }
+
+  async rawResponseHeaders(params?: channels.ResponseRawResponseHeadersParams, metadata?: channels.Metadata): Promise<channels.ResponseRawResponseHeadersResult> {
+    return { headers: await this._object.rawResponseHeaders() };
+  }
 }
 
 export class RouteDispatcher extends Dispatcher<Route, channels.RouteInitializer, channels.RouteEvents> implements channels.RouteChannel {

src/protocol/channels.ts

@@ -802,7 +802,6 @@ export type BrowserContextRequestFinishedEvent = {
   request: RequestChannel,
   response?: ResponseChannel,
   responseEndTiming: number,
-  responseHeaders?: NameValue[],
   requestSizes: RequestSizes,
   page?: PageChannel,
 };
@@ -2690,7 +2689,6 @@ export type ResponseInitializer = {
   url: string,
   status: number,
   statusText: string,
-  requestHeaders: NameValue[],
   headers: NameValue[],
   timing: ResourceTiming,
 };
@@ -2698,6 +2696,8 @@ export interface ResponseChannel extends Channel {
   body(params?: ResponseBodyParams, metadata?: Metadata): Promise<ResponseBodyResult>;
   securityDetails(params?: ResponseSecurityDetailsParams, metadata?: Metadata): Promise<ResponseSecurityDetailsResult>;
   serverAddr(params?: ResponseServerAddrParams, metadata?: Metadata): Promise<ResponseServerAddrResult>;
+  rawRequestHeaders(params?: ResponseRawRequestHeadersParams, metadata?: Metadata): Promise<ResponseRawRequestHeadersResult>;
+  rawResponseHeaders(params?: ResponseRawResponseHeadersParams, metadata?: Metadata): Promise<ResponseRawResponseHeadersResult>;
 }
 export type ResponseBodyParams = {};
 export type ResponseBodyOptions = {};
@@ -2714,6 +2714,16 @@ export type ResponseServerAddrOptions = {};
 export type ResponseServerAddrResult = {
   value?: RemoteAddr,
 };
+export type ResponseRawRequestHeadersParams = {};
+export type ResponseRawRequestHeadersOptions = {};
+export type ResponseRawRequestHeadersResult = {
+  headers: NameValue[],
+};
+export type ResponseRawResponseHeadersParams = {};
+export type ResponseRawResponseHeadersOptions = {};
+export type ResponseRawResponseHeadersResult = {
+  headers: NameValue[],
+};
 
 export interface ResponseEvents {
 }

src/protocol/protocol.yml

@@ -766,9 +766,6 @@ BrowserContext:
         request: Request
         response: Response?
         responseEndTiming: number
-        responseHeaders:
-          type: array?
-          items: NameValue
         requestSizes: RequestSizes
         page: Page?
 
@@ -2197,9 +2194,6 @@ Response:
     url: string
     status: number
     statusText: string
-    requestHeaders:
-      type: array
-      items: NameValue
     headers:
       type: array
       items: NameValue
@@ -2220,6 +2214,18 @@ Response:
       returns:
         value: RemoteAddr?
 
+    rawRequestHeaders:
+      returns:
+        headers:
+          type: array
+          items: NameValue
+
+    rawResponseHeaders:
+      returns:
+        headers:
+          type: array
+          items: NameValue
+
 
 SecurityDetails:
   type: object

src/protocol/validator.ts

@@ -1053,6 +1053,8 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
   scheme.ResponseBodyParams = tOptional(tObject({}));
   scheme.ResponseSecurityDetailsParams = tOptional(tObject({}));
   scheme.ResponseServerAddrParams = tOptional(tObject({}));
+  scheme.ResponseRawRequestHeadersParams = tOptional(tObject({}));
+  scheme.ResponseRawResponseHeadersParams = tOptional(tObject({}));
   scheme.SecurityDetails = tObject({
     issuer: tOptional(tString),
     protocol: tOptional(tString),

src/server/chromium/crNetworkManager.ts

@@ -38,7 +38,6 @@ export class CRNetworkManager {
   private _protocolRequestInterceptionEnabled = false;
   private _requestIdToRequestPausedEvent = new Map<string, Protocol.Fetch.requestPausedPayload>();
   private _eventListeners: RegisteredListener[];
-  private _requestIdToExtraInfo = new Map<string, Protocol.Network.requestWillBeSentExtraInfoPayload>();
   private _responseExtraInfoTracker = new ResponseExtraInfoTracker();
 
   constructor(client: CRSession, page: Page, parentManager: CRNetworkManager | null) {
@@ -133,19 +132,10 @@ export class CRNetworkManager {
     } else {
       this._onRequest(workerFrame, event, null);
     }
-    const extraInfo = this._requestIdToExtraInfo.get(event.requestId);
-    if (extraInfo)
-      this._onRequestWillBeSentExtraInfo(extraInfo);
   }
 
   _onRequestWillBeSentExtraInfo(event: Protocol.Network.requestWillBeSentExtraInfoPayload) {
-    const request = this._requestIdToRequest.get(event.requestId);
-    if (request) {
-      request.request.updateWithRawHeaders(headersObjectToArray(event.headers));
-      this._requestIdToExtraInfo.delete(event.requestId);
-    } else {
-      this._requestIdToExtraInfo.set(event.requestId, event);
-    }
+    this._responseExtraInfoTracker.requestWillBeSentExtraInfo(event);
   }
 
   _onAuthRequired(event: Protocol.Fetch.authRequiredPayload) {
@@ -566,6 +556,7 @@ const errorReasons: { [reason: string]: Protocol.Network.ErrorReason } = {
 
 type RequestInfo = {
   requestId: string,
+  requestWillBeSentExtraInfo: Protocol.Network.requestWillBeSentExtraInfoPayload[],
   responseReceivedExtraInfo: Protocol.Network.responseReceivedExtraInfoPayload[],
   responses: network.Response[],
   loadingFinished?: Protocol.Network.loadingFinishedPayload,
@@ -592,26 +583,18 @@ class ResponseExtraInfoTracker {
 
   requestWillBeSent(event: Protocol.Network.requestWillBeSentPayload) {
     const info = this._requests.get(event.requestId);
-    if (info) {
-      // This is redirect.
+    if (info && event.redirectResponse)
       this._innerResponseReceived(info, event.redirectResponse);
-    } else {
+    else
       this._getOrCreateEntry(event.requestId);
-    }
   }
 
-  _getOrCreateEntry(requestId: string): RequestInfo {
-    let info = this._requests.get(requestId);
-    if (!info) {
-      info = {
-        requestId: requestId,
-        responseReceivedExtraInfo: [],
-        responses: [],
-        sawResponseWithoutConnectionId: false
-      };
-      this._requests.set(requestId, info);
-    }
-    return info;
+  requestWillBeSentExtraInfo(event: Protocol.Network.requestWillBeSentExtraInfoPayload) {
+    const info = this._getOrCreateEntry(event.requestId);
+    if (!info)
+      return;
+    info.requestWillBeSentExtraInfo.push(event);
+    this._patchHeaders(info, info.requestWillBeSentExtraInfo.length - 1);
   }
 
   responseReceived(event: Protocol.Network.responseReceivedPayload) {
@@ -621,8 +604,8 @@ class ResponseExtraInfoTracker {
     this._innerResponseReceived(info, event.response);
   }
 
-  private _innerResponseReceived(info: RequestInfo, response: Protocol.Network.Response | undefined) {
-    if (!response?.connectionId) {
+  private _innerResponseReceived(info: RequestInfo, response: Protocol.Network.Response) {
+    if (!response.connectionId) {
       // Starting with this response we no longer can guarantee that response and extra info correspond to the same index.
       info.sawResponseWithoutConnectionId = true;
     }
@@ -631,7 +614,7 @@ class ResponseExtraInfoTracker {
   responseReceivedExtraInfo(event: Protocol.Network.responseReceivedExtraInfoPayload) {
     const info = this._getOrCreateEntry(event.requestId);
     info.responseReceivedExtraInfo.push(event);
-    this._patchResponseHeaders(info, info.responseReceivedExtraInfo.length - 1);
+    this._patchHeaders(info, info.responseReceivedExtraInfo.length - 1);
     this._checkFinished(info);
   }
 
@@ -648,7 +631,7 @@ class ResponseExtraInfoTracker {
       return;
     response.setWillReceiveExtraHeaders();
     info.responses.push(response);
-    this._patchResponseHeaders(info, info.responses.length - 1);
+    this._patchHeaders(info, info.responses.length - 1);
   }
 
   loadingFinished(event: Protocol.Network.loadingFinishedPayload) {
@@ -667,11 +650,29 @@ class ResponseExtraInfoTracker {
     this._checkFinished(info);
   }
 
-  private _patchResponseHeaders(info: RequestInfo, index: number) {
+  _getOrCreateEntry(requestId: string): RequestInfo {
+    let info = this._requests.get(requestId);
+    if (!info) {
+      info = {
+        requestId: requestId,
+        requestWillBeSentExtraInfo: [],
+        responseReceivedExtraInfo: [],
+        responses: [],
+        sawResponseWithoutConnectionId: false
+      };
+      this._requests.set(requestId, info);
+    }
+    return info;
+  }
+
+  private _patchHeaders(info: RequestInfo, index: number) {
     const response = info.responses[index];
-    const extraInfo = info.responseReceivedExtraInfo[index];
-    if (response && extraInfo)
-      response.extraHeadersReceived(headersObjectToArray(extraInfo.headers));
+    const requestExtraInfo = info.requestWillBeSentExtraInfo[index];
+    if (response && requestExtraInfo)
+      response.setRawRequestHeaders(headersObjectToArray(requestExtraInfo.headers));
+    const responseExtraInfo = info.responseReceivedExtraInfo[index];
+    if (response && responseExtraInfo)
+      response.setRawResponseHeaders(headersObjectToArray(responseExtraInfo.headers));
   }
 
   private _checkFinished(info: RequestInfo) {

src/server/frames.ts

@@ -280,13 +280,6 @@ export class FrameManager {
     this._inflightRequestFinished(request);
     if (request._isFavicon)
       return;
-    this._dispatchRequestFinished(request, response).catch(() => {});
-  }
-
-  private async _dispatchRequestFinished(request: network.Request, response: network.Response | null) {
-    // Avoid unnecessary microtask, we want to report finished early for regular redirects.
-    if (response?.willWaitForExtraHeaders())
-      await response?.waitForExtraHeadersIfNeeded();
     this._page._browserContext.emit(BrowserContext.Events.RequestFinished, { request, response });
   }
 

src/server/network.ts

@@ -19,6 +19,7 @@ import * as types from './types';
 import { assert } from '../utils/utils';
 import { ManualPromise } from '../utils/async';
 import { SdkObject } from './instrumentation';
+import { NameValue } from '../common/types';
 
 export function filterCookies(cookies: types.NetworkCookie[], urls: string[]): types.NetworkCookie[] {
   const parsedURLs = urls.map(s => new URL(s));
@@ -95,7 +96,7 @@ export class Request extends SdkObject {
   private _resourceType: string;
   private _method: string;
   private _postData: Buffer | null;
-  private _headers: types.HeadersArray;
+  readonly _headers: types.HeadersArray;
   private _headersMap = new Map<string, string>();
   private _frame: frames.Frame;
   private _waitForResponsePromise = new ManualPromise<Response | null>();
@@ -150,6 +151,10 @@ export class Request extends SdkObject {
     return this._headersMap.get(name);
   }
 
+  async rawHeaders(): Promise<NameValue[]> {
+    return this._headers;
+  }
+
   response(): PromiseLike<Response | null> {
     return this._waitForResponsePromise;
   }
@@ -187,18 +192,6 @@ export class Request extends SdkObject {
     };
   }
 
-  updateWithRawHeaders(headers: types.HeadersArray) {
-    this._headers = headers;
-    this._headersMap.clear();
-    for (const { name, value } of this._headers)
-      this._headersMap.set(name.toLowerCase(), value);
-    if (!this._headersMap.has('host')) {
-      const host = new URL(this._url).host;
-      this._headers.push({ name: 'host', value: host });
-      this._headersMap.set('host', host);
-    }
-  }
-
   bodySize(): number {
     return this.postDataBuffer()?.length || 0;
   }
@@ -330,7 +323,8 @@ export class Response extends SdkObject {
   private _timing: ResourceTiming;
   private _serverAddrPromise = new ManualPromise<RemoteAddr | undefined>();
   private _securityDetailsPromise = new ManualPromise<SecurityDetails | undefined>();
-  private _extraHeadersPromise: ManualPromise<void> | undefined;
+  private _rawRequestHeadersPromise: ManualPromise<types.HeadersArray> | undefined;
+  private _rawResponseHeadersPromise: ManualPromise<types.HeadersArray> | undefined;
   private _httpVersion: string | undefined;
 
   constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) {
@@ -365,25 +359,6 @@ export class Response extends SdkObject {
     this._httpVersion = httpVersion;
   }
 
-  setWillReceiveExtraHeaders() {
-    this._extraHeadersPromise = new ManualPromise();
-  }
-
-  willWaitForExtraHeaders(): boolean {
-    return !!this._extraHeadersPromise && !this._extraHeadersPromise.isDone();
-  }
-
-  async waitForExtraHeadersIfNeeded(): Promise<void> {
-    await this._extraHeadersPromise;
-  }
-
-  extraHeadersReceived(headers: types.HeadersArray) {
-    this._headers = headers;
-    for (const { name, value } of this._headers)
-      this._headersMap.set(name.toLowerCase(), value);
-    this._extraHeadersPromise?.resolve();
-  }
-
   url(): string {
     return this._url;
   }
@@ -404,6 +379,31 @@ export class Response extends SdkObject {
     return this._headersMap.get(name);
   }
 
+  async rawRequestHeaders(): Promise<NameValue[]> {
+    return this._rawRequestHeadersPromise || Promise.resolve(this._request._headers);
+  }
+
+  async rawResponseHeaders(): Promise<NameValue[]> {
+    return this._rawResponseHeadersPromise || Promise.resolve(this._headers);
+  }
+
+  setWillReceiveExtraHeaders() {
+    this._rawRequestHeadersPromise = new ManualPromise();
+    this._rawResponseHeadersPromise = new ManualPromise();
+  }
+
+  setRawRequestHeaders(headers: types.HeadersArray) {
+    if (!this._rawRequestHeadersPromise)
+      this._rawRequestHeadersPromise = new ManualPromise();
+    this._rawRequestHeadersPromise!.resolve(headers);
+  }
+
+  setRawResponseHeaders(headers: types.HeadersArray) {
+    if (!this._rawResponseHeadersPromise)
+      this._rawResponseHeadersPromise = new ManualPromise();
+    this._rawResponseHeadersPromise!.resolve(headers);
+  }
+
   timing(): ResourceTiming {
     return this._timing;
   }

src/server/supplements/har/harTracer.ts

@@ -246,9 +246,6 @@ export class HarTracer {
       return;
     const request = response.request();
 
-    // Rewrite provisional headers with actual
-    harEntry.request.headers = request.headers().map(header => ({ name: header.name, value: header.value }));
-    harEntry.request.cookies = cookiesForHar(request.headerValue('cookie'), ';');
     harEntry.request.postData = postDataForHar(request, this._options.content);
 
     harEntry.response = {
@@ -259,7 +256,7 @@ export class HarTracer {
       headers: response.headers().map(header => ({ name: header.name, value: header.value })),
       content: {
         size: -1,
-        mimeType: response.headerValue('content-type') || 'x-unknown',
+        mimeType: 'x-unknown',
       },
       headersSize: -1,
       bodySize: -1,
@@ -293,6 +290,19 @@ export class HarTracer {
       if (details)
         harEntry._securityDetails = details;
     }));
+    this._addBarrier(page, response.rawRequestHeaders().then(headers => {
+      for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie'))
+        harEntry.request.cookies.push(...cookiesForHar(header.value, ';'));
+      harEntry.request.headers = headers;
+    }));
+    this._addBarrier(page, response.rawResponseHeaders().then(headers => {
+      for (const header of headers.filter(header => header.name.toLowerCase() === 'set-cookie'))
+        harEntry.response.cookies.push(...cookiesForHar(header.value, '\n'));
+      harEntry.response.headers = headers;
+      const contentType = headers.find(header => header.name.toLowerCase() === 'content-type');
+      if (contentType)
+        harEntry.response.content.mimeType = contentType.value;
+    }));
   }
 
   async flush() {

src/server/webkit/wkPage.ts

@@ -1017,8 +1017,12 @@ export class WKPage implements PageDelegate {
       return;
     this._requestIdToResponseReceivedPayloadEvent.set(request._requestId, event);
     const response = request.createResponse(event.response);
-    if (event.response.requestHeaders && Object.keys(event.response.requestHeaders).length)
-      request.request.updateWithRawHeaders(headersObjectToArray(event.response.requestHeaders));
+    if (event.response.requestHeaders && Object.keys(event.response.requestHeaders).length) {
+      const headers = { ...event.response.requestHeaders };
+      if (!headers['host'])
+        headers['Host'] = new URL(request.request.url()).host;
+      response.setRawRequestHeaders(headersObjectToArray(headers));
+    }
     this._page._frameManager.requestReceivedResponse(response);
 
     if (response.status() === 204) {

src/utils/multimap.ts

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class MultiMap<K, V> {
+  private _map: Map<K, Set<V>>;
+
+  constructor() {
+    this._map = new Map<K, Set<V>>();
+  }
+
+  set(key: K, value: V) {
+    let set = this._map.get(key);
+    if (!set) {
+      set = new Set<V>();
+      this._map.set(key, set);
+    }
+    set.add(value);
+  }
+
+  get(key: K): Set<V> {
+    return this._map.get(key) || new Set();
+  }
+
+  has(key: K): boolean {
+    return this._map.has(key);
+  }
+
+  hasValue(key: K, value: V): boolean {
+    const set = this._map.get(key);
+    if (!set)
+      return false;
+    return set.has(value);
+  }
+
+  get size(): number {
+    return this._map.size;
+  }
+
+  delete(key: K, value: V): boolean {
+    const values = this.get(key);
+    const result = values.delete(value);
+    if (!values.size)
+      this._map.delete(key);
+    return result;
+  }
+
+  deleteAll(key: K) {
+    this._map.delete(key);
+  }
+
+  keys(): IterableIterator<K> {
+    return this._map.keys();
+  }
+
+  values(): Iterable<V> {
+    const result: V[] = [];
+    for (const key of this.keys())
+      result.push(...Array.from(this.get(key)!));
+    return result;
+  }
+
+  clear() {
+    this._map.clear();
+  }
+}

tests/page/page-network-request.spec.ts

@@ -83,18 +83,20 @@ it('should return headers', async ({page, server, browserName}) => {
 
 it('should get the same headers as the server', async ({ page, server, browserName, platform }) => {
   it.fail(browserName === 'webkit' && platform === 'win32', 'Curl does not show accept-encoding and accept-language');
-  it.fixme(browserName === 'chromium', 'Flaky, see https://github.com/microsoft/playwright/issues/6690');
-
   let serverRequest;
   server.setRoute('/empty.html', (request, response) => {
     serverRequest = request;
     response.end('done');
   });
   const response = await page.goto(server.PREFIX + '/empty.html');
-  expect(response.request().headers()).toEqual(serverRequest.headers);
+  const headers = await response.request().rawHeaders();
+  const result = {};
+  for (const header of headers.headers())
+    result[header.name.toLowerCase()] = header.value;
+  expect(result).toEqual(serverRequest.headers);
 });
 
-it('should get the same headers as the server CORP', async ({page, server, browserName, platform}) => {
+it('should get the same headers as the server CORS', async ({page, server, browserName, platform}) => {
   it.fail(browserName === 'webkit' && platform === 'win32', 'Curl does not show accept-encoding and accept-language');
 
   await page.goto(server.PREFIX + '/empty.html');
@@ -109,9 +111,14 @@ it('should get the same headers as the server CORP', async ({page, server, brows
     const data = await fetch(url);
     return data.text();
   }, server.CROSS_PROCESS_PREFIX + '/something');
-  const response = await responsePromise;
   expect(text).toBe('done');
-  expect(response.request().headers()).toEqual(serverRequest.headers);
+  const response = await responsePromise;
+  const headers = await response.request().rawHeaders();
+  const result = {};
+  for (const header of headers.headers())
+    result[header.name.toLowerCase()] = header.value;
+
+  expect(result).toEqual(serverRequest.headers);
 });
 
 it('should return postData', async ({page, server, isAndroid}) => {
@@ -274,7 +281,7 @@ it('should set bodySize and headersSize', async ({page, server,browserName, plat
   ]);
   await (await request.response()).finished();
   expect(request.sizes().requestBodySize).toBe(5);
-  expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(300);
+  expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(250);
 });
 
 it('should should set bodySize to 0 if there was no body', async ({page, server,browserName, platform}) => {
@@ -285,7 +292,7 @@ it('should should set bodySize to 0 if there was no body', async ({page, server,
   ]);
   await (await request.response()).finished();
   expect(request.sizes().requestBodySize).toBe(0);
-  expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(228);
+  expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(200);
 });
 
 it('should should set bodySize, headersSize, and transferSize', async ({page, server, browserName, platform}) => {
@@ -315,6 +322,19 @@ it('should should set bodySize to 0 when there was no response body', async ({pa
   expect(response.request().sizes().responseTransferSize).toBeGreaterThanOrEqual(160);
 });
 
+it('should report raw headers', async ({ page, server, browserName }) => {
+  const response = await page.goto(server.EMPTY_PAGE);
+  const requestHeaders = await response.request().rawHeaders();
+  expect(requestHeaders.headerNames().map(h => h.toLowerCase())).toContain('accept');
+  expect(requestHeaders.getAll('host')).toHaveLength(1);
+  expect(requestHeaders.get('host')).toBe(`localhost:${server.PORT}`);
+
+  const responseHeaders = await response.rawHeaders();
+  expect(responseHeaders.headerNames().map(h => h.toLowerCase())).toContain('content-type');
+  expect(responseHeaders.getAll('content-type')).toHaveLength(1);
+  expect(responseHeaders.get('content-type')).toBe('text/html; charset=utf-8');
+});
+
 it('should report raw response headers in redirects', async ({ page, server, browserName }) => {
   it.skip(browserName === 'webkit', `WebKit won't give us raw headers for redirects`);
   server.setExtraHeaders('/redirect/1.html', { 'sec-test-header': '1.html' });
@@ -327,14 +347,13 @@ it('should report raw response headers in redirects', async ({ page, server, bro
   const expectedHeaders = ['1.html', '2.html', 'empty.html'];
 
   const response = await page.goto(server.PREFIX + '/redirect/1.html');
-  await response.finished();
-
   const redirectChain = [];
   const headersChain = [];
   for (let req = response.request(); req; req = req.redirectedFrom()) {
     redirectChain.unshift(req.url());
     const res = await req.response();
-    headersChain.unshift(res.headers()['sec-test-header']);
+    const headers = await res.rawHeaders();
+    headersChain.unshift(headers.get('sec-test-header'));
   }
 
   expect(redirectChain).toEqual(expectedUrls);

types/types.d.ts

@@ -12667,6 +12667,32 @@ export interface FileChooser {
   }): Promise<void>;
 }
 
+/**
+ * HTTP request and response raw headers collection.
+ */
+export interface Headers {
+  /**
+   * @param name
+   */
+  get(name: string): string|null;
+
+  /**
+   * Returns all header values for the given header name.
+   * @param name
+   */
+  getAll(name: string): Array<string>;
+
+  /**
+   * Returns all header names in this headers collection.
+   */
+  headerNames(): Array<string>;
+
+  /**
+   * Returns all raw headers.
+   */
+  headers(): Array<{ name: string, value: string }>;
+}
+
 /**
  * Keyboard provides an api for managing a virtual keyboard. The high level api is
  * [keyboard.type(text[, options])](https://playwright.dev/docs/api/class-keyboard#keyboard-type), which takes raw
@@ -13022,7 +13048,8 @@ export interface Request {
   frame(): Frame;
 
   /**
-   * An object with HTTP headers associated with the request. All header names are lower-case.
+   * **DEPRECATED**  Use [request.rawHeaders()](https://playwright.dev/docs/api/class-request#request-raw-headers) instead.
+   * @deprecated
    */
   headers(): { [key: string]: string; };
 
@@ -13054,6 +13081,11 @@ export interface Request {
    */
   postDataJSON(): null|any;
 
+  /**
+   * An object with the raw request HTTP headers associated with the request. All headers are as seen in the network stack.
+   */
+  rawHeaders(): Promise<Headers>;
+
   /**
    * Request that was redirected by the server to this one, if any.
    *
@@ -13229,7 +13261,9 @@ export interface Response {
   frame(): Frame;
 
   /**
-   * Returns the object with HTTP headers associated with the response. All header names are lower-case.
+   * **DEPRECATED**  Use [response.rawHeaders()](https://playwright.dev/docs/api/class-response#response-raw-headers)
+   * instead.
+   * @deprecated
    */
   headers(): { [key: string]: string; };
 
@@ -13245,6 +13279,11 @@ export interface Response {
    */
   ok(): boolean;
 
+  /**
+   * An object with the raw response HTTP headers associated with the request. All headers are as seen in the network stack.
+   */
+  rawHeaders(): Promise<Headers>;
+
   /**
    * Returns the matching [Request] object.
    */