chore: add more tests for Request.sizes() (#8686)

docs/src/api/class-request.md

@@ -192,9 +192,8 @@ Returns the matching [Response] object, or `null` if the response was not receiv
 - returns: <[Object]>
   - `requestBodySize` <[int]> Size of the request body (POST data payload) in bytes. Set to 0 if there was no body.
   - `requestHeadersSize` <[int]> Total number of bytes from the start of the HTTP request message until (and including) the double CRLF before the body.
-  - `responseBodySize` <[int]> Size of the received response body in bytes.
+  - `responseBodySize` <[int]> Size of the received response body (encoded) in bytes.
   - `responseHeadersSize` <[int]> Total number of bytes from the start of the HTTP response message until (and including) the double CRLF before the body.
-  - `responseTransferSize` <[int]> Total number of bytes received for the request.
 
 Returns resource size information for given request. Requires the response to be finished via [`method: Response.finished`]
 to ensure the info is available.

src/client/network.ts

@@ -407,7 +407,6 @@ export type RequestSizes = {
   requestHeadersSize: number;
   responseBodySize: number;
   responseHeadersSize: number;
-  responseTransferSize: number;
 };
 
 export class Response extends ChannelOwner<channels.ResponseChannel, channels.ResponseInitializer> implements api.Response {

src/protocol/channels.ts

@@ -2753,7 +2753,6 @@ export type RequestSizes = {
   requestHeadersSize: number,
   responseBodySize: number,
   responseHeadersSize: number,
-  responseTransferSize: number,
 };
 
 export type RemoteAddr = {

src/protocol/protocol.yml

@@ -2259,7 +2259,6 @@ RequestSizes:
     requestHeadersSize: number
     responseBodySize: number
     responseHeadersSize: number
-    responseTransferSize: number
 
 
 RemoteAddr:

src/protocol/validator.ts

@@ -1072,7 +1072,6 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     requestHeadersSize: tNumber,
     responseBodySize: tNumber,
     responseHeadersSize: tNumber,
-    responseTransferSize: tNumber,
   });
   scheme.RemoteAddr = tObject({
     ipAddress: tString,

src/server/chromium/crNetworkManager.ts

@@ -354,7 +354,7 @@ export class CRNetworkManager {
   _onDataReceived(event: Protocol.Network.dataReceivedPayload) {
     const request = this._requestIdToRequest.get(event.requestId);
     if (request)
-      request.request.responseSize.bodySize += event.dataLength;
+      request.request.responseSize.encodedBodySize += event.encodedDataLength;
   }
 
   _onLoadingFinished(event: Protocol.Network.loadingFinishedPayload) {

src/server/network.ts

@@ -81,7 +81,6 @@ export function stripFragmentFromUrl(url: string): string {
 }
 
 type ResponseSize = {
-  bodySize: number;
   encodedBodySize: number;
   transferSize: number;
 };
@@ -102,7 +101,7 @@ export class Request extends SdkObject {
   private _frame: frames.Frame;
   private _waitForResponsePromise = new ManualPromise<Response | null>();
   _responseEndTiming = -1;
-  readonly responseSize: ResponseSize = { bodySize: 0, encodedBodySize: 0, transferSize: 0 };
+  readonly responseSize: ResponseSize = { encodedBodySize: 0, transferSize: 0 };
 
   constructor(frame: frames.Frame, redirectedFrom: Request | null, documentId: string | undefined,
     url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) {
@@ -281,7 +280,6 @@ export type ResourceSizes = {
   requestHeadersSize: number,
   responseBodySize: number,
   responseHeadersSize: number,
-  responseTransferSize: number,
 };
 
 export type RemoteAddr = {
@@ -461,29 +459,17 @@ export class Response extends SdkObject {
     await this._finishedPromise;
     const requestHeadersSize = await this._requestHeadersSize();
     const responseHeadersSize = await this._responseHeadersSize();
-    let { bodySize, encodedBodySize, transferSize } = this._request.responseSize;
-    if (!bodySize) {
+    let { encodedBodySize } = this._request.responseSize;
+    if (!encodedBodySize) {
       const headers = await this._bestEffortResponseHeaders();
       const contentLength = headers.find(h => h.name.toLowerCase() === 'content-length')?.value;
-      bodySize = contentLength ? +contentLength : 0;
-    }
-    if (!encodedBodySize && transferSize) {
-      // Chromium only populates transferSize
-      // Firefox can return 0 transferSize
-      encodedBodySize = Math.max(0, transferSize - responseHeadersSize);
-      // Firefox only populate transferSize.
-      if (!bodySize)
-        bodySize = encodedBodySize;
-    } else if (!transferSize) {
-      // WebKit does not provide transfer size.
-      transferSize = encodedBodySize + responseHeadersSize;
+      encodedBodySize = contentLength ? +contentLength : 0;
     }
     return {
       requestBodySize: this._request.bodySize(),
       requestHeadersSize,
-      responseBodySize: bodySize,
+      responseBodySize: encodedBodySize,
       responseHeadersSize,
-      responseTransferSize: transferSize,
     };
   }
 }

src/server/supplements/har/harTracer.ts

@@ -22,6 +22,7 @@ import * as har from './har';
 import { calculateSha1, monotonicTime } from '../../../utils/utils';
 import { eventsHelper, RegisteredListener } from '../../../utils/eventsHelper';
 import * as mime from 'mime';
+import { ManualPromise } from '../../../utils/async';
 
 const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
 
@@ -205,9 +206,31 @@ export class HarTracer {
     harEntry.request.httpVersion = httpVersion;
     harEntry.response.httpVersion = httpVersion;
 
+    const compressionCalculationBarrier = {
+      _encodedBodySize: -1,
+      _decodedBodySize: -1,
+      barrier: new ManualPromise<void>(),
+      _check: function() {
+        if (this._encodedBodySize !== -1 && this._decodedBodySize !== -1) {
+          harEntry.response.content.compression = Math.max(0, this._decodedBodySize - this._encodedBodySize);
+          this.barrier.resolve();
+        }
+      },
+      setEncodedBodySize: function(encodedBodySize: number){
+        this._encodedBodySize = encodedBodySize;
+        this._check();
+      },
+      setDecodedBodySize: function(decodedBodySize: number) {
+        this._decodedBodySize = decodedBodySize;
+        this._check();
+      }
+    };
+    this._addBarrier(page, compressionCalculationBarrier.barrier);
+
     const promise = response.body().then(buffer => {
       const content = harEntry.response.content;
       content.size = buffer.length;
+      compressionCalculationBarrier.setDecodedBodySize(buffer.length);
       if (buffer && buffer.length > 0) {
         if (this._options.content === 'embedded') {
           content.text = buffer.toString('base64');
@@ -218,7 +241,9 @@ export class HarTracer {
             this._delegate.onContentBlob(content._sha1, buffer);
         }
       }
-    }).catch(() => {}).then(() => {
+    }).catch(() => {
+      compressionCalculationBarrier.setDecodedBodySize(0);
+    }).then(() => {
       const postData = response.request().postDataBuffer();
       if (postData && harEntry.request.postData && this._options.content === 'sha1') {
         harEntry.request.postData._sha1 = calculateSha1(postData) + '.' + (mime.getExtension(harEntry.request.postData.mimeType) || 'dat');
@@ -229,13 +254,13 @@ export class HarTracer {
         this._delegate.onEntryFinished(harEntry);
     });
     this._addBarrier(page, promise);
-    this._addBarrier(page, response.sizes().then(async sizes => {
+    this._addBarrier(page, response.sizes().then(sizes => {
       harEntry.response.bodySize = sizes.responseBodySize;
       harEntry.response.headersSize = sizes.responseHeadersSize;
-      harEntry.response._transferSize = sizes.responseTransferSize;
+      // Fallback for WebKit by calculating it manually
+      harEntry.response._transferSize = response.request().responseSize.transferSize || (sizes.responseHeadersSize + sizes.responseBodySize);
       harEntry.request.headersSize = sizes.requestHeadersSize;
-      const content = harEntry.response.content;
-      content.compression = Math.max(0, sizes.responseBodySize - sizes.responseTransferSize - sizes.responseHeadersSize);
+      compressionCalculationBarrier.setEncodedBodySize(sizes.responseBodySize);
     }));
   }
 

src/server/webkit/wkPage.ts

@@ -379,7 +379,6 @@ export class WKPage implements PageDelegate {
       eventsHelper.addEventListener(this._session, 'Network.responseReceived', e => this._onResponseReceived(e)),
       eventsHelper.addEventListener(this._session, 'Network.loadingFinished', e => this._onLoadingFinished(e)),
       eventsHelper.addEventListener(this._session, 'Network.loadingFailed', e => this._onLoadingFailed(e)),
-      eventsHelper.addEventListener(this._session, 'Network.dataReceived', e => this._onDataReceived(e)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketCreated', e => this._page._frameManager.onWebSocketCreated(e.requestId, e.url)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketWillSendHandshakeRequest', e => this._page._frameManager.onWebSocketRequest(e.requestId)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketHandshakeResponseReceived', e => this._page._frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
@@ -1083,14 +1082,6 @@ export class WKPage implements PageDelegate {
     this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled'));
   }
 
-  _onDataReceived(event: Protocol.Network.dataReceivedPayload) {
-    const request = this._requestIdToRequest.get(event.requestId);
-    if (!request)
-      return;
-    request.request.responseSize.bodySize += event.dataLength || (event.encodedDataLength === -1 ? 0 : event.encodedDataLength);
-    request.request.responseSize.encodedBodySize += event.encodedDataLength !== -1 ? event.encodedDataLength : event.dataLength;
-  }
-
   async _grantPermissions(origin: string, permissions: string[]) {
     const webPermissionToProtocol = new Map<string, string>([
       ['geolocation', 'geolocation'],

tests/har.spec.ts

@@ -280,7 +280,6 @@ it('should include sizes', async ({ contextFactory, server, asset }, testInfo) =
 });
 
 it('should work with gzip compression', async ({ contextFactory, server, browserName }, testInfo) => {
-  it.fixme(browserName !== 'chromium');
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   server.enableGzip('/simplezip.json');
   const response = await page.goto(server.PREFIX + '/simplezip.json');

tests/page/page-network-sizes.spec.ts

@@ -15,13 +15,16 @@
  * limitations under the License.
  */
 
+import fs from 'fs';
+import zlib from 'zlib';
+
 import { test as it, expect } from './pageTest';
 
-it('should set bodySize and headersSize', async ({page, server,browserName, platform}) => {
+it('should set bodySize and headersSize', async ({ page, server }) => {
   await page.goto(server.EMPTY_PAGE);
   const [request] = await Promise.all([
     page.waitForEvent('request'),
-    page.evaluate(() => fetch('./get', { method: 'POST', body: '12345'}).then(r => r.text())),
+    page.evaluate(() => fetch('./get', { method: 'POST', body: '12345' }).then(r => r.text())),
   ]);
   await (await request.response()).finished();
   const sizes = await request.sizes();
@@ -29,7 +32,7 @@ it('should set bodySize and headersSize', async ({page, server,browserName, plat
   expect(sizes.requestHeadersSize).toBeGreaterThanOrEqual(250);
 });
 
-it('should set bodySize to 0 if there was no body', async ({page, server,browserName, platform}) => {
+it('should set bodySize to 0 if there was no body', async ({ page, server, browserName, platform }) => {
   await page.goto(server.EMPTY_PAGE);
   const [request] = await Promise.all([
     page.waitForEvent('request'),
@@ -40,7 +43,7 @@ it('should set bodySize to 0 if there was no body', async ({page, server,browser
   expect(sizes.requestHeadersSize).toBeGreaterThanOrEqual(200);
 });
 
-it('should set bodySize, headersSize, and transferSize', async ({page, server, browserName, platform}) => {
+it('should set bodySize, headersSize, and transferSize', async ({ page, server }) => {
   server.setRoute('/get', (req, res) => {
     // In Firefox, |fetch| will be hanging until it receives |Content-Type| header
     // from server.
@@ -56,13 +59,88 @@ it('should set bodySize, headersSize, and transferSize', async ({page, server, b
   const sizes = await response.request().sizes();
   expect(sizes.responseBodySize).toBe(6);
   expect(sizes.responseHeadersSize).toBeGreaterThanOrEqual(100);
-  expect(sizes.responseTransferSize).toBeGreaterThanOrEqual(100);
 });
 
-it('should set bodySize to 0 when there was no response body', async ({page, server, browserName, platform}) => {
+it('should set bodySize to 0 when there was no response body', async ({ page, server }) => {
   const response = await page.goto(server.EMPTY_PAGE);
   const sizes = await response.request().sizes();
   expect(sizes.responseBodySize).toBe(0);
   expect(sizes.responseHeadersSize).toBeGreaterThanOrEqual(150);
-  expect(sizes.responseTransferSize).toBeGreaterThanOrEqual(160);
 });
+
+it('should have the correct responseBodySize', async ({ page, server, asset, browserName }) => {
+  const response = await page.goto(server.PREFIX + '/simplezip.json');
+  const sizes = await response.request().sizes();
+  expect(sizes.responseBodySize).toBe(fs.statSync(asset('simplezip.json')).size);
+});
+
+it('should have the correct responseBodySize with gzip compression', async ({ page, server, asset }, testInfo) => {
+  server.enableGzip('/simplezip.json');
+  await page.goto(server.EMPTY_PAGE);
+  const [response] = await Promise.all([
+    page.waitForEvent('response'),
+    page.evaluate(() => fetch('./simplezip.json').then(r => r.text()))
+  ]);
+  const sizes = await response.request().sizes();
+
+  const chunks: Buffer[] = [];
+  const gzip = fs.createReadStream(asset('simplezip.json')).pipe(zlib.createGzip());
+  const done = new Promise(resolve => gzip.on('end', resolve));
+  gzip.on('data', o => chunks.push(o));
+  await done;
+
+  expect(sizes.responseBodySize).toBe(Buffer.concat(chunks).length);
+});
+
+it('should handle redirects', async ({ page, server }) => {
+  server.setRedirect('/foo', '/bar');
+  server.setRoute('/bar', (req, resp) => resp.end('bar'));
+  await page.goto(server.EMPTY_PAGE);
+  const [response] = await Promise.all([
+    page.waitForEvent('response'),
+    page.evaluate(async () => fetch('/foo', {
+      method: 'POST',
+      body: '12345',
+    }).then(r => r.text())),
+  ]);
+  expect((await response.request().sizes()).requestBodySize).toBe(5);
+  const newRequest = response.request().redirectedTo();
+  expect((await newRequest.sizes()).responseBodySize).toBe(3);
+});
+
+it('should throw for failed requests', async ({ page, server }) => {
+  server.setRoute('/one-style.css', (req, res) => {
+    res.setHeader('Content-Type', 'text/css');
+    res.connection.destroy();
+  });
+  await page.goto(server.EMPTY_PAGE);
+  const [request] = await Promise.all([
+    page.waitForEvent('requestfailed'),
+    page.goto(server.PREFIX + '/one-style.html')
+  ]);
+  await expect(request.sizes()).rejects.toThrow('Unable to fetch sizes for failed request');
+});
+
+for (const statusCode of [200, 401, 404, 500]) {
+  it(`should work with ${statusCode} status code`, async ({ page, server }) => {
+    server.setRoute('/foo', (req, resp) => {
+      resp.writeHead(statusCode, {
+        'Content-Type': 'text/plain; charset=utf-8',
+        'Content-Length': '3',
+      });
+      resp.end('bar');
+    });
+    await page.goto(server.EMPTY_PAGE);
+    const [response] = await Promise.all([
+      page.waitForEvent('response'),
+      page.evaluate(async () => fetch('/foo', {
+        method: 'POST',
+        body: '12345',
+      }).then(r => r.text())),
+    ]);
+    expect(response.status()).toBe(statusCode);
+    const sizes = await response.request().sizes();
+    expect(sizes.requestBodySize).toBe(5);
+    expect(sizes.responseBodySize).toBe(3);
+  });
+}

types/types.d.ts

@@ -13176,7 +13176,7 @@ export interface Request {
     requestHeadersSize: number;
 
     /**
-     * Size of the received response body in bytes.
+     * Size of the received response body (encoded) in bytes.
      */
     responseBodySize: number;
 
@@ -13184,11 +13184,6 @@ export interface Request {
      * Total number of bytes from the start of the HTTP response message until (and including) the double CRLF before the body.
      */
     responseHeadersSize: number;
-
-    /**
-     * Total number of bytes received for the request.
-     */
-    responseTransferSize: number;
   }>;
 
   /**