microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-12-02 10:34:27 +03:00
Code Issues Projects Releases Wiki Activity

feat(fetch): support response decompression (#8571)

Browse Source
This commit is contained in:
Yury Semikhatsky 2021-08-31 09:34:58 -07:00 committed by GitHub
parent 4d26fb9ccb
commit 9f8e8444d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 141 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/server/fetch.ts
View File

@ -16,10 +16,12 @@
import { HttpsProxyAgent } from 'https-proxy-agent';
import url from 'url';
import zlib from 'zlib';
import * as http from 'http';
import * as https from 'https';
import { BrowserContext } from './browserContext';
import * as types from './types';
import { pipeline, Readable, Transform } from 'stream';
export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> {
try {
@ -30,7 +32,7 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
}
headers['user-agent'] ??= context._options.userAgent || context._browser.userAgent();
headers['accept'] ??= '*/*';
headers['accept-encoding'] ??= 'gzip,deflate';
headers['accept-encoding'] ??= 'gzip,deflate,br';
if (context._options.extraHTTPHeaders) {
for (const {name, value} of context._options.extraHTTPHeaders)
@ -149,9 +151,31 @@ async function sendRequest(context: BrowserContext, url: URL, options: http.Requ
return;
}
}
response.on('aborted', () => reject(new Error('aborted')));
let body: Readable = response;
let transform: Transform | undefined;
const encoding = response.headers['content-encoding'];
if (encoding === 'gzip' || encoding === 'x-gzip') {
transform = zlib.createGunzip({
flush: zlib.constants.Z_SYNC_FLUSH,
finishFlush: zlib.constants.Z_SYNC_FLUSH
});
} else if (encoding === 'br') {
transform = zlib.createBrotliDecompress();
} else if (encoding === 'deflate') {
transform = zlib.createInflate();
}
if (transform) {
body = pipeline(response, transform, e => {
if (e)
reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`));
});
}
const chunks: Buffer[] = [];
response.on('data', chunk => chunks.push(chunk));
response.on('end', () => {
body.on('data', chunk => chunks.push(chunk));
body.on('end', () => {
const body = Buffer.concat(chunks);
fulfill({
url: response.url || url.toString(),
@ -161,8 +185,7 @@ async function sendRequest(context: BrowserContext, url: URL, options: http.Requ
body
});
});
response.on('aborted', () => reject(new Error('aborted')));
response.on('error',reject);
body.on('error',reject);
});
request.on('error', reject);
if (postData)

115
tests/browsercontext-fetch.spec.ts
View File

@ -15,6 +15,8 @@
*/
import http from 'http';
import zlib from 'zlib';
import { pipeline } from 'stream';
import { contextTest as it, expect } from './config/browserTest';
it.skip(({ mode }) => mode !== 'default');
@ -339,7 +341,7 @@ it('should add default headers', async ({context, server, page}) => {
expect(request.headers['accept']).toBe('*/*');
const userAgent = await page.evaluate(() => navigator.userAgent);
expect(request.headers['user-agent']).toBe(userAgent);
expect(request.headers['accept-encoding']).toBe('gzip,deflate');
expect(request.headers['accept-encoding']).toBe('gzip,deflate,br');
});
it('should add default headers to redirects', async ({context, server, page}) => {
@ -352,7 +354,7 @@ it('should add default headers to redirects', async ({context, server, page}) =>
expect(request.headers['accept']).toBe('*/*');
const userAgent = await page.evaluate(() => navigator.userAgent);
expect(request.headers['user-agent']).toBe(userAgent);
expect(request.headers['accept-encoding']).toBe('gzip,deflate');
expect(request.headers['accept-encoding']).toBe('gzip,deflate,br');
});
it('should allow to override default headers', async ({context, server, page}) => {
@ -444,3 +446,112 @@ it('should resolve url relative to baseURL', async function({browser, server, co
const response = await context._fetch('/empty.html');
expect(response.url()).toBe(server.EMPTY_PAGE);
});
it('should support gzip compression', async function({context, server}) {
server.setRoute('/compressed', (req, res) => {
res.writeHead(200, {
'Content-Encoding': 'gzip',
'Content-Type': 'text/plain',
});
const gzip = zlib.createGzip();
pipeline(gzip, res, err => {
if (err)
console.log(`Server error: ${err}`);
});
gzip.write('Hello, world!');
gzip.end();
});
// @ts-expect-error
const response = await context._fetch(server.PREFIX + '/compressed');
expect(await response.text()).toBe('Hello, world!');
});
it('should throw informatibe error on corrupted gzip body', async function({context, server}) {
server.setRoute('/corrupted', (req, res) => {
res.writeHead(200, {
'Content-Encoding': 'gzip',
'Content-Type': 'text/plain',
});
res.write('Hello, world!');
res.end();
});
// @ts-expect-error
const error = await context._fetch(server.PREFIX + '/corrupted').catch(e => e);
expect(error.message).toContain(`failed to decompress 'gzip' encoding`);
});
it('should support brotli compression', async function({context, server}) {
server.setRoute('/compressed', (req, res) => {
res.writeHead(200, {
'Content-Encoding': 'br',
'Content-Type': 'text/plain',
});
const brotli = zlib.createBrotliCompress();
pipeline(brotli, res, err => {
if (err)
console.log(`Server error: ${err}`);
});
brotli.write('Hello, world!');
brotli.end();
});
// @ts-expect-error
const response = await context._fetch(server.PREFIX + '/compressed');
expect(await response.text()).toBe('Hello, world!');
});
it('should throw informatibe error on corrupted brotli body', async function({context, server}) {
server.setRoute('/corrupted', (req, res) => {
res.writeHead(200, {
'Content-Encoding': 'br',
'Content-Type': 'text/plain',
});
res.write('Hello, world!');
res.end();
});
// @ts-expect-error
const error = await context._fetch(server.PREFIX + '/corrupted').catch(e => e);
expect(error.message).toContain(`failed to decompress 'br' encoding`);
});
it('should support deflate compression', async function({context, server}) {
server.setRoute('/compressed', (req, res) => {
res.writeHead(200, {
'Content-Encoding': 'deflate',
'Content-Type': 'text/plain',
});
const deflate = zlib.createDeflate();
pipeline(deflate, res, err => {
if (err)
console.log(`Server error: ${err}`);
});
deflate.write('Hello, world!');
deflate.end();
});
// @ts-expect-error
const response = await context._fetch(server.PREFIX + '/compressed');
expect(await response.text()).toBe('Hello, world!');
});
it('should throw informatibe error on corrupted deflate body', async function({context, server}) {
server.setRoute('/corrupted', (req, res) => {
res.writeHead(200, {
'Content-Encoding': 'deflate',
'Content-Type': 'text/plain',
});
res.write('Hello, world!');
res.end();
});
// @ts-expect-error
const error = await context._fetch(server.PREFIX + '/corrupted').catch(e => e);
expect(error.message).toContain(`failed to decompress 'deflate' encoding`);
});