mirror of
https://github.com/microsoft/playwright.git
synced 2024-10-26 21:33:38 +03:00
fix: remove node-fetch dependency, use custom fetch implementation (#8486)
This commit is contained in:
Yury Semikhatsky
GitHub
parent
998f2ab959
commit
210ad72228
18
package-lock.json
generated
18
package-lock.json
generated
@ -36,7 +36,6 @@
|
||||
"mime": "^2.4.6",
|
||||
"minimatch": "^3.0.3",
|
||||
"ms": "^2.1.2",
|
||||
"node-fetch": "^2.6.1",
|
||||
"pirates": "^4.0.1",
|
||||
"pixelmatch": "^5.2.1",
|
||||
"pngjs": "^5.0.0",
|
||||
@ -6967,14 +6966,6 @@
|
||||
"integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-libs-browser": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz",
|
||||
@ -8596,7 +8587,6 @@
|
||||
},
|
||||
"node_modules/socksv5/node_modules/ipv6": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ipv6/-/ipv6-3.1.1.tgz",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
@ -8615,7 +8605,6 @@
|
||||
},
|
||||
"node_modules/socksv5/node_modules/ipv6/node_modules/sprintf": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf/-/sprintf-0.1.3.tgz",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"engines": {
|
||||
@ -15961,11 +15950,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
},
|
||||
"node-libs-browser": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz",
|
||||
@ -17275,7 +17259,6 @@
|
||||
"dependencies": {
|
||||
"ipv6": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ipv6/-/ipv6-3.1.1.tgz",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -17286,7 +17269,6 @@
|
||||
"dependencies": {
|
||||
"sprintf": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf/-/sprintf-0.1.3.tgz",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
}
|
||||
|
||||
@ -67,7 +67,6 @@
|
||||
"mime": "^2.4.6",
|
||||
"minimatch": "^3.0.3",
|
||||
"ms": "^2.1.2",
|
||||
"node-fetch": "^2.6.1",
|
||||
"pirates": "^4.0.1",
|
||||
"pixelmatch": "^5.2.1",
|
||||
"pngjs": "^5.0.0",
|
||||
@ -91,7 +90,6 @@
|
||||
"@types/mime": "^2.0.3",
|
||||
"@types/minimatch": "^3.0.3",
|
||||
"@types/node": "^10.17.28",
|
||||
"@types/node-fetch": "^2.5.12",
|
||||
"@types/pixelmatch": "^5.2.1",
|
||||
"@types/pngjs": "^3.4.2",
|
||||
"@types/progress": "^2.0.3",
|
||||
|
||||
@ -72,7 +72,6 @@ const DEPENDENCIES = [
|
||||
'https-proxy-agent',
|
||||
'jpeg-js',
|
||||
'mime',
|
||||
'node-fetch',
|
||||
'pngjs',
|
||||
'progress',
|
||||
'proper-lockfile',
|
||||
|
||||
@ -15,8 +15,9 @@
|
||||
*/
|
||||
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
||||
import nodeFetch from 'node-fetch';
|
||||
import * as url from 'url';
|
||||
import url from 'url';
|
||||
import * as http from 'http';
|
||||
import * as https from 'https';
|
||||
import { BrowserContext } from './browserContext';
|
||||
import * as types from './types';
|
||||
|
||||
@ -45,54 +46,130 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
|
||||
}
|
||||
|
||||
// TODO(https://github.com/microsoft/playwright/issues/8381): set user agent
|
||||
const response = await nodeFetch(params.url, {
|
||||
const {fetchResponse, setCookie} = await sendRequest(new URL(params.url), {
|
||||
method: params.method,
|
||||
headers: params.headers,
|
||||
body: params.postData,
|
||||
agent
|
||||
});
|
||||
const body = await response.buffer();
|
||||
const setCookies = response.headers.raw()['set-cookie'];
|
||||
if (setCookies) {
|
||||
const url = new URL(response.url);
|
||||
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
|
||||
const defaultPath = '/' + url.pathname.split('/').slice(0, -1).join('/');
|
||||
const cookies: types.SetNetworkCookieParam[] = [];
|
||||
for (const header of setCookies) {
|
||||
// Decode cookie value?
|
||||
const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
|
||||
if (!cookie)
|
||||
continue;
|
||||
if (!cookie.domain)
|
||||
cookie.domain = url.hostname;
|
||||
if (!canSetCookie(cookie.domain!, url.hostname))
|
||||
continue;
|
||||
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
|
||||
if (!cookie.path || !cookie.path.startsWith('/'))
|
||||
cookie.path = defaultPath;
|
||||
cookies.push(cookie);
|
||||
}
|
||||
if (cookies.length)
|
||||
await context.addCookies(cookies);
|
||||
}
|
||||
|
||||
const headers: types.HeadersArray = [];
|
||||
for (const [name, value] of response.headers.entries())
|
||||
headers.push({ name, value });
|
||||
return {
|
||||
fetchResponse: {
|
||||
url: response.url,
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers,
|
||||
body
|
||||
}
|
||||
};
|
||||
agent,
|
||||
maxRedirects: 20
|
||||
}, params.postData);
|
||||
if (setCookie)
|
||||
await updateCookiesFromHeader(context, fetchResponse.url, setCookie);
|
||||
return { fetchResponse };
|
||||
} catch (e) {
|
||||
return { error: String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
async function updateCookiesFromHeader(context: BrowserContext, responseUrl: string, setCookie: string[]) {
|
||||
const url = new URL(responseUrl);
|
||||
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
|
||||
const defaultPath = '/' + url.pathname.split('/').slice(0, -1).join('/');
|
||||
const cookies: types.SetNetworkCookieParam[] = [];
|
||||
for (const header of setCookie) {
|
||||
// Decode cookie value?
|
||||
const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
|
||||
if (!cookie)
|
||||
continue;
|
||||
if (!cookie.domain)
|
||||
cookie.domain = url.hostname;
|
||||
if (!canSetCookie(cookie.domain!, url.hostname))
|
||||
continue;
|
||||
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
|
||||
if (!cookie.path || !cookie.path.startsWith('/'))
|
||||
cookie.path = defaultPath;
|
||||
cookies.push(cookie);
|
||||
}
|
||||
if (cookies.length)
|
||||
await context.addCookies(cookies);
|
||||
}
|
||||
|
||||
type Response = {
|
||||
fetchResponse: types.FetchResponse,
|
||||
setCookie?: string[]
|
||||
};
|
||||
|
||||
async function sendRequest(url: URL, options: http.RequestOptions & { maxRedirects: number }, postData?: Buffer): Promise<Response>{
|
||||
return new Promise<Response>((fulfill, reject) => {
|
||||
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
|
||||
= (url.protocol === 'https:' ? https : http).request;
|
||||
const request = requestConstructor(url, options, response => {
|
||||
if (redirectStatus.includes(response.statusCode!)) {
|
||||
if (!options.maxRedirects) {
|
||||
reject(new Error('Max redirect count exceeded'));
|
||||
request.abort();
|
||||
return;
|
||||
}
|
||||
const redirectOptions: http.RequestOptions & { maxRedirects: number } = {
|
||||
method: options.method,
|
||||
headers: { ...options.headers },
|
||||
agent: options.agent,
|
||||
maxRedirects: options.maxRedirects - 1,
|
||||
};
|
||||
|
||||
// HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch)
|
||||
const status = response.statusCode!;
|
||||
const method = redirectOptions.method!;
|
||||
if ((status === 301 || status === 302) && method === 'POST' ||
|
||||
status === 303 && !['GET', 'HEAD'].includes(method)) {
|
||||
redirectOptions.method = 'GET';
|
||||
postData = undefined;
|
||||
delete redirectOptions.headers?.[`content-encoding`];
|
||||
delete redirectOptions.headers?.[`content-language`];
|
||||
delete redirectOptions.headers?.[`content-location`];
|
||||
delete redirectOptions.headers?.[`content-type`];
|
||||
}
|
||||
|
||||
// TODO: set-cookie from response, add cookie from the context.
|
||||
|
||||
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
|
||||
if (response.headers.location) {
|
||||
const locationURL = new URL(response.headers.location, url);
|
||||
fulfill(sendRequest(locationURL, redirectOptions, postData));
|
||||
request.abort();
|
||||
return;
|
||||
}
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
response.on('data', chunk => chunks.push(chunk));
|
||||
response.on('end', () => {
|
||||
const body = Buffer.concat(chunks);
|
||||
fulfill({
|
||||
fetchResponse: {
|
||||
url: response.url || url.toString(),
|
||||
status: response.statusCode || 0,
|
||||
statusText: response.statusMessage || '',
|
||||
headers: flattenHeaders(response.headers),
|
||||
body
|
||||
},
|
||||
setCookie: response.headers['set-cookie']
|
||||
});
|
||||
});
|
||||
response.on('error',reject);
|
||||
});
|
||||
request.on('error', reject);
|
||||
if (postData)
|
||||
request.write(postData);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function flattenHeaders(headers: http.IncomingHttpHeaders): types.HeadersArray {
|
||||
const result: types.HeadersArray = [];
|
||||
for (const [name, values] of Object.entries(headers)) {
|
||||
if (values === undefined)
|
||||
continue;
|
||||
if (typeof values === 'string') {
|
||||
result.push({name, value: values as string});
|
||||
} else {
|
||||
for (const value of values)
|
||||
result.push({name, value});
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const redirectStatus = [301, 302, 303, 307, 308];
|
||||
|
||||
function canSetCookie(cookieDomain: string, hostname: string) {
|
||||
// TODO: check public suffix list?
|
||||
hostname = '.' + hostname;
|
||||
@ -101,7 +178,6 @@ function canSetCookie(cookieDomain: string, hostname: string) {
|
||||
return hostname.endsWith(cookieDomain);
|
||||
}
|
||||
|
||||
|
||||
function parseCookie(header: string) {
|
||||
const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim()));
|
||||
if (!pairs.length)
|
||||
|
||||
@ -379,9 +379,6 @@ export type FetchResponse = {
|
||||
url: string,
|
||||
status: number,
|
||||
statusText: string,
|
||||
headers: {
|
||||
name: string,
|
||||
value: string,
|
||||
}[],
|
||||
headers: HeadersArray,
|
||||
body: Buffer,
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user