mirror of
https://github.com/microsoft/playwright.git
synced 2024-12-14 13:45:36 +03:00
chore: use utils via index export (2) (#13399)
This commit is contained in:
Pavel Feldman
GitHub
parent
dbafe3bb4e
commit
6c1fa62000
@ -30,8 +30,9 @@ import { CRDevTools } from './crDevTools';
|
||||
import type { BrowserOptions, BrowserProcess, PlaywrightOptions } from '../browser';
|
||||
import { Browser } from '../browser';
|
||||
import type * as types from '../types';
|
||||
import type { HTTPRequestParams } from '../../utils/utils';
|
||||
import { debugMode, fetchData, getUserAgent, headersArrayToObject, removeFolders, streamToString, wrapInASCIIBox } from '../../utils/utils';
|
||||
import type { HTTPRequestParams } from '../../utils';
|
||||
import { fetchData } from '../../utils';
|
||||
import { debugMode, getUserAgent, headersArrayToObject, removeFolders, streamToString, wrapInASCIIBox } from '../../utils/utils';
|
||||
import { RecentLogsCollector } from '../../utils/debugLogger';
|
||||
import type { Progress } from '../progress';
|
||||
import { ProgressController } from '../progress';
|
||||
|
||||
@ -19,8 +19,9 @@ import extract from 'extract-zip';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { existsAsync, download, getUserAgent } from '../../utils/utils';
|
||||
import { existsAsync, getUserAgent } from '../../utils/utils';
|
||||
import { debugLogger } from '../../utils/debugLogger';
|
||||
import { download } from './download';
|
||||
|
||||
export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise<boolean> {
|
||||
const progressBarName = `Playwright build of ${title}`;
|
||||
|
||||
146
packages/playwright-core/src/server/registry/download.ts
Normal file
146
packages/playwright-core/src/server/registry/download.ts
Normal file
@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import ProgressBar from 'progress';
|
||||
import { httpRequest } from '../../utils';
|
||||
|
||||
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
|
||||
type DownloadFileLogger = (message: string) => void;
|
||||
type DownloadFileOptions = {
|
||||
progressCallback?: OnProgressCallback,
|
||||
log?: DownloadFileLogger,
|
||||
userAgent?: string
|
||||
};
|
||||
|
||||
function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> {
|
||||
const {
|
||||
progressCallback,
|
||||
log = () => {},
|
||||
} = options;
|
||||
log(`running download:`);
|
||||
log(`-- from url: ${url}`);
|
||||
log(`-- to location: ${destinationPath}`);
|
||||
let fulfill: ({ error }: {error: any}) => void = ({ error }) => {};
|
||||
let downloadedBytes = 0;
|
||||
let totalBytes = 0;
|
||||
|
||||
const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; });
|
||||
|
||||
httpRequest({
|
||||
url,
|
||||
headers: options.userAgent ? {
|
||||
'User-Agent': options.userAgent,
|
||||
} : undefined,
|
||||
}, response => {
|
||||
log(`-- response status code: ${response.statusCode}`);
|
||||
if (response.statusCode !== 200) {
|
||||
const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
|
||||
// consume response data to free up memory
|
||||
response.resume();
|
||||
fulfill({ error });
|
||||
return;
|
||||
}
|
||||
const file = fs.createWriteStream(destinationPath);
|
||||
file.on('finish', () => fulfill({ error: null }));
|
||||
file.on('error', error => fulfill({ error }));
|
||||
response.pipe(file);
|
||||
totalBytes = parseInt(response.headers['content-length'] || '0', 10);
|
||||
log(`-- total bytes: ${totalBytes}`);
|
||||
if (progressCallback)
|
||||
response.on('data', onData);
|
||||
}, (error: any) => fulfill({ error }));
|
||||
return promise;
|
||||
|
||||
function onData(chunk: string) {
|
||||
downloadedBytes += chunk.length;
|
||||
progressCallback!(downloadedBytes, totalBytes);
|
||||
}
|
||||
}
|
||||
|
||||
type DownloadOptions = {
|
||||
progressBarName?: string,
|
||||
retryCount?: number
|
||||
log?: DownloadFileLogger
|
||||
userAgent?: string
|
||||
};
|
||||
|
||||
export async function download(
|
||||
url: string,
|
||||
destination: string,
|
||||
options: DownloadOptions = {}
|
||||
) {
|
||||
const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options;
|
||||
for (let attempt = 1; attempt <= retryCount; ++attempt) {
|
||||
log(
|
||||
`downloading ${progressBarName} - attempt #${attempt}`
|
||||
);
|
||||
const { error } = await downloadFile(url, destination, {
|
||||
progressCallback: getDownloadProgress(progressBarName),
|
||||
log,
|
||||
userAgent,
|
||||
});
|
||||
if (!error) {
|
||||
log(`SUCCESS downloading ${progressBarName}`);
|
||||
break;
|
||||
}
|
||||
const errorMessage = error?.message || '';
|
||||
log(`attempt #${attempt} - ERROR: ${errorMessage}`);
|
||||
if (
|
||||
attempt < retryCount &&
|
||||
(errorMessage.includes('ECONNRESET') ||
|
||||
errorMessage.includes('ETIMEDOUT'))
|
||||
) {
|
||||
// Maximum default delay is 3rd retry: 1337.5ms
|
||||
const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt);
|
||||
log(`sleeping ${millis}ms before retry...`);
|
||||
await new Promise(c => setTimeout(c, millis));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getDownloadProgress(progressBarName: string): OnProgressCallback {
|
||||
let progressBar: ProgressBar;
|
||||
let lastDownloadedBytes = 0;
|
||||
|
||||
return (downloadedBytes: number, totalBytes: number) => {
|
||||
if (!process.stderr.isTTY)
|
||||
return;
|
||||
if (!progressBar) {
|
||||
progressBar = new ProgressBar(
|
||||
`Downloading ${progressBarName} - ${toMegabytes(
|
||||
totalBytes
|
||||
)} [:bar] :percent :etas `,
|
||||
{
|
||||
complete: '=',
|
||||
incomplete: ' ',
|
||||
width: 20,
|
||||
total: totalBytes,
|
||||
}
|
||||
);
|
||||
}
|
||||
const delta = downloadedBytes - lastDownloadedBytes;
|
||||
lastDownloadedBytes = downloadedBytes;
|
||||
progressBar.tick(delta);
|
||||
};
|
||||
}
|
||||
|
||||
function toMegabytes(bytes: number) {
|
||||
const mb = bytes / 1024 / 1024;
|
||||
return `${Math.round(mb * 10) / 10} Mb`;
|
||||
}
|
||||
@ -21,7 +21,8 @@ import * as util from 'util';
|
||||
import * as fs from 'fs';
|
||||
import lockfile from 'proper-lockfile';
|
||||
import { getUbuntuVersion } from '../../utils/ubuntuVersion';
|
||||
import { getFromENV, getAsBooleanFromENV, getClientLanguage, calculateSha1, removeFolders, existsAsync, hostPlatform, canAccessFile, spawnAsync, fetchData, wrapInASCIIBox, transformCommandsForRoot } from '../../utils/utils';
|
||||
import { fetchData } from '../../utils';
|
||||
import { getFromENV, getAsBooleanFromENV, getClientLanguage, calculateSha1, removeFolders, existsAsync, hostPlatform, canAccessFile, spawnAsync, wrapInASCIIBox, transformCommandsForRoot } from '../../utils/utils';
|
||||
import type { DependencyGroup } from './dependencies';
|
||||
import { installDependenciesLinux, installDependenciesWindows, validateDependenciesLinux, validateDependenciesWindows } from './dependencies';
|
||||
import { downloadBrowserWithProgressBar, logPolitely } from './browserFetcher';
|
||||
|
||||
@ -17,3 +17,5 @@
|
||||
export { ManualPromise } from './manualPromise';
|
||||
export { MultiMap } from './multimap';
|
||||
export { raceAgainstTimeout, TimeoutRunner, TimeoutRunnerError } from './timeoutRunner';
|
||||
export type { HTTPRequestParams } from './netUtils';
|
||||
export { httpRequest, fetchData } from './netUtils';
|
||||
|
||||
@ -14,7 +14,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import http from 'http';
|
||||
import https from 'https';
|
||||
import net from 'net';
|
||||
import { getProxyForUrl } from 'proxy-from-env';
|
||||
import * as URL from 'url';
|
||||
|
||||
export async function createSocket(host: string, port: number): Promise<net.Socket> {
|
||||
return new Promise((resolve, reject) => {
|
||||
@ -23,3 +27,85 @@ export async function createSocket(host: string, port: number): Promise<net.Sock
|
||||
socket.on('error', error => reject(error));
|
||||
});
|
||||
}
|
||||
|
||||
// `https-proxy-agent` v5 is written in TypeScript and exposes generated types.
|
||||
// However, as of June 2020, its types are generated with tsconfig that enables
|
||||
// `esModuleInterop` option.
|
||||
//
|
||||
// As a result, we can't depend on the package unless we enable the option
|
||||
// for our codebase. Instead of doing this, we abuse "require" to import module
|
||||
// without types.
|
||||
const ProxyAgent = require('https-proxy-agent');
|
||||
|
||||
export type HTTPRequestParams = {
|
||||
url: string,
|
||||
method?: string,
|
||||
headers?: http.OutgoingHttpHeaders,
|
||||
data?: string | Buffer,
|
||||
timeout?: number,
|
||||
};
|
||||
|
||||
export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) {
|
||||
const parsedUrl = URL.parse(params.url);
|
||||
let options: https.RequestOptions = { ...parsedUrl };
|
||||
options.method = params.method || 'GET';
|
||||
options.headers = params.headers;
|
||||
|
||||
const proxyURL = getProxyForUrl(params.url);
|
||||
if (proxyURL) {
|
||||
if (params.url.startsWith('http:')) {
|
||||
const proxy = URL.parse(proxyURL);
|
||||
options = {
|
||||
path: parsedUrl.href,
|
||||
host: proxy.hostname,
|
||||
port: proxy.port,
|
||||
};
|
||||
} else {
|
||||
const parsedProxyURL = URL.parse(proxyURL);
|
||||
(parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:';
|
||||
|
||||
options.agent = new ProxyAgent(parsedProxyURL);
|
||||
options.rejectUnauthorized = false;
|
||||
}
|
||||
}
|
||||
|
||||
const requestCallback = (res: http.IncomingMessage) => {
|
||||
const statusCode = res.statusCode || 0;
|
||||
if (statusCode >= 300 && statusCode < 400 && res.headers.location)
|
||||
httpRequest({ ...params, url: res.headers.location }, onResponse, onError);
|
||||
else
|
||||
onResponse(res);
|
||||
};
|
||||
const request = options.protocol === 'https:' ?
|
||||
https.request(options, requestCallback) :
|
||||
http.request(options, requestCallback);
|
||||
request.on('error', onError);
|
||||
if (params.timeout !== undefined) {
|
||||
const rejectOnTimeout = () => {
|
||||
onError(new Error(`Request to ${params.url} timed out after ${params.timeout}ms`));
|
||||
request.abort();
|
||||
};
|
||||
if (params.timeout <= 0) {
|
||||
rejectOnTimeout();
|
||||
return;
|
||||
}
|
||||
request.setTimeout(params.timeout, rejectOnTimeout);
|
||||
}
|
||||
request.end(params.data);
|
||||
}
|
||||
|
||||
export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequestParams, response: http.IncomingMessage) => Promise<Error>): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
httpRequest(params, async response => {
|
||||
if (response.statusCode !== 200) {
|
||||
const error = onError ? await onError(params, response) : new Error(`fetch failed: server returned code ${response.statusCode}. URL: ${params.url}`);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
let body = '';
|
||||
response.on('data', (chunk: string) => body += chunk);
|
||||
response.on('error', (error: any) => reject(error));
|
||||
response.on('end', () => resolve(body));
|
||||
}, reject);
|
||||
});
|
||||
}
|
||||
|
||||
@ -14,233 +14,20 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import type stream from 'stream';
|
||||
import removeFolder from 'rimraf';
|
||||
import * as crypto from 'crypto';
|
||||
import os from 'os';
|
||||
import http from 'http';
|
||||
import https from 'https';
|
||||
import type { SpawnOptions } from 'child_process';
|
||||
import { spawn, execSync } from 'child_process';
|
||||
import { getProxyForUrl } from 'proxy-from-env';
|
||||
import { execSync, spawn } from 'child_process';
|
||||
import * as crypto from 'crypto';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import removeFolder from 'rimraf';
|
||||
import type stream from 'stream';
|
||||
import * as URL from 'url';
|
||||
import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion';
|
||||
import type { NameValue } from '../protocol/channels';
|
||||
import ProgressBar from 'progress';
|
||||
|
||||
// `https-proxy-agent` v5 is written in TypeScript and exposes generated types.
|
||||
// However, as of June 2020, its types are generated with tsconfig that enables
|
||||
// `esModuleInterop` option.
|
||||
//
|
||||
// As a result, we can't depend on the package unless we enable the option
|
||||
// for our codebase. Instead of doing this, we abuse "require" to import module
|
||||
// without types.
|
||||
const ProxyAgent = require('https-proxy-agent');
|
||||
import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion';
|
||||
|
||||
export const existsAsync = (path: string): Promise<boolean> => new Promise(resolve => fs.stat(path, err => resolve(!err)));
|
||||
|
||||
export type HTTPRequestParams = {
|
||||
url: string,
|
||||
method?: string,
|
||||
headers?: http.OutgoingHttpHeaders,
|
||||
data?: string | Buffer,
|
||||
timeout?: number,
|
||||
};
|
||||
|
||||
function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) {
|
||||
const parsedUrl = URL.parse(params.url);
|
||||
let options: https.RequestOptions = { ...parsedUrl };
|
||||
options.method = params.method || 'GET';
|
||||
options.headers = params.headers;
|
||||
|
||||
const proxyURL = getProxyForUrl(params.url);
|
||||
if (proxyURL) {
|
||||
if (params.url.startsWith('http:')) {
|
||||
const proxy = URL.parse(proxyURL);
|
||||
options = {
|
||||
path: parsedUrl.href,
|
||||
host: proxy.hostname,
|
||||
port: proxy.port,
|
||||
};
|
||||
} else {
|
||||
const parsedProxyURL = URL.parse(proxyURL);
|
||||
(parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:';
|
||||
|
||||
options.agent = new ProxyAgent(parsedProxyURL);
|
||||
options.rejectUnauthorized = false;
|
||||
}
|
||||
}
|
||||
|
||||
const requestCallback = (res: http.IncomingMessage) => {
|
||||
const statusCode = res.statusCode || 0;
|
||||
if (statusCode >= 300 && statusCode < 400 && res.headers.location)
|
||||
httpRequest({ ...params, url: res.headers.location }, onResponse, onError);
|
||||
else
|
||||
onResponse(res);
|
||||
};
|
||||
const request = options.protocol === 'https:' ?
|
||||
https.request(options, requestCallback) :
|
||||
http.request(options, requestCallback);
|
||||
request.on('error', onError);
|
||||
if (params.timeout !== undefined) {
|
||||
const rejectOnTimeout = () => {
|
||||
onError(new Error(`Request to ${params.url} timed out after ${params.timeout}ms`));
|
||||
request.abort();
|
||||
};
|
||||
if (params.timeout <= 0) {
|
||||
rejectOnTimeout();
|
||||
return;
|
||||
}
|
||||
request.setTimeout(params.timeout, rejectOnTimeout);
|
||||
}
|
||||
request.end(params.data);
|
||||
}
|
||||
|
||||
export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequestParams, response: http.IncomingMessage) => Promise<Error>): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
httpRequest(params, async response => {
|
||||
if (response.statusCode !== 200) {
|
||||
const error = onError ? await onError(params, response) : new Error(`fetch failed: server returned code ${response.statusCode}. URL: ${params.url}`);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
let body = '';
|
||||
response.on('data', (chunk: string) => body += chunk);
|
||||
response.on('error', (error: any) => reject(error));
|
||||
response.on('end', () => resolve(body));
|
||||
}, reject);
|
||||
});
|
||||
}
|
||||
|
||||
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
|
||||
type DownloadFileLogger = (message: string) => void;
|
||||
type DownloadFileOptions = {
|
||||
progressCallback?: OnProgressCallback,
|
||||
log?: DownloadFileLogger,
|
||||
userAgent?: string
|
||||
};
|
||||
|
||||
function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> {
|
||||
const {
|
||||
progressCallback,
|
||||
log = () => {},
|
||||
} = options;
|
||||
log(`running download:`);
|
||||
log(`-- from url: ${url}`);
|
||||
log(`-- to location: ${destinationPath}`);
|
||||
let fulfill: ({ error }: {error: any}) => void = ({ error }) => {};
|
||||
let downloadedBytes = 0;
|
||||
let totalBytes = 0;
|
||||
|
||||
const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; });
|
||||
|
||||
httpRequest({
|
||||
url,
|
||||
headers: options.userAgent ? {
|
||||
'User-Agent': options.userAgent,
|
||||
} : undefined,
|
||||
}, response => {
|
||||
log(`-- response status code: ${response.statusCode}`);
|
||||
if (response.statusCode !== 200) {
|
||||
const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
|
||||
// consume response data to free up memory
|
||||
response.resume();
|
||||
fulfill({ error });
|
||||
return;
|
||||
}
|
||||
const file = fs.createWriteStream(destinationPath);
|
||||
file.on('finish', () => fulfill({ error: null }));
|
||||
file.on('error', error => fulfill({ error }));
|
||||
response.pipe(file);
|
||||
totalBytes = parseInt(response.headers['content-length'] || '0', 10);
|
||||
log(`-- total bytes: ${totalBytes}`);
|
||||
if (progressCallback)
|
||||
response.on('data', onData);
|
||||
}, (error: any) => fulfill({ error }));
|
||||
return promise;
|
||||
|
||||
function onData(chunk: string) {
|
||||
downloadedBytes += chunk.length;
|
||||
progressCallback!(downloadedBytes, totalBytes);
|
||||
}
|
||||
}
|
||||
|
||||
type DownloadOptions = {
|
||||
progressBarName?: string,
|
||||
retryCount?: number
|
||||
log?: DownloadFileLogger
|
||||
userAgent?: string
|
||||
};
|
||||
|
||||
export async function download(
|
||||
url: string,
|
||||
destination: string,
|
||||
options: DownloadOptions = {}
|
||||
) {
|
||||
const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options;
|
||||
for (let attempt = 1; attempt <= retryCount; ++attempt) {
|
||||
log(
|
||||
`downloading ${progressBarName} - attempt #${attempt}`
|
||||
);
|
||||
const { error } = await downloadFile(url, destination, {
|
||||
progressCallback: getDownloadProgress(progressBarName),
|
||||
log,
|
||||
userAgent,
|
||||
});
|
||||
if (!error) {
|
||||
log(`SUCCESS downloading ${progressBarName}`);
|
||||
break;
|
||||
}
|
||||
const errorMessage = error?.message || '';
|
||||
log(`attempt #${attempt} - ERROR: ${errorMessage}`);
|
||||
if (
|
||||
attempt < retryCount &&
|
||||
(errorMessage.includes('ECONNRESET') ||
|
||||
errorMessage.includes('ETIMEDOUT'))
|
||||
) {
|
||||
// Maximum default delay is 3rd retry: 1337.5ms
|
||||
const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt);
|
||||
log(`sleeping ${millis}ms before retry...`);
|
||||
await new Promise(c => setTimeout(c, millis));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getDownloadProgress(progressBarName: string): OnProgressCallback {
|
||||
let progressBar: ProgressBar;
|
||||
let lastDownloadedBytes = 0;
|
||||
|
||||
return (downloadedBytes: number, totalBytes: number) => {
|
||||
if (!process.stderr.isTTY)
|
||||
return;
|
||||
if (!progressBar) {
|
||||
progressBar = new ProgressBar(
|
||||
`Downloading ${progressBarName} - ${toMegabytes(
|
||||
totalBytes
|
||||
)} [:bar] :percent :etas `,
|
||||
{
|
||||
complete: '=',
|
||||
incomplete: ' ',
|
||||
width: 20,
|
||||
total: totalBytes,
|
||||
}
|
||||
);
|
||||
}
|
||||
const delta = downloadedBytes - lastDownloadedBytes;
|
||||
lastDownloadedBytes = downloadedBytes;
|
||||
progressBar.tick(delta);
|
||||
};
|
||||
}
|
||||
|
||||
function toMegabytes(bytes: number) {
|
||||
const mb = bytes / 1024 / 1024;
|
||||
return `${Math.round(mb * 10) / 10} Mb`;
|
||||
}
|
||||
|
||||
export function spawnAsync(cmd: string, args: string[], options: SpawnOptions = {}): Promise<{stdout: string, stderr: string, code: number | null, error?: Error}> {
|
||||
const process = spawn(cmd, args, Object.assign({ windowsHide: true }, options));
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user