feat(registry): implement download registry (#1979)

install-from-github.js

@@ -41,9 +41,7 @@ const rmAsync = util.promisify(require('rimraf'));
   if (outdatedFiles.some(Boolean)) {
     console.log(`Rebuilding playwright...`);
     try {
-      execSync('npm run build', {
-        stdio: 'ignore'
-      });
+      execSync('npm run build');
     } catch (e) {
     }
   }
@@ -63,21 +61,17 @@ async function listFiles(dirpath) {
 }
 
 async function downloadAllBrowsersAndGenerateProtocolTypes() {
-  const { downloadBrowserWithProgressBar } = require('./lib/install/browserFetcher');
+  const { installBrowsersWithProgressBar } = require('./lib/install/installer');
   const protocolGenerator = require('./utils/protocol-types-generator');
   const browserPaths = require('./lib/install/browserPaths');
+  const browsersPath = browserPaths.browsersPath(__dirname);
   const browsers = require('./browsers.json')['browsers'];
+  await installBrowsersWithProgressBar(__dirname);
   for (const browser of browsers) {
-    if (await downloadBrowserWithProgressBar(__dirname, browser))
-      await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(__dirname, browser)).catch(console.warn);
+    const browserPath = browserPaths.browserDirectory(browsersPath, browser);
+    await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(browserPath, browser)).catch(console.warn);
   }
 
-  // Cleanup stale revisions.
-  const directories = new Set(await readdirAsync(browserPaths.browsersPath(__dirname)));
-  for (const browser of browsers)
-    directories.delete(browserPaths.browserDirectory(__dirname, browser));
-  await Promise.all([...directories].map(directory => rmAsync(directory)));
-
   try {
     console.log('Generating types...');
     execSync('npm run generate-types');

packages/playwright-chromium/install.js

@@ -14,6 +14,6 @@
  * limitations under the License.
  */
 
-const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
+const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
 
-downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
+installBrowsersWithProgressBar(__dirname);

packages/playwright-firefox/install.js

@@ -14,6 +14,6 @@
  * limitations under the License.
  */
 
-const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
+const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
 
-downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
+installBrowsersWithProgressBar(__dirname);

packages/playwright-webkit/install.js

@@ -14,6 +14,6 @@
  * limitations under the License.
  */
 
-const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
+const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
 
-downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
+installBrowsersWithProgressBar(__dirname);

packages/playwright/install.js

@@ -14,6 +14,6 @@
  * limitations under the License.
  */
 
-const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
+const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
 
-downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
+installBrowsersWithProgressBar(__dirname);

src/install/browserFetcher.ts

@@ -75,12 +75,6 @@ function getDownloadUrl(browserName: BrowserName, platform: BrowserPlatform): st
   }
 }
 
-export type DownloadOptions = {
-  browser: BrowserDescriptor,
-  packagePath: string,
-  serverHost?: string,
-};
-
 function revisionURL(browser: BrowserDescriptor, platform = browserPaths.hostPlatform): string {
   const serverHost = getFromENV('PLAYWRIGHT_DOWNLOAD_HOST') || DEFAULT_DOWNLOAD_HOSTS[browser.name];
   const urlTemplate = getDownloadUrl(browser.name, platform);
@@ -88,19 +82,9 @@ function revisionURL(browser: BrowserDescriptor, platform = browserPaths.hostPla
   return util.format(urlTemplate, serverHost, browser.revision);
 }
 
-export async function downloadBrowsersWithProgressBar(packagePath: string, browsers: BrowserDescriptor[]) {
-  if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
-    logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
-    return false;
-  }
-  for (const browser of browsers)
-    await downloadBrowserWithProgressBar(packagePath, browser);
-}
-
-export async function downloadBrowserWithProgressBar(packagePath: string, browser: BrowserDescriptor): Promise<boolean> {
+export async function downloadBrowserWithProgressBar(browserPath: string, browser: BrowserDescriptor): Promise<boolean> {
   const progressBarName = `${browser.name} v${browser.revision}`;
-  const targetDir = browserPaths.browserDirectory(packagePath, browser);
-  if (await existsAsync(targetDir)) {
+  if (await existsAsync(browserPath)) {
     // Already downloaded.
     return false;
   }
@@ -126,8 +110,8 @@ export async function downloadBrowserWithProgressBar(packagePath: string, browse
   const zipPath = path.join(os.tmpdir(), `playwright-download-${browser.name}-${browserPaths.hostPlatform}-${browser.revision}.zip`);
   try {
     await downloadFile(url, zipPath, progress);
-    await extract(zipPath, {dir: targetDir});
-    await chmodAsync(browserPaths.executablePath(packagePath, browser), 0o755);
+    await extract(zipPath, { dir: browserPath});
+    await chmodAsync(browserPaths.executablePath(browserPath, browser)!, 0o755);
   } catch (e) {
     process.exitCode = 1;
     throw e;
@@ -135,7 +119,7 @@ export async function downloadBrowserWithProgressBar(packagePath: string, browse
     if (await existsAsync(zipPath))
       await unlinkAsync(zipPath);
   }
-  logPolitely(`${progressBarName} downloaded to ${targetDir}`);
+  logPolitely(`${progressBarName} downloaded to ${browserPath}`);
   return true;
 }
 

src/install/browserPaths.ts

@@ -18,7 +18,7 @@
 import { execSync } from 'child_process';
 import * as os from 'os';
 import * as path from 'path';
-import { assert, getFromENV } from '../helper';
+import { getFromENV } from '../helper';
 
 export type BrowserName = 'chromium'|'webkit'|'firefox';
 export type BrowserPlatform = 'win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'linux';
@@ -40,9 +40,10 @@ export const hostPlatform = ((): BrowserPlatform => {
   return platform as BrowserPlatform;
 })();
 
-function getRelativeExecutablePath(browserName: BrowserName): string[] | undefined {
-  if (browserName === 'chromium') {
-    return new Map<BrowserPlatform, string[]>([
+export function executablePath(browserPath: string, browser: BrowserDescriptor): string | undefined {
+  let tokens: string[] | undefined;
+  if (browser.name === 'chromium') {
+    tokens = new Map<BrowserPlatform, string[]>([
       ['linux', ['chrome-linux', 'chrome']],
       ['mac10.13', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']],
       ['mac10.14', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']],
@@ -52,8 +53,8 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin
     ]).get(hostPlatform);
   }
 
-  if (browserName === 'firefox') {
-    return new Map<BrowserPlatform, string[]>([
+  if (browser.name === 'firefox') {
+    tokens = new Map<BrowserPlatform, string[]>([
       ['linux', ['firefox', 'firefox']],
       ['mac10.13', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']],
       ['mac10.14', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']],
@@ -63,8 +64,8 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin
     ]).get(hostPlatform);
   }
 
-  if (browserName === 'webkit') {
-    return new Map<BrowserPlatform, string[] | undefined>([
+  if (browser.name === 'webkit') {
+    tokens = new Map<BrowserPlatform, string[] | undefined>([
       ['linux', ['pw_run.sh']],
       ['mac10.13', undefined],
       ['mac10.14', ['pw_run.sh']],
@@ -73,6 +74,7 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin
       ['win64', ['Playwright.exe']],
     ]).get(hostPlatform);
   }
+  return tokens ? path.join(browserPath, ...tokens) : undefined;
 }
 
 export function browsersPath(packagePath: string): string {
@@ -80,12 +82,11 @@ export function browsersPath(packagePath: string): string {
   return result || path.join(packagePath, '.local-browsers');
 }
 
-export function browserDirectory(packagePath: string, browser: BrowserDescriptor): string {
-  return path.join(browsersPath(packagePath), `${browser.name}-${browser.revision}`);
+export function browserDirectory(browsersPath: string, browser: BrowserDescriptor): string {
+  return path.join(browsersPath, `${browser.name}-${browser.revision}`);
 }
 
-export function executablePath(packagePath: string, browser: BrowserDescriptor): string {
-  const relativePath = getRelativeExecutablePath(browser.name);
-  assert(relativePath, `Unsupported platform for ${browser.name}: ${hostPlatform}`);
-  return path.join(browserDirectory(packagePath, browser), ...relativePath);
+export function isBrowserDirectory(browserPath: string): boolean {
+  const baseName = path.basename(browserPath);
+  return baseName.startsWith('chromium-') || baseName.startsWith('firefox-') || baseName.startsWith('webkit-');
 }

src/install/installer.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as crypto from 'crypto';
+import { getFromENV, logPolitely } from '../helper';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as util from 'util';
+import * as removeFolder from 'rimraf';
+import * as browserPaths from '../install/browserPaths';
+import * as browserFetcher from '../install/browserFetcher';
+
+const fsMkdirAsync = util.promisify(fs.mkdir.bind(fs));
+const fsExistsAsync = (path: string) => new Promise(f => fs.exists(path, f));
+const fsReaddirAsync = util.promisify(fs.readdir.bind(fs));
+const fsReadFileAsync = util.promisify(fs.readFile.bind(fs));
+const fsUnlinkAsync = util.promisify(fs.unlink.bind(fs));
+const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs));
+const removeFolderAsync = util.promisify(removeFolder);
+
+export async function installBrowsersWithProgressBar(packagePath: string) {
+  const browsersPath = browserPaths.browsersPath(packagePath);
+  const linksDir = path.join(browsersPath, '.links');
+
+  if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
+    logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
+    return false;
+  }
+  if (!await fsExistsAsync(browsersPath))
+    await fsMkdirAsync(browsersPath);
+  if (!await fsExistsAsync(linksDir))
+    await fsMkdirAsync(linksDir);
+
+  await fsWriteFileAsync(path.join(linksDir, sha1(packagePath)), packagePath);
+  await validateCache(browsersPath, linksDir);
+}
+
+async function validateCache(browsersPath: string, linksDir: string) {
+  // 1. Collect unused downloads and package descriptors.
+  const allBrowsers: browserPaths.BrowserDescriptor[] = [];
+  for (const fileName of await fsReaddirAsync(linksDir)) {
+    const linkPath = path.join(linksDir, fileName);
+    try {
+      const linkTarget = (await fsReadFileAsync(linkPath)).toString();
+      const browsers = JSON.parse((await fsReadFileAsync(path.join(linkTarget, 'browsers.json'))).toString())['browsers'];
+      allBrowsers.push(...browsers);
+    } catch (e) {
+      logPolitely('Failed to process descriptor at ' + fileName);
+      await fsUnlinkAsync(linkPath).catch(e => {});
+    }
+  }
+
+  // 2. Delete all unused browsers.
+  let downloadedBrowsers = (await fsReaddirAsync(browsersPath)).map(file => path.join(browsersPath, file));
+  downloadedBrowsers = downloadedBrowsers.filter(file => browserPaths.isBrowserDirectory(file));
+  const directories = new Set<string>(downloadedBrowsers);
+  directories.delete(path.join(browsersPath, '.links'));
+  for (const browser of allBrowsers)
+    directories.delete(browserPaths.browserDirectory(browsersPath, browser));
+  for (const directory of directories) {
+    logPolitely('Removing unused browser at ' + directory);
+    await removeFolderAsync(directory).catch(e => {});
+  }
+
+  // 3. Install missing browsers.
+  for (const browser of allBrowsers) {
+    const browserPath = browserPaths.browserDirectory(browsersPath, browser);
+    await browserFetcher.downloadBrowserWithProgressBar(browserPath, browser);
+  }
+}
+
+function sha1(data: string): string {
+  const sum = crypto.createHash('sha1');
+  sum.update(data);
+  return sum.digest('hex');
+}

src/server/browserType.ts

@@ -54,14 +54,18 @@ export interface BrowserType<Browser> {
 
 export abstract class AbstractBrowserType<Browser> implements BrowserType<Browser> {
   private _name: string;
-  private _executablePath: string;
+  private _executablePath: string | undefined;
 
   constructor(packagePath: string, browser: browserPaths.BrowserDescriptor) {
     this._name = browser.name;
-    this._executablePath = browserPaths.executablePath(packagePath, browser);
+    const browsersPath = browserPaths.browsersPath(packagePath);
+    const browserPath = browserPaths.browserDirectory(browsersPath, browser);
+    this._executablePath = browserPaths.executablePath(browserPath, browser);
   }
 
   executablePath(): string {
+    if (!this._executablePath)
+      throw new Error('Browser is not supported on current platform');
     return this._executablePath;
   }