chore: generate protocol during browser roll (#2719)

We now commit protocol.ts files during the roll.
New utils/roll_browser.js helps with that.

This makes our installation very shallow:
- build installer;
- download browsers.

.appveyor.yml

@@ -7,6 +7,5 @@ build: off
 install:
   - ps: Install-Product node $env:nodejs_version
   - npm ci
-  - npm run ctest
-  - npm run wtest
-  - npm run ftest
+  - npm run build
+  - npm run test

.circleci/config.yml

@@ -7,18 +7,19 @@ jobs:
       - checkout
 
       - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-{{ checksum "browsers.json" }}
 
       - run:
           command: npm ci
 
+      - run:
+          command: npm run build
+
       - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-{{ checksum "browsers.json" }}
           paths:
-            - ./.local-browsers
+            - $HOME/.cache/ms-playwright
 
       - run:
           command: |
-            xvfb-run --auto-servernum npm run ctest
-            xvfb-run --auto-servernum npm run wtest
-            xvfb-run --auto-servernum npm run ftest
+            xvfb-run --auto-servernum npm run test

.cirrus.yml

@@ -1,47 +0,0 @@
-env:
-  DISPLAY: :99.0
-  FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992]
-  FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME}
-  FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/task/${CIRRUS_TASK_ID}
-
-task:
-  matrix:
-    - name: Chromium (node8 + linux)
-      container:
-        dockerfile: .ci/node8/Dockerfile.linux
-    - name: Chromium (node10 + linux)
-      container:
-        dockerfile: .ci/node10/Dockerfile.linux
-    - name: Chromium (node12 + linux)
-      container:
-        dockerfile: .ci/node12/Dockerfile.linux
-  xvfb_start_background_script: Xvfb :99 -ac -screen 0 1024x768x24
-  install_script: npm install --unsafe-perm
-  lint_script: npm run lint
-  coverage_script: npm run coverage
-  test_doclint_script: npm run test-doclint
-  test_types_script: npm run test-types
-
-task:
-  matrix:
-    - name: Firefox (node8 + linux)
-      container:
-        dockerfile: .ci/node8/Dockerfile.linux
-      xvfb_start_background_script: Xvfb :99 -ac -screen 0 1024x768x24
-  install_script: npm install --unsafe-perm
-  test_script: npm run ftest
-
-task:
-  osx_instance:
-    image: high-sierra-base
-  name: Chromium (node8 + macOS)
-  env:
-    HOMEBREW_NO_AUTO_UPDATE: 1
-  node_install_script:
-    - brew install node@8
-    - brew link --force node@8
-  install_script: npm install --unsafe-perm
-  lint_script: npm run lint
-  coverage_script: npm run coverage
-  test_doclint_script: npm run test-doclint
-  test_types_script: npm run test-types

.github/workflows/infra.yml

@@ -28,4 +28,5 @@ jobs:
         sudo apt-get install libgbm-dev
         sudo apt-get install xvfb
     - run: npm ci
+    - run: npm run build
     - run: npm run lint

.github/workflows/publish_canary.yml

@@ -21,6 +21,7 @@ jobs:
         registry-url: 'https://registry.npmjs.org'
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: node utils/update_version.js --next
     - run: utils/publish_all_packages.sh --tip-of-tree
       env:

.github/workflows/publish_release.yml

@@ -19,6 +19,7 @@ jobs:
         registry-url: 'https://registry.npmjs.org'
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: utils/publish_all_packages.sh --release
       env:
         NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/tests.yml

@@ -26,6 +26,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -61,6 +62,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ctest 2>./chromium-mac-testrun.log
       env:
         DEBUG: "*"
@@ -85,6 +87,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ctest 2>./chromium-win-testrun.log
       shell: bash
       env:
@@ -110,6 +113,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -145,6 +149,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run wtest 2>./webkit-mac-testrun.log
       env:
         DEBUG: "*,-pw:wrapped*"
@@ -169,6 +174,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run wtest 2>./webkit-win-testrun.log
       shell: bash
       env:
@@ -194,6 +200,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -229,6 +236,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ftest 2>./firefox-mac-testrun.log
       env:
         DEBUG: "*"
@@ -253,6 +261,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ftest 2>./firefox-win-testrun.log
       shell: bash
       env:
@@ -283,6 +292,7 @@ jobs:
         node-version: ${{ matrix.node_version }}
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: bash packages/installation-tests/installation-tests.sh
 
   headful_linux:
@@ -295,6 +305,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -326,6 +337,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'

.gitignore

@@ -12,8 +12,5 @@
 yarn.lock
 /node6
 /src/generated/*
-/src/chromium/protocol.ts
-/src/firefox/protocol.ts
-/src/webkit/protocol.ts
 lib/
 /types/*

.travis.yml

@@ -39,16 +39,12 @@ jobs:
           # this is needed for running headful tests
           - xvfb
       script:
-        - npm run lint
-        - xvfb-run --auto-servernum npm run ctest
-        - xvfb-run --auto-servernum npm run ftest
-        - xvfb-run --auto-servernum npm run wtest
+        - npm run build
+        - xvfb-run --auto-servernum npm run test
     - os: osx
       osx_image: xcode11.3
       node_js: '12'
       script:
-        - npm run lint
-        - npm run ctest
-        - npm run ftest
-        - npm run wtest
+        - npm run build
+        - npm run test
 

browser_patches/README.md

@@ -58,12 +58,8 @@ Send a PR to the Playwright repo to be reviewed.
 
 ## 4. Rolling Playwright to the new browser build
 
-Once the patch has been committed, the build bots will kick in, compile and upload a new browser version to all the platforms.
-
-You can check the CDN status:
+Once the patch has been committed, the build bots will kick in, compile and upload a new browser version to all the platforms. Then you can roll the browser:
 
 ```sh
-$ ./browser_patches/tools/check_cdn.sh
+$ node utils/roll_browser.js chromium 123456
 ```
-
-As the builds appear, you can roll to a new browser version in the `./browsers.json` file.

browsers.json

@@ -1,4 +1,5 @@
 {
+  "comment": "Do not edit this file, use utils/roll_browser.js",
   "browsers": [
     {
       "name": "chromium",

install-from-github.js

@@ -18,67 +18,15 @@
 // This file is only run when someone installs via the github repo
 
 const {execSync} = require('child_process');
-const path = require('path');
-const fs = require('fs');
-const util = require('util');
-const rmAsync = util.promisify(require('rimraf'));
 
-(async () => {
-  const SRC_FOLDER = path.join(__dirname, 'src');
-  const LIB_FOLDER = path.join(__dirname, 'lib');
-  const srcTypeScriptFiles = (await listFiles(path.join(__dirname, 'src'))).filter(filepath => filepath.toLowerCase().endsWith('.ts'));
-  const outdatedFiles = await Promise.all(srcTypeScriptFiles.map(async srcFile => {
-    const libFileTS = path.join(LIB_FOLDER, path.relative(SRC_FOLDER, srcFile));
-    const libFile = libFileTS.substring(0, libFileTS.lastIndexOf('.')) + '.js';
-    try {
-      const [srcStat, libStat] = await Promise.all([fs.promises.stat(srcFile), fs.promises.stat(libFile)]);
-      return srcStat.ctimeMs > libStat.ctimeMs;
-    } catch (e) {
-      // Either `.ts` of `.js` file is missing - rebuild is required.
-      return true;
-    }
-  }));
-  if (outdatedFiles.some(Boolean)) {
-    console.log(`Rebuilding playwright...`);
-    try {
-      execSync('npm run build');
-    } catch (e) {
-    }
-  }
-  await downloadAllBrowsersAndGenerateProtocolTypes();
-})();
-
-async function listFiles(dirpath) {
-  const files = [];
-  await dfs(dirpath);
-  return files;
-
-  async function dfs(dirpath) {
-    const entries = await fs.promises.readdir(dirpath, {withFileTypes: true});
-    files.push(...entries.filter(entry => entry.isFile()).map(entry => path.join(dirpath, entry.name)));
-    await Promise.all(entries.filter(entry => entry.isDirectory()).map(entry => dfs(path.join(dirpath, entry.name))));
-  }
+console.log(`Rebuilding installer...`);
+try {
+  execSync('npm run tsc-installer');
+} catch (e) {
 }
 
-async function downloadAllBrowsersAndGenerateProtocolTypes() {
-  const { installBrowsersWithProgressBar } = require('./lib/install/installer');
-  const protocolGenerator = require('./utils/protocol-types-generator');
-  const browserPaths = require('./lib/install/browserPaths');
-  const browsersPath = browserPaths.browsersPath(__dirname);
-  const browsers = require(path.join(__dirname, 'browsers.json'))['browsers'];
-  await installBrowsersWithProgressBar(__dirname);
-  for (const browser of browsers) {
-    const browserPath = browserPaths.browserDirectory(browsersPath, browser);
-    await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(browserPath, browser)).catch(console.warn);
-  }
+console.log(`Downloading browsers...`);
+const { installBrowsersWithProgressBar } = require('./lib/install/installer');
+installBrowsersWithProgressBar(__dirname);
 
-  try {
-    console.log('Generating types...');
-    execSync('npm run generate-types');
-  } catch (e) {
-  }
-
-  async function readdirAsync(dirpath) {
-    return fs.promises.readdir(dirpath).then(dirs => dirs.map(dir => path.join(dirpath, dir)));
-  }
-}
+console.log(`Done. Use "npm run watch" to compile.`);

package.json

@@ -26,13 +26,14 @@
     "coverage": "cross-env COVERAGE=true BROWSER=all node --unhandled-rejections=strict test/test.js",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
     "tsc": "tsc -p .",
+    "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",
     "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js && node utils/testrunner/test/test.js",
     "lint": "npm run eslint && npm run tsc && npm run doc && npm run test-types && npm run test-infra",
     "debug-test": "node --inspect-brk test/test.js",
     "clean": "rimraf lib && rimraf types",
     "prepare": "node install-from-github.js",
-    "build": "node utils/runWebpack.js --mode='development' && tsc -p .",
+    "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types",
     "watch": "node utils/runWebpack.js --mode='development' --watch --silent | tsc -w -p .",
     "test-types": "npm run generate-types && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json",
     "generate-types": "node utils/generate_types/"

src/install/installer.ts

@@ -20,8 +20,8 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as util from 'util';
 import * as removeFolder from 'rimraf';
-import * as browserPaths from '../install/browserPaths';
-import * as browserFetcher from '../install/browserFetcher';
+import * as browserPaths from './browserPaths';
+import * as browserFetcher from './browserFetcher';
 
 const fsMkdirAsync = util.promisify(fs.mkdir.bind(fs));
 const fsReaddirAsync = util.promisify(fs.readdir.bind(fs));

src/install/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  // This config exists so that we can quickly compile just an installer.
+  "extends": "../../tsconfig.json",
+  "include": ["./*.ts"],
+  "exclude": ["node_modules"]
+}

utils/protocol-types-generator/index.js

@@ -4,7 +4,6 @@ const fs = require('fs');
 const StreamZip = require('node-stream-zip');
 const vm = require('vm');
 const os = require('os');
-const util = require('util');
 
 async function generateProtocol(name, executablePath) {
   if (name === 'chromium')

utils/roll_browser.js

@@ -0,0 +1,75 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const protocolGenerator = require('./protocol-types-generator');
+
+const SCRIPT_NAME = path.basename(__filename);
+const ROOT_PATH = path.resolve(path.join(__dirname, '..'));
+
+function usage() {
+  return `
+usage: ${SCRIPT_NAME} <browser> <revision>
+
+Roll the <browser> to a specific <revision> and generate new protocol.
+Supported browsers: chromium, firefox, webkit.
+
+Example:
+  ${SCRIPT_NAME} chromium 123456
+`;
+}
+
+(async () => {
+  // 1. Parse CLI arguments
+  const args = process.argv.slice(2);
+  if (args.some(arg => arg === '--help')) {
+    console.log(usage());
+    process.exit(1);
+  } else if (args.length < 1) {
+    console.log(`Please specify the browser name, e.g. 'chromium'.`);
+    console.log(`Try running ${SCRIPT_NAME} --help`);
+    process.exit(1);
+  } else if (args.length < 2) {
+    console.log(`Please specify the revision`);
+    console.log(`Try running ${SCRIPT_NAME} --help`);
+    process.exit(1);
+  }
+  const browserName = args[0].toLowerCase();
+  if (!['chromium', 'firefox', 'webkit'].includes(browserName)) {
+    console.log(`Unknown browser "${browserName}"`);
+    console.log(`Try running ${SCRIPT_NAME} --help`);
+    process.exit(1);
+  }
+  const revision = args[1];
+
+  // 2. Update browsers.json.
+  const browsersJSON = require(path.join(ROOT_PATH, 'browsers.json'));
+  browsersJSON.browsers.find(b => b.name === browserName).revision = String(revision);
+  fs.writeFileSync(path.join(ROOT_PATH, 'browsers.json'), JSON.stringify(browsersJSON, null, 2) + '\n');
+
+  // 3. Download new browser.
+  const { installBrowsersWithProgressBar } = require('../lib/install/installer');
+  await installBrowsersWithProgressBar(ROOT_PATH);
+
+  // 4. Generate types.
+  const browser = { name: browserName, revision };
+  const browserPaths = require('../lib/install/browserPaths');
+  const browserDir = browserPaths.browserDirectory(browserPaths.browsersPath(ROOT_PATH), browser);
+  const executablePath = browserPaths.executablePath(browserDir, browser);
+  await protocolGenerator.generateProtocol(browserName, executablePath).catch(console.warn);
+})();