chore: unify workspace helper scripts (#11925)

This patch unifies a variety of different workspace
scripts into a single `//utils/workspace.js`.

Fixes #11362

package.json

@@ -27,7 +27,7 @@
     "build-installer": "babel -s --extensions \".ts\" --out-dir packages/playwright-core/lib/utils/ packages/playwright-core/src/utils",
     "doc": "node utils/doclint/cli.js",
     "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && node utils/generate_channels.js && node utils/generate_types/ --check-clean && npm run lint-tests && npm run test-types && npm run lint-packages",
-    "lint-packages": "node utils/prepare_packages.js --check-clean",
+    "lint-packages": "node utils/workspace.js --ensure-consistent",
     "lint-tests": "node utils/lint_tests.js",
     "flint": "concurrently \"npm run eslint\" \"npm run tsc\" \"npm run doc\" \"npm run check-deps\" \"node utils/generate_channels.js\" \"node utils/generate_types/ --check-clean\" \"npm run lint-tests\" \"npm run test-types\" \"npm run lint-packages\"",
     "clean": "rimraf packages/playwright-core/lib && rimraf packages/playwright-test/lib && rimraf packages/playwright-core/src/generated/",

utils/build/build.js

@@ -20,7 +20,7 @@ const child_process = require('child_process');
 const path = require('path');
 const chokidar = require('chokidar');
 const fs = require('fs');
-const { packages } = require('../list_packages');
+const { workspace } = require('../workspace');
 
 /**
  * @typedef {{
@@ -189,8 +189,8 @@ for (const file of webPackFiles) {
 }
 
 // Run Babel.
-for (const packageDir of packages) {
+for (const pkg of workspace.packages()) {
-  if (!fs.existsSync(path.join(packageDir, 'src')))
+  if (!fs.existsSync(path.join(pkg.path, 'src')))
     continue;
   steps.push({
     command: 'npx',
@@ -198,9 +198,9 @@ for (const packageDir of packages) {
       'babel',
       ...(watchMode ? ['-w', '--source-maps'] : []),
       '--extensions', '.ts',
-      '--out-dir', quotePath(path.join(packageDir, 'lib')),
+      '--out-dir', quotePath(path.join(pkg.path, 'lib')),
       '--ignore', '"packages/playwright-core/src/server/injected/**/*"',
-      quotePath(path.join(packageDir, 'src'))],
+      quotePath(path.join(pkg.path, 'src'))],
     shell: true,
   });
 }

utils/build/clean.js

@@ -1,6 +1,6 @@
-const { packages } = require("../list_packages");
+const { workspace } = require('../workspace');
 const path = require('path');
 const rimraf = require('rimraf');
-for (const packageDir of packages) {
+for (const pkg of workspace.packages()) {
-  rimraf.sync(path.join(packageDir, 'lib'));
+  rimraf.sync(path.join(pkg.path, 'lib'));
-} 
+}

utils/build/update_canary_version.js

@@ -17,6 +17,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const { workspace } = require('../workspace.js');
 const { execSync } = require('child_process');
 
 const packageJSON = require('../../package.json');
@@ -47,4 +48,4 @@ if (process.argv[3] === '--today-date') {
   throw new Error('This script must be run with either --commit-timestamp or --today-date parameter');
 }
 console.log('Setting version to ' + newVersion);
-execSync(`node utils/bump_package_versions.js ${newVersion}`, { stdio: 'inherit' });
+workspace.setVersion(newVersion);

utils/bump_package_versions.js

@@ -1,65 +0,0 @@
-#!/usr/bin/env node
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-//@ts-check
-
-const path = require('path');
-const fs = require('fs');
-const { execSync } = require('child_process');
-
-const { packages, packagesToPublish } = require('./list_packages.js');
-
-(async () => {
-  const version = process.argv[2];
-  if (!version)
-    throw new Error('Please specify version! See --help for more information.');
-  if (version.startsWith('v'))
-    throw new Error('Version must not start with "v"');
-  if (process.argv[2] === '--help')
-    throw new Error(`Usage: node ${path.relative(process.cwd(), __filename)} <version>`);
-  const rootDir = path.join(__dirname, '..');
-
-  // 1. update the package.json (playwright-internal) with the new version
-  execSync(`npm version --no-git-tag-version ${version}`, {
-    stdio: 'inherit',
-    cwd: rootDir,
-  });
-  // 2. Distribute new version to all packages and its dependencies
-  execSync(`node ${path.join(__dirname, 'prepare_packages.js')}`, {
-    stdio: 'inherit',
-    cwd: rootDir,
-  });
-
-  // 3. update the package-lock.json (playwright-internal) with the new version.
-  // Workaround for: https://github.com/npm/cli/issues/3940
-  {
-    const packageLockPath = path.join(rootDir, 'package-lock.json');
-    const packageLock = JSON.parse(fs.readFileSync(packageLockPath, 'utf8'));
-    const publicPackages = new Set(packagesToPublish.map(package => path.basename(package)));
-    for (const package of packages.map(package => path.basename(package))) {
-      const playwrightCorePackages = packageLock['packages']['packages/' + package];
-      if (publicPackages.has(package))
-        playwrightCorePackages.version = version;
-      if (playwrightCorePackages.dependencies && playwrightCorePackages.dependencies['playwright-core'])
-        packageLock['packages']['packages/playwright-test']['dependencies']['playwright-core'] = version;
-    }
-    fs.writeFileSync(packageLockPath, JSON.stringify(packageLock, null, 2) + '\n');
-  }
-
-})().catch(err => {
-  console.error(err);
-  process.exit(1);
-})

utils/list_packages.js

@@ -1,44 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const packageDir = path.join(__dirname, '..', 'packages');
-const packages = fs.readdirSync(packageDir)
-  .filter(packageDir => !packageDir.startsWith('.'))
-  .map(name => path.join(packageDir, name));
-
-const packagePathToJSON = new Map();
-const packageNameToPath = new Map();
-const packagePathToDependencies = new Map();
-for (const packagePath of packages) {
-  const packageJSON = require(path.join(packagePath, 'package.json'));
-  packageNameToPath.set(packageJSON.name, packagePath);
-  packagePathToJSON.set(packagePath, packageJSON);
-}
-
-for (const packagePath of packages)
-  packagePathToDependencies.set(packagePath, new Set(internalDependencies(packagePath)));
-
-// Sort packages by their interdependence.
-packages.sort((a, b) => {
-  if (packagePathToDependencies.get(a).has(b))
-    return 1;
-  if (packagePathToDependencies.get(b).has(a))
-    return -1;
-  return 0;
-});
-
-function* internalDependencies(packagePath) {
-  yield packagePath;
-  for (const dependency of Object.keys(packagePathToJSON.get(packagePath).dependencies || {})) {
-    const dependencyPath = packageNameToPath.get(dependency);
-    if (dependencyPath)
-      yield* internalDependencies(dependencyPath);
-  }
-}
-
-const packagesToPublish = packages.filter(packagePath => !packagePathToJSON.get(packagePath).private);
-
-module.exports = {
-  packages,
-  packageNameToPath,
-  packagesToPublish,
-};

utils/prepare_packages.js

@@ -1,127 +0,0 @@
-#!/usr/bin/env node
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-//@ts-check
-const fs = require('fs');
-const path = require('path');
-const ncp = require('ncp');
-const util = require('util');
-const { packageNameToPath } = require('./list_packages');
-
-const cpAsync = util.promisify(ncp);
-
-const ROOT_PATH = path.join(__dirname, '..');
-
-const LICENSE_FILES = ['NOTICE', 'LICENSE'];
-
-
-const PACKAGES = {
-  'playwright': {
-    browsers: ['chromium', 'firefox', 'webkit', 'ffmpeg'],
-    // We copy README.md additionally for Playwright so that it looks nice on NPM.
-    files: [...LICENSE_FILES, 'README.md'],
-  },
-  'playwright-core': {
-    browsers: [],
-    files: LICENSE_FILES,
-  },
-  '@playwright/test': {
-    browsers: ['chromium', 'firefox', 'webkit', 'ffmpeg'],
-    files: LICENSE_FILES,
-    name: '@playwright/test',
-  },
-  'playwright-webkit': {
-    browsers: ['webkit'],
-    files: LICENSE_FILES,
-  },
-  'playwright-firefox': {
-    browsers: ['firefox'],
-    files: LICENSE_FILES,
-  },
-  'playwright-chromium': {
-    browsers: ['chromium', 'ffmpeg'],
-    files: LICENSE_FILES,
-  },
-  'html-reporter': {
-    files: [],
-  }
-};
-
-const dirtyFiles = [];
-
-(async function () {
-  for (const packagePath of require('./list_packages').packages) {
-    const packageJSON = require(path.join(packagePath, 'package.json'));
-    packageNameToPath.set(packageJSON.name, packagePath);
-  }
-  for (const packageName of packageNameToPath.keys())
-    await lintPackage(packageName);
-  for (const file of dirtyFiles) {
-    console.warn('Updated', path.relative(ROOT_PATH, file));
-  }
-  if (dirtyFiles.length && process.argv.includes('--check-clean'))
-    process.exit(1);
-})();
-
-
-/**
- * @param {string} packageName
- */
-async function lintPackage(packageName) {
-  const packagePath = packageNameToPath.get(packageName);
-  const package = PACKAGES[packageName];
-  if (!package) {
-    console.log(`ERROR: unknown package ${packageName}`);
-    process.exit(1);
-  }
-
-  // 3. Copy package files.
-  for (const file of package.files)
-    await copyToPackage(path.join(ROOT_PATH, file), path.join(packagePath, file));
-
-  // 4. Generate package.json
-  const pwInternalJSON = require(path.join(ROOT_PATH, 'package.json'));
-  const currentPackageJSON = require(path.join(packagePath, 'package.json'));
-  if (currentPackageJSON.private)
-    return;
-  currentPackageJSON.version = pwInternalJSON.version;
-  currentPackageJSON.repository = pwInternalJSON.repository;
-  currentPackageJSON.engines = pwInternalJSON.engines;
-  currentPackageJSON.homepage = pwInternalJSON.homepage;
-  currentPackageJSON.author = pwInternalJSON.author;
-  currentPackageJSON.license = pwInternalJSON.license;
-  for (const name of Object.keys(currentPackageJSON.dependencies || {})) {
-    if (name in PACKAGES)
-      currentPackageJSON.dependencies[name] = pwInternalJSON.version;
-  }
-  await writeToPackage('package.json', JSON.stringify(currentPackageJSON, null, 2) + '\n');
-
-  async function writeToPackage(fileName, content) {
-    const toPath = path.join(packagePath, fileName);
-    const currentContent = await fs.promises.readFile(toPath, 'utf8').catch(e => null);
-    if (currentContent === content)
-      return;
-    dirtyFiles.push(toPath);
-    await fs.promises.writeFile(toPath, content);
-  }
-
-}
-
-async function copyToPackage(fromPath, toPath) {
-  await fs.promises.mkdir(path.dirname(toPath), { recursive: true });
-  await cpAsync(fromPath, toPath);
-}
-

utils/publish_all_packages.sh

@@ -91,8 +91,8 @@ else
 fi
 
 echo "==================== Publishing version ${VERSION} ================"
-node ./utils/prepare_packages.js
+node ./utils/workspace.js --ensure-consistent
-node -e "console.log(require('./utils/list_packages').packagesToPublish.join('\\n'))" | while read package
+node ./utils/workspace.js --list-public-package-paths | while read package
 do
   npm publish ${package} --tag="${NPM_PUBLISH_TAG}"
 done

utils/workspace.js

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// @ts-check
+
+/**
+ * Use the following command to typescheck this file:
+ * npx tsc --target es2020  --watch --checkjs --noemit --moduleResolution node workspace.js
+ */
+const fs = require('fs');
+const path = require('path');
+const util = require('util');
+const url = require('url');
+
+const readJSON = async (filePath) => JSON.parse(await fs.promises.readFile(filePath, 'utf8'));
+const writeJSON = async (filePath, json) => {
+  await fs.promises.writeFile(filePath, JSON.stringify(json, null, 2) + '\n');
+}
+
+class PWPackage {
+  constructor(descriptor) {
+    this.name = descriptor.name;
+    this.path = descriptor.path;
+    this.files = descriptor.files;
+    this.packageJSONPath = path.join(this.path, 'package.json');
+    this.packageJSON = JSON.parse(fs.readFileSync(this.packageJSONPath, 'utf8'));
+    this.isPrivate = !!this.packageJSON.private;
+  }
+}
+
+class Workspace {
+  /**
+   * @param {string} rootDir
+   * @param {PWPackage[]} packages
+   */
+  constructor(rootDir, packages) {
+    this._rootDir = rootDir;
+    this._packages = packages;
+  }
+
+  /**
+   * @returns {PWPackage[]}
+   */
+  packages() {
+    return this._packages;
+  }
+
+  /**
+   * @param {string} version
+   */
+  async setVersion(version) {
+    if (version.startsWith('v'))
+      throw new Error('version must not start with "v"');
+
+    // 1. update workspace's package.json (playwright-internal) with the new version
+    const workspacePackageJSON = await readJSON(path.join(this._rootDir, 'package.json'));
+    workspacePackageJSON.version = version;
+    await writeJSON(path.join(this._rootDir, 'package.json'), workspacePackageJSON);
+    // 2. make workspace consistent.
+    await this.ensureConsistent();
+  }
+
+  async ensureConsistent() {
+    let hasChanges = false;
+
+    const maybeWriteJSON = async (jsonPath, json) => {
+      const oldJson = await readJSON(jsonPath);
+      if (JSON.stringify(json) === JSON.stringify(oldJson))
+        return;
+      hasChanges = true;
+      console.warn('Updated', jsonPath);
+      await writeJSON(jsonPath, json);
+    };
+
+    const workspacePackageJSON = await readJSON(path.join(this._rootDir, 'package.json'));
+    const packageLockPath = path.join(this._rootDir, 'package-lock.json');
+    const packageLock = JSON.parse(await fs.promises.readFile(packageLockPath, 'utf8'));
+    const version = workspacePackageJSON.version;
+
+    // Make sure package-lock version is consistent with root package.json version.
+    packageLock.version = version;
+    packageLock.packages[""].version = version;
+
+    for (const pkg of this._packages) {
+      // 1. Copy package files.
+      for (const file of pkg.files) {
+        const fromPath = path.join(this._rootDir, file);
+        const toPath = path.join(pkg.path, file);
+        await fs.promises.mkdir(path.dirname(pkg.path), { recursive: true });
+        await fs.promises.copyFile(fromPath, toPath);
+      }
+
+      // 2. Make sure package-lock and package's package.json are consistent.
+      //    All manual package-lock management is a workaround for
+      //    https://github.com/npm/cli/issues/3940
+      const pkgLockEntry = packageLock['packages']['packages/' + path.basename(pkg.path)];
+      const depLockEntry = packageLock['dependencies'][pkg.name];
+      if (!pkg.isPrivate) {
+        pkgLockEntry.version = version;
+        pkg.packageJSON.version = version;
+        pkg.packageJSON.repository = workspacePackageJSON.repository;
+        pkg.packageJSON.engines = workspacePackageJSON.engines;
+        pkg.packageJSON.homepage = workspacePackageJSON.homepage;
+        pkg.packageJSON.author = workspacePackageJSON.author;
+        pkg.packageJSON.license = workspacePackageJSON.license;
+      }
+      for (const otherPackage of this._packages) {
+        if (pkgLockEntry.dependencies && pkgLockEntry.dependencies[otherPackage.name])
+          pkgLockEntry.dependencies[otherPackage.name] = version;
+        if (depLockEntry.requires && depLockEntry.requires[otherPackage.name])
+          depLockEntry.requires[otherPackage.name] = version;
+        if (pkg.packageJSON.dependencies && pkg.packageJSON.dependencies[otherPackage.name])
+          pkg.packageJSON.dependencies[otherPackage.name] = version;
+      }
+      await maybeWriteJSON(pkg.packageJSONPath, pkg.packageJSON);
+    }
+    await maybeWriteJSON(packageLockPath, packageLock);
+    return hasChanges;
+  }
+}
+
+const ROOT_PATH = path.join(__dirname, '..');
+const LICENCE_FILES = ['NOTICE', 'LICENSE'];
+const workspace = new Workspace(ROOT_PATH, [
+  new PWPackage({
+    name: 'playwright',
+    path: path.join(ROOT_PATH, 'packages', 'playwright'),
+    // We copy README.md additionally for Playwright so that it looks nice on NPM.
+    files: [...LICENCE_FILES, 'README.md'],
+  }),
+  new PWPackage({
+    name: 'playwright-core',
+    path: path.join(ROOT_PATH, 'packages', 'playwright-core'),
+    files: LICENCE_FILES,
+  }),
+  new PWPackage({
+    name: '@playwright/test',
+    path: path.join(ROOT_PATH, 'packages', 'playwright-test'),
+    files: LICENCE_FILES,
+  }),
+  new PWPackage({
+    name: 'playwright-webkit',
+    path: path.join(ROOT_PATH, 'packages', 'playwright-webkit'),
+    files: LICENCE_FILES,
+  }),
+  new PWPackage({
+    name: 'playwright-firefox',
+    path: path.join(ROOT_PATH, 'packages', 'playwright-firefox'),
+    files: LICENCE_FILES,
+  }),
+  new PWPackage({
+    name: 'playwright-chromium',
+    path: path.join(ROOT_PATH, 'packages', 'playwright-chromium'),
+    files: LICENCE_FILES,
+  }),
+  new PWPackage({
+    name: 'html-reporter',
+    path: path.join(ROOT_PATH, 'packages', 'html-reporter'),
+    files: [],
+  }),
+]);
+
+if (require.main === module) {
+  parseCLI();
+} else {
+  module.exports = {workspace};
+}
+
+function die(message, exitCode = 1) {
+  console.error(message);
+  process.exit(exitCode);
+}
+
+async function parseCLI() {
+  const commands = {
+    '--ensure-consistent': async () => {
+      const hasChanges = await workspace.ensureConsistent();
+      if (hasChanges)
+        die(`\n  ERROR: workspace is inconsistent! Run '//utils/workspace.js --ensure-consistent' and commit changes!`);
+    },
+    '--list-public-package-paths': () => {
+      for (const pkg of workspace.packages()) {
+        if (!pkg.isPrivate)
+          console.log(pkg.path);
+      }
+    },
+    '--set-version': async (version) => {
+      if (!version)
+        die('ERROR: Please specify version! e.g. --set-version 1.99.2');
+      await workspace.setVersion(version);
+    },
+    '--help': () => {
+      console.log([
+        `Available commands:`,
+        ...Object.keys(commands).map(cmd => '  ' + cmd),
+      ].join('\n'));
+    },
+  };
+  const handler = commands[process.argv[2]];
+  if (!handler)
+    die('ERROR: wrong usage! Run with --help to list commands');
+  await handler(process.argv[3]);
+}