test: add installation tests for supposed plugin story (#27331)

.eslintignore

@@ -14,6 +14,7 @@ browser_patches/chromium/output/
 output/
 test-results/
 tests/components/
+tests/installation/fixture-scripts/
 examples/
 DEPS
 .cache/

packages/playwright-core/src/utils/spawnAsync.ts

@@ -24,9 +24,9 @@ export function spawnAsync(cmd: string, args: string[], options: SpawnOptions =
     let stdout = '';
     let stderr = '';
     if (process.stdout)
-      process.stdout.on('data', data => stdout += data);
+      process.stdout.on('data', data => stdout += data.toString());
     if (process.stderr)
-      process.stderr.on('data', data => stderr += data);
+      process.stderr.on('data', data => stderr += data.toString());
     process.on('close', code => resolve({ stdout, stderr, code }));
     process.on('error', error => resolve({ stdout, stderr, code: 0, error }));
   });

tests/installation/fixture-scripts/playwright-test-plugin-types.ts

@@ -0,0 +1,13 @@
+import { test as test1 } from '@playwright/test';
+import type { Page } from '@playwright/test';
+import { test as test2 } from 'playwright-test-plugin';
+
+const test = (test1 as any)._extendTest(test2);
+
+test('sample test', async ({ page, plugin }) => {
+  type IsPage = (typeof page) extends Page ? true : never;
+  const isPage: IsPage = true;
+
+  type IsString = (typeof plugin) extends string ? true : never;
+  const isString: IsString = true;
+});

tests/installation/fixture-scripts/plugin.spec.ts

@@ -0,0 +1,12 @@
+import { test as test1, expect } from '@playwright/test';
+import { test as test2 } from 'playwright-test-plugin';
+
+const test = (test1 as any)._extendTest(test2);
+
+test('sample test', async ({ page, plugin }) => {
+  await page.setContent(`<div>hello</div><span>world</span>`);
+  expect(await page.textContent('span')).toBe('world');
+
+  console.log(`plugin value: ${plugin}`);
+  expect(plugin).toBe('hello from plugin');
+});

tests/installation/globalSetup.ts

@@ -65,6 +65,22 @@ async function globalSetup() {
       build('playwright-browser-webkit', '@playwright/browser-webkit'),
     ]);
 
+    const buildPlaywrightTestPlugin = async () => {
+      const cwd = path.resolve(path.join(__dirname, `playwright-test-plugin`));
+      const tscResult = await spawnAsync('npx', ['tsc', '-p', 'tsconfig.json'], { cwd, shell: process.platform === 'win32' });
+      if (tscResult.code)
+        throw new Error(`Failed to build playwright-test-plugin:\n${tscResult.stderr}\n${tscResult.stdout}`);
+      const packResult = await spawnAsync('npm', ['pack'], { cwd, shell: process.platform === 'win32' });
+      if (packResult.code)
+        throw new Error(`Failed to build playwright-test-plugin:\n${packResult.stderr}\n${packResult.stdout}`);
+      const tgzName = packResult.stdout.trim();
+      const outPath = path.resolve(path.join(outputDir, `playwright-test-plugin.tgz`));
+      await fs.promises.rename(path.join(cwd, tgzName), outPath);
+      console.log('Built playwright-test-plugin');
+      return ['playwright-test-plugin', outPath];
+    };
+    builds.push(await buildPlaywrightTestPlugin());
+
     await fs.promises.writeFile(path.join(__dirname, '.registry.json'), JSON.stringify(Object.fromEntries(builds)));
   }
 

tests/installation/npmTest.ts

@@ -20,10 +20,11 @@ import os from 'os';
 import path from 'path';
 import debugLogger from 'debug';
 import { Registry }  from './registry';
-import { spawnAsync } from './spawnAsync';
 import type { CommonFixtures, CommonWorkerFixtures } from '../config/commonFixtures';
 import { commonFixtures } from '../config/commonFixtures';
 import { removeFolders } from '../../packages/playwright-core/lib/utils/fileUtils';
+import { spawnAsync } from '../../packages/playwright-core/lib/utils/spawnAsync';
+import type { SpawnOptions } from 'child_process';
 
 export const TMP_WORKSPACES = path.join(os.platform() === 'darwin' ? '/tmp' : os.tmpdir(), 'pwt', 'workspaces');
 
@@ -56,7 +57,7 @@ const expect = _expect.extend({
   }
 });
 
-type ExecOptions = { cwd?: string, env?: Record<string, string>, message?: string, expectToExitWithError?: boolean };
+type ExecOptions = SpawnOptions & { message?: string, expectToExitWithError?: boolean };
 type ArgsOrOptions = [] | [...string[]] | [...string[], ExecOptions] | [ExecOptions];
 
 type NPMTestOptions = {
@@ -71,7 +72,7 @@ type NPMTestFixtures = {
   writeConfig: (allowGlobal: boolean) => Promise<void>;
   writeFiles: (nameToContents: Record<string, string>) => Promise<void>;
   exec: (cmd: string, ...argsAndOrOptions: ArgsOrOptions) => Promise<string>;
-  tsc: (...argsAndOrOptions: ArgsOrOptions) => Promise<string>;
+  tsc: (args: string) => Promise<string>;
   registry: Registry;
 };
 
@@ -153,7 +154,7 @@ export const test = _test
 
           args = argsAndOrOptions as string[];
 
-          let result!: Awaited<ReturnType<typeof spawnAsync>>;
+          let result!: {stdout: string, stderr: string, code: number | null, error?: Error};
           await test.step(`exec: ${[cmd, ...args].join(' ')}`, async () => {
             result = await spawnAsync(cmd, args, {
               shell: true,
@@ -197,8 +198,8 @@ export const test = _test
         });
       },
       tsc: async ({ exec }, use) => {
-        await exec('npm i --foreground-scripts typescript@3.8 @types/node@14');
-        await use((...args: ArgsOrOptions) => exec('npx', '-p', 'typescript@4.1.6', 'tsc', ...args));
+        await exec('npm i --foreground-scripts typescript@5.2.2 @types/node@16');
+        await use((args: string) => exec('npx', 'tsc', args, { shell: process.platform === 'win32' }));
       },
     });
 

tests/installation/playwright-test-plugin.spec.ts

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './npmTest';
+import path from 'path';
+import fs from 'fs';
+
+function patchPackageJsonForPreReleaseIfNeeded(tmpWorkspace: string) {
+  // It is not currently possible to declare plugin's peerDependency to match
+  // various pre-release versions, e.g. "1.38.0-next" and "1.39.1-alpha".
+  // See https://github.com/npm/rfcs/pull/397 and https://github.com/npm/node-semver#prerelease-tags.
+  //
+  // Workaround per https://stackoverflow.com/questions/71479750/npm-install-pre-release-versions-for-peer-dependency.
+  const pkg = JSON.parse(fs.readFileSync(path.resolve(tmpWorkspace, 'package.json'), 'utf-8'));
+  if (pkg.dependencies['@playwright/test'].match(/\d+\.\d+-\w+/)) {
+    console.log(`Setting overrides in package.json to make pre-release version of peer dependency work.`);
+    pkg.overrides = { '@playwright/test': '$@playwright/test' };
+    fs.writeFileSync(path.resolve(tmpWorkspace, 'package.json'), JSON.stringify(pkg, null, 2));
+  }
+}
+
+test('npm: @playwright/test plugin should work', async ({ exec, tmpWorkspace }) => {
+  await exec('npm i --foreground-scripts @playwright/test');
+  patchPackageJsonForPreReleaseIfNeeded(tmpWorkspace);
+  await exec('npm i --foreground-scripts playwright-test-plugin');
+  await exec('npx playwright install chromium');
+
+  const output = await exec('npx playwright test -c . --browser=chromium --reporter=line plugin.spec.ts');
+  expect(output).toContain('plugin value: hello from plugin');
+  expect(output).toContain('1 passed');
+
+  await exec('npm i --foreground-scripts typescript@5.2.2 @types/node@16');
+  await exec('npx tsc playwright-test-plugin-types.ts');
+});
+
+test('pnpm: @playwright/test plugin should work', async ({ exec, tmpWorkspace }) => {
+  await exec('pnpm add @playwright/test');
+  patchPackageJsonForPreReleaseIfNeeded(tmpWorkspace);
+  await exec('pnpm add playwright-test-plugin');
+  await exec('pnpm exec playwright install chromium');
+
+  const output = await exec('pnpm exec playwright test -c . --browser=chromium --reporter=line plugin.spec.ts');
+  expect(output).toContain('plugin value: hello from plugin');
+  expect(output).toContain('1 passed');
+
+  await exec('pnpm add typescript@5.2.2 @types/node@16');
+  await exec('pnpm exec tsc playwright-test-plugin-types.ts');
+});
+
+test('yarn: @playwright/test plugin should work', async ({ exec, tmpWorkspace }) => {
+  await exec('yarn add @playwright/test');
+  patchPackageJsonForPreReleaseIfNeeded(tmpWorkspace);
+  await exec('yarn add playwright-test-plugin');
+  await exec('yarn playwright install chromium');
+
+  const output = await exec('yarn playwright test -c . --browser=chromium --reporter=line plugin.spec.ts');
+  expect(output).toContain('plugin value: hello from plugin');
+  expect(output).toContain('1 passed');
+
+  await exec('yarn add typescript@5.2.2 @types/node@16');
+  await exec('yarn tsc playwright-test-plugin-types.ts');
+});

tests/installation/playwright-test-plugin/.gitignore

@@ -0,0 +1,2 @@
+**/*.js
+**/*.d.ts

tests/installation/playwright-test-plugin/.npmignore

@@ -0,0 +1,2 @@
+**/*.ts
+!**/*.d.ts

tests/installation/playwright-test-plugin/index.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test as base } from '@playwright/test';
+
+export const test = base.extend<{ plugin: string }>({
+  plugin: async ({}, use) => {
+    await use('hello from plugin');
+  },
+});

tests/installation/playwright-test-plugin/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "playwright-test-plugin",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "peerDependencies": {
+    "@playwright/test": "1.x"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC"
+}

tests/installation/playwright-test-plugin/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2019",
+    "module": "commonjs",
+    "lib": ["esnext", "dom", "DOM.Iterable"],
+    "esModuleInterop": true,
+    "strict": true,
+    "allowJs": false,
+    "resolveJsonModule": true,
+    "noImplicitOverride": true,
+    "useUnknownInCatchVariables": false,
+    "declaration": true,
+  },
+}

tests/installation/playwright-test-should-work.spec.ts

@@ -18,7 +18,7 @@ import path from 'path';
 
 test('npm: @playwright/test should work', async ({ exec, tmpWorkspace }) => {
   await exec('npm i --foreground-scripts @playwright/test');
-  await exec('npx playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
+  await exec('npx playwright test -c . sample.spec.js', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
 
   await exec('npx playwright install');
   await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });

tests/installation/registry.ts

@@ -19,7 +19,7 @@ import type { Server } from 'http';
 import type http from 'http';
 import https from 'https';
 import path from 'path';
-import { spawnAsync } from './spawnAsync';
+import { spawnAsync } from '../../packages/playwright-core/lib/utils/spawnAsync';
 import { createHttpServer } from '../../packages/playwright-core/lib/utils/network';
 
 const kPublicNpmRegistry = 'https://registry.npmjs.org';

tests/installation/spawnAsync.ts

@@ -1,48 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import type { SpawnOptions } from 'child_process';
-import { spawn } from 'child_process';
-import debugLogger from 'debug';
-
-const debugExec = debugLogger('itest:exec');
-const debugExecStdout = debugLogger('itest:exec:stdout');
-const debugExecStderr = debugLogger('itest:exec:stderr');
-
-export function spawnAsync(cmd: string, args: string[], options: SpawnOptions = {}): Promise<{stdout: string, stderr: string, code: number | null, error?: Error}> {
-  // debugExec(`CWD: ${options.cwd || process.cwd()}`);
-  // debugExec(`ENV: ${Object.entries(options.env || {}).map(([key, value]) => `${key}=${value}`).join(' ')}`);
-  debugExec([cmd, ...args].join(' '));
-  const p = spawn(cmd, args, Object.assign({ windowsHide: true }, options));
-
-  return new Promise(resolve => {
-    let stdout = '';
-    let stderr = '';
-    if (p.stdout) {
-      p.stdout.on('data', data => {
-        debugExecStdout(data.toString());
-        stdout += data;
-      });
-    }
-    if (p.stderr) {
-      p.stderr.on('data', data => {
-        debugExecStderr(data.toString());
-        stderr += data;
-      });
-    }
-    p.on('close', code => resolve({ stdout, stderr, code }));
-    p.on('error', error => resolve({ stdout, stderr, code: 0, error }));
-  });
-}

tests/installation/typescript-types.spec.ts

@@ -37,8 +37,6 @@ test('typescript types should work', async ({ exec, tsc, writeFiles }) => {
 });
 
 test('typescript types should work with module: NodeNext', async ({ exec, tsc, writeFiles }) => {
-  // module: NodeNext got added in TypeScript 4.7
-  await exec('npm i --foreground-scripts typescript@4.7 @types/node@18');
   const libraryPackages = [
     'playwright',
     'playwright-core',
@@ -53,8 +51,8 @@ test('typescript types should work with module: NodeNext', async ({ exec, tsc, w
     await writeFiles({
       [filename]: `import { Page } from '${libraryPackage}';`,
     });
-    await exec('npx', '-p', 'typescript@4.7', 'tsc', '--module nodenext', filename);
+    await tsc(`--module nodenext ${filename}`);
   }
 
-  await exec('npx', '-p', 'typescript@4.7', 'tsc', '--module nodenext', 'playwright-test-types.ts');
+  await tsc('--module nodenext playwright-test-types.ts');
 });

tests/tsconfig.json

@@ -19,5 +19,5 @@
         },
     },
     "include": ["**/*.spec.js", "**/*.ts", "index.d.ts"],
-    "exclude": ["components/"]
+    "exclude": ["components/", "installation/fixture-scripts/"]
 }