chore: fixtures-via-plugin implementation (#13950)

docs/src/test-api/class-testconfig.md

@@ -319,6 +319,9 @@ The directory for each test can be accessed by [`property: TestInfo.snapshotDir`
 
 This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to `'snapshots'`, the [`property: TestInfo.snapshotDir`] would resolve to `snapshots/a.spec.js-snapshots`.
 
+## property: TestConfig.plugins
+- type: ?<[Array]<[TestPlugin]|[string]>>
+
 ## property: TestConfig.preserveOutput
 - type: ?<[PreserveOutput]<"always"|"never"|"failures-only">>
 

docs/src/test-api/class-testplugin.md

@@ -0,0 +1,20 @@
+# class: TestPlugin
+* langs: js
+
+## property: TestPlugin.name
+- type: <[string]>
+
+## optional async method: TestPlugin.setup
+### param: TestPlugin.setup.config
+- `config` <[FullConfig]>
+
+### param: TestPlugin.setup.configDir
+- `configDir` <[string]>
+
+### param: TestPlugin.setup.suite
+- `suite` <[Suite]>
+
+## optional async method: TestPlugin.teardown
+
+## optional property: TestPlugin.fixtures
+- `fixtures` <[any]>

package-lock.json

@@ -5707,7 +5707,7 @@
     },
     "packages/playwright-ct-react": {
       "name": "@playwright/experimental-ct-react",
-      "version": "0.0.5",
+      "version": "0.0.7",
       "license": "Apache-2.0",
       "dependencies": {
         "@vitejs/plugin-react": "^1.0.7",
@@ -5722,7 +5722,7 @@
     },
     "packages/playwright-ct-svelte": {
       "name": "@playwright/experimental-ct-svelte",
-      "version": "0.0.5",
+      "version": "0.0.7",
       "license": "Apache-2.0",
       "dependencies": {
         "@sveltejs/vite-plugin-svelte": "^1.0.0-next.30",
@@ -5737,7 +5737,7 @@
     },
     "packages/playwright-ct-vue": {
       "name": "@playwright/experimental-ct-vue",
-      "version": "0.0.5",
+      "version": "0.0.7",
       "license": "Apache-2.0",
       "dependencies": {
         "@vitejs/plugin-vue": "^2.3.1",

packages/playwright-test/src/loader.ts

@@ -15,7 +15,7 @@
  */
 
 import { installTransform, setCurrentlyLoadingTestFile } from './transform';
-import type { Config, Project, ReporterDescription, FullProjectInternal, FullConfigInternal, Fixtures, FixturesWithLocation } from './types';
+import type { Config, Project, ReporterDescription, FullProjectInternal, FullConfigInternal, Fixtures, FixturesWithLocation, TestPlugin } from './types';
 import { getPackageJsonPath, mergeObjects, errorWithFile } from './util';
 import { setCurrentlyLoadingFileSuite } from './globals';
 import { Suite, type TestCase } from './test';
@@ -63,9 +63,7 @@ export class Loader {
   async loadConfigFile(file: string): Promise<FullConfigInternal> {
     if (this._configFile)
       throw new Error('Cannot load two config files');
-    let config = await this._requireOrImport(file) as Config;
-    if (config && typeof config === 'object' && ('default' in config))
-      config = (config as any)['default'];
+    const config = await this._requireOrImportDefaultObject(file) as Config;
     this._configFile = file;
     await this._processConfigObject(config, path.dirname(file));
     return this._fullConfig;
@@ -125,6 +123,19 @@ export class Loader {
     if (config.snapshotDir !== undefined)
       config.snapshotDir = path.resolve(configDir, config.snapshotDir);
 
+    config.plugins = await Promise.all((config.plugins || []).map(async plugin => {
+      if (typeof plugin === 'string')
+        return (await this._requireOrImportDefaultObject(resolveScript(plugin, configDir))) as TestPlugin;
+      return plugin;
+    }));
+
+    for (const plugin of config.plugins || []) {
+      if (!plugin.fixtures)
+        continue;
+      if (typeof plugin.fixtures === 'string')
+        plugin.fixtures = await this._requireOrImportDefaultObject(resolveScript(plugin.fixtures, configDir));
+    }
+
     this._fullConfig._configDir = configDir;
     this._fullConfig.rootDir = config.testDir || this._configDir;
     this._fullConfig._globalOutputDir = takeFirst(config.outputDir, throwawayArtifactsPath, baseFullConfig._globalOutputDir);
@@ -144,8 +155,9 @@ export class Loader {
     this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots);
     this._fullConfig.workers = takeFirst(config.workers, baseFullConfig.workers);
     this._fullConfig.webServer = takeFirst(config.webServer, baseFullConfig.webServer);
+    this._fullConfig._plugins = takeFirst(config.plugins, baseFullConfig._plugins);
     this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
-    this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, p, throwawayArtifactsPath));
+    this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
   }
 
   async loadTestFile(file: string, environment: 'runner' | 'worker') {
@@ -193,21 +205,11 @@ export class Loader {
   }
 
   async loadGlobalHook(file: string, name: string): Promise<(config: FullConfigInternal) => any> {
-    let hook = await this._requireOrImport(file);
-    if (hook && typeof hook === 'object' && ('default' in hook))
-      hook = hook['default'];
-    if (typeof hook !== 'function')
-      throw errorWithFile(file, `${name} file must export a single function.`);
-    return hook;
+    return this._requireOrImportDefaultFunction(path.resolve(this._fullConfig.rootDir, file), false);
   }
 
   async loadReporter(file: string): Promise<new (arg?: any) => Reporter> {
-    let func = await this._requireOrImport(path.resolve(this._fullConfig.rootDir, file));
-    if (func && typeof func === 'object' && ('default' in func))
-      func = func['default'];
-    if (typeof func !== 'function')
-      throw errorWithFile(file, `reporter file must export a single class.`);
-    return func;
+    return this._requireOrImportDefaultFunction(path.resolve(this._fullConfig.rootDir, file), true);
   }
 
   fullConfig(): FullConfigInternal {
@@ -241,7 +243,7 @@ export class Loader {
     projectConfig.use = mergeObjects(projectConfig.use, this._configCLIOverrides.use);
   }
 
-  private _resolveProject(config: Config, projectConfig: Project, throwawayArtifactsPath: string): FullProjectInternal {
+  private _resolveProject(config: Config, fullConfig: FullConfigInternal, projectConfig: Project, throwawayArtifactsPath: string): FullProjectInternal {
     // Resolve all config dirs relative to configDir.
     if (projectConfig.testDir !== undefined)
       projectConfig.testDir = path.resolve(this._configDir, projectConfig.testDir);
@@ -259,6 +261,7 @@ export class Loader {
     const name = takeFirst(projectConfig.name, config.name, '');
     const screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
     return {
+      _fullConfig: fullConfig,
       _fullyParallel: takeFirst(projectConfig.fullyParallel, config.fullyParallel, undefined),
       _expect: takeFirst(projectConfig.expect, config.expect, {}),
       grep: takeFirst(projectConfig.grep, config.grep, baseFullConfig.grep),
@@ -308,22 +311,38 @@ ${'='.repeat(80)}\n`);
       revertBabelRequire();
     }
   }
+
+  private async _requireOrImportDefaultFunction(file: string, expectConstructor: boolean) {
+    let func = await this._requireOrImport(file);
+    if (func && typeof func === 'object' && ('default' in func))
+      func = func['default'];
+    if (typeof func !== 'function')
+      throw errorWithFile(file, `file must export a single ${expectConstructor ? 'class' : 'function'}.`);
+    return func;
+  }
+
+  private async _requireOrImportDefaultObject(file: string) {
+    let object = await this._requireOrImport(file);
+    if (object && typeof object === 'object' && ('default' in object))
+      object = object['default'];
+    return object;
+  }
 }
 
 class ProjectSuiteBuilder {
-  private _config: FullProjectInternal;
+  private _project: FullProjectInternal;
   private _index: number;
   private _testTypePools = new Map<TestTypeImpl, FixturePool>();
   private _testPools = new Map<TestCase, FixturePool>();
 
   constructor(project: FullProjectInternal, index: number) {
-    this._config = project;
+    this._project = project;
     this._index = index;
   }
 
   private _buildTestTypePool(testType: TestTypeImpl): FixturePool {
     if (!this._testTypePools.has(testType)) {
-      const fixtures = this._applyConfigUseOptions(testType, this._config.use || {});
+      const fixtures = this._applyConfigUseOptions(testType, this._project.use || {});
       const pool = new FixturePool(fixtures);
       this._testTypePools.set(testType, pool);
     }
@@ -335,6 +354,16 @@ class ProjectSuiteBuilder {
     if (!this._testPools.has(test)) {
       let pool = this._buildTestTypePool(test._testType);
 
+      for (const plugin of this._project._fullConfig._plugins) {
+        if (!plugin.fixtures)
+          continue;
+        const pluginFixturesWithLocation: FixturesWithLocation = {
+          fixtures: plugin.fixtures,
+          location: { file: '', line: 0, column: 0 },
+        };
+        pool = new FixturePool([pluginFixturesWithLocation], pool, false);
+      }
+
       const parents: Suite[] = [];
       for (let parent: Suite | undefined = test.parent; parent; parent = parent.parent)
         parents.push(parent);
@@ -366,7 +395,7 @@ class ProjectSuiteBuilder {
         }
       } else {
         const test = entry._clone();
-        test.retries = this._config.retries;
+        test.retries = this._project.retries;
         // We rely upon relative paths being unique.
         // See `getClashingTestsPerSuite()` in `runner.ts`.
         test._id = `${calculateSha1(relativeTitlePath + ' ' + entry.title)}@${entry._requireFile}#run${this._index}-repeat${repeatEachIndex}`;
@@ -624,6 +653,7 @@ export const baseFullConfig: FullConfigInternal = {
   _globalOutputDir: path.resolve(process.cwd()),
   _configDir: '',
   _testGroupsCount: 0,
+  _plugins: [],
 };
 
 function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined {

packages/playwright-test/src/reporters/raw.ts

@@ -167,7 +167,7 @@ class RawReporter {
     const project = suite.project();
     assert(project, 'Internal Error: Invalid project structure');
     const report: JsonReport = {
-      config,
+      config: filterOutPrivateFields(config),
       project: {
         metadata: project.metadata,
         name: project.name,
@@ -317,4 +317,12 @@ function dedupeSteps(steps: JsonTestStep[]): JsonTestStep[] {
   return result;
 }
 
+function filterOutPrivateFields(object: any): any {
+  if (!object || typeof object !== 'object')
+    return object;
+  if (Array.isArray(object))
+    return object.map(filterOutPrivateFields);
+  return Object.fromEntries(Object.entries(object).filter(entry => !entry[0].startsWith('_')).map(entry => [entry[0], filterOutPrivateFields(entry[1])]));
+}
+
 export default RawReporter;

packages/playwright-test/src/runner.ts

@@ -434,7 +434,6 @@ export class Runner {
 
   private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite): Promise<(() => Promise<void>) | undefined> {
     const result: FullResult = { status: 'passed' };
-    const pluginTeardowns: (() => Promise<void>)[] = [];
     let globalSetupResult: any;
 
     const tearDown = async () => {
@@ -449,9 +448,9 @@ export class Runner {
           await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig());
       }, result);
 
-      for (const teardown of pluginTeardowns) {
+      for (const plugin of [...this._plugins, ...config._plugins].reverse()) {
         await this._runAndReportError(async () => {
-          await teardown();
+          await plugin.teardown?.();
         }, result);
       }
     };
@@ -463,11 +462,8 @@ export class Runner {
 
       // First run the plugins, if plugin is a web server we want it to run before the
       // config's global setup.
-      for (const plugin of this._plugins) {
+      for (const plugin of [...this._plugins, ...config._plugins])
         await plugin.setup?.(config, config._configDir, rootSuite);
-        if (plugin.teardown)
-          pluginTeardowns.unshift(plugin.teardown);
-      }
 
       // The do global setup.
       if (config.globalSetup)

packages/playwright-test/src/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import type { Fixtures, TestError, Project } from '../types/test';
+import type { Fixtures, TestError, Project, TestPlugin } from '../types/test';
 import type { Location } from '../types/testReporter';
 import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types';
 export * from '../types/test';
@@ -44,6 +44,7 @@ export interface FullConfigInternal extends FullConfigPublic {
   _globalOutputDir: string;
   _configDir: string;
   _testGroupsCount: number;
+  _plugins: TestPlugin[];
 
   // Overrides the public field.
   projects: FullProjectInternal[];
@@ -54,6 +55,7 @@ export interface FullConfigInternal extends FullConfigPublic {
  * increasing the surface area of the public API type called FullProject.
  */
 export interface FullProjectInternal extends FullProjectPublic {
+  _fullConfig: FullConfigInternal;
   _fullyParallel: boolean;
   _expect: Project['expect'];
   _screenshotsDir: string;

packages/playwright-test/types/test.d.ts

@@ -366,6 +366,22 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
 
 type LiteralUnion<T extends U, U = string> = T | (U & { zz_IGNORE_ME?: never });
 
+/**
+ *
+ */
+export interface TestPlugin {
+  fixtures?: Fixtures;
+  name: string;
+
+  /**
+   * @param config
+   * @param configDir
+   * @param suite
+   */
+  setup?(config: FullConfig, configDir: string, suite: Suite): Promise<void>;
+
+  teardown?(): Promise<void>;}
+
 /**
  * Playwright Test provides many options to configure how your tests are collected and executed, for example `timeout` or
  * `testDir`. These options are described in the [TestConfig] object in the [configuration file](https://playwright.dev/docs/test-configuration).
@@ -459,6 +475,7 @@ interface TestConfig {
    *
    */
   webServer?: TestConfigWebServer;
+  plugins?: TestPlugin[],
   /**
    * Configuration for the `expect` assertion library. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts).
    *

tests/components/ct-react/.gitignore

@@ -11,6 +11,8 @@
 # production
 /build
 
+/dist-pw
+
 # misc
 .DS_Store
 .env.local

tests/config/experimental.d.ts

@@ -17026,6 +17026,22 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
 
 type LiteralUnion<T extends U, U = string> = T | (U & { zz_IGNORE_ME?: never });
 
+/**
+ *
+ */
+export interface TestPlugin {
+  fixtures?: Fixtures;
+  name: string;
+
+  /**
+   * @param config
+   * @param configDir
+   * @param suite
+   */
+  setup?(config: FullConfig, configDir: string, suite: Suite): Promise<void>;
+
+  teardown?(): Promise<void>;}
+
 /**
  * Playwright Test provides many options to configure how your tests are collected and executed, for example `timeout` or
  * `testDir`. These options are described in the [TestConfig] object in the [configuration file](https://playwright.dev/docs/test-configuration).
@@ -17119,6 +17135,7 @@ interface TestConfig {
    *
    */
   webServer?: TestConfigWebServer;
+  plugins?: TestPlugin[],
   /**
    * Configuration for the `expect` assertion library. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts).
    *

tests/playwright-test/global-setup.spec.ts

@@ -184,7 +184,7 @@ test('globalSetup should throw when passed non-function', async ({ runInlineTest
       });
     `,
   });
-  expect(output).toContain(`globalSetup.ts: globalSetup file must export a single function.`);
+  expect(output).toContain(`globalSetup.ts: file must export a single function.`);
 });
 
 test('globalSetup should work with default export and run the returned fn', async ({ runInlineTest }) => {

tests/playwright-test/plugins.spec.ts

@@ -0,0 +1,162 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs';
+import { test, expect } from './playwright-test-fixtures';
+
+test('event order', async ({ runInlineTest }, testInfo) => {
+  const log = testInfo.outputPath('logs.txt');
+  const result = await runInlineTest({
+    'log.ts': `
+      import { appendFileSync } from 'fs';
+      const log = (...args) => appendFileSync('${log.replace(/\\/g, '\\\\')}', args.join(' ') + '\\n');
+      export default log;
+    `,
+    'test.spec.ts': `
+      import log from './log';
+      const { test } = pwt;
+      test('it works', async ({}) => {
+      });
+    `,
+    'playwright.config.ts': `
+      import { myPlugin } from './plugin.ts';
+      module.exports = {
+        plugins: [
+            myPlugin('a'),
+            myPlugin('b'),
+        ],
+        globalSetup: 'globalSetup.ts',
+        globalTeardown: 'globalTeardown.ts',
+      };
+    `,
+    'globalSetup.ts': `
+      import log from './log';
+      const setup = async () => {
+        await new Promise(r => setTimeout(r, 100));
+        log('globalSetup');
+      }
+      export default setup;
+    `,
+    'globalTeardown.ts': `
+      import log from './log';
+      const teardown = async () => {
+        await new Promise(r => setTimeout(r, 100));
+        log('globalTeardown');
+      }
+      export default teardown;
+    `,
+    'plugin.ts': `
+      import log from './log';
+      export const myPlugin = (name: string) => ({
+        setup: async () => {
+          await new Promise(r => setTimeout(r, 100));
+          log(name, 'setup');
+        },
+        teardown: async () => {
+          await new Promise(r => setTimeout(r, 100));
+          log(name, 'teardown');
+        },
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  const logLines = await fs.promises.readFile(log, 'utf8');
+  expect(logLines.split('\n')).toEqual([
+    'a setup',
+    'b setup',
+    'globalSetup',
+    'globalTeardown',
+    'b teardown',
+    'a teardown',
+    '',
+  ]);
+});
+
+test('plugins via require', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('it works', async ({}) => {
+        expect(process.env.PW_CONFIG_DIR).toContain('plugins-via-require');
+      });
+    `,
+    'playwright.config.ts': `
+      export default { plugins: [ 'plugin.ts' ] };
+    `,
+    'plugin.ts': `
+      export function setup(config, configDir, suite) {
+        process.env.PW_CONFIG_DIR = configDir;
+      };
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('fixtures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('it works', async ({ foo }) => {
+        expect(foo).toEqual(42);
+      });
+
+      test('it uses standard fixture', async ({ myBrowserName }) => {
+        expect(myBrowserName).toEqual('chromium');
+      });
+    `,
+    'playwright.config.ts': `
+      import plugin from './plugin.ts';
+      module.exports = {
+        plugins: [ plugin ],
+      };
+    `,
+    'plugin.ts': `
+      export default {
+        fixtures: {
+          foo: 42,
+          myBrowserName: async ({ browserName }, use) => { await use(browserName) }
+        }
+      };
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+});
+
+test('fixtures via require', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('it works', async ({ foo }) => {
+        expect(foo).toEqual(42);
+      });
+    `,
+    'playwright.config.ts': `
+      export default {
+        plugins: [ { fixtures: require.resolve('./fixtures.ts') } ],
+      };
+    `,
+    'fixtures.ts': `
+      //@no-header
+      export default {
+        foo: 42
+      };
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});

utils/generate_types/overrides-test.d.ts

@@ -56,9 +56,14 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
 
 type LiteralUnion<T extends U, U = string> = T | (U & { zz_IGNORE_ME?: never });
 
+export interface TestPlugin {
+  fixtures?: Fixtures;
+}
+
 interface TestConfig {
   reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html', string> | ReporterDescription[];
   webServer?: TestConfigWebServer;
+  plugins?: TestPlugin[],
 }
 
 export interface Config<TestArgs = {}, WorkerArgs = {}> extends TestConfig {