feat: introduce the --ignore-snapshots option (#17004)

This patch introduces `--ignore-snapshots` Playwright Test CLI option,
and `ignoreSnapshots` configuration argument.

docs/src/test-api/class-testconfig.md

@@ -226,6 +226,11 @@ Filter to only run tests with a title **not** matching one of the patterns. This
 
 `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests).
 
+## property: TestConfig.ignoreSnapshots
+* since: v1.26
+- type: ?<[boolean]>
+
+Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await expect(page).toHaveScreenshot()`.
 
 ## property: TestConfig.maxFailures
 * since: v1.10

docs/src/test-cli-js.md

@@ -111,6 +111,8 @@ Complete set of Playwright Test options is available in the [configuration file]
 
 - `--timeout <number>`: Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).
 
+- `--ignore-snapshots` or `-i`: Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots.
+
 - `--update-snapshots` or `-u`: Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed.
 
 - `--workers <number>` or `-j <number>`: The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md).

packages/playwright-test/src/cli.ts

@@ -59,6 +59,7 @@ function addTestCommand(program: Command) {
   command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: run all projects)`);
   command.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`);
   command.option('--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`);
+  command.option('-i, --ignore-snapshots', `Ignore screenshot and snapshot expectations`);
   command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`);
   command.option('-x', `Stop after the first failure`);
   command.action(async (args, opts) => {
@@ -216,6 +217,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
     reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [resolveReporter(r)]) : undefined,
     shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
     timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
+    ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
     updateSnapshots: options.updateSnapshots ? 'all' as const : undefined,
     workers: options.workers ? parseInt(options.workers, 10) : undefined,
   };

packages/playwright-test/src/loader.ts

@@ -94,6 +94,7 @@ export class Loader {
     config.shard = takeFirst(this._configCLIOverrides.shard, config.shard);
     config.timeout = takeFirst(this._configCLIOverrides.timeout, config.timeout);
     config.updateSnapshots = takeFirst(this._configCLIOverrides.updateSnapshots, config.updateSnapshots);
+    config.ignoreSnapshots = takeFirst(this._configCLIOverrides.ignoreSnapshots, config.ignoreSnapshots);
     if (this._configCLIOverrides.projects && config.projects)
       throw new Error(`Cannot use --browser option when configuration file defines projects. Specify browserName in the projects instead.`);
     config.projects = takeFirst(this._configCLIOverrides.projects, config.projects as any);
@@ -139,6 +140,7 @@ export class Loader {
     this._fullConfig.reportSlowTests = takeFirst(config.reportSlowTests, baseFullConfig.reportSlowTests);
     this._fullConfig.quiet = takeFirst(config.quiet, baseFullConfig.quiet);
     this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard);
+    this._fullConfig._ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._ignoreSnapshots);
     this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots);
     this._fullConfig.workers = takeFirst(config.workers, baseFullConfig.workers);
     const webServers = takeFirst(config.webServer, baseFullConfig.webServer);
@@ -553,6 +555,11 @@ function validateConfig(file: string, config: Config) {
       throw errorWithFile(file, `config.shard.current must be a positive number, not greater than config.shard.total`);
   }
 
+  if ('ignoreSnapshots' in config && config.ignoreSnapshots !== undefined) {
+    if (typeof config.ignoreSnapshots !== 'boolean')
+      throw errorWithFile(file, `config.ignoreSnapshots must be a boolean`);
+  }
+
   if ('updateSnapshots' in config && config.updateSnapshots !== undefined) {
     if (typeof config.updateSnapshots !== 'string' || !['all', 'none', 'missing'].includes(config.updateSnapshots))
       throw errorWithFile(file, `config.updateSnapshots must be one of "all", "none" or "missing"`);
@@ -647,6 +654,7 @@ export const baseFullConfig: FullConfigInternal = {
   _globalOutputDir: path.resolve(process.cwd()),
   _configDir: '',
   _testGroupsCount: 0,
+  _ignoreSnapshots: false,
   _workerIsolation: 'isolate-pools',
 };
 

packages/playwright-test/src/matchers/toMatchSnapshot.ts

@@ -251,6 +251,10 @@ export function toMatchSnapshot(
     throw new Error(`toMatchSnapshot() must be called during the test`);
   if (received instanceof Promise)
     throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.');
+
+  if (testInfo.config._ignoreSnapshots)
+    return { pass: !this.isNot, message: () => '' };
+
   const helper = new SnapshotHelper(
       testInfo, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received),
       testInfo.project._expect?.toMatchSnapshot || {},
@@ -292,6 +296,10 @@ export async function toHaveScreenshot(
   const testInfo = currentTestInfo();
   if (!testInfo)
     throw new Error(`toHaveScreenshot() must be called during the test`);
+
+  if (testInfo.config._ignoreSnapshots)
+    return { pass: !this.isNot, message: () => '' };
+
   const config = (testInfo.project._expect as any)?.toHaveScreenshot;
   const snapshotPathResolver = process.env.PWTEST_USE_SCREENSHOTS_DIR_FOR_TEST
     ? testInfo._screenshotPath.bind(testInfo)

packages/playwright-test/src/runner.ts

@@ -73,6 +73,7 @@ export type ConfigCLIOverrides = {
   reporter?: string;
   shard?: { current: number, total: number };
   timeout?: number;
+  ignoreSnapshots?: boolean;
   updateSnapshots?: 'all'|'none'|'missing';
   workers?: number;
   projects?: { name: string, use?: any }[],

packages/playwright-test/src/types.ts

@@ -46,6 +46,7 @@ export interface FullConfigInternal extends FullConfigPublic {
   _configDir: string;
   _testGroupsCount: number;
   _watchMode: boolean;
+  _ignoreSnapshots: boolean;
   _workerIsolation: WorkerIsolation;
   /**
    * If populated, this should also be the first/only entry in _webServers. Legacy singleton `webServer` as well as those provided via an array in the user-facing playwright.config.{ts,js} will be in `_webServers`. The legacy field (`webServer`) field additionally stores the backwards-compatible singleton `webServer` since it had been showing up in globalSetup to the user.

packages/playwright-test/types/test.d.ts

@@ -678,6 +678,12 @@ interface TestConfig {
    */
   grepInvert?: RegExp|Array<RegExp>;
 
+  /**
+   * Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await
+   * expect(page).toHaveScreenshot()`.
+   */
+  ignoreSnapshots?: boolean;
+
   /**
    * The maximum number of test failures for the whole test suite run. After reaching this number, testing will stop and exit
    * with an error. Setting to zero (default) disables this behavior.

tests/playwright-test/config.spec.ts

@@ -373,6 +373,26 @@ test('should inerhit use options in projects', async ({ runInlineTest }) => {
   expect(result.passed).toBe(1);
 });
 
+test('should support ignoreSnapshots config option', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        ignoreSnapshots: true,
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}, testInfo) => {
+        expect('foo').toMatchSnapshot();
+        expect('foo').not.toMatchSnapshot();
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
 test('should work with undefined values and base', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `

tests/playwright-test/golden.spec.ts

@@ -268,6 +268,27 @@ test('should update snapshot with the update-snapshots flag', async ({ runInline
   expect(data.toString()).toBe(ACTUAL_SNAPSHOT);
 });
 
+test('should ignore text snapshot with the ignore-snapshots flag', async ({ runInlineTest }, testInfo) => {
+  const EXPECTED_SNAPSHOT = 'Hello world';
+  const ACTUAL_SNAPSHOT = 'Hello world updated';
+  const result = await runInlineTest({
+    ...files,
+    'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', ({}) => {
+        expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, { 'ignore-snapshots': true });
+
+  expect(result.exitCode).toBe(0);
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(``);
+  const data = fs.readFileSync(snapshotOutputPath);
+  expect(data.toString()).toBe(EXPECTED_SNAPSHOT);
+});
+
 test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => {
   const EXPECTED_SNAPSHOT = 'Hello world';
   const ACTUAL_SNAPSHOT = 'Hello world updated';

tests/playwright-test/to-have-screenshot.spec.ts

@@ -555,6 +555,20 @@ test('should fail on same snapshots with negate matcher', async ({ runInlineTest
   expect(result.output).toContain('Expected result should be different from the actual one.');
 });
 
+test('should not fail if --ignore-snapshots is passed', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
+    '__screenshots__/a.spec.js/snapshot.png': redImage,
+    'a.spec.js': `
+      pwt.test('is a test', async ({ page }) => {
+        await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
+      });
+    `
+  }, { 'ignore-snapshots': true });
+
+  expect(result.exitCode).toBe(0);
+});
+
 test('should write missing expectations locally twice and continue', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     ...playwrightConfig({ screenshotsDir: '__screenshots__' }),