fix(test runner): allow multiple missing snapshots per test (#10621)

Instead of failing right away, continue test execution but mark
the test as failed.

packages/playwright-test/src/matchers/golden.ts

@@ -26,7 +26,7 @@ import { TestInfoImpl, UpdateSnapshots } from '../types';
 import { addSuffixToFilePath } from '../util';
 
 // Note: we require the pngjs version of pixelmatch to avoid version mismatches.
-const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] }));
+const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] })) as typeof import('pngjs');
 
 const extensionToMimeType: { [key: string]: string } = {
   'dat': 'application/octet-string',
@@ -36,14 +36,15 @@ const extensionToMimeType: { [key: string]: string } = {
   'txt': 'text/plain',
 };
 
-const GoldenComparators: { [key: string]: any } = {
+type Comparator = (actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options?: any) => { diff?: Buffer; errorMessage?: string; } | null;
+const GoldenComparators: { [key: string]: Comparator } = {
   'application/octet-string': compareBuffersOrStrings,
   'image/png': compareImages,
   'image/jpeg': compareImages,
   'text/plain': compareText,
 };
 
-function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string): { diff?: object; errorMessage?: string; } | null {
+function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string): { diff?: Buffer; errorMessage?: string; } | null {
   if (typeof actualBuffer === 'string')
     return compareText(actualBuffer, expectedBuffer);
   if (!actualBuffer || !(actualBuffer instanceof Buffer))
@@ -53,7 +54,7 @@ function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer:
   return null;
 }
 
-function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: object; errorMessage?: string; } | null {
+function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: Buffer; errorMessage?: string; } | null {
   if (!actualBuffer || !(actualBuffer instanceof Buffer))
     return { errorMessage: 'Actual result should be a Buffer.' };
 
@@ -69,7 +70,7 @@ function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mi
   return count > 0 ? { diff: PNG.sync.write(diff) } : null;
 }
 
-function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: object; errorMessage?: string; diffExtension?: string; } | null {
+function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: Buffer; errorMessage?: string; diffExtension?: string; } | null {
   if (typeof actual !== 'string')
     return { errorMessage: 'Actual result should be a string' };
   const expected = expectedBuffer.toString('utf-8');
@@ -86,14 +87,13 @@ function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?:
 export function compare(
   actual: Buffer | string,
   pathSegments: string[],
-  snapshotPath: TestInfoImpl['snapshotPath'],
-  outputPath: TestInfoImpl['outputPath'],
+  testInfo: TestInfoImpl,
   updateSnapshots: UpdateSnapshots,
   withNegateComparison: boolean,
   options?: { threshold?: number }
 ): { pass: boolean; message?: string; expectedPath?: string, actualPath?: string, diffPath?: string, mimeType?: string } {
-  const snapshotFile = snapshotPath(...pathSegments);
-  const outputFile = outputPath(...pathSegments);
+  const snapshotFile = testInfo.snapshotPath(...pathSegments);
+  const outputFile = testInfo.outputPath(...pathSegments);
   const expectedPath = addSuffixToFilePath(outputFile, '-expected');
   const actualPath = addSuffixToFilePath(outputFile, '-actual');
   const diffPath = addSuffixToFilePath(outputFile, '-diff');
@@ -116,6 +116,15 @@ export function compare(
       console.log(message);
       return { pass: true, message };
     }
+    if (updateSnapshots === 'missing') {
+      if (testInfo.status === 'passed')
+        testInfo.status = 'failed';
+      if (!('error' in testInfo))
+        testInfo.error = { value: 'Error: ' + message };
+      else if (testInfo.error?.value)
+        testInfo.error.value += '\nError: ' + message;
+      return { pass: true, message };
+    }
     return { pass: false, message };
   }
 

packages/playwright-test/src/matchers/toMatchSnapshot.ts

@@ -51,8 +51,7 @@ export function toMatchSnapshot(this: ReturnType<Expect['getState']>, received:
   const { pass, message, expectedPath, actualPath, diffPath, mimeType } = compare(
       received,
       pathSegments,
-      testInfo.snapshotPath,
-      testInfo.outputPath,
+      testInfo,
       updateSnapshots,
       withNegateComparison,
       options

tests/playwright-test/golden.spec.ts

@@ -155,22 +155,31 @@ test('should fail on same snapshots with negate matcher', async ({ runInlineTest
   expect(result.output).toContain('Expected result should be different from the actual one.');
 });
 
-test('should write missing expectations locally', async ({ runInlineTest }, testInfo) => {
+test('should write missing expectations locally twice and continue', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     ...files,
     'a.spec.js': `
       const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot('snapshot.txt');
+        expect('Hello world2').toMatchSnapshot('snapshot2.txt');
+        console.log('Here we are!');
       });
     `
-  }, {}, { CI: '' });
+  });
 
   expect(result.exitCode).toBe(1);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
-  expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
-  const data = fs.readFileSync(snapshotOutputPath);
-  expect(data.toString()).toBe('Hello world');
+  expect(result.failed).toBe(1);
+
+  const snapshot1OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(`${snapshot1OutputPath} is missing in snapshots, writing actual`);
+  expect(fs.readFileSync(snapshot1OutputPath, 'utf-8')).toBe('Hello world');
+
+  const snapshot2OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot2.txt');
+  expect(result.output).toContain(`${snapshot2OutputPath} is missing in snapshots, writing actual`);
+  expect(fs.readFileSync(snapshot2OutputPath, 'utf-8')).toBe('Hello world2');
+
+  expect(result.output).toContain('Here we are!');
 });
 
 test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => {