feat: implement a new image comparison function (#19166)

This patch implements a new image comparison function, codenamed
"ssim-cie94". The goal of the new comparison function is to cancel out
browser non-determenistic rendering.

To use the new comparison function:

```ts
await expect(page).toHaveScreenshot({
  comparator: 'ssim-cie94',
});
```

As of Nov 30, 2022, we identified the following sources of
non-determenistic rendering for Chromium:
- Anti-aliasing for certain shapes might be different due to the
  way skia rasterizes certain shapes.
- Color blending might be different on `x86` and `aarch64`
architectures.

The new function employs a few heuristics to fight these
differences.

Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`:
1. If the [CIE94] metric is less then 1.0, then we consider these pixels
   **EQUAL**. (The value `1.0` is the [just-noticeable difference] for
   [CIE94].). Otherwise, proceed to next step.
1. If all the 8 neighbors of the first pixel match its color, or
   if the 8 neighbors of the second pixel match its color, then these
   pixels are **DIFFERENT**. (In case of anti-aliasing, some of the
   direct neighbors have to be blended up or down.) Otherwise, proceed
   to next step.
1. If SSIM in some locality around the different pixels is more than
   0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them
   as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be
   very close to 1.0).

[CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94
[just-noticeable difference]:
https://en.wikipedia.org/wiki/Just-noticeable_difference

.gitattributes

@@ -1,5 +1,4 @@
 # text files must be lf for golden file tests to work
 * text=auto eol=lf
-
 # make project show as TS on GitHub
 *.js linguist-detectable=false

docs/src/api/class-locatorassertions.md

@@ -1362,6 +1362,9 @@ Snapshot name.
 ### option: LocatorAssertions.toHaveScreenshot#1.scale = %%-screenshot-option-scale-default-css-%%
 * since: v1.23
 
+### option: LocatorAssertions.toHaveScreenshot#1.comparator = %%-assertions-comparator-%%
+* since: v1.29
+
 ### option: LocatorAssertions.toHaveScreenshot#1.maxDiffPixels = %%-assertions-max-diff-pixels-%%
 * since: v1.23
 
@@ -1405,6 +1408,9 @@ Note that screenshot assertions only work with Playwright test runner.
 ### option: LocatorAssertions.toHaveScreenshot#2.scale = %%-screenshot-option-scale-default-css-%%
 * since: v1.23
 
+### option: LocatorAssertions.toHaveScreenshot#2.comparator = %%-assertions-comparator-%%
+* since: v1.29
+
 ### option: LocatorAssertions.toHaveScreenshot#2.maxDiffPixels = %%-assertions-max-diff-pixels-%%
 * since: v1.23
 

docs/src/api/class-pageassertions.md

@@ -170,6 +170,9 @@ Snapshot name.
 ### option: PageAssertions.toHaveScreenshot#1.scale = %%-screenshot-option-scale-default-css-%%
 * since: v1.23
 
+### option: PageAssertions.toHaveScreenshot#1.comparator = %%-assertions-comparator-%%
+* since: v1.29
+
 ### option: PageAssertions.toHaveScreenshot#1.maxDiffPixels = %%-assertions-max-diff-pixels-%%
 * since: v1.23
 
@@ -218,6 +221,9 @@ Note that screenshot assertions only work with Playwright test runner.
 ### option: PageAssertions.toHaveScreenshot#2.scale = %%-screenshot-option-scale-default-css-%%
 * since: v1.23
 
+### option: PageAssertions.toHaveScreenshot#2.comparator = %%-assertions-comparator-%%
+* since: v1.29
+
 ### option: PageAssertions.toHaveScreenshot#2.maxDiffPixels = %%-assertions-max-diff-pixels-%%
 * since: v1.23
 

docs/src/api/class-snapshotassertions.md

@@ -43,6 +43,9 @@ Note that matching snapshots only work with Playwright test runner.
 
 Snapshot name.
 
+### option: SnapshotAssertions.toMatchSnapshot#1.comparator = %%-assertions-comparator-%%
+* since: v1.29
+
 ### option: SnapshotAssertions.toMatchSnapshot#1.maxDiffPixels = %%-assertions-max-diff-pixels-%%
 * since: v1.22
 
@@ -79,6 +82,9 @@ Learn more about [visual comparisons](../test-snapshots.md).
 
 Note that matching snapshots only work with Playwright test runner.
 
+### option: SnapshotAssertions.toMatchSnapshot#2.comparator = %%-assertions-comparator-%%
+* since: v1.29
+
 ### option: SnapshotAssertions.toMatchSnapshot#2.maxDiffPixels = %%-assertions-max-diff-pixels-%%
 * since: v1.22
 

docs/src/api/params.md

@@ -800,6 +800,12 @@ Time to retry the assertion for.
 
 An acceptable amount of pixels that could be different. Default is configurable with `TestConfig.expect`. Unset by default.
 
+## assertions-comparator
+* langs: js
+- `comparator` <[string]> Either `"pixelmatch"` or `"ssim-cie94"`.
+
+A comparator function to use when comparing images.
+
 ## assertions-max-diff-pixel-ratio
 * langs: js
 - `maxDiffPixelRatio` <[float]>

docs/src/test-api/class-testconfig.md

@@ -39,14 +39,16 @@ export default config;
 - type: ?<[Object]>
   - `timeout` ?<[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms.
   - `toHaveScreenshot` ?<[Object]> Configuration for the [`method: PageAssertions.toHaveScreenshot#1`] method.
-    - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+    - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+    - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`.
     - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default.
     - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
     - `animations` ?<[ScreenshotAnimations]<"allow"|"disabled">> See [`option: animations`] in [`method: Page.screenshot`]. Defaults to `"disabled"`.
     - `caret` ?<[ScreenshotCaret]<"hide"|"initial">> See [`option: caret`] in [`method: Page.screenshot`]. Defaults to `"hide"`.
     - `scale` ?<[ScreenshotScale]<"css"|"device">> See [`option: scale`] in [`method: Page.screenshot`]. Defaults to `"css"`.
   - `toMatchSnapshot` ?<[Object]> Configuration for the [`method: SnapshotAssertions.toMatchSnapshot#1`] method.
-    - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+    - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+    - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`.
     - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default.
     - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
 

docs/src/test-api/class-testproject.md

@@ -110,14 +110,16 @@ export default config;
 - type: ?<[Object]>
   - `timeout` ?<[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms.
   - `toHaveScreenshot` ?<[Object]> Configuration for the [`method: PageAssertions.toHaveScreenshot#1`] method.
-    - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+    - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+    - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`.
     - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default.
     - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
     - `animations` ?<[ScreenshotAnimations]<"allow"|"disabled">> See [`option: animations`] in [`method: Page.screenshot`]. Defaults to `"disabled"`.
     - `caret` ?<[ScreenshotCaret]<"hide"|"initial">> See [`option: caret`] in [`method: Page.screenshot`]. Defaults to `"hide"`.
     - `scale` ?<[ScreenshotScale]<"css"|"device">> See [`option: scale`] in [`method: Page.screenshot`]. Defaults to `"css"`.
   - `toMatchSnapshot` ?<[Object]> Configuration for the [`method: SnapshotAssertions.toMatchSnapshot#1`] method.
-    - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+    - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+    - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`.
     - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default.
     - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
 

package-lock.json

@@ -58,6 +58,7 @@
         "react-dom": "^18.1.0",
         "rimraf": "^3.0.2",
         "socksv5": "0.0.6",
+        "ssim.js": "^3.5.0",
         "typescript": "^4.7.3",
         "vite": "^3.2.1",
         "ws": "^8.5.0",
@@ -4876,6 +4877,12 @@
       "license": "BSD-3-Clause",
       "optional": true
     },
+    "node_modules/ssim.js": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz",
+      "integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==",
+      "dev": true
+    },
     "node_modules/stack-trace": {
       "version": "0.0.10",
       "dev": true,
@@ -9174,6 +9181,12 @@
       "dev": true,
       "optional": true
     },
+    "ssim.js": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz",
+      "integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==",
+      "dev": true
+    },
     "stack-trace": {
       "version": "0.0.10",
       "dev": true

package.json

@@ -93,6 +93,7 @@
     "react-dom": "^18.1.0",
     "rimraf": "^3.0.2",
     "socksv5": "0.0.6",
+    "ssim.js": "^3.5.0",
     "typescript": "^4.7.3",
     "vite": "^3.2.1",
     "ws": "^8.5.0",

packages/playwright-core/package.json

@@ -23,6 +23,10 @@
     "./lib/grid/gridServer": "./lib/grid/gridServer.js",
     "./lib/outofprocess": "./lib/outofprocess.js",
     "./lib/utils": "./lib/utils/index.js",
+    "./lib/image_tools/stats": "./lib/image_tools/stats.js",
+    "./lib/image_tools/compare": "./lib/image_tools/compare.js",
+    "./lib/image_tools/imageChannel": "./lib/image_tools/imageChannel.js",
+    "./lib/image_tools/colorUtils": "./lib/image_tools/colorUtils.js",
     "./lib/common/userAgent": "./lib/common/userAgent.js",
     "./lib/containers/docker": "./lib/containers/docker.js",
     "./lib/utils/comparators": "./lib/utils/comparators.js",

packages/playwright-core/src/image_tools/colorUtils.ts

@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function blendWithWhite(c: number, a: number): number {
+  return 255 + (c - 255) * a;
+}
+
+export function rgb2gray(r: number, g: number, b: number): number {
+  // NOTE: this is the exact integer formula from SSIM.js.
+  // See https://github.com/obartra/ssim/blob/ca8e3c6a6ff5f4f2e232239e0c3d91806f3c97d5/src/matlab/rgb2gray.ts#L56
+  return (77 * r + 150 * g + 29 * b + 128) >> 8;
+}
+
+// Percieved color difference defined by CIE94.
+// See https://en.wikipedia.org/wiki/Color_difference#CIE94
+//
+// The result of 1.0 is a "just-noticiable difference".
+//
+// Other results interpretation (taken from http://zschuessler.github.io/DeltaE/learn/):
+//   < 1.0     Not perceptible by human eyes.
+//   1-2        Perceptible through close observation.
+//   2-10       Perceptible at a glance.
+//   11-49      Colors are more similar than opposite
+//   100        Colors are exact opposite
+export function colorDeltaE94(rgb1: number[], rgb2: number[]) {
+  const [l1, a1, b1] = xyz2lab(srgb2xyz(rgb1));
+  const [l2, a2, b2] = xyz2lab(srgb2xyz(rgb2));
+  const deltaL = l1 - l2;
+  const deltaA = a1 - a2;
+  const deltaB = b1 - b2;
+  const c1 = Math.sqrt(a1 ** 2 + b1 ** 2);
+  const c2 = Math.sqrt(a2 ** 2 + b2 ** 2);
+  const deltaC = c1 - c2;
+  let deltaH = deltaA ** 2 + deltaB ** 2 - deltaC ** 2;
+  deltaH = deltaH < 0 ? 0 : Math.sqrt(deltaH);
+  // The k1, k2, kL, kC, kH values for "graphic arts" applications.
+  // See https://en.wikipedia.org/wiki/Color_difference#CIE94
+  const k1 = 0.045;
+  const k2 = 0.015;
+  const kL = 1;
+  const kC = 1;
+  const kH = 1;
+
+  const sC = 1.0 + k1 * c1;
+  const sH = 1.0 + k2 * c1;
+  const sL = 1;
+
+  return Math.sqrt((deltaL / sL / kL) ** 2 + (deltaC / sC / kC) ** 2 + (deltaH / sH / kH) ** 2);
+}
+
+// sRGB -> 1-normalized XYZ (i.e. Y ∈ [0, 1]) with D65 illuminant
+// See https://en.wikipedia.org/wiki/SRGB#From_sRGB_to_CIE_XYZ
+export function srgb2xyz(rgb: number[]): number[] {
+  let r = rgb[0] / 255;
+  let g = rgb[1] / 255;
+  let b = rgb[2] / 255;
+  r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
+  g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
+  b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;
+  return [
+    (r * 0.4124 + g * 0.3576 + b * 0.1805),
+    (r * 0.2126 + g * 0.7152 + b * 0.0722),
+    (r * 0.0193 + g * 0.1192 + b * 0.9505),
+  ];
+}
+
+const sigma_pow2 = 6 * 6 / 29 / 29;
+const sigma_pow3 = 6 * 6 * 6 / 29 / 29 / 29;
+
+// 1-normalized CIE XYZ with D65 to L*a*b*
+// See https://en.wikipedia.org/wiki/CIELAB_color_space#From_CIEXYZ_to_CIELAB
+export function xyz2lab(xyz: number[]): number[] {
+  const x = xyz[0] / 0.950489;
+  const y = xyz[1];
+  const z = xyz[2] / 1.088840;
+
+  const fx = x > sigma_pow3 ? x ** (1 / 3) : x / 3 / sigma_pow2 + 4 / 29;
+  const fy = y > sigma_pow3 ? y ** (1 / 3) : y / 3 / sigma_pow2 + 4 / 29;
+  const fz = z > sigma_pow3 ? z ** (1 / 3) : z / 3 / sigma_pow2 + 4 / 29;
+
+  const l = 116 * fy - 16;
+  const a = 500 * (fx - fy);
+  const b = 200 * (fy - fz);
+
+  return [l, a, b];
+}

packages/playwright-core/src/image_tools/compare.ts

@@ -0,0 +1,107 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { blendWithWhite, colorDeltaE94, rgb2gray } from './colorUtils';
+import { ImageChannel } from './imageChannel';
+import { ssim, FastStats } from './stats';
+
+const SSIM_WINDOW_RADIUS = 5;
+const VARIANCE_WINDOW_RADIUS = 1;
+
+function drawPixel(width: number, data: Buffer, x: number, y: number, r: number, g: number, b: number) {
+  const idx = (y * width + x) * 4;
+  data[idx + 0] = r;
+  data[idx + 1] = g;
+  data[idx + 2] = b;
+  data[idx + 3] = 255;
+}
+
+type CompareOptions = {
+  maxColorDeltaE94: number;
+};
+
+export function compare(actual: Buffer, expected: Buffer, diff: Buffer, width: number, height: number, options: CompareOptions) {
+  const {
+    maxColorDeltaE94
+  } = options;
+  const [r1, g1, b1] = ImageChannel.intoRGB(width, height, expected);
+  const [r2, g2, b2] = ImageChannel.intoRGB(width, height, actual);
+
+  const drawRedPixel = (x: number, y: number) => drawPixel(width, diff, x, y, 255, 0, 0);
+  const drawYellowPixel = (x: number, y: number) => drawPixel(width, diff, x, y, 255, 255, 0);
+  const drawGrayPixel = (x: number, y: number) => {
+    const gray = rgb2gray(r1.get(x, y), g1.get(x, y), b1.get(x, y));
+    const value = blendWithWhite(gray, 0.1);
+    drawPixel(width, diff, x, y, value, value, value);
+  };
+
+  let fastR, fastG, fastB;
+
+  let diffCount = 0;
+  for (let y = 0; y < height; ++y){
+    for (let x = 0; x < width; ++x) {
+      // Fast-path: equal pixels.
+      if (r1.get(x, y) === r2.get(x, y) && g1.get(x, y) === g2.get(x, y) && b1.get(x, y) === b2.get(x, y)) {
+        drawGrayPixel(x, y);
+        continue;
+      }
+
+      // Compare pixel colors using the dE94 color difference formulae.
+      // The dE94 is normalized so that the value of 1.0 is the "just-noticeable-difference".
+      // Color difference below 1.0 is not noticeable to a human eye, so we can disregard it.
+      // See https://en.wikipedia.org/wiki/Color_difference
+      const delta = colorDeltaE94(
+          [r1.get(x, y), g1.get(x, y), b1.get(x, y)],
+          [r2.get(x, y), g2.get(x, y), b2.get(x, y)]
+      );
+
+      if (delta <= maxColorDeltaE94) {
+        drawGrayPixel(x, y);
+        continue;
+      }
+
+      // if this pixel is a part of a flood fill of a 3x3 square then it cannot be
+      // anti-aliasing pixel so it must be a pixel difference.
+      if (!fastR || !fastG || !fastB) {
+        fastR = new FastStats(r1, r2);
+        fastG = new FastStats(g1, g2);
+        fastB = new FastStats(b1, b2);
+      }
+      const [varX1, varY1] = r1.boundXY(x - VARIANCE_WINDOW_RADIUS, y - VARIANCE_WINDOW_RADIUS);
+      const [varX2, varY2] = r1.boundXY(x + VARIANCE_WINDOW_RADIUS, y + VARIANCE_WINDOW_RADIUS);
+      const var1 = fastR.varianceC1(varX1, varY1, varX2, varY2) + fastG.varianceC1(varX1, varY1, varX2, varY2) + fastB.varianceC1(varX1, varY1, varX2, varY2);
+      const var2 = fastR.varianceC2(varX1, varY1, varX2, varY2) + fastG.varianceC2(varX1, varY1, varX2, varY2) + fastB.varianceC2(varX1, varY1, varX2, varY2);
+      if (var1 === 0 && var2 === 0) {
+        drawRedPixel(x, y);
+        ++diffCount;
+        continue;
+      }
+
+      const [ssimX1, ssimY1] = r1.boundXY(x - SSIM_WINDOW_RADIUS, y - SSIM_WINDOW_RADIUS);
+      const [ssimX2, ssimY2] = r1.boundXY(x + SSIM_WINDOW_RADIUS, y + SSIM_WINDOW_RADIUS);
+      const ssimRGB = (ssim(fastR, ssimX1, ssimY1, ssimX2, ssimY2) + ssim(fastG, ssimX1, ssimY1, ssimX2, ssimY2) + ssim(fastB, ssimX1, ssimY1, ssimX2, ssimY2)) / 3.0;
+      const isAntialiassed = ssimRGB >= 0.99;
+      if (isAntialiassed) {
+        drawYellowPixel(x, y);
+      } else {
+        drawRedPixel(x, y);
+        ++diffCount;
+      }
+    }
+  }
+
+  return diffCount;
+}

packages/playwright-core/src/image_tools/imageChannel.ts

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { blendWithWhite } from './colorUtils';
+
+export class ImageChannel {
+  data: Uint8Array;
+  width: number;
+  height: number;
+
+  static intoRGB(width: number, height: number, data: Buffer): ImageChannel[] {
+    const r = new Uint8Array(width * height);
+    const g = new Uint8Array(width * height);
+    const b = new Uint8Array(width * height);
+    for (let y = 0; y < height; ++y) {
+      for (let x = 0; x < width; ++x) {
+        const index = y * width + x;
+        const offset = index * 4;
+        const alpha = data[offset + 3] === 255 ? 1 : data[offset + 3] / 255;
+        r[index] = blendWithWhite(data[offset], alpha);
+        g[index] = blendWithWhite(data[offset + 1], alpha);
+        b[index] = blendWithWhite(data[offset + 2], alpha);
+      }
+    }
+    return [
+      new ImageChannel(width, height, r),
+      new ImageChannel(width, height, g),
+      new ImageChannel(width, height, b),
+    ];
+  }
+
+  constructor(width: number, height: number, data: Uint8Array) {
+    this.data = data;
+    this.width = width;
+    this.height = height;
+  }
+
+  get(x: number, y: number) {
+    return this.data[y * this.width + x];
+  }
+
+  boundXY(x: number, y: number) {
+    return [
+      Math.min(Math.max(x, 0), this.width - 1),
+      Math.min(Math.max(y, 0), this.height - 1),
+    ];
+  }
+}

packages/playwright-core/src/image_tools/stats.ts

@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { ImageChannel } from './imageChannel';
+
+export interface Stats {
+  c1: ImageChannel;
+  c2: ImageChannel;
+
+  // Compute mean value. See https://en.wikipedia.org/wiki/Mean
+  meanC1(x1: number, y1: number, x2: number, y2: number): number;
+  meanC2(x1: number, y1: number, x2: number, y2: number): number;
+  // Compute **population** (not sample) variance. See https://en.wikipedia.org/wiki/Variance
+  varianceC1(x1: number, y1: number, x2: number, y2: number): number;
+  varianceC2(x1: number, y1: number, x2: number, y2: number): number;
+  // Compute covariance. See https://en.wikipedia.org/wiki/Covariance
+  covariance(x1: number, y1: number, x2: number, y2: number): number;
+}
+
+// Image channel has a 8-bit depth.
+const DYNAMIC_RANGE = 2 ** 8 - 1;
+
+export function ssim(stats: Stats, x1: number, y1: number, x2: number, y2: number): number {
+  const mean1 = stats.meanC1(x1, y1, x2, y2);
+  const mean2 = stats.meanC2(x1, y1, x2, y2);
+  const var1 = stats.varianceC1(x1, y1, x2, y2);
+  const var2 = stats.varianceC2(x1, y1, x2, y2);
+  const cov = stats.covariance(x1, y1, x2, y2);
+  const c1 = (0.01 * DYNAMIC_RANGE) ** 2;
+  const c2 = (0.03 * DYNAMIC_RANGE) ** 2;
+  return (2 * mean1 * mean2 + c1) * (2 * cov + c2) / (mean1 ** 2 + mean2 ** 2 + c1) / (var1 + var2 + c2);
+}
+
+export class FastStats implements Stats {
+  c1: ImageChannel;
+  c2: ImageChannel;
+
+  private _partialSumC1: number[];
+  private _partialSumC2: number[];
+  private _partialSumMult: number[];
+  private _partialSumSq1: number[];
+  private _partialSumSq2: number[];
+
+  constructor(c1: ImageChannel, c2: ImageChannel) {
+    this.c1 = c1;
+    this.c2 = c2;
+    const { width, height } = c1;
+
+    this._partialSumC1 = new Array(width * height);
+    this._partialSumC2 = new Array(width * height);
+    this._partialSumSq1 = new Array(width * height);
+    this._partialSumSq2 = new Array(width * height);
+    this._partialSumMult = new Array(width * height);
+
+    const recalc = (mx: number[], idx: number, initial: number, x: number, y: number) => {
+      mx[idx] = initial;
+      if (y > 0)
+        mx[idx] += mx[(y - 1) * width + x];
+      if (x > 0)
+        mx[idx] += mx[y * width + x - 1];
+      if (x > 0 && y > 0)
+        mx[idx] -= mx[(y - 1) * width + x - 1];
+    };
+
+    for (let y = 0; y < height; ++y) {
+      for (let x = 0; x < width; ++x) {
+        const idx = y * width + x;
+        recalc(this._partialSumC1, idx, this.c1.data[idx], x, y);
+        recalc(this._partialSumC2, idx, this.c2.data[idx], x, y);
+        recalc(this._partialSumSq1, idx, this.c1.data[idx] * this.c1.data[idx], x, y);
+        recalc(this._partialSumSq2, idx, this.c2.data[idx] * this.c2.data[idx], x, y);
+        recalc(this._partialSumMult, idx, this.c1.data[idx] * this.c2.data[idx], x, y);
+      }
+    }
+  }
+
+  _sum(partialSum: number[], x1: number, y1: number, x2: number, y2: number): number {
+    const width = this.c1.width;
+    let result = partialSum[y2 * width + x2];
+    if (y1 > 0)
+      result -= partialSum[(y1 - 1) * width + x2];
+    if (x1 > 0)
+      result -= partialSum[y2 * width + x1 - 1];
+    if (x1 > 0 && y1 > 0)
+      result += partialSum[(y1 - 1) * width + x1 - 1];
+    return result;
+  }
+
+  meanC1(x1: number, y1: number, x2: number, y2: number): number {
+    const N = (y2 - y1 + 1) * (x2 - x1 + 1);
+    return this._sum(this._partialSumC1, x1, y1, x2, y2) / N;
+  }
+
+  meanC2(x1: number, y1: number, x2: number, y2: number): number {
+    const N = (y2 - y1 + 1) * (x2 - x1 + 1);
+    return this._sum(this._partialSumC2, x1, y1, x2, y2) / N;
+  }
+
+  varianceC1(x1: number, y1: number, x2: number, y2: number): number {
+    const N = (y2 - y1 + 1) * (x2 - x1 + 1);
+    return (this._sum(this._partialSumSq1, x1, y1, x2, y2) - (this._sum(this._partialSumC1, x1, y1, x2, y2) ** 2) / N) / N;
+  }
+
+  varianceC2(x1: number, y1: number, x2: number, y2: number): number {
+    const N = (y2 - y1 + 1) * (x2 - x1 + 1);
+    return (this._sum(this._partialSumSq2, x1, y1, x2, y2) - (this._sum(this._partialSumC2, x1, y1, x2, y2) ** 2) / N) / N;
+  }
+
+  covariance(x1: number, y1: number, x2: number, y2: number): number {
+    const N = (y2 - y1 + 1) * (x2 - x1 + 1);
+    return (this._sum(this._partialSumMult, x1, y1, x2, y2) - this._sum(this._partialSumC1, x1, y1, x2, y2) * this._sum(this._partialSumC2, x1, y1, x2, y2) / N) / N;
+  }
+}
+

packages/playwright-core/src/protocol/validator.ts

@@ -987,6 +987,7 @@ scheme.PageExpectScreenshotParams = tObject({
     selector: tString,
   })),
   comparatorOptions: tOptional(tObject({
+    comparator: tOptional(tString),
     maxDiffPixels: tOptional(tNumber),
     maxDiffPixelRatio: tOptional(tNumber),
     threshold: tOptional(tNumber),

packages/playwright-core/src/utils/DEPS.list

@@ -2,5 +2,6 @@
 ./
 ../third_party/diff_match_patch
 ../third_party/pixelmatch
+../image_tools/compare.ts
 ../utilsBundle.ts
 ../zipBundle.ts

packages/playwright-core/src/utils/comparators.ts

@@ -17,26 +17,17 @@
 
 import { colors, jpegjs } from '../utilsBundle';
 import pixelmatch from '../third_party/pixelmatch';
+import { compare } from '../image_tools/compare';
 import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
 import { PNG } from '../utilsBundle';
 
-export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number };
+export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number, comparator?: string };
 export type ComparatorResult = { diff?: Buffer; errorMessage: string; } | null;
 export type Comparator = (actualBuffer: Buffer | string, expectedBuffer: Buffer, options?: any) => ComparatorResult;
 
-let customPNGComparator: Comparator | undefined;
-if (process.env.PW_CUSTOM_PNG_COMPARATOR) {
-  try {
-    customPNGComparator = require(process.env.PW_CUSTOM_PNG_COMPARATOR);
-    if (typeof customPNGComparator !== 'function')
-      customPNGComparator = undefined;
-  } catch (e) {
-  }
-}
-
 export function getComparator(mimeType: string): Comparator {
   if (mimeType === 'image/png')
-    return customPNGComparator ?? compareImages.bind(null, 'image/png');
+    return compareImages.bind(null, 'image/png');
   if (mimeType === 'image/jpeg')
     return compareImages.bind(null, 'image/jpeg');
   if (mimeType === 'text/plain')
@@ -68,9 +59,18 @@ function compareImages(mimeType: string, actualBuffer: Buffer | string, expected
     };
   }
   const diff = new PNG({ width: expected.width, height: expected.height });
-  const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {
-    threshold: options.threshold ?? 0.2,
-  });
+  let count;
+  if (options.comparator === 'ssim-cie94') {
+    count = compare(expected.data, actual.data, diff.data, expected.width, expected.height, {
+      maxColorDeltaE94: (options.threshold ?? 0.01) * 100,
+    });
+  } else if ((options.comparator ?? 'pixelmatch') === 'pixelmatch') {
+    count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {
+      threshold: options.threshold ?? 0.2,
+    });
+  } else {
+    throw new Error(`Configuration specifies unknown comparator "${options.comparator}"`);
+  }
 
   const maxDiffPixels1 = options.maxDiffPixels;
   const maxDiffPixels2 = options.maxDiffPixelRatio !== undefined ? expected.width * expected.height * options.maxDiffPixelRatio : undefined;

packages/playwright-test/src/matchers/toMatchSnapshot.ts

@@ -145,6 +145,7 @@ class SnapshotHelper<T extends ImageComparatorOptions> {
       maxDiffPixels: options.maxDiffPixels,
       maxDiffPixelRatio: options.maxDiffPixelRatio,
       threshold: options.threshold,
+      comparator: options.comparator,
     };
     this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot';
   }
@@ -305,6 +306,7 @@ export async function toHaveScreenshot(
   const helper = new SnapshotHelper(
       testInfo, snapshotPathResolver, 'png',
       {
+        comparator: config?.comparator,
         maxDiffPixels: config?.maxDiffPixels,
         maxDiffPixelRatio: config?.maxDiffPixelRatio,
         threshold: config?.threshold,

packages/playwright-test/types/test.d.ts

@@ -534,8 +534,16 @@ interface TestConfig {
      */
     toHaveScreenshot?: {
       /**
-       * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the
-       * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+       * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+       */
+      comparator?: string;
+
+      /**
+       * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and
+       * `1` (lax). `"pixelmatch"` comparator computes color difference in
+       * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"`
+       * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults
+       * `threshold` value to `0.01`.
        */
       threshold?: number;
 
@@ -576,8 +584,16 @@ interface TestConfig {
      */
     toMatchSnapshot?: {
       /**
-       * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the
-       * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+       * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+       */
+      comparator?: string;
+
+      /**
+       * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and
+       * `1` (lax). `"pixelmatch"` comparator computes color difference in
+       * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"`
+       * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults
+       * `threshold` value to `0.01`.
        */
       threshold?: number;
 
@@ -3845,6 +3861,11 @@ interface LocatorAssertions {
      */
     caret?: "hide"|"initial";
 
+    /**
+     * A comparator function to use when comparing images.
+     */
+    comparator?: string;
+
     /**
      * Specify locators that should be masked when the screenshot is taken. Masked elements will be overlaid with a pink
      * box `#FF00FF` that completely covers its bounding box.
@@ -3922,6 +3943,11 @@ interface LocatorAssertions {
      */
     caret?: "hide"|"initial";
 
+    /**
+     * A comparator function to use when comparing images.
+     */
+    comparator?: string;
+
     /**
      * Specify locators that should be masked when the screenshot is taken. Masked elements will be overlaid with a pink
      * box `#FF00FF` that completely covers its bounding box.
@@ -4170,6 +4196,11 @@ interface PageAssertions {
       height: number;
     };
 
+    /**
+     * A comparator function to use when comparing images.
+     */
+    comparator?: string;
+
     /**
      * When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to
      * `false`.
@@ -4277,6 +4308,11 @@ interface PageAssertions {
       height: number;
     };
 
+    /**
+     * A comparator function to use when comparing images.
+     */
+    comparator?: string;
+
     /**
      * When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to
      * `false`.
@@ -4407,6 +4443,11 @@ interface SnapshotAssertions {
    * @param options
    */
   toMatchSnapshot(name: string|Array<string>, options?: {
+    /**
+     * A comparator function to use when comparing images.
+     */
+    comparator?: string;
+
     /**
      * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`. Default is
      * configurable with `TestConfig.expect`. Unset by default.
@@ -4455,6 +4496,11 @@ interface SnapshotAssertions {
    * @param options
    */
   toMatchSnapshot(options?: {
+    /**
+     * A comparator function to use when comparing images.
+     */
+    comparator?: string;
+
     /**
      * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`. Default is
      * configurable with `TestConfig.expect`. Unset by default.
@@ -4581,8 +4627,16 @@ interface TestProject {
      */
     toHaveScreenshot?: {
       /**
-       * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the
-       * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+       * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+       */
+      comparator?: string;
+
+      /**
+       * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and
+       * `1` (lax). `"pixelmatch"` comparator computes color difference in
+       * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"`
+       * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults
+       * `threshold` value to `0.01`.
        */
       threshold?: number;
 
@@ -4623,8 +4677,16 @@ interface TestProject {
      */
     toMatchSnapshot?: {
       /**
-       * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the
-       * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+       * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`.
+       */
+      comparator?: string;
+
+      /**
+       * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and
+       * `1` (lax). `"pixelmatch"` comparator computes color difference in
+       * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"`
+       * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults
+       * `threshold` value to `0.01`.
        */
       threshold?: number;
 

packages/protocol/src/channels.ts

@@ -1825,6 +1825,7 @@ export type PageExpectScreenshotParams = {
     selector: string,
   },
   comparatorOptions?: {
+    comparator?: string,
     maxDiffPixels?: number,
     maxDiffPixelRatio?: number,
     threshold?: number,
@@ -1850,6 +1851,7 @@ export type PageExpectScreenshotOptions = {
     selector: string,
   },
   comparatorOptions?: {
+    comparator?: string,
     maxDiffPixels?: number,
     maxDiffPixelRatio?: number,
     threshold?: number,

packages/protocol/src/protocol.yml

@@ -1266,6 +1266,7 @@ Page:
         comparatorOptions:
           type: object?
           properties:
+            comparator: string?
             maxDiffPixels: number?
             maxDiffPixelRatio: number?
             threshold: number?

tests/image_tools/fixtures.spec.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from '../playwright-test/stable-test-runner';
+import { PNG } from 'playwright-core/lib/utilsBundle';
+import { compare } from 'playwright-core/lib/image_tools/compare';
+import fs from 'fs';
+import path from 'path';
+
+function listFixtures(root: string, fixtures: Set<string> = new Set()) {
+  for (const item of fs.readdirSync(root, { withFileTypes: true })) {
+    const p = path.join(root, item.name);
+    if (item.isDirectory())
+      listFixtures(p, fixtures);
+    else if (item.isFile() && p.endsWith('-actual.png'))
+      fixtures.add(p.substring(0, p.length - '-actual.png'.length));
+  }
+  return fixtures;
+}
+
+const FIXTURES_DIR = path.join(__dirname, 'fixtures');
+
+function declareFixtureTest(fixtureRoot: string, fixtureName: string, shouldMatch: boolean) {
+  test(path.relative(fixtureRoot, fixtureName), async ({}, testInfo) => {
+    const [actual, expected] = await Promise.all([
+      fs.promises.readFile(fixtureName + '-actual.png'),
+      fs.promises.readFile(fixtureName + '-expected.png'),
+    ]);
+    testInfo.attach(fixtureName + '-actual.png', {
+      body: actual,
+      contentType: 'image/png',
+    });
+    testInfo.attach(fixtureName + '-expected.png', {
+      body: expected,
+      contentType: 'image/png',
+    });
+    const actualPNG = PNG.sync.read(actual);
+    const expectedPNG = PNG.sync.read(expected);
+    expect(actualPNG.width).toBe(expectedPNG.width);
+    expect(actualPNG.height).toBe(expectedPNG.height);
+
+    const diffPNG = new PNG({ width: actualPNG.width, height: actualPNG.height });
+    const diffCount = compare(actualPNG.data, expectedPNG.data, diffPNG.data, actualPNG.width, actualPNG.height, {
+      maxColorDeltaE94: 1.0,
+    });
+
+    testInfo.attach(fixtureName + '-diff.png', {
+      body: PNG.sync.write(diffPNG),
+      contentType: 'image/png',
+    });
+
+    if (shouldMatch)
+      expect(diffCount).toBe(0);
+    else
+      expect(diffCount).not.toBe(0);
+  });
+}
+
+test.describe('basic fixtures', () => {
+  test.describe.configure({ mode: 'parallel' });
+
+  for (const fixtureName of listFixtures(path.join(FIXTURES_DIR, 'should-match')))
+    declareFixtureTest(FIXTURES_DIR, fixtureName, true /* shouldMatch */);
+  for (const fixtureName of listFixtures(path.join(FIXTURES_DIR, 'should-fail')))
+    declareFixtureTest(FIXTURES_DIR, fixtureName, false /* shouldMatch */);
+});
+
+const customImageToolsFixtures = process.env.IMAGE_TOOLS_FIXTURES;
+if (customImageToolsFixtures) {
+  test.describe('custom fixtures', () => {
+    test.describe.configure({ mode: 'parallel' });
+
+    for (const fixtureName of listFixtures(path.join(customImageToolsFixtures, 'should-match')))
+      declareFixtureTest(customImageToolsFixtures, fixtureName, true /* shouldMatch */);
+    for (const fixtureName of listFixtures(path.join(customImageToolsFixtures, 'should-fail')))
+      declareFixtureTest(customImageToolsFixtures, fixtureName, false /* shouldMatch */);
+  });
+}

tests/image_tools/fixtures/should-fail/julia-ssim-trap/README.md

@@ -0,0 +1,11 @@
+# Julia SSIM trap
+
+[SSIM](https://en.wikipedia.org/wiki/Structural_similarity) is a metric used to compare image similarity.
+
+While original SSIM is computed against the luma channel (i.e. in a gray-scale),
+the Julia language [computes a weighted combination of per-channel SSIM's](https://github.com/JuliaImages/ImageQualityIndexes.jl/blob/e014cee9bef7023a1047b6eb0cbe49fbf28f2fed/src/ssim.jl#L39-L41).
+
+This sample is a white image and a gray image that are reported equal by Julia SSIM.
+It also traps all the suggestions for color-weighted SSIM given here:
+https://dsp.stackexchange.com/questions/75187/how-to-apply-the-ssim-measure-on-rgb-images
+

tests/image_tools/fixtures/should-fail/original-ssim-trap/README.md

@@ -0,0 +1,8 @@
+# Original SSIM trap
+
+[SSIM](https://en.wikipedia.org/wiki/Structural_similarity) is a metric used to compare image similarity.
+
+The sample provides two different images. However, since the original SSIM implementation
+[converts images into
+gray-scale](https://github.com/obartra/ssim/blob/ca8e3c6a6ff5f4f2e232239e0c3d91806f3c97d5/src/index.ts#L104),
+SSIM metric will yield a perfect match for these images.

tests/image_tools/fixtures/should-fail/trivial/README.md

@@ -0,0 +1,3 @@
+# Trivial failing examples
+
+Some trivial failing examples

tests/image_tools/fixtures/should-match/crbug-919955/README.md

@@ -0,0 +1,5 @@
+# Chrome non-determenistic rendering
+
+Reported by: https://bugs.chromium.org/p/chromium/issues/detail?id=919955
+
+

tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/README.md

@@ -0,0 +1,5 @@
+# Tiny anti-aliasing sample
+
+This is a 10x10 image sample with a 3 anti-aliased pixels in-between. This is actually
+a cropped down snapshot of one of the [ubuntu-x86-vs-ubunu-arm samples](../ubuntu-x86-vs-ubuntu-arm/samples/stylings_stories-Stylings-stories-Texture-bar-should-use-custom-path-chrome/stylings-stories/texture/bar/should-use-custom-path-actual.png) handy for debugging
+purposes.

tests/image_tools/fixtures/should-match/trivial/README.md

@@ -0,0 +1,3 @@
+# Equal small images
+
+Simple equal images.

tests/image_tools/unit.spec.ts

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test } from '../playwright-test/stable-test-runner';
+import { ssim, FastStats } from 'playwright-core/lib/image_tools/stats';
+import { ImageChannel } from 'playwright-core/lib/image_tools/imageChannel';
+import { srgb2xyz, xyz2lab, colorDeltaE94 } from 'playwright-core/lib/image_tools/colorUtils';
+import referenceSSIM from 'ssim.js';
+import { randomPNG, assertEqual, grayChannel } from './utils';
+
+test('srgb to lab conversion should work', async () => {
+  const srgb = [123, 81, 252];
+  const [x, y, z] = srgb2xyz(srgb);
+  // Values obtained with http://colormine.org/convert/rgb-to-xyz
+  assertEqual(x, 0.28681495837305815);
+  assertEqual(y, 0.17124087944445404);
+  assertEqual(z, 0.938890585081072);
+  const [l, a, b] = xyz2lab([x, y, z]);
+  // Values obtained with http://colormine.org/convert/rgb-to-lab
+  assertEqual(l, 48.416007793699535);
+  assertEqual(a, 57.71275605467668);
+  assertEqual(b, -79.29993619401066);
+});
+
+test('colorDeltaE94 should work', async () => {
+  const rgb1 = [123, 81, 252];
+  const rgb2 = [43, 201, 100];
+  // Value obtained with http://colormine.org/delta-e-calculator/cie94
+  assertEqual(colorDeltaE94(rgb1, rgb2), 71.2159);
+});
+
+test('fast stats and naive computation should match', async () => {
+  const N = 13, M = 17;
+  const png1 = randomPNG(N, M, 239);
+  const png2 = randomPNG(N, M, 261);
+  const [r1] = ImageChannel.intoRGB(png1.width, png1.height, png1.data);
+  const [r2] = ImageChannel.intoRGB(png2.width, png2.height, png2.data);
+  const fastStats = new FastStats(r1, r2);
+
+  for (let x1 = 0; x1 < png1.width; ++x1) {
+    for (let y1 = 0; y1 < png1.height; ++y1) {
+      for (let x2 = x1; x2 < png1.width; ++x2) {
+        for (let y2 = y1; y2 < png1.height; ++y2) {
+          assertEqual(fastStats.meanC1(x1, y1, x2, y2), computeMean(r1, x1, y1, x2, y2));
+          assertEqual(fastStats.varianceC1(x1, y1, x2, y2), computeVariance(r1, x1, y1, x2, y2));
+          assertEqual(fastStats.covariance(x1, y1, x2, y2), computeCovariance(r1, r2, x1, y1, x2, y2));
+        }
+      }
+    }
+  }
+});
+
+test('ssim + fastStats should match "weber" algorithm from ssim.js', async () => {
+  const N = 200;
+  const png1 = randomPNG(N, N, 239);
+  const png2 = randomPNG(N, N, 261);
+  const windowRadius = 5;
+  const refSSIM = referenceSSIM(png1 as any, png2 as any, {
+    downsample: false,
+    ssim: 'weber',
+    windowSize: windowRadius * 2 + 1,
+  });
+  const gray1 = grayChannel(png1);
+  const gray2 = grayChannel(png2);
+  const fastStats = new FastStats(gray1, gray2);
+  for (let y = windowRadius; y < N - windowRadius; ++y) {
+    for (let x = windowRadius; x < N - windowRadius; ++x) {
+      const customSSIM = ssim(fastStats, x - windowRadius, y - windowRadius, x + windowRadius, y + windowRadius);
+      const reference = refSSIM.ssim_map.data[(y - windowRadius) * refSSIM.ssim_map.width + x - windowRadius];
+      assertEqual(customSSIM, reference);
+    }
+  }
+});
+
+function computeMean(c: ImageChannel, x1: number, y1: number, x2: number, y2: number) {
+  let result = 0;
+  const N = (x2 - x1 + 1) * (y2 - y1 + 1);
+  for (let y = y1; y <= y2; ++y) {
+    for (let x = x1; x <= x2; ++x)
+      result += c.get(x, y);
+  }
+  return result / N;
+}
+
+function computeVariance(c: ImageChannel, x1: number, y1: number, x2: number, y2: number) {
+  let result = 0;
+  const mean = computeMean(c, x1, y1, x2, y2);
+  const N = (x2 - x1 + 1) * (y2 - y1 + 1);
+  for (let y = y1; y <= y2; ++y) {
+    for (let x = x1; x <= x2; ++x)
+      result += (c.get(x, y) - mean) ** 2;
+  }
+  return result / N;
+}
+
+function computeCovariance(c1: ImageChannel, c2: ImageChannel, x1: number, y1: number, x2: number, y2: number) {
+  const N = (x2 - x1 + 1) * (y2 - y1 + 1);
+  const mean1 = computeMean(c1, x1, y1, x2, y2);
+  const mean2 = computeMean(c2, x1, y1, x2, y2);
+  let result = 0;
+  for (let y = y1; y <= y2; ++y) {
+    for (let x = x1; x <= x2; ++x)
+      result += (c1.get(x, y) - mean1) * (c2.get(x, y) - mean2);
+  }
+  return result / N;
+}

tests/image_tools/utils.ts

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PNG } from 'playwright-core/lib/utilsBundle';
+import { ImageChannel } from 'playwright-core/lib/image_tools/imageChannel';
+
+// mulberry32
+export function createRandom(seed) {
+  return function() {
+    let t = seed += 0x6D2B79F5;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+
+export function randomPNG(width, height, seed) {
+  const random = createRandom(seed);
+  const png = new PNG({ width, height });
+  for (let i = 0; i < height; ++i) {
+    for (let j = 0; j < width; ++j) {
+      for (let k = 0; k < 4; ++k)
+        png.data[(i * width + j) * 4 + k] = (random() * 255) | 0;
+    }
+  }
+  return png;
+}
+
+export function assertEqual(value1, value2) {
+  if (Math.abs(value1 - value2) >= 1e-3)
+    throw new Error(`ERROR: ${value1} is not equal to ${value2}`);
+}
+
+// NOTE: this is exact formula from SSIM.js and it DOES NOT include alpha.
+// We use it to better compare with original SSIM implementation.
+export function grayChannel(image: PNG) {
+  const width = image.width;
+  const height = image.height;
+  const gray = new Uint8Array(image.width * image.height);
+  for (let y = 0; y < image.height; ++y) {
+    for (let x = 0; x < image.width; ++x) {
+      const index = y * image.width + x;
+      const offset = index * 4;
+      gray[index] = (77 * image.data[offset] + 150 * image.data[offset + 1] + 29 * image.data[offset + 2] + 128) >> 8;
+    }
+  }
+  return new ImageChannel(width, height, gray);
+}

tests/playwright-test/golden.spec.ts

@@ -620,6 +620,80 @@ test('should respect project threshold', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(0);
 });
 
+test('should respect comparator name', async ({ runInlineTest }) => {
+  const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png'));
+  const actual = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png'));
+  const result = await runInlineTest({
+    ...files,
+    'a.spec.js-snapshots/snapshot.png': expected,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('should pass', ({}) => {
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', {
+          threshold: 0,
+          comparator: 'ssim-cie94',
+        });
+      });
+      test('should fail', ({}) => {
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', {
+          threshold: 0,
+          comparator: 'pixelmatch',
+        });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.report.suites[0].specs[0].title).toBe('should pass');
+  expect(result.report.suites[0].specs[0].ok).toBe(true);
+  expect(result.report.suites[0].specs[1].title).toBe('should fail');
+  expect(result.report.suites[0].specs[1].ok).toBe(false);
+});
+
+test('should respect comparator in config', async ({ runInlineTest }) => {
+  const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png'));
+  const actual = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png'));
+  const result = await runInlineTest({
+    ...files,
+    'playwright.config.ts': `
+      module.exports = {
+        snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}',
+        projects: [
+          {
+            name: 'should-pass',
+            expect: {
+              toMatchSnapshot: {
+                comparator: 'ssim-cie94',
+              }
+            },
+          },
+          {
+            name: 'should-fail',
+            expect: {
+              toMatchSnapshot: {
+                comparator: 'pixelmatch',
+              }
+            },
+          },
+        ],
+      };
+    `,
+    '__screenshots__/a.spec.js/snapshot.png': expected,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('test', ({}) => {
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', {
+          threshold: 0,
+        });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.report.suites[0].specs[0].tests[0].projectName).toBe('should-pass');
+  expect(result.report.suites[0].specs[0].tests[0].status).toBe('expected');
+  expect(result.report.suites[0].specs[0].tests[1].projectName).toBe('should-fail');
+  expect(result.report.suites[0].specs[0].tests[1].status).toBe('unexpected');
+});
+
 test('should sanitize snapshot name when passed as string', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     ...files,

tests/playwright-test/playwright.config.ts

@@ -22,15 +22,20 @@ import * as path from 'path';
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const config: Config = {
-  testDir: __dirname,
-  testIgnore: ['assets/**', 'stable-test-runner/**'],
   timeout: 30000,
   forbidOnly: !!process.env.CI,
   workers: process.env.CI ? 2 : undefined,
   preserveOutput: process.env.CI ? 'failures-only' : 'always',
   projects: [
     {
-      name: 'playwright-test'
+      name: 'playwright-test',
+      testDir: __dirname,
+      testIgnore: ['assets/**', 'stable-test-runner/**'],
+    },
+    {
+      name: 'image_tools',
+      testDir: path.join(__dirname, '../image_tools'),
+      testIgnore: [path.join(__dirname, '../fixtures/**')],
     },
   ],
   reporter: process.env.CI ? [

tests/playwright-test/to-have-screenshot.spec.ts

@@ -1022,6 +1022,78 @@ test('should update expectations with retries', async ({ runInlineTest }, testIn
   expect(comparePNGs(data, whiteImage)).toBe(null);
 });
 
+test('should respect comparator name', async ({ runInlineTest }) => {
+  const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png'));
+  const actualURL = pathToFileURL(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png'));
+  const result = await runInlineTest({
+    ...playwrightConfig({
+      snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}',
+    }),
+    '__screenshots__/a.spec.js/snapshot.png': expected,
+    'a.spec.js': `
+      pwt.test('should pass', async ({ page }) => {
+        await page.goto('${actualURL}');
+        await expect(page.locator('img')).toHaveScreenshot('snapshot.png', {
+          threshold: 0,
+          comparator: 'ssim-cie94',
+        });
+      });
+      pwt.test('should fail', async ({ page }) => {
+        await page.goto('${actualURL}');
+        await expect(page.locator('img')).toHaveScreenshot('snapshot.png', {
+          threshold: 0,
+          comparator: 'pixelmatch',
+        });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.report.suites[0].specs[0].title).toBe('should pass');
+  expect(result.report.suites[0].specs[0].ok).toBe(true);
+  expect(result.report.suites[0].specs[1].title).toBe('should fail');
+  expect(result.report.suites[0].specs[1].ok).toBe(false);
+});
+
+test('should respect comparator in config', async ({ runInlineTest }) => {
+  const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png'));
+  const actualURL = pathToFileURL(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png'));
+  const result = await runInlineTest({
+    ...playwrightConfig({
+      snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}',
+      projects: [
+        {
+          name: 'should-pass',
+          expect: {
+            toHaveScreenshot: {
+              comparator: 'ssim-cie94',
+            }
+          },
+        },
+        {
+          name: 'should-fail',
+          expect: {
+            toHaveScreenshot: {
+              comparator: 'pixelmatch',
+            }
+          },
+        },
+      ],
+    }),
+    '__screenshots__/a.spec.js/snapshot.png': expected,
+    'a.spec.js': `
+      pwt.test('test', async ({ page }) => {
+        await page.goto('${actualURL}');
+        await expect(page.locator('img')).toHaveScreenshot('snapshot.png', { threshold: 0, });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.report.suites[0].specs[0].tests[0].projectName).toBe('should-pass');
+  expect(result.report.suites[0].specs[0].tests[0].status).toBe('expected');
+  expect(result.report.suites[0].specs[0].tests[1].projectName).toBe('should-fail');
+  expect(result.report.suites[0].specs[0].tests[1].status).toBe('unexpected');
+});
+
 function playwrightConfig(obj: any) {
   return {
     'playwright.config.js': `