chore: add step attachments into the expect trace event (#22543)

packages/playwright-core/src/utils/traceUtils.ts

@@ -97,9 +97,7 @@ export async function mergeTraceFiles(fileName: string, temporaryTraceFiles: str
 }
 
 export async function saveTraceFile(fileName: string, traceEvents: TraceEvent[], saveSources: boolean) {
-  const lines: string[] = traceEvents.map(e => JSON.stringify(e));
   const zipFile = new yazl.ZipFile();
-  zipFile.addBuffer(Buffer.from(lines.join('\n')), 'trace.trace');
 
   if (saveSources) {
     const sourceFiles = new Set<string>();
@@ -115,6 +113,25 @@ export async function saveTraceFile(fileName: string, traceEvents: TraceEvent[],
       }).catch(() => {});
     }
   }
+
+  const sha1s = new Set<string>();
+  for (const event of traceEvents.filter(e => e.type === 'after') as AfterActionTraceEvent[]) {
+    for (const attachment of (event.attachments || []).filter(a => !!a.path)) {
+      await fs.promises.readFile(attachment.path!).then(content => {
+        const sha1 = calculateSha1(content);
+        if (sha1s.has(sha1))
+          return;
+        sha1s.add(sha1);
+        zipFile.addBuffer(content, 'resources/' + sha1);
+        attachment.sha1 = sha1;
+        delete attachment.path;
+      }).catch();
+    }
+  }
+
+  const traceContent = Buffer.from(traceEvents.map(e => JSON.stringify(e)).join('\n'));
+  zipFile.addBuffer(traceContent, 'trace.trace');
+
   await new Promise(f => {
     zipFile.end(undefined, () => {
       zipFile.outputStream.pipe(fs.createWriteStream(fileName)).on('close', f);
@@ -136,12 +153,13 @@ export function createBeforeActionTraceEventForExpect(callId: string, apiName: s
   };
 }
 
-export function createAfterActionTraceEventForExpect(callId: string, error?: SerializedError['error']): AfterActionTraceEvent {
+export function createAfterActionTraceEventForExpect(callId: string, attachments: AfterActionTraceEvent['attachments'], error?: SerializedError['error']): AfterActionTraceEvent {
   return {
     type: 'after',
     callId,
     endTime: monotonicTime(),
     log: [],
+    attachments,
     error,
   };
 }

packages/playwright-test/src/matchers/expect.ts

@@ -48,7 +48,7 @@ import {
   toPass
 } from './matchers';
 import { toMatchSnapshot, toHaveScreenshot, toHaveScreenshotStepTitle } from './toMatchSnapshot';
-import type { Expect } from '../../types/test';
+import type { Expect, TestInfo } from '../../types/test';
 import { currentTestInfo, currentExpectTimeout, setCurrentExpectConfigureTimeout } from '../common/globals';
 import { filteredStackTrace, serializeError, stringifyStackFrames, trimLongString } from '../util';
 import {
@@ -58,6 +58,7 @@ import {
   printReceived,
 } from '../common/expectBundle';
 import { zones } from 'playwright-core/lib/utils';
+import type { AfterActionTraceEvent } from '../../../trace/src/trace';
 
 // from expect/build/types
 export type SyncExpectationResult = {
@@ -243,6 +244,7 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
 
       const defaultTitle = `expect${this._info.isPoll ? '.poll' : ''}${this._info.isSoft ? '.soft' : ''}${this._info.isNot ? '.not' : ''}.${matcherName}${argsSuffix}`;
       const wallTime = Date.now();
+      const initialAttachments = new Set(testInfo.attachments.slice());
       const step = testInfo._addStep({
         location: stackFrames[0],
         category: 'expect',
@@ -281,7 +283,7 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
         const serializerError = serializeError(jestError);
         if (generateTraceEvent) {
           const error = { name: jestError.name, message: jestError.message, stack: jestError.stack };
-          testInfo._traceEvents.push(createAfterActionTraceEventForExpect(`expect@${callId}`, error));
+          testInfo._traceEvents.push(createAfterActionTraceEventForExpect(`expect@${callId}`, serializeAttachments(testInfo.attachments, initialAttachments), error));
         }
         step.complete({ error: serializerError });
         if (this._info.isSoft)
@@ -292,7 +294,7 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
 
       const finalizer = () => {
         if (generateTraceEvent)
-          testInfo._traceEvents.push(createAfterActionTraceEventForExpect(`expect@${callId}`));
+          testInfo._traceEvents.push(createAfterActionTraceEventForExpect(`expect@${callId}`, serializeAttachments(testInfo.attachments, initialAttachments)));
         step.complete({});
       };
 
@@ -363,4 +365,15 @@ function computeArgsSuffix(matcherName: string, args: any[]) {
   return value ? `(${value})` : '';
 }
 
+function serializeAttachments(attachments: TestInfo['attachments'], initialAttachments: Set<TestInfo['attachments'][0]>): AfterActionTraceEvent['attachments'] {
+  return attachments.filter(a => !initialAttachments.has(a)).map(a => {
+    return {
+      name: a.name,
+      contentType: a.contentType,
+      path: a.path,
+      body: a.body?.toString('base64'),
+    };
+  });
+}
+
 expectLibrary.extend(customMatchers);

packages/trace-viewer/src/traceModel.ts

@@ -170,6 +170,7 @@ export class TraceModel {
         existing!.log = event.log;
         existing!.result = event.result;
         existing!.error = event.error;
+        existing!.attachments = event.attachments;
         break;
       }
       case 'action': {

packages/trace-viewer/src/ui/callTab.tsx

@@ -23,6 +23,8 @@ import { CopyToClipboard } from './copyToClipboard';
 import { asLocator } from '@isomorphic/locatorGenerators';
 import type { Language } from '@isomorphic/locatorGenerators';
 import { ErrorMessage } from '@web/components/errorMessage';
+import { ImageDiffView } from '@web/components/imageDiffView';
+import type { TestAttachment } from '@web/components/imageDiffView';
 
 export const CallTab: React.FunctionComponent<{
   action: ActionTraceEvent | undefined,
@@ -38,7 +40,19 @@ export const CallTab: React.FunctionComponent<{
   const paramKeys = Object.keys(params);
   const wallTime = action.wallTime ? new Date(action.wallTime).toLocaleString() : null;
   const duration = action.endTime ? msToString(action.endTime - action.startTime) : 'Timed Out';
+
+  const expected = action.attachments?.find(a => a.name.endsWith('-expected.png') && (a.path || a.sha1)) as TestAttachment | undefined;
+  const actual = action.attachments?.find(a => a.name.endsWith('-actual.png') && (a.path || a.sha1)) as TestAttachment | undefined;
+  const diff = action.attachments?.find(a => a.name.endsWith('-diff.png') && (a.path || a.sha1)) as TestAttachment | undefined;
+
   return <div className='call-tab'>
+    { expected && actual && <div className='call-section'>Image diff</div> }
+    { expected && actual && <ImageDiffView imageDiff={{
+      name: 'Image diff',
+      expected: { attachment: { ...expected, path: attachmentURL(expected) }, title: 'Expected' },
+      actual: { attachment: { ...actual, path: attachmentURL(actual) } },
+      diff: diff ? { attachment: { ...diff, path: attachmentURL(diff) } } : undefined,
+    }} /> }
     {!!error && <ErrorMessage error={error} />}
     {!!error && <div className='call-section'>Call</div>}
     <div className='call-line'>{action.apiName}</div>
@@ -146,3 +160,15 @@ function parseSerializedValue(value: SerializedValue, handles: any[] | undefined
   }
   return '<object>';
 }
+
+function attachmentURL(attachment: {
+  name: string;
+  contentType: string;
+  path?: string;
+  sha1?: string;
+  body?: string;
+}) {
+  if (attachment.sha1)
+    return 'sha1/' + attachment.sha1;
+  return 'file?path=' + attachment.path;
+}

packages/trace-viewer/src/ui/modelUtil.ts

@@ -109,6 +109,8 @@ function dedupeAndSortActions(actions: ActionTraceEvent[]) {
     if (callAction) {
       if (expectAction.error)
         callAction.error = expectAction.error;
+      if (expectAction.attachments)
+        callAction.attachments = expectAction.attachments;
       continue;
     }
     result.push(expectAction);

packages/trace/src/trace.ts

@@ -79,6 +79,13 @@ export type AfterActionTraceEvent = {
   afterSnapshot?: string;
   log: string[];
   error?: SerializedError['error'];
+  attachments?: {
+    name: string;
+    contentType: string;
+    path?: string;
+    sha1?: string;
+    body?: string;  // base64
+  }[];
   result?: any;
 };
 

packages/web/src/components/imageDiffView.css

@@ -0,0 +1,46 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+.image-diff-view .image-wrapper img {
+  flex: auto;
+  box-shadow: var(--box-shadow-thick);
+  margin: 24px auto;
+  min-width: 200px;
+  max-width: 80%;
+}
+
+.image-diff-view .image-wrapper {
+  flex: auto;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.image-diff-view .image-wrapper div {
+  flex: none;
+  align-self: stretch;
+  height: 2em;
+  font-weight: 500;
+  padding-top: 1em;
+  display: flex;
+  flex-direction: row;
+}
+
+.image-diff-view .modes > div {
+  margin: 10px;
+  cursor: pointer;
+  user-select: none;
+}

packages/web/src/components/imageDiffView.tsx

@@ -0,0 +1,178 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import * as React from 'react';
+import './imageDiffView.css';
+
+export type TestAttachment = {
+  name: string;
+  contentType: string;
+  path: string;
+};
+
+export type ImageDiff = {
+  name: string,
+  expected?: { attachment: TestAttachment, title: string },
+  actual?: { attachment: TestAttachment },
+  diff?: { attachment: TestAttachment },
+};
+
+export const ImageDiffView: React.FunctionComponent<{
+ imageDiff: ImageDiff,
+}> = ({ imageDiff: diff }) => {
+  // Pre-select a tab called "diff", if any.
+  const [mode, setMode] = React.useState<'diff' | 'actual' | 'expected'>(diff.diff ? 'diff' : 'actual');
+  const diffElement = React.useRef<HTMLDivElement>(null);
+  const imageElement = React.useRef<HTMLImageElement>(null);
+  const [sliderPosition, setSliderPosition] = React.useState<number>(0);
+  const onImageLoaded = (side?: 'left' | 'right') => {
+    if (diffElement.current)
+      diffElement.current.style.minHeight = diffElement.current.offsetHeight + 'px';
+    if (side && diffElement.current && imageElement.current) {
+      const gap = Math.max(0, (diffElement.current.offsetWidth - imageElement.current.offsetWidth) / 2 - 20);
+      if (side === 'left')
+        setSliderPosition(gap);
+      else if (side === 'right')
+        setSliderPosition(diffElement.current.offsetWidth - gap);
+    }
+  };
+
+  return <div className='vbox image-diff-view'>
+    <div className='hbox modes'>
+      {diff.diff && <div onClick={() => setMode('diff')}>Diff</div>}
+      <div onClick={() => setMode('actual')}>Actual</div>
+      <div onClick={() => setMode('expected')}>Expected</div>
+    </div>
+    <div style={{ position: 'relative' }} ref={diffElement}>
+      {diff.diff && mode === 'diff' && <ImageWithSize src={diff.diff!.attachment.path!} onLoad={() => onImageLoaded()} />}
+      {diff.diff && mode === 'actual' && <ImageDiffSlider sliderPosition={sliderPosition} setSliderPosition={setSliderPosition}>
+        <ImageWithSize src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('right')} imageRef={imageElement} style={{ boxShadow: 'none' }} />
+        <ImageWithSize src={diff.actual!.attachment.path!} />
+      </ImageDiffSlider>}
+      {diff.diff && mode === 'expected' && <ImageDiffSlider sliderPosition={sliderPosition} setSliderPosition={setSliderPosition}>
+        <ImageWithSize src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('left')} imageRef={imageElement} />
+        <ImageWithSize src={diff.actual!.attachment.path!} style={{ boxShadow: 'none' }} />
+      </ImageDiffSlider>}
+      {!diff.diff && mode === 'actual' && <ImageWithSize src={diff.actual!.attachment.path!} onLoad={() => onImageLoaded()} />}
+      {!diff.diff && mode === 'expected' && <ImageWithSize src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded()} />}
+    </div>
+  </div>;
+};
+
+export const ImageDiffSlider: React.FC<React.PropsWithChildren<{
+  sliderPosition: number,
+  setSliderPosition: (position: number) => void,
+}>> = ({ children, sliderPosition, setSliderPosition }) => {
+  const [resizing, setResizing] = React.useState<{ offset: number, size: number } | null>(null);
+  const size = sliderPosition;
+
+  const childrenArray = React.Children.toArray(children);
+  document.body.style.userSelect = resizing ? 'none' : 'inherit';
+
+  const gripStyle: React.CSSProperties = {
+    ...absolute,
+    zIndex: 100,
+    cursor: 'ew-resize',
+    left: resizing ? 0 : size - 4,
+    right: resizing ? 0 : undefined,
+    width: resizing ? 'initial' : 8,
+  };
+
+  return <>
+    {childrenArray[0]}
+    <div style={{ ...absolute }}>
+      <div style={{
+        ...absolute,
+        display: 'flex',
+        zIndex: 50,
+        clip: `rect(0, ${size}px, auto, 0)`,
+        backgroundColor: 'var(--vscode-panel-background)',
+      }}>
+        {childrenArray[1]}
+      </div>
+      <div
+        style={gripStyle}
+        onMouseDown={event => setResizing({ offset: event.clientX, size })}
+        onMouseUp={() => setResizing(null)}
+        onMouseMove={event => {
+          if (!event.buttons) {
+            setResizing(null);
+          } else if (resizing) {
+            const offset = event.clientX;
+            const delta = offset - resizing.offset;
+            const newSize = resizing.size + delta;
+
+            const splitView = (event.target as HTMLElement).parentElement!;
+            const rect = splitView.getBoundingClientRect();
+            const size = Math.min(Math.max(0, newSize), rect.width);
+            setSliderPosition(size);
+          }
+        }}
+      ></div>
+      <div data-testid='test-result-image-mismatch-grip' style={{
+        ...absolute,
+        left: size - 1,
+        width: 20,
+        zIndex: 80,
+        margin: '10px -10px',
+        pointerEvents: 'none',
+        display: 'flex',
+      }}>
+        <div style={{
+          position: 'absolute',
+          top: 0,
+          bottom: 0,
+          left: 9,
+          width: 2,
+          backgroundColor: 'var(--vscode-panel-border)',
+        }}>
+        </div>
+        <svg style={{ fill: 'var(--vscode-panel-border)' }} viewBox="0 0 27 20"><path d="M9.6 0L0 9.6l9.6 9.6z"></path><path d="M17 19.2l9.5-9.6L16.9 0z"></path></svg>
+      </div>
+    </div>
+  </>;
+};
+
+const ImageWithSize: React.FunctionComponent<{
+  src: string,
+  onLoad?: () => void,
+  imageRef?: React.RefObject<HTMLImageElement>,
+  style?: React.CSSProperties,
+}> = ({ src, onLoad, imageRef, style }) => {
+  const newRef = React.useRef<HTMLImageElement>(null);
+  const ref = imageRef ?? newRef;
+  const [size, setSize] = React.useState<{ width: number, height: number } | null>(null);
+  return <div className='image-wrapper'>
+    <div>
+      <span style={{ flex: '1 1 0', textAlign: 'end' }}>{ size ? size.width : ''}</span>
+      <span style={{ flex: 'none', margin: '0 5px' }}>x</span>
+      <span style={{ flex: '1 1 0', textAlign: 'start' }}>{ size ? size.height : ''}</span>
+    </div>
+    <img src={src} onLoad={() => {
+      onLoad?.();
+      if (ref.current)
+        setSize({ width: ref.current.naturalWidth, height: ref.current.naturalHeight });
+    }} ref={ref} style={style} />
+  </div>;
+};
+
+const absolute: React.CSSProperties = {
+  position: 'absolute',
+  top: 0,
+  right: 0,
+  bottom: 0,
+  left: 0,
+};