feat(codgen): support positioned clicks in a canvas (#9503)

packages/playwright-core/src/server/supplements/injected/recorder.ts

@@ -249,6 +249,7 @@ export class Recorder {
     this._performAction({
       name: 'click',
       selector: this._hoveredModel!.selector,
+      position: positionForEvent(event),
       signals: [],
       button: buttonForEvent(event),
       modifiers: modifiersForEvent(event),
@@ -567,6 +568,17 @@ function buttonForEvent(event: MouseEvent): 'left' | 'middle' | 'right' {
   return 'left';
 }
 
+function positionForEvent(event: MouseEvent): Point |undefined {
+  const targetElement = (event.target as HTMLElement);
+  if (targetElement.nodeName !== 'CANVAS')
+    return;
+  const rect = targetElement.getBoundingClientRect();
+  return {
+    x: event.clientX - rect.left,
+    y: event.clientY - rect.top,
+  };
+}
+
 function consumeEvent(e: Event) {
   e.preventDefault();
   e.stopPropagation();

packages/playwright-core/src/server/supplements/recorder/csharp.ts

@@ -106,6 +106,8 @@ export class CSharpLanguageGenerator implements LanguageGenerator {
           options.modifiers = modifiers;
         if (action.clickCount > 2)
           options.clickCount = action.clickCount;
+        if (action.position)
+          options.position = action.position;
         if (!Object.entries(options).length)
           return `${method}Async(${quote(action.selector)})`;
         const optionsString = formatObject(options, '    ', (isPage ? 'Page' : 'Frame') + method + 'Options');

packages/playwright-core/src/server/supplements/recorder/java.ts

@@ -102,6 +102,8 @@ export class JavaLanguageGenerator implements LanguageGenerator {
           options.modifiers = modifiers;
         if (action.clickCount > 2)
           options.clickCount = action.clickCount;
+        if (action.position)
+          options.position = action.position;
         const optionsText = formatClickOptions(options, isPage);
         return `${method}(${quote(action.selector)}${optionsText ? ', ' : ''}${optionsText})`;
       }
@@ -222,6 +224,8 @@ function formatClickOptions(options: MouseClickOptions, isPage: boolean) {
     lines.push(`  .setModifiers(Arrays.asList(${options.modifiers.map(m => `KeyboardModifier.${m.toUpperCase()}`).join(', ')}))`);
   if (options.clickCount)
     lines.push(`  .setClickCount(${options.clickCount})`);
+  if (options.position)
+    lines.push(`  .setPosition(${options.position.x}, ${options.position.y})`);
   if (!lines.length)
     return '';
   lines.unshift(`new ${isPage ? 'Page' : 'Frame'}.ClickOptions()`);

packages/playwright-core/src/server/supplements/recorder/javascript.ts

@@ -120,6 +120,8 @@ export class JavaScriptLanguageGenerator implements LanguageGenerator {
           options.modifiers = modifiers;
         if (action.clickCount > 2)
           options.clickCount = action.clickCount;
+        if (action.position)
+          options.position = action.position;
         const optionsString = formatOptions(options);
         return `${method}(${quote(action.selector)}${optionsString})`;
       }

packages/playwright-core/src/server/supplements/recorder/python.ts

@@ -111,6 +111,8 @@ export class PythonLanguageGenerator implements LanguageGenerator {
           options.modifiers = modifiers;
         if (action.clickCount > 2)
           options.clickCount = action.clickCount;
+        if (action.position)
+          options.position = action.position;
         const optionsString = formatOptions(options, true);
         return `${method}(${quote(action.selector)}${optionsString})`;
       }
@@ -198,6 +200,8 @@ function formatValue(value: any): string {
     return `[${value.map(formatValue).join(', ')}]`;
   if (typeof value === 'string')
     return quote(value);
+  if (typeof value === 'object')
+    return JSON.stringify(value);
   return String(value);
 }
 

packages/playwright-core/src/server/supplements/recorder/recorderActions.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import type { Point } from '../../../common/types';
+
 export type ActionName =
   'check' |
   'click' |
@@ -37,6 +39,7 @@ export type ClickAction = ActionBase & {
   button: 'left' | 'middle' | 'right',
   modifiers: number,
   clickCount: number,
+  position?: Point,
 };
 
 export type CheckAction = ActionBase & {

packages/playwright-core/src/server/supplements/recorder/utils.ts

@@ -31,6 +31,8 @@ export function toClickOptions(action: actions.ClickAction): { method: 'click' |
     options.modifiers = modifiers;
   if (action.clickCount > 2)
     options.clickCount = action.clickCount;
+  if (action.position)
+    options.position = action.position;
   return { method, options };
 }
 

tests/config/browserTest.ts

@@ -69,6 +69,7 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
       proxy,
       args,
       handleSIGINT: false,
+      devtools: process.env.DEVTOOLS === '1',
     });
   }, { scope: 'worker' } ],
 

tests/inspector/cli-codegen-1.spec.ts

@@ -88,6 +88,66 @@ test.describe('cli codegen', () => {
     expect(message.text()).toBe('click');
   });
 
+  test('should make a positioned click on a canvas', async ({ page, openRecorder }) => {
+    const recorder = await openRecorder();
+
+    await recorder.setContentAndWait(`
+      <canvas width="500" height="500" style="margin: 42px"/>
+      <script>
+      document.querySelector("canvas").addEventListener("click", event => {
+        const rect = event.target.getBoundingClientRect();
+        console.log("click", event.clientX - rect.left, event.clientY - rect.top);
+      })
+      </script>
+    `);
+
+    const selector = await recorder.waitForHighlight(() => recorder.page.hover('canvas', {
+      position: { x: 250, y: 250 },
+    }));
+    expect(selector).toBe('canvas');
+    const [message, sources] = await Promise.all([
+      page.waitForEvent('console', msg => msg.type() !== 'error'),
+      recorder.waitForOutput('JavaScript', 'click'),
+      recorder.page.click('canvas', {
+        position: { x: 250, y: 250 },
+      })
+    ]);
+
+    expect(sources.get('JavaScript').text).toContain(`
+  // Click canvas
+  await page.click('canvas', {
+    position: {
+      x: 250,
+      y: 250
+    }
+  });`);
+
+    expect(sources.get('Python').text).toContain(`
+    # Click canvas
+    page.click("canvas", position={"x":250,"y":250})`);
+
+    expect(sources.get('Python Async').text).toContain(`
+    # Click canvas
+    await page.click("canvas", position={"x":250,"y":250})`);
+
+    expect(sources.get('Java').text).toContain(`
+      // Click canvas
+      page.click("canvas", new Page.ClickOptions()
+        .setPosition(250, 250));`);
+
+    expect(sources.get('C#').text).toContain(`
+        // Click canvas
+        await page.ClickAsync("canvas", new PageClickOptions
+        {
+            Position = new Position
+            {
+                X = 250,
+                Y = 250,
+            },
+        });`);
+    expect(message.text()).toBe('click 250 250');
+  });
+
   test('should work with TrustedTypes', async ({ page, openRecorder }) => {
     const recorder = await openRecorder();