chore: move element coordinates handling to common (#139)

Browser now implement boundingBox(), contentQuads() and layoutViewport().
This commit is contained in:
Dmitry Gozman 2019-12-05 09:54:50 -08:00 committed by GitHub
parent 3f554b3273
commit d4f0084f67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 212 additions and 244 deletions

View File

@ -58,14 +58,10 @@ export class DOMWorldDelegate implements dom.DOMWorldDelegate {
return (remoteObject as Protocol.Runtime.RemoteObject).subtype === 'node';
}
private _getBoxModel(handle: dom.ElementHandle): Promise<void | Protocol.DOM.getBoxModelReturnValue> {
return this._client.send('DOM.getBoxModel', {
objectId: toRemoteObject(handle).objectId
}).catch(error => debugError(error));
}
async boundingBox(handle: dom.ElementHandle): Promise<types.Rect | null> {
const result = await this._getBoxModel(handle);
const result = await this._client.send('DOM.getBoxModel', {
objectId: toRemoteObject(handle).objectId
}).catch(debugError);
if (!result)
return null;
const quad = result.model.border;
@ -76,120 +72,30 @@ export class DOMWorldDelegate implements dom.DOMWorldDelegate {
return {x, y, width, height};
}
async contentQuads(handle: dom.ElementHandle): Promise<types.Quad[] | null> {
const result = await this._client.send('DOM.getContentQuads', {
objectId: toRemoteObject(handle).objectId
}).catch(debugError);
if (!result)
return null;
return result.quads.map(quad => [
{ x: quad[0], y: quad[1] },
{ x: quad[2], y: quad[3] },
{ x: quad[4], y: quad[5] },
{ x: quad[6], y: quad[7] }
]);
}
async layoutViewport(): Promise<{ width: number, height: number }> {
const layoutMetrics = await this._client.send('Page.getLayoutMetrics');
return { width: layoutMetrics.layoutViewport.clientWidth, height: layoutMetrics.layoutViewport.clientHeight };
}
screenshot(handle: dom.ElementHandle, options: ScreenshotOptions = {}): Promise<string | Buffer> {
const page = this._frameManager.page();
return page._screenshotter.screenshotElement(page, handle, options);
}
async ensurePointerActionPoint(handle: dom.ElementHandle, relativePoint?: types.Point): Promise<types.Point> {
await handle._scrollIntoViewIfNeeded();
if (!relativePoint)
return this._clickablePoint(handle);
let r = await this._viewportPointAndScroll(handle, relativePoint);
if (r.scrollX || r.scrollY) {
const error = await handle.evaluate((element, scrollX, scrollY) => {
if (!element.ownerDocument || !element.ownerDocument.defaultView)
return 'Node does not have a containing window';
element.ownerDocument.defaultView.scrollBy(scrollX, scrollY);
return false;
}, r.scrollX, r.scrollY);
if (error)
throw new Error(error);
r = await this._viewportPointAndScroll(handle, relativePoint);
if (r.scrollX || r.scrollY)
throw new Error('Failed to scroll relative point into viewport');
}
return r.point;
}
private async _clickablePoint(handle: dom.ElementHandle): Promise<types.Point> {
const fromProtocolQuad = (quad: number[]): types.Point[] => {
return [
{x: quad[0], y: quad[1]},
{x: quad[2], y: quad[3]},
{x: quad[4], y: quad[5]},
{x: quad[6], y: quad[7]}
];
};
const intersectQuadWithViewport = (quad: types.Point[], width: number, height: number): types.Point[] => {
return quad.map(point => ({
x: Math.min(Math.max(point.x, 0), width),
y: Math.min(Math.max(point.y, 0), height),
}));
};
const computeQuadArea = (quad: types.Point[]) => {
// Compute sum of all directed areas of adjacent triangles
// https://en.wikipedia.org/wiki/Polygon#Simple_polygons
let area = 0;
for (let i = 0; i < quad.length; ++i) {
const p1 = quad[i];
const p2 = quad[(i + 1) % quad.length];
area += (p1.x * p2.y - p2.x * p1.y) / 2;
}
return Math.abs(area);
};
const [result, layoutMetrics] = await Promise.all([
this._client.send('DOM.getContentQuads', {
objectId: toRemoteObject(handle).objectId
}).catch(debugError),
this._client.send('Page.getLayoutMetrics'),
]);
if (!result || !result.quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Filter out quads that have too small area to click into.
const { clientWidth, clientHeight } = layoutMetrics.layoutViewport;
const quads = result.quads.map(fromProtocolQuad)
.map(quad => intersectQuadWithViewport(quad, clientWidth, clientHeight))
.filter(quad => computeQuadArea(quad) > 1);
if (!quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Return the middle point of the first quad.
const quad = quads[0];
let x = 0;
let y = 0;
for (const point of quad) {
x += point.x;
y += point.y;
}
return {
x: x / 4,
y: y / 4
};
}
async _viewportPointAndScroll(handle: dom.ElementHandle, relativePoint: types.Point): Promise<{point: types.Point, scrollX: number, scrollY: number}> {
const model = await this._getBoxModel(handle);
let point: types.Point;
if (!model) {
point = relativePoint;
} else {
// Use padding quad to be compatible with offsetX/offsetY properties.
const quad = model.model.padding;
const x = Math.min(quad[0], quad[2], quad[4], quad[6]);
const y = Math.min(quad[1], quad[3], quad[5], quad[7]);
point = {
x: x + relativePoint.x,
y: y + relativePoint.y,
};
}
const metrics = await this._client.send('Page.getLayoutMetrics');
// Give one extra pixel to avoid any issues on viewport edge.
let scrollX = 0;
if (point.x < 1)
scrollX = point.x - 1;
if (point.x > metrics.layoutViewport.clientWidth - 1)
scrollX = point.x - metrics.layoutViewport.clientWidth + 1;
let scrollY = 0;
if (point.y < 1)
scrollY = point.y - 1;
if (point.y > metrics.layoutViewport.clientHeight - 1)
scrollY = point.y - metrics.layoutViewport.clientHeight + 1;
return { point, scrollX, scrollY };
}
async setInputFiles(handle: dom.ElementHandle, files: input.FilePayload[]): Promise<void> {
await handle.evaluate(input.setFileInputFunction, files);
}

View File

@ -8,7 +8,7 @@ import * as types from './types';
import * as injectedSource from './generated/injectedSource';
import * as cssSelectorEngineSource from './generated/cssSelectorEngineSource';
import * as xpathSelectorEngineSource from './generated/xpathSelectorEngineSource';
import { assert, helper } from './helper';
import { assert, helper, debugError } from './helper';
import Injected from './injected/injected';
import { SelectorRoot } from './injected/selectorEngine';
@ -19,9 +19,10 @@ export interface DOMWorldDelegate {
isJavascriptEnabled(): boolean;
isElement(remoteObject: any): boolean;
contentFrame(handle: ElementHandle): Promise<frames.Frame | null>;
contentQuads(handle: ElementHandle): Promise<types.Quad[] | null>;
layoutViewport(): Promise<{ width: number, height: number }>;
boundingBox(handle: ElementHandle): Promise<types.Rect | null>;
screenshot(handle: ElementHandle, options?: any): Promise<string | Buffer>;
ensurePointerActionPoint(handle: ElementHandle, relativePoint?: types.Point): Promise<types.Point>;
setInputFiles(handle: ElementHandle, files: input.FilePayload[]): Promise<void>;
adoptElementHandle(handle: ElementHandle, to: DOMWorld): Promise<ElementHandle>;
}
@ -190,8 +191,101 @@ export class ElementHandle extends js.JSHandle {
throw new Error(error);
}
private async _ensurePointerActionPoint(relativePoint?: types.Point): Promise<types.Point> {
await this._scrollIntoViewIfNeeded();
if (!relativePoint)
return this._clickablePoint();
let r = await this._viewportPointAndScroll(relativePoint);
if (r.scrollX || r.scrollY) {
const error = await this.evaluate((element, scrollX, scrollY) => {
if (!element.ownerDocument || !element.ownerDocument.defaultView)
return 'Node does not have a containing window';
element.ownerDocument.defaultView.scrollBy(scrollX, scrollY);
return false;
}, r.scrollX, r.scrollY);
if (error)
throw new Error(error);
r = await this._viewportPointAndScroll(relativePoint);
if (r.scrollX || r.scrollY)
throw new Error('Failed to scroll relative point into viewport');
}
return r.point;
}
private async _clickablePoint(): Promise<types.Point> {
const intersectQuadWithViewport = (quad: types.Quad): types.Quad => {
return quad.map(point => ({
x: Math.min(Math.max(point.x, 0), metrics.width),
y: Math.min(Math.max(point.y, 0), metrics.height),
})) as types.Quad;
};
const computeQuadArea = (quad: types.Quad) => {
// Compute sum of all directed areas of adjacent triangles
// https://en.wikipedia.org/wiki/Polygon#Simple_polygons
let area = 0;
for (let i = 0; i < quad.length; ++i) {
const p1 = quad[i];
const p2 = quad[(i + 1) % quad.length];
area += (p1.x * p2.y - p2.x * p1.y) / 2;
}
return Math.abs(area);
};
const [quads, metrics] = await Promise.all([
this._world.delegate.contentQuads(this),
this._world.delegate.layoutViewport(),
]);
if (!quads || !quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
const filtered = quads.map(quad => intersectQuadWithViewport(quad)).filter(quad => computeQuadArea(quad) > 1);
if (!filtered.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Return the middle point of the first quad.
const result = { x: 0, y: 0 };
for (const point of filtered[0]) {
result.x += point.x / 4;
result.y += point.y / 4;
}
return result;
}
private async _viewportPointAndScroll(relativePoint: types.Point): Promise<{point: types.Point, scrollX: number, scrollY: number}> {
const [box, border] = await Promise.all([
this.boundingBox(),
this.evaluate((e: Element) => {
const style = e.ownerDocument.defaultView.getComputedStyle(e);
return { x: parseInt(style.borderLeftWidth, 10), y: parseInt(style.borderTopWidth, 10) };
}).catch(debugError),
]);
const point = { x: relativePoint.x, y: relativePoint.y };
if (box) {
point.x += box.x;
point.y += box.y;
}
if (border) {
// Make point relative to the padding box to align with offsetX/offsetY.
point.x += border.x;
point.y += border.y;
}
const metrics = await this._world.delegate.layoutViewport();
// Give one extra pixel to avoid any issues on viewport edge.
let scrollX = 0;
if (point.x < 1)
scrollX = point.x - 1;
if (point.x > metrics.width - 1)
scrollX = point.x - metrics.width + 1;
let scrollY = 0;
if (point.y < 1)
scrollY = point.y - 1;
if (point.y > metrics.height - 1)
scrollY = point.y - metrics.height + 1;
return { point, scrollX, scrollY };
}
async _performPointerAction(action: (point: types.Point) => Promise<void>, options?: input.PointerActionOptions): Promise<void> {
const point = await this._world.delegate.ensurePointerActionPoint(this, options ? options.relativePoint : undefined);
const point = await this._ensurePointerActionPoint(options ? options.relativePoint : undefined);
let restoreModifiers: input.Modifier[] | undefined;
if (options && options.modifiers)
restoreModifiers = await this._world.delegate.keyboard._ensureModifiers(options.modifiers);

View File

@ -60,10 +60,36 @@ export class DOMWorldDelegate implements dom.DOMWorldDelegate {
}
async boundingBox(handle: dom.ElementHandle): Promise<types.Rect | null> {
return await this._session.send('Page.getBoundingBox', {
const quads = await this.contentQuads(handle);
if (!quads || !quads.length)
return null;
let minX = Infinity;
let maxX = -Infinity;
let minY = Infinity;
let maxY = -Infinity;
for (const quad of quads) {
for (const point of quad) {
minX = Math.min(minX, point.x);
maxX = Math.max(maxX, point.x);
minY = Math.min(minY, point.y);
maxY = Math.max(maxY, point.y);
}
}
return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
}
async contentQuads(handle: dom.ElementHandle): Promise<types.Quad[] | null> {
const result = await this._session.send('Page.getContentQuads', {
frameId: this._frameId,
objectId: toRemoteObject(handle).objectId,
});
}).catch(debugError);
if (!result)
return null;
return result.quads.map(quad => [ quad.p1, quad.p2, quad.p3, quad.p4 ]);
}
async layoutViewport(): Promise<{ width: number, height: number }> {
return this._frameManager._page.evaluate(() => ({ width: innerWidth, height: innerHeight }));
}
async screenshot(handle: dom.ElementHandle, options: any = {}): Promise<string | Buffer> {
@ -87,53 +113,6 @@ export class DOMWorldDelegate implements dom.DOMWorldDelegate {
}));
}
async ensurePointerActionPoint(handle: dom.ElementHandle, relativePoint?: types.Point): Promise<types.Point> {
await handle._scrollIntoViewIfNeeded();
if (!relativePoint)
return this._clickablePoint(handle);
const box = await this.boundingBox(handle);
return { x: box.x + relativePoint.x, y: box.y + relativePoint.y };
}
private async _clickablePoint(handle: dom.ElementHandle): Promise<types.Point> {
type Quad = {p1: types.Point, p2: types.Point, p3: types.Point, p4: types.Point};
const computeQuadArea = (quad: Quad) => {
// Compute sum of all directed areas of adjacent triangles
// https://en.wikipedia.org/wiki/Polygon#Simple_polygons
let area = 0;
const points = [quad.p1, quad.p2, quad.p3, quad.p4];
for (let i = 0; i < points.length; ++i) {
const p1 = points[i];
const p2 = points[(i + 1) % points.length];
area += (p1.x * p2.y - p2.x * p1.y) / 2;
}
return Math.abs(area);
};
const computeQuadCenter = (quad: Quad) => {
let x = 0, y = 0;
for (const point of [quad.p1, quad.p2, quad.p3, quad.p4]) {
x += point.x;
y += point.y;
}
return {x: x / 4, y: y / 4};
};
const result = await this._session.send('Page.getContentQuads', {
frameId: this._frameId,
objectId: toRemoteObject(handle).objectId,
}).catch(debugError);
if (!result || !result.quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Filter out quads that have too small area to click into.
const quads = result.quads.filter(quad => computeQuadArea(quad) > 1);
if (!quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Return the middle point of the first quad.
return computeQuadCenter(quads[0]);
}
async setInputFiles(handle: dom.ElementHandle, files: input.FilePayload[]): Promise<void> {
await handle.evaluate(input.setFileInputFunction, files);
}

View File

@ -17,6 +17,7 @@ export type EvaluateHandleOn = <Args extends any[]>(pageFunction: PageFunctionOn
export type Rect = { x: number, y: number, width: number, height: number };
export type Point = { x: number, y: number };
export type Quad = [ Point, Point, Point, Point ];
export type TimeoutOptions = { timeout?: number };

View File

@ -55,7 +55,40 @@ export class DOMWorldDelegate implements dom.DOMWorldDelegate {
}
async boundingBox(handle: dom.ElementHandle): Promise<types.Rect | null> {
throw new Error('boundingBox() is not implemented');
const quads = await this.contentQuads(handle);
if (!quads || !quads.length)
return null;
let minX = Infinity;
let maxX = -Infinity;
let minY = Infinity;
let maxY = -Infinity;
for (const quad of quads) {
for (const point of quad) {
minX = Math.min(minX, point.x);
maxX = Math.max(maxX, point.x);
minY = Math.min(minY, point.y);
maxY = Math.max(maxY, point.y);
}
}
return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
}
async contentQuads(handle: dom.ElementHandle): Promise<types.Quad[] | null> {
const result = await this._client.send('DOM.getContentQuads', {
objectId: toRemoteObject(handle).objectId
}).catch(debugError);
if (!result)
return null;
return result.quads.map(quad => [
{ x: quad[0], y: quad[1] },
{ x: quad[2], y: quad[3] },
{ x: quad[4], y: quad[5] },
{ x: quad[6], y: quad[7] }
]);
}
async layoutViewport(): Promise<{ width: number, height: number }> {
return this._frameManager._page.evaluate(() => ({ width: innerWidth, height: innerHeight }));
}
async screenshot(handle: dom.ElementHandle, options: any = {}): Promise<string | Buffer> {
@ -70,72 +103,6 @@ export class DOMWorldDelegate implements dom.DOMWorldDelegate {
return buffer;
}
async ensurePointerActionPoint(handle: dom.ElementHandle, relativePoint?: types.Point): Promise<types.Point> {
await handle._scrollIntoViewIfNeeded();
if (!relativePoint)
return this._clickablePoint(handle);
const box = await this.boundingBox(handle);
return { x: box.x + relativePoint.x, y: box.y + relativePoint.y };
}
private async _clickablePoint(handle: dom.ElementHandle): Promise<types.Point> {
const fromProtocolQuad = (quad: number[]): types.Point[] => {
return [
{x: quad[0], y: quad[1]},
{x: quad[2], y: quad[3]},
{x: quad[4], y: quad[5]},
{x: quad[6], y: quad[7]}
];
};
const intersectQuadWithViewport = (quad: types.Point[], width: number, height: number): types.Point[] => {
return quad.map(point => ({
x: Math.min(Math.max(point.x, 0), width),
y: Math.min(Math.max(point.y, 0), height),
}));
};
const computeQuadArea = (quad: types.Point[]) => {
// Compute sum of all directed areas of adjacent triangles
// https://en.wikipedia.org/wiki/Polygon#Simple_polygons
let area = 0;
for (let i = 0; i < quad.length; ++i) {
const p1 = quad[i];
const p2 = quad[(i + 1) % quad.length];
area += (p1.x * p2.y - p2.x * p1.y) / 2;
}
return Math.abs(area);
};
const [result, viewport] = await Promise.all([
this._client.send('DOM.getContentQuads', {
objectId: toRemoteObject(handle).objectId
}).catch(debugError),
handle.evaluate(() => ({ clientWidth: innerWidth, clientHeight: innerHeight })),
]);
if (!result || !result.quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Filter out quads that have too small area to click into.
const {clientWidth, clientHeight} = viewport;
const quads = result.quads.map(fromProtocolQuad)
.map(quad => intersectQuadWithViewport(quad, clientWidth, clientHeight))
.filter(quad => computeQuadArea(quad) > 1);
if (!quads.length)
throw new Error('Node is either not visible or not an HTMLElement');
// Return the middle point of the first quad.
const quad = quads[0];
let x = 0;
let y = 0;
for (const point of quad) {
x += point.x;
y += point.y;
}
return {
x: x / 4,
y: y / 4
};
}
async setInputFiles(handle: dom.ElementHandle, files: input.FilePayload[]): Promise<void> {
const objectId = toRemoteObject(handle).objectId;
await this._client.send('DOM.setInputFiles', { objectId, files });

View File

@ -1,6 +1,10 @@
<link rel='stylesheet' href='./style.css'>
<script src='./script.js' type='text/javascript'></script>
<style>
body {
height: 100;
margin: 8px;
}
div {
line-height: 18px;
}

View File

@ -1,6 +1,8 @@
<style>
body {
display: flex;
height: 100%;
margin: 8px;
}
body iframe {

View File

@ -2,6 +2,8 @@
body {
display: flex;
flex-direction: column;
height: 100%;
margin: 8px;
}
body iframe {

View File

@ -283,14 +283,30 @@ module.exports.addTests = function({testRunner, expect, playwright, FFOX, CHROME
expect(await frame.evaluate(() => window.result)).toBe('Clicked');
});
it.skip(FFOX || WEBKIT)('should click the button with relative point', async({page, server}) => {
it('should click the button with relative point', async({page, server}) => {
await page.goto(server.PREFIX + '/input/button.html');
await page.click('button', { relativePoint: { x: 20, y: 10 } });
expect(await page.evaluate(() => result)).toBe('Clicked');
expect(await page.evaluate(() => offsetX)).toBe(20);
expect(await page.evaluate(() => offsetY)).toBe(10);
});
it.skip(FFOX || WEBKIT)('should click a very large button with relative point', async({page, server}) => {
it('should click the button with px border with relative point', async({page, server}) => {
await page.goto(server.PREFIX + '/input/button.html');
await page.$eval('button', button => button.style.borderWidth = '2px');
await page.click('button', { relativePoint: { x: 20, y: 10 } });
expect(await page.evaluate(() => result)).toBe('Clicked');
expect(await page.evaluate(() => offsetX)).toBe(20);
expect(await page.evaluate(() => offsetY)).toBe(10);
});
it('should click the button with em border with relative point', async({page, server}) => {
await page.goto(server.PREFIX + '/input/button.html');
await page.$eval('button', button => button.style.borderWidth = '2em');
await page.click('button', { relativePoint: { x: 20, y: 10 } });
expect(await page.evaluate(() => result)).toBe('Clicked');
expect(await page.evaluate(() => offsetX)).toBe(20);
expect(await page.evaluate(() => offsetY)).toBe(10);
});
it('should click a very large button with relative point', async({page, server}) => {
await page.goto(server.PREFIX + '/input/button.html');
await page.$eval('button', button => button.style.height = button.style.width = '2000px');
await page.click('button', { relativePoint: { x: 1900, y: 1910 } });

View File

@ -21,7 +21,7 @@ module.exports.addTests = function({testRunner, expect, FFOX, CHROME, WEBKIT}) {
const {it, fit, xit} = testRunner;
const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
describe.skip(WEBKIT)('ElementHandle.boundingBox', function() {
describe('ElementHandle.boundingBox', function() {
it('should work', async({page, server}) => {
await page.setViewport({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
@ -32,13 +32,10 @@ module.exports.addTests = function({testRunner, expect, FFOX, CHROME, WEBKIT}) {
it('should handle nested frames', async({page, server}) => {
await page.setViewport({width: 500, height: 500});
await page.goto(server.PREFIX + '/frames/nested-frames.html');
const nestedFrame = page.frames()[1].childFrames()[1];
const nestedFrame = page.frames().find(frame => frame.name() === 'dos');
const elementHandle = await nestedFrame.$('div');
const box = await elementHandle.boundingBox();
if (CHROME)
expect(box).toEqual({ x: 28, y: 260, width: 264, height: 18 });
else
expect(box).toEqual({ x: 28, y: 182, width: 247, height: 18 });
expect(box).toEqual({ x: 28, y: 276, width: 264, height: 18 });
});
it('should return null for invisible elements', async({page, server}) => {
await page.setContent('<div style="display:none">hi</div>');

View File

@ -175,7 +175,7 @@ function checkSources(sources) {
const properties = [].concat(...types.map(type => type.properties));
return new Documentation.Type(name.replace(/false\|true/g, 'boolean'), properties);
}
if (type.typeArguments) {
if (type.typeArguments && type.symbol) {
const properties = [];
const innerTypeNames = [];
for (const typeArgument of type.typeArguments) {