feat(engines): introduce xpath engine, switch $x to use it (#64)

package.json

@@ -23,8 +23,8 @@
     "doc": "node utils/doclint/cli.js",
     "coverage": "cross-env COVERAGE=true npm run unit",
     "tsc": "tsc -p .",
-    "build": "npx webpack --config src/injected/cssSelectorEngine.webpack.config.js --mode='production' && npx webpack --config src/injected/injected.webpack.config.js --mode='production' && tsc -p .",
-    "watch": "npx webpack --config src/injected/cssSelectorEngine.webpack.config.js --mode='development' --watch --silent | npx webpack --config src/injected/injected.webpack.config.js --mode='development' --watch --silent | tsc -w -p .",
+    "build": "node utils/runWebpack.js --mode='production' && tsc -p .",
+    "watch": "node utils/runWebpack.js --mode='development' --watch --silent | tsc -w -p .",
     "apply-next-version": "node utils/apply_next_version.js",
     "bundle": "npx browserify -r ./index.js:playwright -o utils/browser/playwright-web.js",
     "test-types": "node utils/doclint/generate_types && npx -p typescript@2.1 tsc -p utils/doclint/generate_types/test/",

src/chromium/ExecutionContext.ts

@@ -24,6 +24,7 @@ import { createJSHandle, ElementHandle, JSHandle } from './JSHandle';
 import { Protocol } from './protocol';
 import * as injectedSource from '../generated/injectedSource';
 import * as cssSelectorEngineSource from '../generated/cssSelectorEngineSource';
+import * as xpathSelectorEngineSource from '../generated/xpathSelectorEngineSource';
 
 export const EVALUATION_SCRIPT_URL = '__playwright_evaluation_script__';
 const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
@@ -164,7 +165,7 @@ export class ExecutionContext {
 
   _injected(): Promise<JSHandle> {
     if (!this._injectedPromise) {
-      const engineSources = [cssSelectorEngineSource.source];
+      const engineSources = [cssSelectorEngineSource.source, xpathSelectorEngineSource.source];
       const source = `
         new (${injectedSource.source})([
           ${engineSources.join(',\n')}

src/chromium/JSHandle.ts

@@ -454,16 +454,8 @@ export class ElementHandle extends JSHandle {
 
   async $x(expression: string): Promise<ElementHandle[]> {
     const arrayHandle = await this.evaluateHandle(
-        (element, expression) => {
-          const document = element.ownerDocument || element;
-          const iterator = document.evaluate(expression, element, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);
-          const array = [];
-          let item;
-          while ((item = iterator.iterateNext()))
-            array.push(item);
-          return array;
-        },
-        expression
+      (root: SelectorRoot, expression: string, injected: Injected) => injected.querySelectorAll('xpath=' + expression, root),
+      expression, await this._context._injected()
     );
     const properties = await arrayHandle.getProperties();
     await arrayHandle.dispose();

src/firefox/ExecutionContext.ts

@@ -20,6 +20,7 @@ import {JSHandle, createHandle} from './JSHandle';
 import { Frame } from './FrameManager';
 import * as injectedSource from '../generated/injectedSource';
 import * as cssSelectorEngineSource from '../generated/cssSelectorEngineSource';
+import * as xpathSelectorEngineSource from '../generated/xpathSelectorEngineSource';
 
 export class ExecutionContext {
   _session: any;
@@ -120,7 +121,7 @@ export class ExecutionContext {
 
   _injected(): Promise<JSHandle> {
     if (!this._injectedPromise) {
-      const engineSources = [cssSelectorEngineSource.source];
+      const engineSources = [cssSelectorEngineSource.source, xpathSelectorEngineSource.source];
       const source = `
         new (${injectedSource.source})([
           ${engineSources.join(',\n')}

src/firefox/JSHandle.ts

@@ -253,16 +253,8 @@ export class ElementHandle extends JSHandle {
 
   async $x(expression: string): Promise<Array<ElementHandle>> {
     const arrayHandle = await this._frame.evaluateHandle(
-        (element, expression) => {
-          const document = element.ownerDocument || element;
-          const iterator = document.evaluate(expression, element, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);
-          const array = [];
-          let item;
-          while ((item = iterator.iterateNext()))
-            array.push(item);
-          return array;
-        },
-        this, expression
+      (root: SelectorRoot, expression: string, injected: Injected) => injected.querySelectorAll('xpath=' + expression, root),
+      this, expression, await this._context._injected()
     );
     const properties = await arrayHandle.getProperties();
     await arrayHandle.dispose();

src/injected/cssSelectorEngine.ts

@@ -3,7 +3,7 @@
 
 import { SelectorEngine, SelectorRoot } from './selectorEngine';
 
-export const CSSEngine: SelectorEngine = {
+const CSSEngine: SelectorEngine = {
   name: 'css',
 
   create(root: SelectorRoot, targetElement: Element): string | undefined {

src/injected/injected.ts

@@ -6,7 +6,7 @@ import { Utils } from './utils';
 
 type ParsedSelector = { engine: SelectorEngine, selector: string }[];
 
-export class Injected {
+class Injected {
   readonly utils: Utils;
   readonly engines: Map<string, SelectorEngine>;
 

src/injected/webpack-inline-source-plugin.js

@@ -18,6 +18,7 @@ module.exports = class InlineSource {
       if (source.endsWith(';'))
         source = source.substring(0, source.length - 1);
       source = '(' + source + ').default';
+      fs.mkdirSync(path.dirname(this.outFile), { recursive: true });
       const newSource = 'export const source = ' + JSON.stringify(source) + ';';
       fs.writeFileSync(this.outFile, newSource);
       callback();

src/injected/xpathSelectorEngine.ts

@@ -0,0 +1,179 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { SelectorEngine, SelectorType, SelectorRoot } from './selectorEngine';
+
+const maxTextLength = 80;
+const minMeaningfulSelectorLegth = 100;
+
+const XPathEngine: SelectorEngine = {
+  name: 'xpath',
+
+  create(root: SelectorRoot, targetElement: Element, type: SelectorType): string | undefined {
+    const document = root instanceof Document ? root : root.ownerDocument;
+    if (!document)
+      return;
+
+    const xpathCache = new Map<string, Element[]>();
+    if (type === 'notext')
+      return createNoText(root, targetElement);
+
+    const tokens: string[] = [];
+
+    function evaluateXPath(expression: string): Element[] {
+      let nodes: Element[] | undefined = xpathCache.get(expression);
+      if (!nodes) {
+        nodes = [];
+        try {
+          const result = document.evaluate(expression, root, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);
+          for (let node = result.iterateNext(); node; node = result.iterateNext()) {
+            if (node.nodeType === Node.ELEMENT_NODE)
+              nodes.push(node as Element);
+          }
+        } catch (e) {
+        }
+        xpathCache.set(expression, nodes);
+      }
+      return nodes;
+    }
+
+    function uniqueXPathSelector(prefix?: string): string | undefined {
+      const path = tokens.slice();
+      if (prefix)
+        path.unshift(prefix);
+      let selector = '//' + path.join('/');
+      while (selector.includes('///'))
+        selector = selector.replace('///', '//');
+      if (selector.endsWith('/'))
+        selector = selector.substring(0, selector.length - 1);
+      const nodes: Element[] = evaluateXPath(selector);
+      if (nodes[nodes.length - 1] === targetElement)
+        return selector;
+
+      // If we are looking at a small set of elements with long selector, fall back to ordinal.
+      if (nodes.length < 5 && selector.length > minMeaningfulSelectorLegth) {
+        const index = nodes.indexOf(targetElement);
+        if (index !== -1)
+          return `(${selector})[${index + 1}]`;
+      }
+      return undefined;
+    }
+
+    function escapeAndCap(text: string) {
+      text = text.substring(0, maxTextLength);
+      // XPath 1.0 does not support quote escaping.
+      // 1. If there are no single quotes - use them.
+      if (text.indexOf(`'`) === -1)
+        return `'${text}'`;
+      // 2. If there are no double quotes - use them to enclose text.
+      if (text.indexOf(`"`) === -1)
+        return `"${text}"`;
+      // 3. Otherwise, use popular |concat| trick.
+      const Q = `'`;
+      return `concat(${text.split(Q).map(token => Q + token + Q).join(`, "'", `)})`;
+    }
+
+    const defaultAttributes = new Set([ 'title', 'aria-label', 'disabled', 'role' ]);
+    const importantAttributes = new Map<string, string[]>([
+      [ 'form', [ 'action' ] ],
+      [ 'img', [ 'alt' ] ],
+      [ 'input', [ 'placeholder', 'type', 'name', 'value' ] ],
+    ]);
+
+    let usedTextConditions = false;
+    for (let element: Element | null = targetElement; element && element !== root; element = element.parentElement) {
+      const nodeName = element.nodeName.toLowerCase();
+      const tag = nodeName === 'svg' ? '*' : nodeName;
+
+      const tagConditions = [];
+      if (nodeName === 'svg')
+        tagConditions.push('local-name()="svg"');
+
+      const attrConditions: string[] = [];
+      const importantAttrs = [ ...defaultAttributes, ...(importantAttributes.get(tag) || []) ];
+      for (const attr of importantAttrs) {
+        const value = element.getAttribute(attr);
+        if (value && value.length < maxTextLength)
+          attrConditions.push(`normalize-space(@${attr})=${escapeAndCap(value)}`);
+        else if (value)
+          attrConditions.push(`starts-with(normalize-space(@${attr}), ${escapeAndCap(value)})`);
+      }
+
+      const text = document.evaluate('normalize-space(.)', element).stringValue;
+      const textConditions = [];
+      if (tag !== 'select' && text.length && !usedTextConditions) {
+        if (text.length < maxTextLength)
+          textConditions.push(`normalize-space(.)=${escapeAndCap(text)}`);
+        else
+          textConditions.push(`starts-with(normalize-space(.), ${escapeAndCap(text)})`);
+        usedTextConditions = true;
+      }
+
+      // Always retain the last tag.
+      const conditions = [ ...tagConditions, ...textConditions, ...attrConditions ];
+      const token = conditions.length ? `${tag}[${conditions.join(' and ')}]` : (tokens.length ? '' : tag);
+      const selector = uniqueXPathSelector(token);
+      if (selector)
+        return selector;
+
+      // Ordinal is the weakest signal.
+      const parent = element.parentElement;
+      let tagWithOrdinal = tag;
+      if (parent) {
+        const siblings = Array.from(parent.children);
+        const sameTagSiblings = siblings.filter(sibling => (sibling as Element).nodeName.toLowerCase() === nodeName);
+        if (sameTagSiblings.length > 1)
+          tagWithOrdinal += `[${1 + siblings.indexOf(element)}]`;
+      }
+
+      // Do not include text into this token, only tag / attributes.
+      // Topmost node will get all the text.
+      const nonTextConditions = [ ...tagConditions, ...attrConditions ];
+      const levelToken = nonTextConditions.length ? `${tagWithOrdinal}[${nonTextConditions.join(' and ')}]` : tokens.length ? '' : tagWithOrdinal;
+      tokens.unshift(levelToken);
+    }
+    return uniqueXPathSelector();
+  },
+
+  query(root: SelectorRoot, selector: string): Element | undefined {
+    const document = root instanceof Document ? root : root.ownerDocument;
+    if (!document)
+      return;
+    const it = document.evaluate(selector, root, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);
+    for (let node = it.iterateNext(); node; node = it.iterateNext()) {
+      if (node.nodeType === Node.ELEMENT_NODE)
+        return node as Element;
+    }
+  },
+
+  queryAll(root: SelectorRoot, selector: string): Element[] {
+    const result: Element[] = [];
+    const document = root instanceof Document ? root : root.ownerDocument;
+    if (!document)
+      return result;
+    const it = document.evaluate(selector, root, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);
+    for (let node = it.iterateNext(); node; node = it.iterateNext()) {
+      if (node.nodeType === Node.ELEMENT_NODE)
+        result.push(node as Element);
+    }
+    return result;
+  }
+};
+
+function createNoText(root: SelectorRoot, targetElement: Element): string {
+  const steps = [];
+  for (let element: Element | null = targetElement; element && element !== root; element = element.parentElement) {
+    if (element.getAttribute('id')) {
+      steps.unshift(`//*[@id="${element.getAttribute('id')}"]`);
+      return steps.join('/');
+    }
+    const siblings = element.parentElement ? Array.from(element.parentElement.children) : [];
+    const similarElements: Element[] = siblings.filter(sibling => element!.nodeName === sibling.nodeName);
+    const index = similarElements.length === 1 ? 0 : similarElements.indexOf(element) + 1;
+    steps.unshift(index ? `${element.nodeName}[${index}]` : element.nodeName);
+  }
+
+  return '/' + steps.join('/');
+}
+
+export default XPathEngine;

src/injected/xpathSelectorEngine.webpack.config.js

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+const path = require('path');
+const InlineSource = require('./webpack-inline-source-plugin.js');
+
+module.exports = {
+  entry: path.join(__dirname, 'xpathSelectorEngine.ts'),
+  devtool: 'source-map',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        loader: 'ts-loader',
+        options: {
+          transpileOnly: true
+        },
+        exclude: /node_modules/
+      }
+    ]
+  },
+  resolve: {
+    extensions: [ '.tsx', '.ts', '.js' ]
+  },
+  output: {
+    filename: 'xpathSelectorEngineSource.js',
+    path: path.resolve(__dirname, '../../lib/injected/generated')
+  },
+  plugins: [
+    new InlineSource(path.join(__dirname, '..', 'generated', 'xpathSelectorEngineSource.ts')),
+  ]
+};

src/webkit/ExecutionContext.ts

@@ -23,6 +23,7 @@ import { createJSHandle, JSHandle } from './JSHandle';
 import { Protocol } from './protocol';
 import * as injectedSource from '../generated/injectedSource';
 import * as cssSelectorEngineSource from '../generated/cssSelectorEngineSource';
+import * as xpathSelectorEngineSource from '../generated/xpathSelectorEngineSource';
 
 export const EVALUATION_SCRIPT_URL = '__playwright_evaluation_script__';
 const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
@@ -305,7 +306,7 @@ export class ExecutionContext {
 
   _injected(): Promise<JSHandle> {
     if (!this._injectedPromise) {
-      const engineSources = [cssSelectorEngineSource.source];
+      const engineSources = [cssSelectorEngineSource.source, xpathSelectorEngineSource.source];
       const source = `
         new (${injectedSource.source})([
           ${engineSources.join(',\n')}

src/webkit/JSHandle.ts

@@ -333,16 +333,8 @@ export class ElementHandle extends JSHandle {
 
   async $x(expression: string): Promise<ElementHandle[]> {
     const arrayHandle = await this.evaluateHandle(
-        (element, expression) => {
-          const document = element.ownerDocument || element;
-          const iterator = document.evaluate(expression, element, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);
-          const array = [];
-          let item;
-          while ((item = iterator.iterateNext()))
-            array.push(item);
-          return array;
-        },
-        expression
+      (root: SelectorRoot, expression: string, injected: Injected) => injected.querySelectorAll('xpath=' + expression, root),
+      expression, await this._context._injected()
     );
     const properties = await arrayHandle.getProperties();
     await arrayHandle.dispose();

utils/runWebpack.js

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+const child_process = require('child_process');
+const path = require('path');
+
+const files = [
+  path.join('src', 'injected', 'cssSelectorEngine.webpack.config.js'),
+  path.join('src', 'injected', 'xpathSelectorEngine.webpack.config.js'),
+  path.join('src', 'injected', 'injected.webpack.config.js'),
+];
+
+function runOne(runner, file) {
+  return runner('npx', ['webpack', '--config', file, ...process.argv.slice(2)], { stdio: 'inherit', shell: true });
+}
+
+const args = process.argv.slice(2);
+if (args.includes('--watch')) {
+  const spawns = files.map(file => runOne(child_process.spawn, file));
+  process.on('exit', () => spawns.forEach(s => s.kill()));
+} else {
+  for (const file of files) {
+    const out = runOne(child_process.spawnSync, file);
+    if (out.status)
+      process.exit(out.status);
+  }
+}