microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-10-28 06:07:53 +03:00
Code Issues Projects Releases Wiki Activity

feat(adb): expose a11y tree (#4694)

Browse Source
This commit is contained in:
Pavel Feldman 2020-12-13 08:14:32 -08:00 committed by GitHub
parent 1b7fb7d56a
commit 7c89ec051a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 96 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
bin/android-driver.apk
View File

Binary file not shown.

6
src/client/android.ts
View File

@ -185,6 +185,12 @@ export class AndroidDevice extends ChannelOwner<channels.AndroidDeviceChannel, c
});
}
async tree(): Promise<apiInternal.AndroidElementInfo> {
return await this._wrapApiCall('androidDevice.tree', async () => {
return (await this._channel.tree()).tree;
});
}
async close() {
return this._wrapApiCall('androidDevice.close', async () => {
await this._channel.close();

4
src/dispatchers/androidDispatcher.ts
View File

@ -102,6 +102,10 @@ export class AndroidDeviceDispatcher extends Dispatcher<AndroidDevice, channels.
return { info: await this._object.send('info', params) };
}
async tree(params: channels.AndroidDeviceTreeParams): Promise<channels.AndroidDeviceTreeResult> {
return { tree: await this._object.send('tree', params) };
}
async inputType(params: channels.AndroidDeviceInputTypeParams) {
const text = params.text;
const keyCodes: number[] = [];

7
src/protocol/channels.ts
View File

@ -2461,6 +2461,7 @@ export interface AndroidDeviceChannel extends Channel {
scroll(params: AndroidDeviceScrollParams, metadata?: Metadata): Promise<AndroidDeviceScrollResult>;
swipe(params: AndroidDeviceSwipeParams, metadata?: Metadata): Promise<AndroidDeviceSwipeResult>;
info(params: AndroidDeviceInfoParams, metadata?: Metadata): Promise<AndroidDeviceInfoResult>;
tree(params?: AndroidDeviceTreeParams, metadata?: Metadata): Promise<AndroidDeviceTreeResult>;
inputType(params: AndroidDeviceInputTypeParams, metadata?: Metadata): Promise<AndroidDeviceInputTypeResult>;
inputPress(params: AndroidDeviceInputPressParams, metadata?: Metadata): Promise<AndroidDeviceInputPressResult>;
inputTap(params: AndroidDeviceInputTapParams, metadata?: Metadata): Promise<AndroidDeviceInputTapResult>;
@ -2594,6 +2595,11 @@ export type AndroidDeviceInfoOptions = {
export type AndroidDeviceInfoResult = {
info: AndroidElementInfo,
};
export type AndroidDeviceTreeParams = {};
export type AndroidDeviceTreeOptions = {};
export type AndroidDeviceTreeResult = {
tree: AndroidElementInfo,
};
export type AndroidDeviceInputTypeParams = {
text: string,
};
@ -2802,6 +2808,7 @@ export type AndroidSelector = {
};
export type AndroidElementInfo = {
children?: AndroidElementInfo[],
clazz: string,
desc: string,
res: string,

7
src/protocol/protocol.yml
View File

@ -2200,6 +2200,10 @@ AndroidDevice:
returns:
info: AndroidElementInfo
tree:
returns:
tree: AndroidElementInfo
inputType:
parameters:
text: string
@ -2369,6 +2373,9 @@ AndroidSelector:
AndroidElementInfo:
type: object
properties:
children:
type: array?
items: AndroidElementInfo
clazz: string
desc: string
res: string

2
src/protocol/validator.ts
View File

@ -965,6 +965,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
scheme.AndroidDeviceInfoParams = tObject({
selector: tType('AndroidSelector'),
});
scheme.AndroidDeviceTreeParams = tOptional(tObject({}));
scheme.AndroidDeviceInputTypeParams = tObject({
text: tString,
});
@ -1074,6 +1075,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
text: tOptional(tString),
});
scheme.AndroidElementInfo = tObject({
children: tOptional(tArray(tType('AndroidElementInfo'))),
clazz: tString,
desc: tString,
res: tString,

78
src/server/android/driver/app/src/androidTest/java/com/microsoft/playwright/androiddriver/InstrumentedTest.java
View File

@ -16,11 +16,13 @@
package com.microsoft.playwright.androiddriver;
import android.bluetooth.BluetoothClass;
import android.graphics.Point;
import android.graphics.Rect;
import android.net.LocalServerSocket;
import android.net.LocalSocket;
import android.os.Bundle;
import android.view.accessibility.AccessibilityNodeInfo;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SdkSuppress;
@ -30,6 +32,7 @@ import androidx.test.uiautomator.BySelector;
import androidx.test.uiautomator.Direction;
import androidx.test.uiautomator.UiDevice;
import androidx.test.uiautomator.UiObject2;
import androidx.test.uiautomator.UiSelector;
import androidx.test.uiautomator.Until;
import org.json.JSONArray;
@ -42,7 +45,10 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.regex.Pattern;
/**
@ -221,21 +227,25 @@ public class InstrumentedTest {
wait(device, params).swipe(parseDirection(params), params.getInt("percent"));
}
private static JSONObject serializeRect(Rect rect) throws JSONException {
JSONObject rectObject = new JSONObject();
rectObject.put("x", rect.left);
rectObject.put("y", rect.top);
rectObject.put("width", rect.width());
rectObject.put("height", rect.height());
return rectObject;
}
private static JSONObject info(UiDevice device, JSONObject params) throws JSONException {
JSONObject info = new JSONObject();
UiObject2 object = device.findObject(parseSelector(params));
Rect bounds = object.getVisibleBounds();
JSONObject boundsObject = new JSONObject();
boundsObject.put("x", bounds.left);
boundsObject.put("y", bounds.top);
boundsObject.put("width", bounds.width());
boundsObject.put("height", bounds.height());
JSONObject info = new JSONObject();
info.put("clazz", object.getClassName());
info.put("pkg", object.getApplicationPackage());
info.put("desc", object.getContentDescription());
info.put("res", object.getResourceName());
info.put("text", object.getText());
info.put("bounds", boundsObject);
info.put("bounds", serializeRect(object.getVisibleBounds()));
info.put("checkable", object.isCheckable());
info.put("checked", object.isChecked());
info.put("clickable", object.isClickable());
@ -248,6 +258,26 @@ public class InstrumentedTest {
return info;
}
private static JSONObject info(AccessibilityNodeInfo node) throws JSONException {
JSONObject info = new JSONObject();
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
info.put("desc", node.getContentDescription());
info.put("res", node.getViewIdResourceName());
info.put("text", node.getText());
info.put("bounds", serializeRect(bounds));
info.put("checkable", node.isCheckable());
info.put("checked", node.isChecked());
info.put("clickable", node.isClickable());
info.put("enabled", node.isEnabled());
info.put("focusable", node.isFocusable());
info.put("focused", node.isFocused());
info.put("longClickable", node.isLongClickable());
info.put("scrollable", node.isScrollable());
info.put("selected", node.isSelected());
return info;
}
private static void inputPress(UiDevice device, JSONObject params) throws JSONException {
device.pressKeyCode(params.getInt("keyCode"));
}
@ -273,6 +303,35 @@ public class InstrumentedTest {
device.drag(from.x, from.y, to.x, to.y, params.getInt("steps"));
}
private static JSONObject tree(UiDevice device) throws JSONException {
return serializeA11yNode(getRootA11yNode(device));
}
private static AccessibilityNodeInfo getRootA11yNode(UiDevice device) {
try {
Method getQueryController = UiDevice.class.getDeclaredMethod("getQueryController");
getQueryController.setAccessible(true);
Object queryController = getQueryController.invoke(device);
Method getRootNode = queryController.getClass().getDeclaredMethod("getRootNode");
getRootNode.setAccessible(true);
return (AccessibilityNodeInfo) getRootNode.invoke(queryController);
} catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
return null;
}
}
private static JSONObject serializeA11yNode(AccessibilityNodeInfo node) throws JSONException {
JSONObject object = info(node);
if (node.getChildCount() == 0)
return object;
JSONArray children = new JSONArray();
object.put("children", children);
for (int i = 0; i < node.getChildCount(); ++i)
children.put(serializeA11yNode(node.getChild(i)));
return object;
}
@Test
public void main() {
UiDevice device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation());
@ -352,6 +411,9 @@ public class InstrumentedTest {
case "inputDrag":
inputDrag(device, params);
break;
case "tree":
response.put("result", tree(device));
break;
default:
}