diff --git a/bin/android-driver.apk b/bin/android-driver.apk index bb51655417..66cc5a2c0a 100644 Binary files a/bin/android-driver.apk and b/bin/android-driver.apk differ diff --git a/src/client/android.ts b/src/client/android.ts index 6957511ddd..339c48180f 100644 --- a/src/client/android.ts +++ b/src/client/android.ts @@ -185,6 +185,12 @@ export class AndroidDevice extends ChannelOwner { + return await this._wrapApiCall('androidDevice.tree', async () => { + return (await this._channel.tree()).tree; + }); + } + async close() { return this._wrapApiCall('androidDevice.close', async () => { await this._channel.close(); diff --git a/src/dispatchers/androidDispatcher.ts b/src/dispatchers/androidDispatcher.ts index ecb1083f2d..a952a8d32e 100644 --- a/src/dispatchers/androidDispatcher.ts +++ b/src/dispatchers/androidDispatcher.ts @@ -102,6 +102,10 @@ export class AndroidDeviceDispatcher extends Dispatcher { + return { tree: await this._object.send('tree', params) }; + } + async inputType(params: channels.AndroidDeviceInputTypeParams) { const text = params.text; const keyCodes: number[] = []; diff --git a/src/protocol/channels.ts b/src/protocol/channels.ts index 9c3b040019..dd60c8adbd 100644 --- a/src/protocol/channels.ts +++ b/src/protocol/channels.ts @@ -2461,6 +2461,7 @@ export interface AndroidDeviceChannel extends Channel { scroll(params: AndroidDeviceScrollParams, metadata?: Metadata): Promise; swipe(params: AndroidDeviceSwipeParams, metadata?: Metadata): Promise; info(params: AndroidDeviceInfoParams, metadata?: Metadata): Promise; + tree(params?: AndroidDeviceTreeParams, metadata?: Metadata): Promise; inputType(params: AndroidDeviceInputTypeParams, metadata?: Metadata): Promise; inputPress(params: AndroidDeviceInputPressParams, metadata?: Metadata): Promise; inputTap(params: AndroidDeviceInputTapParams, metadata?: Metadata): Promise; @@ -2594,6 +2595,11 @@ export type AndroidDeviceInfoOptions = { export type AndroidDeviceInfoResult = { info: AndroidElementInfo, }; +export type AndroidDeviceTreeParams = {}; +export type AndroidDeviceTreeOptions = {}; +export type AndroidDeviceTreeResult = { + tree: AndroidElementInfo, +}; export type AndroidDeviceInputTypeParams = { text: string, }; @@ -2802,6 +2808,7 @@ export type AndroidSelector = { }; export type AndroidElementInfo = { + children?: AndroidElementInfo[], clazz: string, desc: string, res: string, diff --git a/src/protocol/protocol.yml b/src/protocol/protocol.yml index c243c87b0d..ad82b20c6d 100644 --- a/src/protocol/protocol.yml +++ b/src/protocol/protocol.yml @@ -2200,6 +2200,10 @@ AndroidDevice: returns: info: AndroidElementInfo + tree: + returns: + tree: AndroidElementInfo + inputType: parameters: text: string @@ -2369,6 +2373,9 @@ AndroidSelector: AndroidElementInfo: type: object properties: + children: + type: array? + items: AndroidElementInfo clazz: string desc: string res: string diff --git a/src/protocol/validator.ts b/src/protocol/validator.ts index 013598a0a8..16c8363dc5 100644 --- a/src/protocol/validator.ts +++ b/src/protocol/validator.ts @@ -965,6 +965,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { scheme.AndroidDeviceInfoParams = tObject({ selector: tType('AndroidSelector'), }); + scheme.AndroidDeviceTreeParams = tOptional(tObject({})); scheme.AndroidDeviceInputTypeParams = tObject({ text: tString, }); @@ -1074,6 +1075,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { text: tOptional(tString), }); scheme.AndroidElementInfo = tObject({ + children: tOptional(tArray(tType('AndroidElementInfo'))), clazz: tString, desc: tString, res: tString, diff --git a/src/server/android/driver/app/src/androidTest/java/com/microsoft/playwright/androiddriver/InstrumentedTest.java b/src/server/android/driver/app/src/androidTest/java/com/microsoft/playwright/androiddriver/InstrumentedTest.java index 4c9cb6d7af..f8e930e61c 100644 --- a/src/server/android/driver/app/src/androidTest/java/com/microsoft/playwright/androiddriver/InstrumentedTest.java +++ b/src/server/android/driver/app/src/androidTest/java/com/microsoft/playwright/androiddriver/InstrumentedTest.java @@ -16,11 +16,13 @@ package com.microsoft.playwright.androiddriver; +import android.bluetooth.BluetoothClass; import android.graphics.Point; import android.graphics.Rect; import android.net.LocalServerSocket; import android.net.LocalSocket; import android.os.Bundle; +import android.view.accessibility.AccessibilityNodeInfo; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SdkSuppress; @@ -30,6 +32,7 @@ import androidx.test.uiautomator.BySelector; import androidx.test.uiautomator.Direction; import androidx.test.uiautomator.UiDevice; import androidx.test.uiautomator.UiObject2; +import androidx.test.uiautomator.UiSelector; import androidx.test.uiautomator.Until; import org.json.JSONArray; @@ -42,7 +45,10 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.regex.Pattern; /** @@ -221,21 +227,25 @@ public class InstrumentedTest { wait(device, params).swipe(parseDirection(params), params.getInt("percent")); } + private static JSONObject serializeRect(Rect rect) throws JSONException { + JSONObject rectObject = new JSONObject(); + rectObject.put("x", rect.left); + rectObject.put("y", rect.top); + rectObject.put("width", rect.width()); + rectObject.put("height", rect.height()); + return rectObject; + } + private static JSONObject info(UiDevice device, JSONObject params) throws JSONException { - JSONObject info = new JSONObject(); UiObject2 object = device.findObject(parseSelector(params)); - Rect bounds = object.getVisibleBounds(); - JSONObject boundsObject = new JSONObject(); - boundsObject.put("x", bounds.left); - boundsObject.put("y", bounds.top); - boundsObject.put("width", bounds.width()); - boundsObject.put("height", bounds.height()); + + JSONObject info = new JSONObject(); info.put("clazz", object.getClassName()); info.put("pkg", object.getApplicationPackage()); info.put("desc", object.getContentDescription()); info.put("res", object.getResourceName()); info.put("text", object.getText()); - info.put("bounds", boundsObject); + info.put("bounds", serializeRect(object.getVisibleBounds())); info.put("checkable", object.isCheckable()); info.put("checked", object.isChecked()); info.put("clickable", object.isClickable()); @@ -248,6 +258,26 @@ public class InstrumentedTest { return info; } + private static JSONObject info(AccessibilityNodeInfo node) throws JSONException { + JSONObject info = new JSONObject(); + Rect bounds = new Rect(); + node.getBoundsInScreen(bounds); + info.put("desc", node.getContentDescription()); + info.put("res", node.getViewIdResourceName()); + info.put("text", node.getText()); + info.put("bounds", serializeRect(bounds)); + info.put("checkable", node.isCheckable()); + info.put("checked", node.isChecked()); + info.put("clickable", node.isClickable()); + info.put("enabled", node.isEnabled()); + info.put("focusable", node.isFocusable()); + info.put("focused", node.isFocused()); + info.put("longClickable", node.isLongClickable()); + info.put("scrollable", node.isScrollable()); + info.put("selected", node.isSelected()); + return info; + } + private static void inputPress(UiDevice device, JSONObject params) throws JSONException { device.pressKeyCode(params.getInt("keyCode")); } @@ -273,6 +303,35 @@ public class InstrumentedTest { device.drag(from.x, from.y, to.x, to.y, params.getInt("steps")); } + private static JSONObject tree(UiDevice device) throws JSONException { + return serializeA11yNode(getRootA11yNode(device)); + } + + private static AccessibilityNodeInfo getRootA11yNode(UiDevice device) { + try { + Method getQueryController = UiDevice.class.getDeclaredMethod("getQueryController"); + getQueryController.setAccessible(true); + Object queryController = getQueryController.invoke(device); + + Method getRootNode = queryController.getClass().getDeclaredMethod("getRootNode"); + getRootNode.setAccessible(true); + return (AccessibilityNodeInfo) getRootNode.invoke(queryController); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + return null; + } + } + + private static JSONObject serializeA11yNode(AccessibilityNodeInfo node) throws JSONException { + JSONObject object = info(node); + if (node.getChildCount() == 0) + return object; + JSONArray children = new JSONArray(); + object.put("children", children); + for (int i = 0; i < node.getChildCount(); ++i) + children.put(serializeA11yNode(node.getChild(i))); + return object; + } + @Test public void main() { UiDevice device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); @@ -352,6 +411,9 @@ public class InstrumentedTest { case "inputDrag": inputDrag(device, params); break; + case "tree": + response.put("result", tree(device)); + break; default: }