AppAgent 源码 (xml 解析)

1. 数据准备

adb shell uiautomator dump /sdcard/output.xml  # 获取手机ui界面的xml文件
adb pull /sdcard/output.xml output.xml		# 将手机上的xml文件拉取到电脑上

具体的xml文件：

<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0"><node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="" class="android.widget.LinearLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="android:id/content" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="com.miui.home:id/shortcut_menu_layer" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="com.miui.home:id/drag_layer_background" class="android.view.View" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" /><node index="1" text="" resource-id="com.miui.home:id/drag_layer" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="com.miui.home:id/shortcut_menu_blur_view" class="android.view.View" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" /><node index="1" text="" resource-id="com.miui.home:id/drag_view_container" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" /><node index="2" text="" resource-id="com.miui.home:id/screen" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="1" text="" resource-id="com.miui.home:id/scrim_view" class="android.view.View" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" /><node index="2" text="" resource-id="com.miui.home:id/screen_content" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]"><node index="0" text="" resource-id="com.miui.home:id/workspace" class="com.miui.home.launcher.ScreenView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2660]"><node index="1" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,1959][1220,2133]"><node index="0" text="" resource-id="" class="com.miui.home.launcher.ScreenView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,1959][1220,2133]"><node index="0" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="App vault" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[451,2019][501,2069]" /><node index="1" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="1 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[504,2019][557,2072]" /><node index="2" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="2 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="true" bounds="[557,2019][610,2072]" /><node index="3" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="3 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[610,2019][663,2072]" /><node index="4" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="4 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[663,2019][716,2072]" /><node index="5" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="5 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[716,2019][769,2072]" /></node></node><node index="2" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,243][1220,2660]"><node index="0" text="" resource-id="" class="android.view.ViewGroup" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,243][1220,2660]"><node index="0" text="Amap" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Amap" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,1101][324,1387]" /><node index="1" text="Clock" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Clock" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,1387][324,1673]" /><node index="2" text="Notes" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Notes" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[896,1387][1182,1673]" /><node index="3" text="Calculator" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Calculator" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,1673][324,1959]" /><node index="4" text="Settings" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Settings" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[324,1673][610,1959]" /><node index="5" text="Calendar" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Calendar" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[610,1673][896,1959]" /><node index="6" text="Weather" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Weather" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[896,1673][1182,1959]" /></node></node></node><node index="1" text="" resource-id="com.miui.home:id/hotseat" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,2133][1220,2419]"><node index="0" text="" resource-id="com.miui.home:id/hotseat_content_screen" class="com.miui.home.launcher.ScreenView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[12,2133][1208,2419]"><node index="0" text="Phone" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Phone" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,2133][324,2419]" /><node index="1" text="Messaging" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Messaging" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[324,2133][610,2419]" /><node index="2" text="Camera" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Camera" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[610,2133][896,2419]" /><node index="3" text="夸克" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="夸克" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[896,2133][1182,2419]" /></node></node></node><node index="3" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" /><node index="4" text="" resource-id="com.miui.home:id/search_bar_container" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,2418][1220,2712]"><node index="0" text="" resource-id="com.miui.home:id/search_bar" class="android.widget.FrameLayout" package="com.miui.home" content-desc="Search bar" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[91,2418][1129,2611]"><node index="0" text="" resource-id="com.miui.home:id/search_bar_desktop_layout" class="android.widget.RelativeLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[91,2446][1129,2583]"><node index="0" text="" resource-id="com.miui.home:id/search_bar_extra_layout" class="android.widget.ViewAnimator" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[91,2446][1129,2583]"><node index="0" text="" resource-id="com.miui.home:id/search_bar_xiaoai_layout" class="android.widget.FrameLayout" package="com.miui.home" content-desc="Mi AI" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[976,2446][1129,2583]"><node index="0" text="" resource-id="com.miui.home:id/xiaoai_button" class="android.widget.ImageView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[1015,2446][1090,2583]" /></node></node><node index="1" text="" resource-id="com.miui.home:id/search_bar_content_icon_layout" class="android.widget.FrameLayout" package="com.miui.home" content-desc="Search" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[91,2446][244,2583]"><node index="0" text="" resource-id="com.miui.home:id/search_bar_content_icon" class="android.widget.ImageView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[130,2446][205,2583]" /></node></node></node></node></node><node index="3" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1,2712]"><node index="0" text="" resource-id="com.miui.home:id/default_position" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="true" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,90][1,2712]" /></node><node index="4" text="" resource-id="com.miui.home:id/drop_target_bar" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,985]" /></node></node></node></node></node></node>
</hierarchy>

原始的xml文件没有格式化，可以通过这个网址将其格式化，方便阅读：
https://tool.12333si.com/xmlformat/

2. xml 数据解析

1. 导入必要的库

首先，我们需要导入 xml.etree.ElementTree 模块来解析 XML 文件，并使用 dataclasses 模块来定义数据结构。

import xml.etree.ElementTree as ET
from dataclasses import dataclass, field

2. 定义 AndroidElement 类

原始的代码为：

class AndroidElement:def __init__(self, uid, bbox, attrib):self.uid = uidself.bbox = bboxself.attrib = attrib

但是这样不方便打印输出。

我们使用 dataclass 装饰器定义一个 AndroidElement 类，用于存储每个元素的唯一标识符 (uid)、边界框 (bbox) 和属性 (attrib)。

@dataclass
class AndroidElement:uid: strbbox: tupleattrib: dict = field(default_factory=dict)

3. 配置参数

我们定义了一个全局配置字典 configs，其中包含一个最小距离参数 MIN_DIST，用于判断两个元素是否过于接近。

configs = {"MIN_DIST": 30}

4. 获取元素的唯一标识符

get_id_from_element 函数用于生成每个元素的唯一标识符。它根据元素的 resource-id、class、content-desc 以及元素的宽度和高度来生成标识符。

def get_id_from_element(elem):bounds = elem.attrib["bounds"][1:-1].split("][")x1, y1 = map(int, bounds[0].split(","))x2, y2 = map(int, bounds[1].split(","))elem_w, elem_h = x2 - x1, y2 - y1if "resource-id" in elem.attrib and elem.attrib["resource-id"]:elem_id = elem.attrib["resource-id"].replace(":", ".").replace("/", "_")else:elem_id = f"{elem.attrib['class']}_{elem_w}_{elem_h}"if ("content-desc" in elem.attriband elem.attrib["content-desc"]and len(elem.attrib["content-desc"]) < 20):content_desc = (elem.attrib["content-desc"].replace("/", "_").replace(" ", "").replace(":", "_"))elem_id += f"_{content_desc}"return elem_id

5. 遍历 XML 树

traverse_tree 函数用于遍历 XML 树，并提取符合条件的元素。它会检查每个元素的属性，并根据配置的最小距离参数判断是否将其添加到列表中。

def traverse_tree(xml_path, elem_list, attrib, add_index=False):path = []for event, elem in ET.iterparse(xml_path, ["start", "end"]):if event == "start":path.append(elem)if attrib in elem.attrib and elem.attrib[attrib] == "true":parent_prefix = ""if len(path) > 1:parent_prefix = get_id_from_element(path[-2])bounds = elem.attrib["bounds"][1:-1].split("][")x1, y1 = map(int, bounds[0].split(","))x2, y2 = map(int, bounds[1].split(","))center = (x1 + x2) // 2, (y1 + y2) // 2elem_id = get_id_from_element(elem)if parent_prefix:elem_id = parent_prefix + "_" + elem_idif add_index:elem_id += f"_{elem.attrib['index']}"close = Falsefor e in elem_list:bbox = e.bboxcenter_ = (bbox[0][0] + bbox[1][0]) // 2, (bbox[0][1] + bbox[1][1]) // 2dist = (abs(center[0] - center_[0]) ** 2+ abs(center[1] - center_[1]) ** 2) ** 0.5if dist <= configs["MIN_DIST"]:close = Truebreakif not close:elem_list.append(AndroidElement(elem_id, ((x1, y1), (x2, y2)), attrib))if event == "end":path.pop()

6. 解析 XML 文件并输出结果

最后，我们指定 XML 文件的路径，并调用 traverse_tree 函数来解析文件。解析完成后，我们输出提取的元素列表及其长度。

xml_path = "../output.xml"
clickable_list = []traverse_tree(xml_path, clickable_list, "clickable", True)print(clickable_list)
print(len(clickable_list))

7. 输出结果

运行上述代码后，程序将输出所有可点击元素的列表及其数量。例如：

[AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_50_50_Appvault_0', bbox=((451, 2019), (501, 2069)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_1screen_1', bbox=((504, 2019), (557, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_2screen_2', bbox=((557, 2019), (610, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_3screen_3', bbox=((610, 2019), (663, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_4screen_4', bbox=((663, 2019), (716, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_5screen_5', bbox=((716, 2019), (769, 2072)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Amap_0', bbox=((38, 1101), (324, 1387)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Clock_1', bbox=((38, 1387), (324, 1673)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Notes_2', bbox=((896, 1387), (1182, 1673)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Calculator_3', bbox=((38, 1673), (324, 1959)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Settings_4', bbox=((324, 1673), (610, 1959)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Calendar_5', bbox=((610, 1673), (896, 1959)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Weather_6', bbox=((896, 1673), (1182, 1959)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_Phone_0', bbox=((38, 2133), (324, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_Messaging_1', bbox=((324, 2133), (610, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_Camera_2', bbox=((610, 2133), (896, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_夸克_3', bbox=((896, 2133), (1182, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_search_bar_Searchbar_com.miui.home.id_search_bar_desktop_layout_0', bbox=((91, 2446), (1129, 2583)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_search_bar_extra_layout_com.miui.home.id_search_bar_xiaoai_layout_MiAI_0', bbox=((976, 2446), (1129, 2583)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_search_bar_desktop_layout_com.miui.home.id_search_bar_content_icon_layout_Search_1', bbox=((91, 2446), (244, 2583)), attrib='clickable')]
20