使用pywinauto库实现PC桌面应用程序自动化查找元素时遇到以下问题：
翻阅pywinauto官方文档发现一个可用的基于pywinauto的元素定位工具项目，修改代码以实现以下需求:
取自csdn，见文末链接。
def window_capture(hwnd): # 后台截图,保存到内存
hwndDC = win32gui.GetWindowDC(hwnd) # 返回句柄窗口的设备环境、覆盖整个窗口，包括非客户区，标题栏，菜单，边框
mfcDC = win32ui.CreateDCFromHandle(hwndDC) # 创建设备描述表
saveDC = mfcDC.CreateCompatibleDC() # 创建内存设备描述表
rctA = win32gui.GetWindowRect(hwnd) # 获取句柄窗口的大小信息
w = rctA[2] - rctA[0]
h = rctA[3] - rctA[1]
saveBitMap = win32ui.CreateBitmap() # 创建位图对象
saveBitMap.CreateCompatibleBitmap(mfcDC, w, h)
saveDC.SelectObject(saveBitMap)
saveDC.BitBlt((0, 0), (w, h), mfcDC, (0, 0), win32con.SRCCOPY) # 截图至内存设备描述表
signedIntsArray = saveBitMap.GetBitmapBits(True)
img = np.frombuffer(signedIntsArray, dtype="uint8")
img.shape = (h, w, 4)
win32gui.DeleteObject(saveBitMap.GetHandle())
mfcDC.DeleteDC()
saveDC.DeleteDC()
return cv2.cvtColor(img, cv2.COLOR_RGBA2RGB)
返回的是numpy.ndarray，后续转换为qicon显示到TableWidgetItem中。
部分控件无法靠该方法截图，返回的为黑屏数据。
取自csdn，见文末链接。
def draw_outline(rect: tuple[int, int, int, int], color: tuple[int, int, int] = (0, 255, 0), thickness: int = 2,
handle=None):
"""
绘制矩形
:param rect: 矩形范围
:param color: rgb元组
:param thickness: 线条粗细
:param handle:要刷新的窗口句柄
:return:
"""
hwnd = win32gui.GetDesktopWindow()
hPen = win32gui.CreatePen(win32con.PS_SOLID, thickness, win32api.RGB(color[0], color[1], color[2])) # 定义框颜色
if not handle:
handle = win32gui.WindowFromPoint((rect[0], rect[1]))
if handle:
win32gui.InvalidateRect(handle, None, True)
win32gui.UpdateWindow(handle)
win32gui.RedrawWindow(hwnd, None, None,
win32con.RDW_FRAME | win32con.RDW_INVALIDATE | win32con.RDW_UPDATENOW | win32con.RDW_ALLCHILDREN)
hwndDC = win32gui.GetDC(hwnd) # 根据窗口句柄获取窗口的设备上下文DC（Divice Context）
win32gui.SelectObject(hwndDC, hPen)
hbrush = win32gui.GetStockObject(win32con.NULL_BRUSH) # 定义透明画刷
prebrush = win32gui.SelectObject(hwndDC, hbrush)
win32gui.Rectangle(hwndDC, rect[0], rect[1], rect[2], rect[3]) # 左上到右下的坐标
win32gui.SaveDC(hwndDC)
win32gui.SelectObject(hwndDC, prebrush)
# 回收资源
win32gui.DeleteObject(hPen)
win32gui.DeleteObject(hbrush)
win32gui.DeleteObject(prebrush)
win32gui.ReleaseDC(hwnd, hwndDC)
win32gui.InvalidateRect(handle, None, True)
win32gui.UpdateWindow(handle)
靠这两行代码刷新窗口去除矩形框，传入对应的窗口句柄
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import sys
import warnings
import cv2
import numpy as np
import win32api
import win32con
import win32gui
import win32ui
from PyQt5.QtCore import QCoreApplication, QSize, QModelIndex
from PyQt5.QtCore import QLocale
from PyQt5.QtCore import QSettings
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QStandardItem, QFont, QIcon, QImage, QPixmap
from PyQt5.QtGui import QStandardItemModel
from PyQt5.QtWidgets import QApplication, QPushButton, QComboBox, QGridLayout, QTreeView, QWidget, \
QTableWidget, QTableWidgetItem, QAbstractItemView, QLineEdit
warnings.simplefilter("ignore", UserWarning)
sys.coinit_flags = 2
from pywinauto import backend
def window_capture(hwnd): # 后台截图,保存到内存
hwndDC = win32gui.GetWindowDC(hwnd) # 返回句柄窗口的设备环境、覆盖整个窗口，包括非客户区，标题栏，菜单，边框
mfcDC = win32ui.CreateDCFromHandle(hwndDC) # 创建设备描述表
saveDC = mfcDC.CreateCompatibleDC() # 创建内存设备描述表
rctA = win32gui.GetWindowRect(hwnd) # 获取句柄窗口的大小信息
w = rctA[2] - rctA[0]
h = rctA[3] - rctA[1]
saveBitMap = win32ui.CreateBitmap() # 创建位图对象
saveBitMap.CreateCompatibleBitmap(mfcDC, w, h)
saveDC.SelectObject(saveBitMap)
saveDC.BitBlt((0, 0), (w, h), mfcDC, (0, 0), win32con.SRCCOPY) # 截图至内存设备描述表
signedIntsArray = saveBitMap.GetBitmapBits(True)
img = np.frombuffer(signedIntsArray, dtype="uint8")
img.shape = (h, w, 4)
win32gui.DeleteObject(saveBitMap.GetHandle())
mfcDC.DeleteDC()
saveDC.DeleteDC()
return cv2.cvtColor(img, cv2.COLOR_RGBA2RGB)
def draw_outline(rect: tuple[int, int, int, int], color: tuple[int, int, int] = (0, 255, 0), thickness: int = 2,
handle=None):
"""
绘制矩形
:param rect: 矩形范围
:param color: rgb元组
:param thickness: 线条粗细
:param handle:要刷新的窗口句柄
:return:
"""
hwnd = win32gui.GetDesktopWindow()
hPen = win32gui.CreatePen(win32con.PS_SOLID, thickness, win32api.RGB(color[0], color[1], color[2])) # 定义框颜色
if not handle:
handle = win32gui.WindowFromPoint((rect[0], rect[1]))
if handle:
win32gui.InvalidateRect(handle, None, True)
win32gui.UpdateWindow(handle)
win32gui.RedrawWindow(hwnd, None, None,
win32con.RDW_FRAME | win32con.RDW_INVALIDATE | win32con.RDW_UPDATENOW | win32con.RDW_ALLCHILDREN)
hwndDC = win32gui.GetDC(hwnd) # 根据窗口句柄获取窗口的设备上下文DC（Divice Context）
win32gui.SelectObject(hwndDC, hPen)
hbrush = win32gui.GetStockObject(win32con.NULL_BRUSH) # 定义透明画刷
prebrush = win32gui.SelectObject(hwndDC, hbrush)
win32gui.Rectangle(hwndDC, rect[0], rect[1], rect[2], rect[3]) # 左上到右下的坐标
win32gui.SaveDC(hwndDC)
win32gui.SelectObject(hwndDC, prebrush)
# 回收资源
win32gui.DeleteObject(hPen)
win32gui.DeleteObject(hbrush)
win32gui.DeleteObject(prebrush)
win32gui.ReleaseDC(hwnd, hwndDC)
class SpyWindow(QWidget):
def __init__(self, parent=None, target_name=None):
super(SpyWindow, self).__init__(parent)
self.setMinimumSize(800, 800)
self.setLocale(QLocale(QLocale.English, QLocale.UnitedStates))
self.setWindowTitle(QCoreApplication.translate("MainWindow", "AutoSpy"))
self.settings = QSettings('AutoSpy', 'MainWindow')
# Main layout
self.mainLayout = QGridLayout()
# Backend combobox
self.pushButton = QPushButton("Refresh")
self.comboBox = QComboBox()
self.comboBox.setMouseTracking(False)
self.comboBox.setMaxVisibleItems(5)
self.comboBox.setObjectName("comboBox")
for _backend in backend.registry.backends.keys():
self.comboBox.addItem(_backend)
self.target = QLineEdit(target_name)
if not target_name:
self.target.setPlaceholderText("Add selection title")
# Add top widgets to main window
self.mainLayout.addWidget(self.target, 0, 0, 1, 1)
self.mainLayout.addWidget(self.pushButton, 0, 1, 1, 1)
self.mainLayout.addWidget(self.comboBox, 1, 1, 1, 1)
self.tree_view = QTreeView()
self.tree_view.setFont(QFont("Consolas", 11, 2))
self.tree_view.setColumnWidth(0, 150)
self.comboBox.setCurrentText('uia')
self.__initialize_calc()
self.tableWidget = QTableWidget()
self.tableWidget.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
self.tableWidget.setColumnCount(2)
self.tableWidget.horizontalHeader().setStretchLastSection(True)
self.tableWidget.verticalHeader().setVisible(False)
self.tableWidget.setFont(QFont("Consolas", 11, 2))
self.tableWidget.setHorizontalHeaderLabels(['assign', 'value'])
self.tableWidget.setIconSize(QSize(550, 400))
self.comboBox.activated[str].connect(self.__show_tree)
# Add center widgets to main window
self.mainLayout.addWidget(self.tree_view, 2, 0, 1, 1)
self.mainLayout.addWidget(self.tableWidget, 2, 1, 1, 1)
self.setLayout(self.mainLayout)
geometry = self.settings.value('Geometry', bytes('', 'utf-8'))
self.restoreGeometry(geometry)
self.pushButton.clicked.connect(lambda: self.__show_tree('uia'))
def __initialize_calc(self, _backend=None):
if not _backend:
if sys.platform.startswith("linux"):
_backend = 'atspi'
else:
_backend = 'uia'
self.element_info = backend.registry.backends[_backend].element_info_class()
self.tree_model = MyTreeModel(self.element_info, _backend, self.target.text())
self.tree_model.setHeaderData(0, Qt.Horizontal, 'Controls')
self.tree_view.setModel(self.tree_model)
self.tree_view.clicked.connect(self.__show_property)
def __show_tree(self, text):
backend = text
self.__initialize_calc(backend)
self.tree_view.expand(self.tree_view.model().index(0, 0))
def __show_property(self, index: QModelIndex = None):
data = index.data() + self.tree_view.model().itemData(index).get(3, "")
hwnd = None
left, top, right, bottom = 0, 0, win32api.GetSystemMetrics(win32con.SM_CXSCREEN), win32api.GetSystemMetrics(
win32con.SM_CYSCREEN)
def get_hwnd(target):
for p_msg in self.tree_model.props_dict.get(
target.data() + self.tree_view.model().itemData(target).get(3, "")):
if p_msg[0] == 'handle':
handle = eval(p_msg[1])
if handle:
hwnd = handle
return hwnd
else:
return None
parent_index = index.parent()
parent_data = parent_index.data()
if parent_data:
while parent_data:
target_index = parent_index
parent_index = parent_index.parent()
parent_data = parent_index.data()
hwnd = get_hwnd(target_index)
if hwnd:
break
else:
target_index = index
hwnd = get_hwnd(target_index)
for msg in self.tree_model.props_dict.get(data):
if msg[0] == 'handle':
handle = eval(msg[1])
if handle:
hwnd = handle
if msg[0] == 'rectangle':
text = msg[1]
left, top, right, bottom = text[1:-1].replace(" ", "").split(',')
left = int(left[1:])
top = int(top[1:])
right = int(right[1:])
bottom = int(bottom[1:])
draw_outline((left, top, right, bottom), handle=hwnd)
break
img1 = window_capture(hwnd)
rect1 = win32gui.GetWindowRect(hwnd)
l1 = rect1[0]
t1 = rect1[1]
img2 = img1[top - t1:bottom - t1, left - l1:right - l1]
im = QImage(img2.tobytes(), img2.shape[1], img2.shape[0], img2.shape[1] * 3, QImage.Format_BGR888)
self.tableWidget.setRowCount(0)
data_list = self.tree_model.props_dict.get(data)
for i, data in enumerate(data_list):
self.tableWidget.insertRow(self.tableWidget.rowCount())
self.tableWidget.setItem(i, 0, QTableWidgetItem(data[0]))
self.tableWidget.setItem(i, 1, QTableWidgetItem(data[1]))
self.tableWidget.insertRow(self.tableWidget.rowCount())
self.tableWidget.setItem(self.tableWidget.rowCount() - 1, 0, QTableWidgetItem("image"))
self.tableWidget.setItem(self.tableWidget.rowCount() - 1, 1, QTableWidgetItem(QIcon(QPixmap(im)), ""))
self.tableWidget.resizeColumnsToContents()
self.tableWidget.resizeRowsToContents()
def closeEvent(self, event):
geometry = self.saveGeometry()
self.settings.setValue('Geometry', geometry)
super(SpyWindow, self).closeEvent(event)
class MyTreeModel(QStandardItemModel):
def __init__(self, element_info, backend, target):
QStandardItemModel.__init__(self)
root_node = self.invisibleRootItem()
self.props_dict = {}
self.backend = backend
self.__generate_props_dict(element_info)
for child in element_info.children():
if child.name == target or target == "":
self.__generate_props_dict(child)
node_value = self.__node_name(child)
if isinstance(node_value, tuple):
child_item = QStandardItem(node_value[0])
child_item.setToolTip(str(node_value[1]))
else:
child_item = QStandardItem(node_value)
child_item.setEditable(False)
root_node.appendRow(child_item)
self.__get_next(child, child_item)
if target:
break
def __get_next(self, element_info, parent):
for child in element_info.children():
self.__generate_props_dict(child)
node_value = self.__node_name(child)
if isinstance(node_value, tuple):
child_item = QStandardItem(node_value[0])
child_item.setToolTip(str(node_value[1]))
else:
child_item = QStandardItem(node_value)
child_item.setEditable(False)
parent.appendRow(child_item)
self.__get_next(child, child_item)
def __node_name(self, element_info):
if 'uia' == self.backend:
auto_id = str(element_info.automation_id)
if auto_id:
return '[%s] "%s"' % (auto_id[auto_id.rfind('.') + 1:], str(element_info.name)), id(element_info)
else:
return '"%s" (%s)' % (str(element_info.name), id(element_info))
elif 'atspi' == self.backend:
return '%s "%s" (%s)' % (str(element_info.control_type),
str(element_info.name),
id(element_info))
return '"%s" (%s)' % (str(element_info.name), id(element_info))
def __generate_props_dict(self, element_info):
props = [
['control_id', str(element_info.control_id)],
['class_name', str(element_info.class_name)],
['enabled', str(element_info.enabled)],
['handle', str(element_info.handle)],
['name', str(element_info.name)],
['process_id', str(element_info.process_id)],
['rectangle', str(element_info.rectangle)],
['rich_text', str(element_info.rich_text)],
['visible', str(element_info.visible)]
]
props_win32 = [
] if (self.backend == 'win32') else []
props_uia = [
['automation_id', str(element_info.automation_id)],
['control_type', str(element_info.control_type)],
['element', str(element_info.element)],
['framework_id', str(element_info.framework_id)],
['runtime_id', str(element_info.runtime_id)]
] if (self.backend == 'uia') else []
props_atspi = [
['control_type', str(element_info.control_type)],
['runtime_id', str(element_info.runtime_id)]
] if (self.backend == 'atspi') else []
props.extend(props_uia)
props.extend(props_win32)
props.extend(props_atspi)
node_value = self.__node_name(element_info)
if isinstance(node_value, tuple):
node_value = node_value[0] + str(node_value[1])
node_dict = {node_value: props}
self.props_dict.update(node_dict)
if __name__ == "__main__":
app = QApplication(sys.argv)
app.setStyle('Fusion')
w = SpyWindow()
w.show()
sys.exit(app.exec_())
pyqt界面:
因篇幅问题不能全部显示，请点此查看更多更全内容
Copyright © 2019- yrrf.cn 版权所有 赣ICP备2024042794号-2
违法及侵权请联系：TEL:199 1889 7713 E-MAIL:2724546146@qq.com
本站由北京市万商天勤律师事务所王兴未律师提供法律服务