UIAutomation实战：自动回复

1、导入可能需要用到的第三方库

import time
from datetime import datetime
import uiautomation as auto
import pyperclip
import pyautogui
import pygetwindow
import sys
import subprocess
import SparkApi
import AskSpark

2、初始化一些想要自动化控制的软件或者程序

def initialization_qn():
    # subprocess.Popen(['千牛工作台.exe'], shell=False)
    # 避免使用shell=True：在某些情况下，将 shell=True 设置为 subprocess.Popen 可能导致字符编码问题。尝试删除 shell=True，并将命令和参数分开传递
    floating_bar = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowToolSaveBits')
    if floating_bar:
        print("千牛已经完成初始化")
    else:
        print("打开千牛工作台")
        app_path = r"D:\Program Files\AliWorkbench\AliWorkbench.exe"
        args = [app_path, "/run:desktop"]
        subprocess.Popen(args, shell=False)
        time.sleep(30)
    print("打开千牛工作台接待中心")
    floating_bar = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowToolSaveBits')
    if not floating_bar:
        print("千牛工作台未完成初始化，程序有问题")
        sys.exit()
    print(floating_bar.Name)
    floating_bar.Click(x=110, y=55, simulateMove=True)
    print("切换到接待中心联系中名单")
    rc_window = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowIcon')
    rc_window.Click(x=106, y=170, simulateMove=True)
    # 150 290
    print("切换到第一个联系人")
    rc_window.Click(x=150, y=290, simulateMove=True)
    # 535 500
    print("移动鼠标聚焦聊天信息框")
    rc_window.MoveCursorToInnerPos(x=535, y=500, simulateMove=True)

3、自定义一些可能用得到的UIautomation库的一些常用自定义方法

获取所有同级控件

def get_all_siblings(start_control):
    # 创建一个列表来存储同级控件
    siblings = []

    # 首先获取第一个同级控件
    sibling = start_control.GetNextSiblingControl()

    while sibling is not None and sibling != auto.GetRootControl() and sibling:
        # 添加找到的同级控件到列表中
        siblings.append(sibling)

        # 获取下一个同级控件，直到没有更多
        sibling = sibling.GetNextSiblingControl()

    # 返回包含所有同级控件的列表
    return siblings

获取光标下的控件详细信息

def get_control_from_cursor():

    time.sleep(3)

    idea_control = auto.ControlFromCursor()

    if idea_control is not None and idea_control != auto.GetRootControl():
        print("**********")
        print("能够识别光标下的UI控件")
        auto.EnumAndLogControlAncestors(control=idea_control, showAllName=True, showPid=True)
        if idea_control.Name != "":
            print("**********")
            print(f"该控件有名字：{idea_control.Name}")

        all_children = []
        all_children = idea_control.GetChildren()
        if all_children:
            print("**********")
            print("该控件有子控件：")
            for item in all_children:
                if item:
                    print(auto.EnumAndLogControlAncestors(control=item, showAllName=True, showPid=True))
        else:
            print("**********")
            print("该控件没有子控件")

        all_siblings = []
        all_siblings = get_all_siblings(idea_control)
        if all_siblings:
            print("**********")
            print("该控件有同级控件：")
            for item in all_siblings:
                print(auto.EnumAndLogControlAncestors(control=item, showAllName=True, showPid=True))
        else:
            print("**********")
            print("该控件没有同级控件")

        return idea_control
    else:
        print("**********")
        print("无法识别光标下的UI控件")
        return None

获取光标下的控件精简信息

def get_control_from_cursor2():
    time.sleep(3)
    idea_control = auto.ControlFromCursor()
    if idea_control:
        print(idea_control)
        print(idea_control.Name)
        rect = (idea_control.BoundingRectangle.left, idea_control.BoundingRectangle.top, idea_control.BoundingRectangle.right, idea_control.BoundingRectangle.bottom)
        rect_wh = (idea_control.BoundingRectangle.width(), idea_control.BoundingRectangle.height())
        print(rect)
        print(rect_wh)
        return idea_control, rect, rect_wh

获取区域内的有效文本

def get_text_area(start_x, start_y, end_x, end_y):
    auto.WheelUp(1)
    auto.WheelDown(3)
    auto.MoveTo(start_x, start_y)
    auto.PressMouse(start_x, start_y)
    auto.MoveTo(end_x, end_y)
    auto.ReleaseMouse()
    auto.SendKeys('{Ctrl}c')
    # auto.Click()
    text = ""
    text = pyperclip.paste()
    message = text.split('\n')
    # 使用filter函数过滤掉空字符串
    message = list(filter(lambda x: x.strip() != '', message))
    # # 使用列表推导过滤掉空字符串
    # message = [line for line in message if line.strip() != '']
    # 从最后一个子项开始向前遍历
    m_message = ""
    for i in range(-1, -len(message) - 1, -1):
        if message[i] != "已读" and message[i] != "未读" and message[i]:
            m_message = message[i]
            break
    if m_message:
        print("**********")
        print(m_message)
        return m_message

示例

if __name__ == '__main__':
    # idea_c = get_control_from_cursor()
    initialization_qn()
    ic, rect, rect_wh = get_control_from_cursor2()
    ms = get_text_area(rect[0], rect[1], rect[2], rect[3])
    print(f"发送问题：{ms}")
    q = AskSpark.ask(ms)
    print(f"AI回答：{q}")
    print("激活发送消息栏")
    rc_window = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowIcon')
    # 1200 950
    rc_window.Click(x=530, y=650, simulateMove=True)
    auto.SendKeys(q)
    auto.SendKey(0x0D)
    # auto.SendKey('{Enter}')

4、完整Demo：

# pygetwindow, pyautogui, pyperclip, uiautomation
import time
from datetime import datetime
import uiautomation as auto
import pyperclip
import pyautogui
import pygetwindow
import sys
import subprocess
import SparkApi
import AskSpark


def initialization_qn():
    # subprocess.Popen(['千牛工作台.exe'], shell=False)
    # 避免使用shell=True：在某些情况下，将 shell=True 设置为 subprocess.Popen 可能导致字符编码问题。尝试删除 shell=True，并将命令和参数分开传递
    floating_bar = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowToolSaveBits')
    if floating_bar:
        print("千牛已经完成初始化")
    else:
        print("打开千牛工作台")
        app_path = r"D:\Program Files\AliWorkbench\AliWorkbench.exe"
        args = [app_path, "/run:desktop"]
        subprocess.Popen(args, shell=False)
        time.sleep(30)
    print("打开千牛工作台接待中心")
    floating_bar = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowToolSaveBits')
    if not floating_bar:
        print("千牛工作台未完成初始化，程序有问题")
        sys.exit()
    print(floating_bar.Name)
    floating_bar.Click(x=110, y=55, simulateMove=True)
    print("切换到接待中心联系中名单")
    rc_window = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowIcon')
    rc_window.Click(x=106, y=170, simulateMove=True)
    # 150 290
    print("切换到第一个联系人")
    rc_window.Click(x=150, y=290, simulateMove=True)
    # 535 500
    print("移动鼠标聚焦聊天信息框")
    rc_window.MoveCursorToInnerPos(x=535, y=500, simulateMove=True)


def get_all_siblings(start_control):
    # 创建一个列表来存储同级控件
    siblings = []

    # 首先获取第一个同级控件
    sibling = start_control.GetNextSiblingControl()

    while sibling is not None and sibling != auto.GetRootControl() and sibling:
        # 添加找到的同级控件到列表中
        siblings.append(sibling)

        # 获取下一个同级控件，直到没有更多
        sibling = sibling.GetNextSiblingControl()

    # 返回包含所有同级控件的列表
    return siblings


def get_control_from_cursor():

    time.sleep(3)

    idea_control = auto.ControlFromCursor()

    if idea_control is not None and idea_control != auto.GetRootControl():
        print("**********")
        print("能够识别光标下的UI控件")
        auto.EnumAndLogControlAncestors(control=idea_control, showAllName=True, showPid=True)
        if idea_control.Name != "":
            print("**********")
            print(f"该控件有名字：{idea_control.Name}")

        all_children = []
        all_children = idea_control.GetChildren()
        if all_children:
            print("**********")
            print("该控件有子控件：")
            for item in all_children:
                if item:
                    print(auto.EnumAndLogControlAncestors(control=item, showAllName=True, showPid=True))
        else:
            print("**********")
            print("该控件没有子控件")

        all_siblings = []
        all_siblings = get_all_siblings(idea_control)
        if all_siblings:
            print("**********")
            print("该控件有同级控件：")
            for item in all_siblings:
                print(auto.EnumAndLogControlAncestors(control=item, showAllName=True, showPid=True))
        else:
            print("**********")
            print("该控件没有同级控件")

        return idea_control
    else:
        print("**********")
        print("无法识别光标下的UI控件")
        return None


def get_control_from_cursor2():
    time.sleep(3)
    idea_control = auto.ControlFromCursor()
    if idea_control:
        print(idea_control)
        print(idea_control.Name)
        rect = (idea_control.BoundingRectangle.left, idea_control.BoundingRectangle.top, idea_control.BoundingRectangle.right, idea_control.BoundingRectangle.bottom)
        rect_wh = (idea_control.BoundingRectangle.width(), idea_control.BoundingRectangle.height())
        print(rect)
        print(rect_wh)
        return idea_control, rect, rect_wh


def get_text_area(start_x, start_y, end_x, end_y):
    auto.WheelUp(1)
    auto.WheelDown(3)
    auto.MoveTo(start_x, start_y)
    auto.PressMouse(start_x, start_y)
    auto.MoveTo(end_x, end_y)
    auto.ReleaseMouse()
    auto.SendKeys('{Ctrl}c')
    # auto.Click()
    text = ""
    text = pyperclip.paste()
    message = text.split('\n')
    # 使用filter函数过滤掉空字符串
    message = list(filter(lambda x: x.strip() != '', message))
    # # 使用列表推导过滤掉空字符串
    # message = [line for line in message if line.strip() != '']
    # 从最后一个子项开始向前遍历
    m_message = ""
    for i in range(-1, -len(message) - 1, -1):
        if message[i] != "已读" and message[i] != "未读" and message[i]:
            m_message = message[i]
            break
    if m_message:
        print("**********")
        print(m_message)
        return m_message


if __name__ == '__main__':
    # idea_c = get_control_from_cursor()
    initialization_qn()
    ic, rect, rect_wh = get_control_from_cursor2()
    ms = get_text_area(rect[0], rect[1], rect[2], rect[3])
    print(f"发送问题：{ms}")
    q = AskSpark.ask(ms)
    print(f"AI回答：{q}")
    print("激活发送消息栏")
    rc_window = auto.WindowControl(searchDepth=1, ClassName='Qt5152QWindowIcon')
    # 1200 950
    rc_window.Click(x=530, y=650, simulateMove=True)
    auto.SendKeys(q)
    auto.SendKey(0x0D)
    # auto.SendKey('{Enter}')