python图像匹配实现弹窗自动关闭、跳转(刷课助手)

python图像匹配实现弹窗自动关闭、跳转

背景:

总是有些不得不刷的网课,还会时不时的跳出弹窗确认是否在观看,

使用浏览器播放的还可以使用js脚本实现自动连播,

而有些课程会有自己的客户端,或者自己封装的浏览器,插入脚本就很难实现

所以想到了使用opencv的模板匹配,基本思路就是定时截取电脑屏幕,判断截取的图像中是否有弹窗,如果有则

获取弹窗位置,使用pyautogui.click(x, y)模拟鼠标点击关闭或下一节课等按键.

关键代码:cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)

其中的image就是定时截取的电脑屏幕图片,template就是弹窗中需要点击的按键截图

目前能够实现基本的出现弹窗后自动跳转功能

import os
import cv2
import time
import pyautogui
import sys
import argparse  # argparse解析命令行参数
from os import path
def capture_screen():
    # 截取电脑桌面
    screenshot = pyautogui.screenshot()
    screenshot.save('main.png')
    return screenshot

def click_at_position(x, y):
    # 使用鼠标点击指定位置
    pyautogui.click(x, y)

def cv_show(title,img):
    cv2.imshow(title,img)
    cv2.waitKey(0) 
    cv2.destroyAllWindows()
    return

def is_image_contained(main_image_path, sub_image_path, threshold=0.6):
    # 读取主图和子图
    main_image = cv2.imread(main_image_path)
    sub_image = cv2.imread(sub_image_path)
    
    # 将子图匹配到主图
    result = cv2.matchTemplate(main_image, sub_image, cv2.TM_CCOEFF_NORMED)

    # 获取匹配结果中的最大值和位置
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)

    # 如果最大值超过设定的阈值，则判断子图在主图中存在
    if max_val > threshold:
        return True,max_loc
    else:
        return False,None
def deal_result(main_image_path,sub_image_path,isShow):
    result, top_left = is_image_contained(main_image_path, sub_image_path)
    current_time = time.gmtime()
    formatted_time = time.strftime('%Y-%m-%d %H:%M:%S', current_time)
    if result:        
        print("屏幕中有弹窗",top_left,formatted_time)
        main_image = cv2.imread(main_image_path)
        #通过左上角点的坐标和h，w计算右下角点的坐标
        bottom_right = (top_left[0]+100, top_left[1]+40)#[0]为横坐标，[1]为纵坐标，横加宽纵加高就是右下角的点坐标
        print('1s后关闭弹窗',formatted_time)
        time.sleep(1)
        click_at_position(top_left[0]+20,top_left[1]+20)
        #绘制矩形
        if isShow=='true':
            resoult_img = cv2.rectangle(main_image.copy(),top_left,bottom_right,255,5)
            cv_show('img',resoult_img)
        return True
    else:
        print("无弹窗",formatted_time)
        return False
if __name__ == "__main__":
    # # 检查是否包含 SIFT 模块
    # sift_exists = cv2.SIFT_create()

    # if sift_exists:
    #   print("SIFT 模块存在")
    # else:
    #   print("SIFT 模块不存在")
    # 示例：获取 main_image.png 中包含 sub_image.png 的具体位置
    ap = argparse.ArgumentParser()
    ap.add_argument("-s", "--show",default="false", help="Is the annotated image displayed")
    ap.add_argument("-t", "--test",default="false", help="use test image")
    args = vars(ap.parse_args())
    script_dir = path.dirname(path.realpath(sys.argv[0]))
    main_image_path = script_dir+ '\main.png'
    sub_image_path = script_dir+'\closebutton.png'
    current_time = time.localtime()
    formatted_time = time.strftime('%Y-%m-%d %H:%M:%S', current_time)
    print('开始执行程序',formatted_time)
    num=0
    if args["test"]=='false':
        try:
            while True:
                # 截取屏幕
                screenshot = capture_screen()
                if deal_result(main_image_path,sub_image_path,args["test"]):
                    num+=1
                print('共点击',num,'次!')
                time.sleep(10)

        except KeyboardInterrupt:
            print("程序被终止")
    else:
        main_image_path = script_dir+ '\/test.png'
        sub_image_path = script_dir+'\subtest.png'
        deal_result(main_image_path,sub_image_path,args["test"])
        print('end--')
        try:
          input()
        except KeyboardInterrupt:
          print("程序被终止")

待优化:

​ 未实现多尺寸匹配,要求template图片的尺寸与image中的匹配的区域尺寸一致.

【腾讯云】语音识别、人脸融合等热门AI产品新用户14.9元起，共享AI领域应用场景和解决方案