python项目学习 opencv抓取手部姿态mediapipe学习识别后触发下滑指令 手势识别刷视频

import time
import numpy
import cv2
import mediapipe as mp
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
from mediapipe.tasks.python.vision import HandLandmarkerOptions, HandLandmarkerResult
from mediapipe import solutions
from mediapipe.framework.formats import landmark_pb2

class HandGestureCapture:
    def __init__(self):
        self.open_camera()#调用摄像头
        self.hands_detector()#调用手部检测模型
        self.run()
    def open_camera(self):#摄像头初始化
        self.capture = cv2.VideoCapture(0)#打开笔记本摄像头
    def hands_detector(self):#创建手部检测任务
        hand_options = HandLandmarkerOptions(#训练模型
            base_options=python.BaseOptions(model_asset_path='./hand_landmarker.task'),#导入学习模型
            running_mode=vision.RunningMode.LIVE_STREAM,#任务运行模式 视频流模式
            result_callback=self.audio_monitor#结果监听器 用于返回值
        )
        self.hand_detector = vision.HandLandmarker.create_from_options(hand_options)#创建检测任务
        self.hand_result = None#初始化监听器返回值
        self.finger_index_to_angles = dict()#存储手部关键向量角度
        self.last_trigger_cmd_time = 0#记录触发时间
    def audio_monitor(self, result, output_image, timestamp_ms):#回调函数,用于处理检测结果。当检测到手部信息时,会将其存储在self.hand_result
        self.hand_result = result#检测到的手部信息
        #print(f'hand landmarker result: consume {self.get_cur_time() - timestamp_ms} ms, {self.hand_result}')

        current_time = self.get_current_time()#当前时间戳
        if len(self.hand_result.hand_world_landmarks) > 0:
            self.calculate_all_fingers_angles()
            # print(self.finger_index_to_angles)
            if self.check_hands_open() and current_time - self.last_trigger_cmd_time > 1000:#检测手部姿态 与上次触发时间间隔1000ms
                print(f'{current_time} Open')
                import pyautogui#指令控制鼠标键盘 ide以管理员身份运行
                pyautogui.press('down')
                self.last_trigger_cmd_time = current_time
    def check_hands_open(self):#检测手部姿态有无打开
        angle_threashold = 120#设置阈值
        for i in range(4):
            if self.finger_index_to_angles[5 + 4 * i] < angle_threashold or self.finger_index_to_angles[6 + 4 * i] < angle_threashold:
                return False#手部握拳
        #循环完四根手指均是舒展状态再作open判断
        return True
    def calculate_all_fingers_angles(self):#存储关键向量
        hand_landmarks = self.hand_result.hand_world_landmarks[0]#只识别一只手
        self.finger_index_to_angles.clear()
        for i in range(4):#计算食指，中指，无名指，小指关键向量角度
            #一根手指计算靠近手掌的两个角度
            self.finger_index_to_angles[5 + i * 4] = self.calculate_finger_angle(0, 5 + i * 4, 6 + i * 4, hand_landmarks)
            self.finger_index_to_angles[6 + i * 4] = self.calculate_finger_angle(5 + i * 4, 6 + i * 4, 7 + i * 4, hand_landmarks)
    def calculate_finger_angle(self, root_index, middle_index, end_index, hand_landmarks):#计算向量角度
        root = hand_landmarks[root_index]#索引关键点信息
        middle = hand_landmarks[middle_index]
        end = hand_landmarks[end_index]

        vector1 = numpy.array([root.x - middle.x, root.y - middle.y, root.z - middle.z])
        vector2 = numpy.array([end.x - middle.x, end.y - middle.y, end.z - middle.z])
        vector1_norm = self.normalize_vector(vector1)#单位向量
        vector2_norm = self.normalize_vector(vector2)
        dot_product_result = numpy.dot(vector1_norm, vector2_norm)#cos(angle)=单位向量的点乘
        angle = numpy.rad2deg(numpy.arccos(dot_product_result))#angle=arccos(cos(angle))
        return angle
    def normalize_vector(self, vector):#计算单位向量
        length = numpy.linalg.norm(vector)#向量的模长

        return vector / length
    def get_current_time(self):#当前时间戳
        return int(time.time() * 1000)#当前时间转化为ms
    def run(self):#运行摄像头，接收视频流
        while True:
            success, image = self.capture.read()#参数success 为True 或者False,代表有没有读取到图片 参数image表示截取到一帧的图片
            image = cv2.flip(image, 1)  # 镜像翻转image
            if success:
                image_numpy_array = numpy.asarray(image)#image转化为numpy数组
                mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data = image_numpy_array)#numpy数组转化为mp_image
                self.hand_detector.detect_async(mp_image, self.get_current_time())

                if self.hand_result:
                    image = self.draw_landmarks_on_image(image, self.hand_result)

                cv2.imshow('control', image)
                key = cv2.waitKey(1)#cv.waitKey()是一个键盘绑定函数 以ms为单位
                if key == 27:#等待esc键 退出
                    break
        self.exit()
    def draw_landmarks_on_image(self, rgb_image, detection_result):#关键点关键线绘制
        hand_landmarks_list = detection_result.hand_landmarks
        handedness_list = detection_result.handedness
        annotated_image = numpy.copy(rgb_image)

        for idx in range(len(hand_landmarks_list)):
            hand_landmarks = hand_landmarks_list[idx]
            handedness = handedness_list[idx]

            # Draw the hand landmarks.
            hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
            hand_landmarks_proto.landmark.extend([
                landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
            ])
            solutions.drawing_utils.draw_landmarks(
                annotated_image,
                hand_landmarks_proto,
                solutions.hands.HAND_CONNECTIONS,#关键点连线
                solutions.drawing_styles.get_default_hand_landmarks_style(),#关键点绘制形式
                solutions.drawing_styles.get_default_hand_connections_style())#连线绘制形式

        return annotated_image
    def exit(self):#程序退出
        self.capture.release()#释放摄像头

if __name__ == '__main__':
    Capture = HandGestureCapture()
    print('Finish')

运行效果

ide以管理员身份运行，运行程序，识别到手掌打开且与上次触发时间间隔超过1000ms时，会触发键盘向下的方向键，借此可手势识别隔空刷视频