使用Python和MediaPipe实现手势虚拟鼠标控制

概述

使用Python实现虚拟鼠标控制，利用手势识别来替代传统鼠标操作。这一实现依赖于计算机视觉库OpenCV、手势识别库MediaPipe以及其他辅助库如PyAutoGUI和Pynput。

环境配置

在开始之前，请确保已安装以下Python库：

pip install opencv-python mediapipe pynput pyautogui numpy pillow

模块介绍

1. utils.py

utils.py包含一个Utils类，主要提供在图像上添加中文文本的功能。这对于在实时视频流中显示信息非常有用。

代码解析

import cv2
import numpy as np
from PIL import Image, ImageDraw, ImageFont

class Utils:
    def __init__(self):
        pass

    def cv2AddChineseText(self, img, text, position, textColor=(0, 255, 0), textSize=30):
        if isinstance(img, np.ndarray):  # 判断是否OpenCV图片类型
            img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
        draw = ImageDraw.Draw(img)
        fontStyle = ImageFont.truetype("./fonts/simsun.ttc", textSize, encoding="utf-8")
        draw.text(position, text, textColor, font=fontStyle)
        return cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR)

2. handProcess.py

handProcess.py负责手势的识别和处理。该模块使用MediaPipe库来检测和跟踪手部的关键点，并根据手势的不同动作触发相应的鼠标操作。

代码解析

import cv2
import mediapipe as mp
import time
import math
import numpy as np
from utils import Utils

class HandProcess:
    def __init__(self, static_image_mode=False, max_num_hands=2):
        self.mp_drawing = mp.solutions.drawing_utils
        self.mp_hands = mp.solutions.hands
        self.hands = self.mp_hands.Hands(static_image_mode=static_image_mode,
                                         min_detection_confidence=0.7,
                                         min_tracking_confidence=0.5,
                                         max_num_hands=max_num_hands)
        self.landmark_list = []
        self.action_labels = {
            'none': '无',
            'move': '鼠标移动',
            'click_single_active': '触发单击',
            'click_single_ready': '单击准备',
            'click_right_active': '触发右击',
            'click_right_ready': '右击准备',
            'scroll_up': '向上滑页',
            'scroll_down': '向下滑页',
            'drag': '鼠标拖拽'
        }
        self.action_deteted = ''

    def checkHandsIndex(self, handedness):
        if len(handedness) == 1:
            handedness_list = [handedness[0].classification[0].label]
        else:
            handedness_list = [handedness[0].classification[0].label, handedness[1].classification[0].label]
        return handedness_list

    def getDistance(self, pointA, pointB):
        return math.hypot((pointA[0] - pointB[0]), (pointA[1] - pointB[1]))

    def getFingerXY(self, index):
        return (self.landmark_list[index][1], self.landmark_list[index][2])

    def drawInfo(self, img, action):
        thumbXY, indexXY, middleXY = map(self.getFingerXY, [4, 8, 12])
        if action == 'move':
            img = cv2.circle(img, indexXY, 20, (255, 0, 255), -1)
        elif action == 'click_single_active':
            middle_point = int((indexXY[0] + thumbXY[0]) / 2), int((indexXY[1] + thumbXY[1]) / 2)
            img = cv2.circle(img, middle_point, 30, (0, 255, 0), -1)
        elif action == 'click_single_ready':
            img = cv2.circle(img, indexXY, 20, (255, 0, 255), -1)
            img = cv2.circle(img, thumbXY, 20, (255, 0, 255), -1)
            img = cv2.line(img, indexXY, thumbXY, (255, 0, 255), 2)
        elif action == 'click_right_active':
            middle_point = int((indexXY[0] + middleXY[0]) / 2), int((indexXY[1] + middleXY[1]) / 2)
            img = cv2.circle(img, middle_point, 30, (0, 255, 0), -1)
        elif action == 'click_right_ready':
            img = cv2.circle(img, indexXY, 20, (255, 0, 255), -1)
            img = cv2.circle(img, middleXY, 20, (255, 0, 255), -1)
            img = cv2.line(img, indexXY, middleXY, (255, 0, 255), 2)
        return img

    def checkHandAction(self, img, drawKeyFinger=True):
        upList = self.checkFingersUp()
        action = 'none'
        if len(upList) == 0:
            return img, action, None
        dete_dist = 100
        key_point = self.getFingerXY(8)
        if upList == [0, 1, 0, 0, 0]:
            action = 'move'
        if upList == [1, 1, 0, 0, 0]:
            l1 = self.getDistance(self.getFingerXY(4), self.getFingerXY(8))
            action = 'click_single_active' if l1 < dete_dist else 'click_single_ready'
        if upList == [0, 1, 1, 0, 0]:
            l1 = self.getDistance(self.getFingerXY(8), self.getFingerXY(12))
            action = 'click_right_active' if l1 < dete_dist else 'click_right_ready'
        if upList == [1, 1, 1, 1, 1]:
            action = 'scroll_up'
        if upList == [0, 1, 1, 1, 1]:
            action = 'scroll_down'
        if upList == [0, 0, 1, 1, 1]:
            key_point = self.getFingerXY(12)
            action = 'drag'
        img = self.drawInfo(img, action) if drawKeyFinger else img
        self.action_deteted = self.action_labels[action]
        return img, action, key_point

    def checkFingersUp(self):
        fingerTipIndexs = [4, 8, 12, 16, 20]
        upList = []
        if len(self.landmark_list) == 0:
            return upList
        if self.landmark_list[fingerTipIndexs[0]][1] < self.landmark_list[fingerTipIndexs[0] - 1][1]:
            upList.append(1)
        else:
            upList.append(0)
        for i in range(1, 5):
            if self.landmark_list[fingerTipIndexs[i]][2] < self.landmark_list[fingerTipIndexs[i] - 2][2]:
                upList.append(1)
            else:
                upList.append(0)
        return upList

    def processOneHand(self, img, drawBox=True, drawLandmarks=True):
        utils = Utils()
        results = self.hands.process(img)
        self.landmark_list = []
        if results.multi_hand_landmarks:
            for hand_index, hand_landmarks in enumerate(results.multi_hand_landmarks):
                if drawLandmarks:
                    self.mp_drawing.draw_landmarks(img, hand_landmarks, self.mp_hands.HAND_CONNECTIONS,
                                                   self.mp_drawing_styles.get_default_hand_landmarks_style(),
                                                   self.mp_drawing_styles.get_default_hand_connections_style())
                for landmark_id, finger_axis in enumerate(hand_landmarks.landmark):
                    h, w, c = img.shape
                    p_x, p_y = math.ceil(finger_axis.x * w), math.ceil(finger_axis.y * h)
                    self.landmark_list.append([landmark_id, p_x, p_y, finger_axis.z])
                if drawBox:
                    x_min, x_max = min(self.landmark_list, key=lambda i: i[1])[1], max(self.landmark_list, key=lambda i: i[1])[1]
                    y_min, y_max = min(self.landmark_list, key=lambda i: i[2])[2], max(self.landmark_list, key=lambda i: i[2])[2]
                    img = cv2.rectangle(img, (x_min - 30, y_min - 30), (x_max + 30, y_max + 30), (0, 255, 

0), 2)
                    img = utils.cv2AddChineseText(img, self.action_deteted, (x_min - 20, y_min - 120), textColor=(255, 0, 255), textSize=60)
        return img

3. virtual_mouse.py

virtual_mouse.py是主程序模块，整合了手势识别和鼠标控制功能，实现了通过手势控制鼠标移动、点击和滚动的功能。

代码解析

import cv2
import handProcess
import time
import numpy as np
import pyautogui
from utils import Utils
from pynput.mouse import Button, Controller

class VirtualMouse:
    def __init__(self):
        self.image = None
        self.mouse = Controller()

    def recognize(self):
        handprocess = handProcess.HandProcess(False, 1)
        utils = Utils()
        fpsTime = time.time()
        cap = cv2.VideoCapture(0)
        resize_w = 960
        resize_h = 720
        frameMargin = 100
        screenWidth, screenHeight = pyautogui.size()
        stepX, stepY = 0, 0
        finalX, finalY = 0, 0
        smoothening = 7
        action_trigger_time = {
            'single_click': 0,
            'double_click': 0,
            'right_click': 0
        }
        mouseDown = False

        while cap.isOpened():
            action_zh = ''
            success, self.image = cap.read()
            self.image = cv2.resize(self.image, (resize_w, resize_h))
            if not success:
                print("空帧")
                continue
            self.image.flags.writeable = False
            self.image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
            self.image = cv2.flip(self.image, 1)
            self.image = handprocess.processOneHand(self.image)
            cv2.rectangle(self.image, (frameMargin, frameMargin), (resize_w - frameMargin, resize_h - frameMargin), (255, 0, 255), 2)
            self.image, action, key_point = handprocess.checkHandAction(self.image, drawKeyFinger=True)
            action_zh = handprocess.action_labels[action]
            if key_point:
                x3 = np.interp(key_point[0], (frameMargin, resize_w - frameMargin), (0, screenWidth))
                y3 = np.interp(key_point[1], (frameMargin, resize_h - frameMargin), (0, screenHeight))
                finalX = stepX + (x3 - stepX) / smoothening
                finalY = stepY + (y3 - stepY) / smoothening
                now = time.time()
                if action_zh == '鼠标拖拽':
                    if not mouseDown:
                        self.mouse.press(Button.left)
                        mouseDown = True
                    self.mouse.position = (finalX, finalY)
                else:
                    if mouseDown:
                        self.mouse.release(Button.left)
                    mouseDown = False
                if action_zh == '鼠标移动':
                    self.mouse.position = (finalX, finalY)
                elif action_zh == '单击准备':
                    pass
                elif action_zh == '触发单击' and (now - action_trigger_time['single_click'] > 0.3):
                    self.mouse.click(Button.left, 1)
                    action_trigger_time['single_click'] = now
                elif action_zh == '右击准备':
                    pass
                elif action_zh == '触发右击' and (now - action_trigger_time['right_click'] > 2):
                    self.mouse.click(Button.right, 1)
                    action_trigger_time['right_click'] = now
                elif action_zh == '向上滑页':
                    pyautogui.scroll(30)
                elif action_zh == '向下滑页':
                    pyautogui.scroll(-30)
                stepX, stepY = finalX, finalY
            self.image.flags.writeable = True
            self.image = cv2.cvtColor(self.image, cv2.COLOR_RGB2BGR)
            cTime = time.time()
            fps_text = 1 / (cTime - fpsTime)
            fpsTime = cTime
            self.image = utils.cv2AddChineseText(self.image, "帧率: " + str(int(fps_text)), (10, 30), textColor=(255, 0, 255), textSize=50)
            self.image = cv2.resize(self.image, (resize_w // 2, resize_h // 2))
            cv2.imshow('virtual mouse', self.image)
            if cv2.waitKey(5) & 0xFF == 27:
                break
        cap.release()

control = VirtualMouse()
control.recognize()

4. 功能列表

在这个虚拟鼠标控制项目中，通过识别不同的手势来触发相应的鼠标操作。以下是该项目中实现的主要功能及其对应的手势：

1. 鼠标移动

   • 手势：食指竖起（其他手指收回）。
   • 描述：食指指尖的移动映射到屏幕上的鼠标光标移动。

2. 单击准备

   • 手势：拇指和食指都竖起且未接触。
   • 描述：准备触发单击。

3. 触发单击

   • 手势：拇指和食指接触（捏合）。
   • 描述：触发一次鼠标左键单击。

4. 右击准备

   • 手势：食指和中指都竖起且未接触。
   • 描述：准备触发右击。

5. 触发右击

   • 手势：食指和中指接触（捏合）。
   • 描述：触发一次鼠标右键单击。

6. 鼠标拖拽

   • 手势：中指、无名指和小指竖起（拇指和食指收回）。
   • 描述：模拟鼠标左键按住并拖动。

7. 向上滚动

   • 手势：五指全部竖起。
   • 描述：触发页面向上滚动。

8. 向下滚动

   • 手势：除了拇指外，其他四指竖起。
   • 描述：触发页面向下滚动。

5.测试