AI计算机视觉进阶项目（二）——手势虚拟拖拽项目实战

本文链接：https://blog.csdn.net/bigData1994pb/article/details/128302642

本文档介绍了一个使用OpenCV和Mediapipe库实现的手势虚拟拖拽方块的项目。通过读取摄像头视频流，绘制方块，并利用Mediapipe获取手指关键点坐标，检测食指是否在方块上以实现拖拽效果。当食指与中指间距离超过阈值时，退出拖拽状态，同时为方块添加半透明处理，以便观察手指动作。

摘要由CSDN通过智能技术生成

一、功能概述

实现实时的手势虚拟拖拽方块

二、设计方法

OpenCV读取摄像头视频流并显示
在画面画一个方块
用第三方库获取手指坐标
手指移动到方块上激活，并实现拖拽

三、设计步骤

OpenCv视频流获取

在画面画一个方块

通过mediapipe获取手指关键点坐标

判断手指是否在方块上

如果在方块上，实现方块随着手指移动

四、实现流程

4.1获取视频流

import cv2
import numpy as np

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1) 

  
    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()

4.2 画一个方块

import cv2
import numpy as np

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)

# 方块的参数
square_x = 100
square_y = 100
square_w = 100


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1) 

    # 画一个方块
    cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)

    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()

4.3通过mediapipe获取手指关键点坐标

mediapipe官网在：Hands | mediapipe

import cv2
import numpy as np
import mediapipe as mp

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

hands = mp_hands.Hands(
    model_complexity = 0,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)

# 方块的参数
square_x = 100
square_y = 100
square_w = 100


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1)  

    # mediapipe颜色处理
    frame.flags.writeable = False
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)

    frame.flags.writeable = True
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)

    # 解析results将关键点绘制出来
    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            mp_drawing.draw_landmarks(
                frame,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS,
                mp_drawing_styles.get_default_hand_landmarks_style(),
                mp_drawing_styles.get_default_hand_connections_style()
            )


    # 画一个方块
    cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)

    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()

4.4检测食指是否在方块上，若在进行移动

由官网可以知道，食指指尖在8号位置。

import cv2
import numpy as np
import mediapipe as mp

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

hands = mp_hands.Hands(
    model_complexity = 0,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)

# 获取画面的宽度和高度
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

# 方块的参数
square_x = 100
square_y = 100
square_w = 100

L1 = 0
L2 = 0

on_squre = False # 判断食指是否在方块上


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1)  

    # mediapipe颜色处理
    frame.flags.writeable = False
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)

    frame.flags.writeable = True
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)

    # 解析results将关键点绘制出来
    if results.multi_hand_landmarks:  # 判断是否出现手
        # 解析遍历每一双手
        for hand_landmarks in results.multi_hand_landmarks:

            # 绘制21个关键点
            mp_drawing.draw_landmarks(
                frame,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS,
                mp_drawing_styles.get_default_hand_landmarks_style(),
                mp_drawing_styles.get_default_hand_connections_style()
            )
        

        # 保存21个关键点的x和y坐标
        x_list = []
        y_list = []
        for landmark in hand_landmarks.landmark:
            # 添加x坐标
            x_list.append(landmark.x)
            # 添加y坐标
            y_list.append(landmark.y)


        # 获取食指指尖位置
        index_finger_x = int(x_list[8] * width)
        index_finger_y = int(y_list[8] * height)

        # # 坐标位置画一个圆进行验证
        # cv2.circle(frame, (index_finger_x, index_finger_y), 20, (255, 0, 255), -1)
        # print(index_finger_x, index_finger_y)

        # 判读食指之间在不在方块上
        if (index_finger_x > square_x) and (index_finger_x < (square_x + square_w)) and (index_finger_y >square_y) and (index_finger_y<square_y + square_w):
           if on_squre == False:
                # 如果手指在方块上只需要更新rectangle函数中的(square_x, square_y)即可
                L1 = abs(index_finger_x - square_x)
                L2 = abs(index_finger_y - square_y)
                on_squre = True
                print("在方块上")
        else:
            print("不在方块上")
        if on_squre: # 如果手指在方块上就对方块的坐标进行刷新
            square_x = index_finger_x - L1
            square_y = index_finger_y - L2

    # 画一个方块
    cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)

    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()

这样便实现了方块随着手的移动而移动，但是有一个问题就是手不能摆脱方块了，一直在随着手的移动而移动

4.5 加入退出机制

通过计算中指与食指之间的距离来进行退出，如果距离大于一定阈值就不进行拖拽

import cv2
import numpy as np
import mediapipe as mp
import math

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

hands = mp_hands.Hands(
    model_complexity = 0,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)

# 获取画面的宽度和高度
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

# 方块的参数
square_x = 100
square_y = 100
square_w = 100

L1 = 0
L2 = 0

on_squre = False # 判断食指是否在方块上


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1)  

    # mediapipe颜色处理
    frame.flags.writeable = False
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)

    frame.flags.writeable = True
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)

    # 解析results将关键点绘制出来
    if results.multi_hand_landmarks:  # 判断是否出现手
        # 解析遍历每一双手
        for hand_landmarks in results.multi_hand_landmarks:

            # 绘制21个关键点
            mp_drawing.draw_landmarks(
                frame,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS,
                mp_drawing_styles.get_default_hand_landmarks_style(),
                mp_drawing_styles.get_default_hand_connections_style()
            )
        

        # 保存21个关键点的x和y坐标
        x_list = []
        y_list = []
        for landmark in hand_landmarks.landmark:
            # 添加x坐标
            x_list.append(landmark.x)
            # 添加y坐标
            y_list.append(landmark.y)


        # 获取食指指尖x，y位置
        index_finger_x = int(x_list[8] * width)
        index_finger_y = int(y_list[8] * height)

        
        # 获取中指指尖x，y位置
        middle_finger_x = int(x_list[12] * width)
        middle_finger_y = int(y_list[12] * height)

        # 计算食指指尖与中指之间的距离（勾股定理）
        finger_len = math.hypot((index_finger_x-middle_finger_x),(index_finger_y-middle_finger_y))
        print(finger_len)

        # 如果距离小于30就激活手指拖拽，或者就不进行拖拽
        if finger_len<60:

            # # 坐标位置画一个圆进行验证
            # cv2.circle(frame, (index_finger_x, index_finger_y), 20, (255, 0, 255), -1)
            # print(index_finger_x, index_finger_y)

            # 判读食指之间在不在方块上
            if (index_finger_x > square_x) and (index_finger_x < (square_x + square_w)) and (index_finger_y >square_y) and (index_finger_y<square_y + square_w):
                if on_squre == False:
                    # 如果手指在方块上只需要更新rectangle函数中的(square_x, square_y)即可
                    L1 = abs(index_finger_x - square_x)
                    L2 = abs(index_finger_y - square_y)
                    on_squre = True
                    # print("在方块上")
            else:
                # print("不在方块上")
                pass
        else:
            # 取消激活
            on_squre = False
        if on_squre: # 如果手指在方块上就对方块的坐标进行刷新
            square_x = index_finger_x - L1
            square_y = index_finger_y - L2

    # 画一个方块
    cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)

    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()

4.6 半透明处理

为了可以观看方块后面的手指，进行半透明处理

import cv2
import numpy as np
import mediapipe as mp
import math

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

hands = mp_hands.Hands(
    model_complexity = 0,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)

# 获取画面的宽度和高度
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

# 方块的参数
square_x = 100
square_y = 100
square_w = 100

L1 = 0
L2 = 0

on_squre = False # 判断食指是否在方块上


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1)  

    # mediapipe颜色处理
    frame.flags.writeable = False
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)

    frame.flags.writeable = True
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)

    # 解析results将关键点绘制出来
    if results.multi_hand_landmarks:  # 判断是否出现手
        # 解析遍历每一双手
        for hand_landmarks in results.multi_hand_landmarks:

            # 绘制21个关键点
            mp_drawing.draw_landmarks(
                frame,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS,
                mp_drawing_styles.get_default_hand_landmarks_style(),
                mp_drawing_styles.get_default_hand_connections_style()
            )
        

        # 保存21个关键点的x和y坐标
        x_list = []
        y_list = []
        for landmark in hand_landmarks.landmark:
            # 添加x坐标
            x_list.append(landmark.x)
            # 添加y坐标
            y_list.append(landmark.y)


        # 获取食指指尖x，y位置
        index_finger_x = int(x_list[8] * width)
        index_finger_y = int(y_list[8] * height)

        
        # 获取中指指尖x，y位置
        middle_finger_x = int(x_list[12] * width)
        middle_finger_y = int(y_list[12] * height)

        # 计算食指指尖与中指之间的距离（勾股定理）
        finger_len = math.hypot((index_finger_x-middle_finger_x),(index_finger_y-middle_finger_y))
        print(finger_len)

        # 如果距离小于30就激活手指拖拽，或者就不进行拖拽
        if finger_len<60:

            # # 坐标位置画一个圆进行验证
            # cv2.circle(frame, (index_finger_x, index_finger_y), 20, (255, 0, 255), -1)
            # print(index_finger_x, index_finger_y)

            # 判读食指之间在不在方块上
            if (index_finger_x > square_x) and (index_finger_x < (square_x + square_w)) and (index_finger_y >square_y) and (index_finger_y<square_y + square_w):
                if on_squre == False:
                    # 如果手指在方块上只需要更新rectangle函数中的(square_x, square_y)即可
                    L1 = abs(index_finger_x - square_x)
                    L2 = abs(index_finger_y - square_y)
                    on_squre = True
                    # print("在方块上")
            else:
                # print("不在方块上")
                pass
        else:
            # 取消激活
            on_squre = False
        if on_squre: # 如果手指在方块上就对方块的坐标进行刷新
            square_x = index_finger_x - L1
            square_y = index_finger_y - L2

    # # 画一个方块
    # cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)

    # 画一个半透明的方块
    overlay = frame.copy()
    cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)
    frame = cv2.addWeighted(overlay, 0.5, frame, 0.5, 0)

    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()

五、完整程序

import cv2
import numpy as np
import mediapipe as mp
import math

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

hands = mp_hands.Hands(
    model_complexity = 0,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

# 获取摄像头的视频流
cap = cv2.VideoCapture(0)

# 获取画面的宽度和高度
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

# 方块的参数
square_x = 100
square_y = 100
square_w = 100

L1 = 0
L2 = 0

on_squre = False # 判断食指是否在方块上


while True:
    # 读取每一帧
    ret, frame = cap.read()

    # 对图像进行处理
    frame = cv2.flip(frame, 1)  

    # mediapipe颜色处理
    frame.flags.writeable = False
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)

    frame.flags.writeable = True
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)

    # 解析results将关键点绘制出来
    if results.multi_hand_landmarks:  # 判断是否出现手
        # 解析遍历每一双手
        for hand_landmarks in results.multi_hand_landmarks:

            # 绘制21个关键点
            mp_drawing.draw_landmarks(
                frame,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS,
                mp_drawing_styles.get_default_hand_landmarks_style(),
                mp_drawing_styles.get_default_hand_connections_style()
            )
        

        # 保存21个关键点的x和y坐标
        x_list = []
        y_list = []
        for landmark in hand_landmarks.landmark:
            # 添加x坐标
            x_list.append(landmark.x)
            # 添加y坐标
            y_list.append(landmark.y)


        # 获取食指指尖x，y位置
        index_finger_x = int(x_list[8] * width)
        index_finger_y = int(y_list[8] * height)

        
        # 获取中指指尖x，y位置
        middle_finger_x = int(x_list[12] * width)
        middle_finger_y = int(y_list[12] * height)

        # 计算食指指尖与中指之间的距离（勾股定理）
        finger_len = math.hypot((index_finger_x-middle_finger_x),(index_finger_y-middle_finger_y))
        print(finger_len)

        # 如果距离小于30就激活手指拖拽，或者就不进行拖拽
        if finger_len<60:

            # # 坐标位置画一个圆进行验证
            # cv2.circle(frame, (index_finger_x, index_finger_y), 20, (255, 0, 255), -1)
            # print(index_finger_x, index_finger_y)

            # 判读食指之间在不在方块上
            if (index_finger_x > square_x) and (index_finger_x < (square_x + square_w)) and (index_finger_y >square_y) and (index_finger_y<square_y + square_w):
                if on_squre == False:
                    # 如果手指在方块上只需要更新rectangle函数中的(square_x, square_y)即可
                    L1 = abs(index_finger_x - square_x)
                    L2 = abs(index_finger_y - square_y)
                    on_squre = True
                    # print("在方块上")
            else:
                # print("不在方块上")
                pass
        else:
            # 取消激活
            on_squre = False
        if on_squre: # 如果手指在方块上就对方块的坐标进行刷新
            square_x = index_finger_x - L1
            square_y = index_finger_y - L2

    # # 画一个方块
    # cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)

    # 画一个半透明的方块
    overlay = frame.copy()
    cv2.rectangle(frame, (square_x, square_y), (square_x+square_w, square_y+square_w), (255, 0, 0), -1)
    frame = cv2.addWeighted(overlay, 0.5, frame, 0.5, 0)

    cv2.imshow("Virtual drag", frame)

    # 退出条件
    if cv2.waitKey(10) & 0xff==27:
        break

cap.release()
cv2.destroyAllWindows()