基于mediapipe的手势识别

安装opencv：pip install opencv-python

安装mediapipe：pip install mediapipe

draw_utils.py:

import cv2
import numpy as np


def draw_line(img, width, height,  hand, start_index, stop_index):
    for i in range(start_index, stop_index):
        x1, y1 = int(hand.landmark[i].x * width), int(hand.landmark[i].y * height)
        x2, y2 = int(hand.landmark[i + 1].x * width), int(hand.landmark[i + 1].y * height)
        cv2.line(img, (x1, y1), (x2, y2), (255, 255, 255), 2)


def draw_hand(img, width, height, hand):
    # 画圆
    for i in range(21):
        pos_x = hand.landmark[i].x * width  # hand.landmark[i].x为归一化后的坐标
        pos_y = hand.landmark[i].y * height
        cv2.circle(img, (int(pos_x), int(pos_y)), 5, (0, 0, 255), -1)
    # 画线
    draw_line(img, width, height, hand, 0, 4)
    draw_line(img, width, height, hand, 5, 8)
    draw_line(img, width, height, hand, 9, 12)
    draw_line(img, width, height, hand, 13, 16)
    draw_line(img, width, height, hand, 17, 20)
    index = [0, 5, 9, 13, 17, 0]
    for i in range(5):
        pt1 = (int(hand.landmark[index[i]].x * width), int(hand.landmark[index[i]].y * height))
        pt2 = (int(hand.landmark[index[i + 1]].x * width), int(hand.landmark[index[i + 1]].y * height))
        cv2.line(img, pt1, pt2, (255, 255, 255), 2)


def get_angle(v1, v2):
    angle = np.dot(v1, v2) / (np.sqrt(np.sum(v1 * v1)) * np.sqrt(np.sum(v2 * v2)))
    angle = np.arccos(angle) * 180 / 3.14
    return angle


def get_str_gesture(out_fingers, list_lms):
    if len(out_fingers) == 1 and out_fingers[0] == 8:
        v1 = list_lms[6] - list_lms[7]
        v2 = list_lms[8] - list_lms[7]
        angle = get_angle(v1, v2)
        if angle < 160:
            str_gesture = '9'
        else:
            str_gesture = '1'
    elif len(out_fingers) == 1 and out_fingers[0] == 4:
        str_gesture = 'Good'
    elif len(out_fingers) == 1 and out_fingers[0] == 20:
        str_gesture = 'Bad'
    elif len(out_fingers) == 2 and out_fingers[0] == 8 and out_fingers[1] == 12:
        str_gesture = '2'
    elif len(out_fingers) == 2 and out_fingers[0] == 4 and out_fingers[1] == 20:
        str_gesture = '6'
    elif len(out_fingers) == 2 and out_fingers[0] == 4 and out_fingers[1] == 8:
            str_gesture = '8'
    elif len(out_fingers) == 3 and out_fingers[0] == 8 and out_fingers[1] == 12 and out_fingers[2] == 16:
        str_gesture = '3'
    elif len(out_fingers) == 3 and out_fingers[0] == 4 and out_fingers[1] == 8 and out_fingers[2] == 12:
        str_gesture = '7'
    elif len(out_fingers) == 4 and out_fingers[0] == 8 and out_fingers[1] == 12 and out_fingers[2] == 16 and out_fingers[3] == 20:
        str_gesture = '4'
    elif len(out_fingers) == 5:
        str_gesture = '5'
    elif len(out_fingers) == 0:
        str_gesture = '10'
    else:
        str_gesture = ''
    return str_gesture

import mediapipe as mp
import cv2
import numpy as np
from draw_utils import *

cap = cv2.VideoCapture(0)
mpHands = mp.solutions.hands
hands = mpHands.Hands()
mpDraw = mp.solutions.drawing_utils

while True:
    # 读取一帧图像
    ret, img = cap.read()
    height, width, channels = img.shape
    # 转换为RGB
    imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # 得到检测结果
    results = hands.process(imgRGB)

    # if results.multi_hand_landmarks:
    #     for hand in results.multi_hand_landmarks:
    #         mpDraw.draw_landmarks(img, hand, mpHands.HAND_CONNECTIONS)
    #         # draw_hand(img, width, height, hand)

    if results.multi_hand_landmarks:
        hand = results.multi_hand_landmarks[0]
        mpDraw.draw_landmarks(img, hand, mpHands.HAND_CONNECTIONS)

        # 采集所有关键点坐标
        list_lms = []
        for i in range(21):
            pos_x = int(hand.landmark[i].x * width)
            pos_y = int(hand.landmark[i].y * height)
            list_lms.append([pos_x, pos_y])

        # 构造凸包点
        list_lms = np.array(list_lms, dtype=np.int32)
        hull_index = [0, 1, 2, 3, 6, 10, 14, 19, 18, 17]
        hull = cv2.convexHull(list_lms[hull_index], True)
        # cv2.polylines(img, [hull], True, (0, 255, 0), 2)

        # 查找外部的点数
        ll = [4, 8, 12, 16, 20]
        out_fingers = []
        for i in ll:
            pt = (int(list_lms[i][0]), int(list_lms[i][1]))
            dist = cv2.pointPolygonTest(hull, pt, True)
            if dist < 0:
                out_fingers.append(i)

        str_gesture = get_str_gesture(out_fingers, list_lms)
        cv2.putText(img, str_gesture, (100,100), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 255, 0), 4, cv2.LINE_AA)
        for i in ll:
            pos_x = int(hand.landmark[i].x * width)
            pos_y = int(hand.landmark[i].y * height)
            cv2.circle(img, (pos_x, pos_y), 3, (0, 255, 255), -1)

    cv2.imshow('hands', img)
    key = cv2.waitKey(1)
    if key == ord('q'):
        break
cap.release()

运行结果：

 

 

 

 

 

 

 

 

 

 

 

第二种思路：识别单根手指是否弯曲，然后根据五根手指的弯曲程度判断手势所对应的数字。

基于mediapipe的手势数字识别

 基于mediapipe的手势识别