树莓派上利用python+opencv+dlib+pygame实现嘴唇检测

书接上回树莓派上利用python+opencv+dlib实现嘴唇检测

我发现卡顿主要原因就是dlib检测过程，但是实力不够还改变不了哈哈，那么我就想着要不然做一个界面让这个卡顿在人们的操作习惯中不那么卡顿，做界面主要使用pygame。

项目的主要目标是加入pygame来定义按钮，让检测操作仅仅发生在点击按钮之后，效果如下：

1、安装pygame

python3 -m pip install -U pygame --user

或者是直接写后面的也可以

pip install -U pygame --user

2、安装python3-sdl2

2.1 说明

这个安装是在树莓派上面的，在windows上面不需要安装这个

2.2 原因

在树莓派上运行，发现pygame中字体文件也就是ttf文件读取不出来，会存在如下错误：ImportError: libSDL_ttf-2.0.so.0: cannot open shared object file: No such file or directory

2.3 解决方法

网络上的主要解决方法有三种，大家都可以尝试一下看看哪一种合适，我的是要安装python3-sdl2

# 第一种
sudo apt-get install libsdl-ttf2.0-0
# 第二种
sudo apt-get install libsdl2-mixer-2.0-0
# 第三种
sudo apt-get install python3-sdl2

3、代码

# from gpiozero import LED
from time import sleep
from subprocess import check_call
import cv2 as cv
import numpy as np
import dlib
import threading
import pygame
from pygame.locals import *
import sys


# 声明界面中按钮的界面
# 输入参数
# screen：主界面，按钮界面将绘制于主界面之上
# font：pygame字体全局变量
# color：字体颜色
# button_size：按钮形状，要是元组中包含三个参数则是圆形，四个是矩形
# text：按钮中的文字
# x：按钮界面位于主界面中x的相对位置
# y：按钮界面位于主界面中y的相对位置
class SurfaceButton:
    def __init__(self, screen, font, color, button_size, text, x=0, y=0):
        self.screen = screen
        self.color = color
        self.button_size = button_size
        self.text = font.render(text, True, (158, 16, 16))
        self.surface = pygame.Surface(screen.get_size())
        # 配合下面代码使用，使得创建界面默认的黑色背景最终是透明的
        self.surface.set_colorkey((0, 0, 0))
        self.surfaceX = x
        self.surfaceY = y
        # 用于调试中显示输入文字
        # self.text_show = text

    # 把按钮界面绘制到主界面上
    def draw(self):
        # 绘制矩形按钮
        if len(self.button_size) == 4:
            pygame.draw.rect(self.surface, self.color, pygame.Rect(self.button_size), width=0)
            self.surface.blit(self.text, (
                self.button_size[0] - self.text.get_width() // 2, self.button_size[1] - self.text.get_height() // 2))
            self.surface.convert_alpha()
            self.screen.blit(self.surface, (self.surfaceX, self.surfaceY))
        # 绘制圆形按钮
        elif len(self.button_size) == 3:
            pygame.draw.circle(self.surface, self.color, self.button_size[:2], self.button_size[2], width=0)
            self.surface.blit(self.text, (
                self.button_size[0] - self.text.get_width() // 2, self.button_size[1] - self.text.get_height() // 2))
            self.surface.convert_alpha()
            self.screen.blit(self.surface, (self.surfaceX, self.surfaceY))


# 多线程操作
class MyThread(threading.Thread):
    def __init__(self, func, args=()):
        super(MyThread, self).__init__()
        self.func = func
        self.args = args

    # 在执行函数的同时，把结果赋值给result,然后通过get_result函数获取返回的结果
    def run(self):
        self.result = self.func(*self.args)

    def get_result(self):
        try:
            return self.result
        except Exception as e:
            return e


# 利用dlib进行人脸和嘴唇检测并绘制
# 输入参数
# screen：主界面
# img：用于嘴唇检测的图片，格式是opencv的图片格式（BGR）
def lip_detector(screen, img):
    landmarks_lip = []
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    rects = detector(gray, 1)
    print('检测到了 %d 个人脸' % len(rects))
    for (i, rect) in enumerate(rects):
        # 标记人脸中的68个landmark点
        landmarks = predictor(gray, rect)
        landmarks_lip_tem = []
        for n in range(48, 68):
            x = landmarks.part(n).x
            y = landmarks.part(n).y
            landmarks_lip_tem.append((x, y))
            # cv.circle(img=img, center=(x, y), radius=3, color=(0, 255, 0), thickness=-1)
        landmarks_lip.append(landmarks_lip_tem)
    draw_lip(screen, img, landmarks_lip)


# 绘制嘴唇线条
def draw_lip(screen, img, landmarks_lip):
    img_makeup = img.copy()
    for i in range(np.array(landmarks_lip).shape[0]):
        for m in range(2):
            for n in range(lip_order_num - 1):
                cv.line(img_makeup, landmarks_lip[i][lip_order_dlib[m][n]], landmarks_lip[i][lip_order_dlib[m][n + 1]],
                        color=(0, 255, 0),
                        thickness=2, lineType=8)
    screen.blit(cvimage_to_pygame(cv.cvtColor(img_makeup, cv.COLOR_BGR2RGB)), (0, 0))


# 获取摄像头序号
def search_cap_num():
    for i in range(2000):
        cap = cv.VideoCapture(i)
        cap_opened = cap.isOpened()
        if cap_opened == True:
            return i


# 获取摄像头所能拍摄的像素
# 注意这个地方输出的shape是高度，宽度，深度，所以要转化一下
def set_screen_size():
    ret, img = cap.read()
    return img.shape[1::-1]


# 将opencv图片转化为pygame中图片
def cvimage_to_pygame(image):
    """Convert cvimage into a pygame image"""
    return pygame.image.frombuffer(image.tobytes(), image.shape[1::-1], "RGB")


# 处理pygame按钮界面的点击事件
def deal_with_event():
    global button_makeup_clicked, button_quit_clicked, \
        button_circle_quit, button_circle_makeup
    for event in pygame.event.get():
        if event.type == QUIT:
            # 接收到退出事件后退出程序
            sys.exit()
        if event.type == MOUSEBUTTONDOWN:
            pressed_array = pygame.mouse.get_pressed()
            if pressed_array[0]:
                mouse_pos = pygame.mouse.get_pos()
                mouse_x = mouse_pos[0]
                mouse_y = mouse_pos[1]
                # print(mouse_pos)
                # print(button_circle.button_size[0]-button_circle.button_size[2],button_circle.button_size[0]+button_circle.button_size[2],button_circle.button_size[1] - button_circle.button_size[2],button_circle.button_size[1] + button_circle.button_size[2])

                if (not button_makeup_clicked) and \
                        button_circle_makeup.button_size[0] - button_circle_makeup.button_size[2] <= mouse_x <= \
                        button_circle_makeup.button_size[0] + button_circle_makeup.button_size[2] and \
                        button_circle_makeup.button_size[1] - button_circle_makeup.button_size[2] <= mouse_y <= \
                        button_circle_makeup.button_size[1] + button_circle_makeup.button_size[2]:
                    button_makeup_clicked = True
                    button_quit_clicked = False
                elif (not button_quit_clicked) and \
                        button_circle_quit.button_size[0] - button_circle_quit.button_size[2] <= mouse_x <= \
                        button_circle_quit.button_size[0] + button_circle_quit.button_size[2] and \
                        button_circle_quit.button_size[1] - button_circle_quit.button_size[2] <= mouse_y <= \
                        button_circle_quit.button_size[1] + button_circle_quit.button_size[2]:
                    button_quit_clicked = True
                    button_makeup_clicked = False


cap_num = search_cap_num()
cap = cv.VideoCapture(cap_num)

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
lip_order_dlib = np.array([[48, 49, 50, 51, 52, 53, 54, 64, 63, 62, 61, 60, 48],
                           [48, 59, 58, 57, 56, 55, 54, 64, 65, 66, 67, 60, 48]]) - 48
lip_order_num = lip_order_dlib.shape[1]

# 初始化pygame,为使用硬件做准备
pygame.init()
# 创建一个窗口
screen_video = pygame.display.set_mode(set_screen_size(), 0, 32)
screen_width = screen_video.get_size()[0]
screen_height = screen_video.get_size()[1]

white = (255, 255, 255)
button_circle_pos = [(screen_width - screen_width // 6), screen_height - screen_height // 7]
button_circle_raduis = [screen_height // 10]
button_circle_size = tuple(button_circle_pos + button_circle_raduis)

button_makeup_clicked = False
button_quit_clicked = True
# 字体
font = pygame.font.Font("font.ttf", 24)
# 字体
# font = pygame.font.SysFont(None, 24)

button_circle_makeup = SurfaceButton(screen_video, font, white, button_circle_size, "化妆")
button_circle_quit = SurfaceButton(screen_video, font, white, button_circle_size, "退出")
# 设置窗口标题
pygame.display.set_caption("Hello, World!")
img = {}

# 主循环
while cap.isOpened():
    if button_makeup_clicked:
        lip_detector(screen_video, img)
        button_circle_quit.draw()
        pygame.display.flip()
        while True:
            deal_with_event()
            if button_quit_clicked:
                break

    elif button_quit_clicked:
        ret, img = cap.read()
        img_rgb = cv.cvtColor(img, cv.COLOR_BGR2RGB)
        screen_video.blit(cvimage_to_pygame(img_rgb), (0, 0))
        button_circle_makeup.draw()
        deal_with_event()
        pygame.display.flip()

3.1 代码逻辑说明

其中MyThread类不需要管，我一开始想尝试多线程，后来觉得没有必要。
上述代码主要实现的功能是加入两个按钮，一个是化妆，一个是退出。

一开始的时候按钮是化妆，这个时候界面上显示的是利用opencv获取的图像信息，整个过程不会用到dlib中有关人脸检测和嘴唇特征点检测的代码。

当点击了化妆之后，按钮变成了退出，我们获取此时的一张图片，这个图片用来检测，我们只检测一次，绘制出图片上面存在的嘴唇特征，然后就开始等待退出按钮被点击。

当退出按钮点击之后，按钮变成化妆，界面上重新显示的是利用opencv获取的图像信息。