深度学习速成版03---实战Keras+ CNN+搭建表情识别系统

实战Keras+ CNN+搭建表情识别系统

环境搭建

conda create -n emotion_detect python=3.7

进入环境
source activate emotion_detect
conda activate emotion_detect
 
退出环境
source deactivate
conda deactivate
列出环境
conda env list
conda info --env
conda info -e
删除环境
conda remove -n emotion_detect --all

导出当前工程依赖的python包清单

pip3 freeze > requirements.txt
pip3 install -r requirements.txt

fer2013人脸表情数据集简介

fer2013人脸表情数据集由35886张人脸表情图片组成，其中，测试图（Training）28708张，公共验证图（PublicTest）和私有验证图（PrivateTest）各3589张，每张图片是由大小固定为48×48的灰度图像组成，共有7种表情，分别对应于数字标签0-6，具体表情对应的标签和中英文如下：0 anger 生气； 1 disgust 厌恶； 2 fear 恐惧； 3 happy 开心； 4 sad 伤心；5 surprised 惊讶； 6 normal 中性。

但是，数据集并没有直接给出图片，而是将表情、图片数据、用途的数据保存到csv文件中，如下图所示，

如上图所示，第一张图是csv文件的开头，第一行是表头，说明每列数据的含义，第一列表示表情标签，第二列即为图片数据，这里是原始的图片数据，最后一列为用途。

将表情图片提取出来

知道数据结构以后，就好办了，使用pandas解析csv文件，再将原始图片数据保存为jpg文件，并根据用途和标签标签进行分类，分别保存到对应文件夹下，代码比较简单，并且做了详细备注，直接给完整代码如下：

import cv2
# pip install opencv-python -i https://mirrors.aliyun.com/pypi/simple/
import pandas as pd
import numpy as np
import os


emotions = {
    '0': 'anger',  # 生气
    '1': 'disgust',  # 厌恶
    '2': 'fear',  # 恐惧
    '3': 'happy',  # 开心
    '4': 'sad',  # 伤心
    '5': 'surprised',  # 惊讶
    '6': 'normal',  # 中性
}
#创建文件夹
def createDir(dir):
    if os.path.exists(dir) == False:
        os.makedirs(dir)
createDir("../imgs")
def saveImageFromFer2013(file):
    #读取csv文件
    faces_data = pd.read_csv(file)
    # print(faces_data.head(10))
    # print(faces_data.columns) #['emotion', 'pixels', 'Usage']
    # print(faces_data.shape)(35887, 3)
    # print(len(faces_data)) # 35887
    #遍历csv文件内容，并将图片数据按分类保存
    print(faces_data.head())
    for index in range(len(faces_data)):
        #解析每一行csv文件内容
        emotion_data = faces_data.iloc[index, 0]
        # print(emotion_data)
        image_data = faces_data.iloc[index, 1]
        # print(image_data)
        # print(type(image_data))
        usage_data = faces_data.iloc[index, 2]

        #  ['222', '222', '210'] 变成[222, 222, 210]
        data = list(map(float, image_data.split()))
        #将图片数据转换成48*48
        image = np.array(data).reshape(48, 48)
        print(image.shape)
        #选择分类，并创建文件名
        dirName = usage_data
        emotionName = emotions[str(emotion_data)]
        # print(emotionName)
        #图片要保存的文件夹
        image_path = os.path.join(dirName, emotionName)
        print(image_path)
        # 创建“用途文件夹”和“表情”文件夹
        createDir(image_path)
        #图片文件名
        image_Name = os.path.join(image_path, str(index) + '.jpg')
        cv2.imwrite(image_Name, image)  #使用cv2实现图片与numpy数组的相互转化


saveImageFromFer2013('../datasets/fer2013/fer2013.csv')

运行结果，

运行完上面的代码后，得到3个文件夹，文件下有相应的表情的子文件夹，

子文件夹下又有相应的图片，

搭建CNN模型

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple tensorflow==1.14.0

新建model.py

from keras.models import Sequential
from keras.layers.advanced_activations import PReLU

from keras.layers import Convolution2D, BatchNormalization, AveragePooling2D, Dropout, Flatten, Dense, Activation
def simple_CNN(input_shape, num_classes):
    model = Sequential()
    model.add(Convolution2D(16, 7, 7, padding='same', input_shape=input_shape))
    model.add(PReLU())
    model.add(BatchNormalization())
    model.add(AveragePooling2D(pool_size=(5, 5),strides=(2, 2),padding='same'))
    model.add(Dropout(.5))



    model.add(Convolution2D(32, 5, 5, padding='same', input_shape=input_shape))
    model.add(PReLU())
    model.add(BatchNormalization())
    model.add(AveragePooling2D(pool_size=(3, 3),strides=(2, 2), padding='same'))
    model.add(Dropout(.5))



    model.add(Convolution2D(32, 3, 3, padding='same', input_shape=input_shape))
    model.add(PReLU())
    model.add(BatchNormalization())
    model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
    model.add(Dropout(.5))

    # 展平
    model.add(Flatten())
    model.add(Dense(1028))
    model.add(PReLU())
    model.add(Dropout(.5))
    model.add(Dense(1028))
    model.add(PReLU())
    model.add(Dropout(.5))
    model.add(Dense(num_classes))
    model.add(Activation('softmax'))
    return model
if __name__ == '__main__':
    input_shape = (48, 48, 1)
    num_classes = 7
    model = simple_CNN(input_shape, num_classes)
    model.summary()

• 新建utils.py工具类

import pandas as pd
import numpy as np
def load_data(data_file):
    faces_data = pd.read_csv(data_file)
    pixels = faces_data['pixels'].to_list()
    # 对数据进行 one_hot 编码
    df = pd.get_dummies(faces_data['emotion'])
    emotions = df.values
    w, h = 48, 48
    faces = []
    for pixel_seq in pixels:
        face = list(map(int, pixel_seq.split()))
        face = np.array(face).reshape(w, h)
        faces.append(face)
    faces = np.array(faces)
    # print(faces.shape) # (35887, 48, 48)
    # 增加一个维度
    faces = np.expand_dims(faces, -1)
    # print(faces.shape) # (35887, 48, 48)

    return faces, emotions
def preproces_input(data):
    x = np.array(data, dtype=np.float32)
    x = x/255.0
    return x

进行train

import os
from keras.callbacks import ModelCheckpoint, CSVLogger
from models import simple_CNN
from utils import load_data, preprocess_input


data_path = 'datasets/fer2013/fer2013.csv'
model_save_path = 'trained_models/simpler_CNN2.hdf5'

#加载人脸表情训练数据和对应表情标签
faces, emotions = load_data(data_path)

#人脸数据归一化，将像素值从0-255映射到0-1之间
faces = preprocess_input(faces)
#得到表情分类个数
num_classes = emotions.shape[1]

#(48, 48, 1)
image_size = faces.shape[1:]


batch_size = 128
num_epochs = 1000

model = simple_CNN(image_size, num_classes)

#断点续训
if os.path.exists(model_save_path):
    model.load_weights(model_save_path)
    # 若成功加载前面保存的参数，输出下列信息
    print("checkpoint_loaded")

#编译模型，categorical_crossentropy多分类选用
model.compile(optimizer='adam', loss='categorical_crossentropy',
                                        metrics=['accuracy'])

#记录日志
csv_logger = CSVLogger('training.log')

#保存检查点
model_checkpoint = ModelCheckpoint(model_save_path,
                                    'val_accuracy', verbose=1,
                                    save_best_only=True)

model_callbacks = [model_checkpoint, csv_logger]

#训练模型
model.fit(faces,emotions,batch_size,num_epochs,verbose=1,
                                    callbacks=model_callbacks,
                                    validation_split=.1,
                                    shuffle=True)

进行图片表情是识别

准备好人脸识别模型和表情识别模型

编写辅助工具类 utils.py

import cv2
import numpy as np
import pandas as pd
from keras.preprocessing import image

def load_image(image_path, grayscale=False, target_size=None):
    color_mode = 'grayscale'
    if grayscale==False:
        color_mode = 'rgb'
    else:
        grayscale=False
    pill_image = image.load_img(image_path, grayscale, color_mode, target_size)
    return image.img_to_array(pill_image)
def detect_faces(detect_model, gray_image_array):
    return detect_model.detectMultiScale(gray_image_array, 1.3, 5)

def get_coordinates(face_coordinates):
    x, y, width, height = face_coordinates
    return (x, x +width , y, y+height)

def draw_bounding_box(face_coordinates, image_array, color):
    x, y, width, height = face_coordinates
    cv2.rectangle(image_array, (x, y), (x + width, y + height), color, 2)
def draw_text(face_coordinates, image_array,  text, color, x_offset=0, y_offset=0,
                                                font_scale=2, thickness=2):
    x, y = face_coordinates[:2]

    cv2.putText(image_array, text, (x + x_offset, y + y_offset),
                cv2.FONT_HERSHEY_SIMPLEX, font_scale, color, thickness, cv2.LINE_AA)

• 编写 image.py
  输入一张图片可以识别图片中所有人脸及表情

import cv2
from keras.models import load_model
from utils import load_image, detect_faces, get_coordinates, preproces_input, draw_bounding_box,draw_text
import numpy as np
image_path = 'images/4.jpg'
detection_model_path = 'train_model/haarcascade_frontalface_default.xml'
emotion_model_path = 'train_model/simple_CNN.985-0.66.hdf5'
emotion_labels = {0:'angry',1:'disgust',2:'sad',3:'happy',
                4:'sad',5:'surprise',6:'neutral'}
# 加载人脸模型
face_decttion = cv2.CascadeClassifier(detection_model_path)

emotion_classifier = load_model(emotion_model_path, compile=False)
# 获取模型输入图像的宽和高尺寸
emotion_target_size = emotion_classifier.input_shape[1:3]

# 加载原始图像
rgb_image = load_image(image_path,grayscale=False)
gray_image = load_image(image_path,grayscale=True)
# 去掉维度为1的维度, (只留下宽和高, 去掉灰度维度)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
# 检测到了所有的人脸
faces = detect_faces(face_decttion, gray_image)
# 处理每一个脸
for face_coordinates in faces:
    x1, x2, y1, y2  = get_coordinates(face_coordinates)
    # 抠出 人脸  数组
    gray_face = gray_image[y1:y2, x1:x2]

    try:
        gray_face = cv2.resize(gray_face, (emotion_target_size))

    except:
        print("转换失败")
        continue
    # 归一化
    gray_face = preproces_input(gray_face)

    gray_face = np.expand_dims(gray_face, 0)

    # (1, 48, 48, 1)  # (图片数量, 高, 宽, 通道数)

    gray_face = np.expand_dims(gray_face, -1)

    emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))

    emotion_text = emotion_labels[emotion_label_arg]

    print('emotion_text = ', emotion_text)

    # 画边框
    color = (255, 0, 0)
    draw_bounding_box(face_coordinates, rgb_image, color)
    draw_text(face_coordinates, rgb_image,emotion_text, color, 0, face_coordinates[3] + 30, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('images/predict4.jpg', bgr_image)

输入
输出

实时识别表情的制作

首先先编写camera.py 目的是搞懂 opencv是如何调用摄像头的

import cv2
# 创建一个VideoCapture对象, 会调取摄像头
cap = cv2.VideoCapture(0)
while True:
    # 逐帧捕获
    ret, frame = cap.read()
    # ret 为布尔值, 代表有没有读取到图片, frame 为截取到的一帧率=的图片
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    cv2.imshow("frame", gray)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
#结束循环释放VideoCapture对象
cap.release()
cv2.destroyAllWindows()

编写video_test.py

import cv2
from keras.models import load_model
import numpy as np
from statistics import mode
from utils import preprocess_input

#opencv人脸识别模型
detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
#我们训练好的表情识别模型
classification_model_path = 'trained_models/emotion_models/simple_CNN.985-0.66.hdf5'

frame_window = 10

emotion_labels = {0:'angry',1:'disgust',2:'sad',3:'happy',
                    4:'sad',5:'surprise',6:'neutral'}

#加载人脸检测模型
face_detection = cv2.CascadeClassifier(detection_model_path)

#加载表情识别
emotion_classifier = load_model(classification_model_path)

emotion_window = []
#调起摄像头
video_capture = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.startWindowThread()
cv2.namedWindow('window_frame')

while True:
    #读取一帧
    _, frame = video_capture.read()
    #获得灰度图，并且在内存中创建一个图像对象
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #获取当前帧中的全部人脸
    faces = face_detection.detectMultiScale(gray,1.3,5)
    #对于所有发现的人脸
    for (x,y,w,h) in faces:
        #在脸周围画一个矩形框，(255,0,0)是颜色，2是线宽
        cv2.rectangle(gray,(x,y),(x+w,y+h),(255,0,0),2)

        #获取人脸图像
        face = gray[y:y+h,x:x+w]

        try:
            # shape变为(48,48)
            face = cv2.resize(face,(48,48))
        except:
            continue

        #扩充维度，shape变为(1,48,48,1)
        face = np.expand_dims(face,0)
        face = np.expand_dims(face,-1)
        #人脸数据归一化，将像素值从0-255映射到0-1之间
        face = preprocess_input(face)

        #调用我们训练好的表情识别模型，预测分类
        emotion_arg = np.argmax(emotion_classifier.predict(face))
        emotion = emotion_labels[emotion_arg]

        emotion_window.append(emotion)

        if len(emotion_window) >= frame_window:
            emotion_window.pop(0)

        try:
            #获得出现次数最多的分类
            emotion_mode = mode(emotion_window)
        except:
            continue

        #在矩形框上部，输出分类文字
        cv2.putText(gray,emotion_mode,(x,y-30), font, .7,(255,0,0),1,cv2.LINE_AA)


    try:
        #将图片从内存中显示到屏幕上
        cv2.imshow('window_frame', gray)
    except:
        continue

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

video_capture.release()
cv2.destroyAllWindows()