基于CNN+OpenCV的微表情识别项目

微表情识别

使用 OpenCV 和 PyTorch 从头开始​​实时微表情识别

Environment

• Python 3.9.7
• PyTorch 1.10.1
• OpenCV 4.5.4

使用给出的示例代码 - 使用网络摄像头或视频进行尝试 - meRecognition.py

数据收集和清理

import os
import filecmp
import shutil
Target Folder
duplicates = 0
for sRoot,sDirs,sFiles in os.walk(sourceDir):
    break
for sourceName in sFiles:
    sourceFile = sourceDir+'\\'+sourceName
    for tRoot,tDirs,tFiles in os.walk(targetDir):
        break
    if tFiles == []:
        shutil.copy(sourceFile, targetDir)
    if tFiles != []:
        for targetName in tFiles:
            targetFile = targetDir+'\\'+targetName
            compare = filecmp.cmp(sourceFile,targetFile)
            if compare == True:
                duplicates+=1
        if duplicates == 0: # only copy if there is no duplicate file in the target folder
            shutil.copy(sourceFile, targetDir)
        duplicates = 0

# face detection 
# to remove multiple face images, non face images and corrupt/non convertible images
import os
import cv2
import mediapipe as mp

sourceDir = 'D:\MER\Dataset\Expressions\duplicate_deleted\Surprise' # Source folder
targetDir = 'D:\MER\Dataset\Expressions\Face_crop\Surprise' # Target Folder

imageCount = 0 # for file naming
errorCount = 0

findFace = mp.solutions.face_detection.FaceDetection()
def faceBox(frame):#face bounding box
    try:
        frameRGB = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
    except:
        return []
    results = findFace.process(frameRGB)
    myFaces = []
    if results.detections != None:
        for face in results.detections:
            bBox = face.location_data.relative_bounding_box
            myFaces.append(bBox)
    return myFaces

for sRoot,sDirs,sFiles in os.walk(sourceDir):
    break
for sourceName in sFiles:
    sourceFile = sourceDir+'\\'+sourceName
    image = cv2.imread(sourceFile)
    faceBoxes = faceBox(image)
    if len(faceBoxes) == 1:
        imageCount+=1
        fileName = 'surprise'+str(imageCount)+'.jpg' #change file name here
        height,width,channel = image.shape
        x,y,w,h = int(faceBoxes[0].xmin*width),int(faceBoxes[0].ymin*height),int(faceBoxes[0].width*width),int(faceBoxes[0].height*height)
        # cv2.rectangle(image,(x,y),(x+w,y+h),(255,0,0),1)
        croppedImage = image[y:y+h,x:x+w]
        try:
            cv2.imwrite(targetDir+'\\'+fileName,croppedImage)
        except:
            errorCount+=1
            print(errorCount)
            continue

数据集

笔记

# CNN Model
class MERCnnModel(ImageClassificationBase):
    def __init__(self):
        super().__init__()
        self.network = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 64 x 40 x 40
    
            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 128 x 20 x 20
    
            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 256 x 10 x 10
    
            nn.Flatten(),
            nn.Linear(256*10*10, 1024),
            nn.ReLU(),
            nn.Linear(1024, 512),
            nn.ReLU(),
            nn.Linear(512, 10))
        
    def forward(self, xb):
        return self.network(xb)

训练后的模型在验证集上给出了 78% 的准确率，在测试集上给出了 81.4% 的准确率。