Dlib+Eyelike疲劳检测

Dms.pro

INCLUDEPATH +=/usr/local/opencv4.6.0/include/ \
              /usr/local/opencv4.6.0/include/opencv4/ \
              /usr/local/opencv4.6.0/include/opencv4/opencv2
INCLUDEPATH +=/usr/local/include/python3.7m/ \
              /usr/local/include/python3.7m  -Wno-unused-result -Wsign-compare  -DNDEBUG -g -fwrapv -O3 -Wall
INCLUDEPATH +=/usr/local/lib/python3.7/site-packages/numpy/core/include/numpy
INCLUDEPATH +=/opt/zzy_space/fdlib/include \
INCLUDEPATH +=/usr/local/lib/python3.7/site-packages*


LIBS +=-L/usr/local/lib/python3.7/config-3.7m-x86_64-linux-gnu/ \
       -L/usr/local/lib -lpython3.7m -lpthread -ldl  -lutil -lm  -Xlinker -export-dynamic
LIBS +=/usr/local/lib/lib*
LIBS +=/usr/local/opencv4.6.0/lib/lib*
LIBS+=/usr/local/opencv4.6.0/lib/libopencv_videoio.so
LIBS+=/usr/local/opencv4.6.0/lib/libopencv_imgcodecs.so

#LIBS +=/opt/zzy_space/dlib-19.14/build/temp.linux-x86_64-3.7/dlib_build/libdlib.a
LIBS +=/opt/zzy_space/dlib-19.14/build/dlib/libdlib.a
LIBS +=/usr/lib/liblapack.so.3


QT       += core gui

greaterThan(QT_MAJOR_VERSION, 4): QT += widgets

CONFIG += c++11

# The following define makes your compiler emit warnings if you use
# any Qt feature that has been marked deprecated (the exact warnings
# depend on your compiler). Please consult the documentation of the
# deprecated API in order to know how to port your code away from it.
DEFINES += QT_DEPRECATED_WARNINGS

# You can also make your code fail to compile if it uses deprecated APIs.
# In order to do so, uncomment the following line.
# You can also select to disable deprecated APIs only up to a certain version of Qt.
#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000    # disables all the APIs deprecated before Qt 6.0.0

SOURCES += \
    Cam.cpp \
    eyelike.cpp \
    main.cpp \
    mainwindow.cpp \
    nThread.cpp \
    yolo.cpp

HEADERS += \
    Cam.h \
    eyelike.h \
    mainwindow.h \
    nThread.h \
    yolo.h

FORMS += \
    mainwindow.ui

# Default rules for deployment.
qnx: target.path = /tmp/$${TARGET}/bin
else: unix:!android: target.path = /opt/$${TARGET}/bin
!isEmpty(target.path): INSTALLS += target

DISTFILES += \
    mydetect.py

mThread.h

#ifndef NTHREAD_H
#define NTHREAD_H
//#include "eyelike.h"
#include <QThread>

class nThread : public QThread
{
public:
    nThread();
    void run();
private:
    //Eyelike e1;
};

#endif // NTHREAD_H

mThread.cpp

#include "nThread.h"
#include <dlib/opencv.h>
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_io.h>
#include <dlib/image_processing.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/imgproc/types_c.h>
#include <opencv2/dnn.hpp>      // 深度学习模块
#include <QDebug>

using namespace std;
using namespace dlib;
using namespace cv;

Mat myframe;
VideoCapture mycap;
frontal_face_detector detector;
shape_predictor pos_modle;


nThread::nThread()
{
    //std::vector<cv::Mat> rgb(3);
    //cv::split(myframe, rgb);
}

void nThread::run()
{

    cout<<myframe.type()<<endl;
    detector = get_frontal_face_detector();
    deserialize("./shape_predictor_68_face_landmarks.dat") >> pos_modle;

    //dlib


}

eyelike.h

#ifndef EYELIKE_H
#define EYELIKE_H

#include "mainwindow.h"
#include "ui_mainwindow.h"
#include <iostream>
//#include <queue>
//#include <math.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/imgproc/types_c.h>


class Eyelike {
public:
    //main
    void dlib68(cv::Mat frame, Ui::MainWindow *cui);
    void detfatigue(cv::Mat frame, cv::VideoCapture cap, Ui::MainWindow *cui);
    void detectAndDisplay(cv::Mat frame, Ui::MainWindow *cui);
    cv::Mat findSkin (cv::Mat &frame);
    void findEyes(cv::Mat frame_gray, cv::Rect face, Ui::MainWindow *cui);

    //Find eye center
    cv::Mat floodKillEdges(cv::Mat &mat);
    cv::Point unscalePoint(cv::Point p, cv::Rect origSize);
    void scaleToFastSize(const cv::Mat &src,cv::Mat &dst);
    cv::Mat computeMatXGradient(const cv::Mat &mat);
    void testPossibleCentersFormula(int x, int y, const cv::Mat &weight,double gx, double gy, cv::Mat &out);
    cv::Point findEyeCenter(cv::Mat face, cv::Rect eye, std::string debugWindow);
    bool floodShouldPushPoint(const cv::Point &np, const cv::Mat &mat);

    //Find eye corner
    void createCornerKernels();
    void releaseCornerKernels();
    cv::Mat eyeCornerMap(const cv::Mat &region, bool left, bool left2);
    cv::Point2f findEyeCorner(cv::Mat region,bool left, bool left2);
    cv::Point2f findSubpixelEyeCorner(cv::Mat region, cv::Point maxP);

    //Tools
    bool rectInImage(cv::Rect rect, cv::Mat image);
    bool inMat(cv::Point p,int rows,int cols);
    cv::Mat matrixMagnitude(const cv::Mat &matX, const cv::Mat &matY);
    double computeDynamicThreshold(const cv::Mat &mat, double stdDevFactor);


public:

    //Ui::MainWindow *cui;
    //cv::Mat frame;
    //cv::VideoCapture cap;

};

#endif // EYELIKE_H

eyelike.c

#include "mainwindow.h"
#include "ui_mainwindow.h"
#include "eyelike.h"

using namespace std;
using namespace cv;
using namespace dlib;
extern frontal_face_detector detector;
extern shape_predictor pos_modle;

int m_counter = 0, m_total = 0;
int e_counter = 0, e_total = 0;
float MAR_THRESH = 0.18, EYE_AR_THRESH = 1.5;
float EYE_AR_CONSEC_FRAMES = 0.65, MOUTH_AR_CONSEC_FRAMES = 3;

#if CV_MAJOR_VERSION >= 4
#define CV_WINDOW_NORMAL                cv::WINDOW_NORMAL
#define CV_BGR2YCrCb                    cv::COLOR_BGR2YCrCb
#define CV_HAAR_SCALE_IMAGE             cv::CASCADE_SCALE_IMAGE
#define CV_HAAR_FIND_BIGGEST_OBJECT     cv::CASCADE_FIND_BIGGEST_OBJECT
#endif

#define kEyeLeft true
#define kEyeRight false

const bool kPlotVectorField = false;

// Size constants
const int kEyePercentTop = 25;
const int kEyePercentSide = 13;
const int kEyePercentHeight = 30;
const int kEyePercentWidth = 35;

// Preprocessing
const bool kSmoothFaceImage = false;
const float kSmoothFaceFactor = 0.005;

// Algorithm Parameters
const int kFastEyeWidth = 50;
const int kWeightBlurSize = 5;
const bool kEnableWeight = true;
const float kWeightDivisor = 1.0;
const double kGradientThreshold = 50.0;

// Postprocessing
const bool kEnablePostProcess = true;
const float kPostProcessThreshold = 0.97;

// Eye Corner
const bool kEnableEyeCorner = false;

cv::String face_cascade_name = "/opt/QTproject/Dms/res/haarcascade_frontalface_alt.xml";
cv::CascadeClassifier face_cascade;
std::string main_window_name = "Capture - Face detection";
std::string face_window_name = "Capture - Face";
cv::RNG rng(12345);
cv::Mat debugImage;
cv::Mat skinCrCbHist = cv::Mat::zeros(cv::Size(256, 256), CV_8UC1);


void findEyes(cv::Mat frame_gray, cv::Rect face);
cv::Point findEyeCenter(cv::Mat face, cv::Rect eye, std::string debugWindow);
cv::Point2f findEyeCorner(cv::Mat region,bool left, bool left2);
cv::Mat floodKillEdges(cv::Mat &mat);
cv::Mat matrixMagnitude(const cv::Mat &matX, const cv::Mat &matY);
double computeDynamicThreshold(const cv::Mat &mat, double stdDevFactor);
bool inMat(cv::Point p,int rows,int cols);
cv::Point2f findSubpixelEyeCorner(cv::Mat region, cv::Point maxP);

//---------------------------main-----------------------------
void Eyelike::detfatigue(Mat frame, VideoCapture cap, Ui::MainWindow *cui)
{
      if( !face_cascade.load( face_cascade_name ) )
            printf("--(!)Error loading face cascade, please change face_cascade_name in source code.\n");

      createCornerKernels();
      ellipse(skinCrCbHist, cv::Point(113, 155), cv::Size(23, 15),
               43.0, 0.0, 360.0, cv::Scalar(255, 255, 255), -1);

       if(cap.isOpened() )
       {
         while( true )
         {
           cap.read(frame);
           // mirror it
           cv::flip(frame, frame, 1);
           frame.copyTo(debugImage);

           // Apply the classifier to the frame
           if( !frame.empty() )
           {
             detectAndDisplay(frame, cui);
           }
           else
           {
             printf(" --(!) No captured frame -- Break!");
             break;
           }
           //imshow(main_window_name, eye.frame);
           //imshow(main_window_name,debugImage);

           int c = cv::waitKey(10);
           if( (char)c == 'q' ) { break; }
           if( (char)c == 'f' ) {
             imwrite("frame.png",frame);
           }

         }
       }

       releaseCornerKernels();

       return ;

}

void Eyelike::dlib68(Mat frame, Ui::MainWindow *cui)
{
    float mnum[2];

    //将图像转化为dlib中的BGR的形式
     dlib::cv_image<bgr_pixel> cimg(frame);
     //cout<<frame.type()<<endl;
    std::vector<dlib::rectangle> faces = detector(cimg);
    std::vector<full_object_detection> shapes;
    unsigned int faceNumber = faces.size();   //获取容器中向量的个数即人脸的个数
    for (unsigned int i = 0; i < faceNumber; i++)
    {
        shapes.push_back(pos_modle(cimg, faces[i]));
    }
    if(shapes.empty())
    {
        cout<<"shapes is empty"<<endl;
    }
    if (!shapes.empty())
    {
        for (int i = 36; i <= 68; i++)
        {       //用来画特征值的点
                cv::circle(frame, cvPoint(shapes[0].part(i).x(), shapes[0].part(i).y()), 1, cv::Scalar(0, 0, 255), -1);
        }

        //mouth points
        unsigned int y_50 = shapes[0].part(50).y();
        unsigned int y_52 = shapes[0].part(52).y();
        unsigned int y_56 = shapes[0].part(56).y();
        unsigned int y_58 = shapes[0].part(58).y();
        unsigned int x_48 = shapes[0].part(48).x();
        unsigned int x_54 = shapes[0].part(54).x();

        int height_mou1 = y_58 - y_50;
        int height_mou2 = y_56 - y_52;
        int length_mou = x_54 - x_48;
        float height_mou = (height_mou1 + height_mou2) / 2;        //眼睛上下距离
        if (height_mou == 0)  //当mouth闭合的时候，距离可能检测为0，宽高比出错
            height_mou = 1;

        float ar_mou = (height_mou / length_mou);
        mnum[0] = ar_mou;
        //在屏幕上显示眼睛的高度及宽高比
        //cout << "mouth宽高比" << ar_mou << endl;


        //Left Eye Points
        unsigned int x_36 = shapes[0].part(36).x();
        unsigned int y_37 = shapes[0].part(37).y();
        unsigned int y_38 = shapes[0].part(38).y();
        unsigned int x_39 = shapes[0].part(39).x();
        unsigned int y_40 = shapes[0].part(40).y();
        unsigned int y_41 = shapes[0].part(41).y();

        //Right Eye Points
        unsigned int x_42 = shapes[0].part(42).x();
        unsigned int y_43 = shapes[0].part(43).y();
        unsigned int y_44 = shapes[0].part(44).y();
        unsigned int x_45 = shapes[0].part(45).x();
        unsigned int y_46 = shapes[0].part(46).y();
        unsigned int y_47 = shapes[0].part(47).y();

        unsigned int height_left_eye1 = y_41 - y_37;            //37到41的纵向距离
        unsigned int height_left_eye2 = y_40 - y_38;            //38到40的纵向距离
        unsigned int height_right_eye1 = y_47 - y_43;            //
        unsigned int height_right_eye2 = y_46 - y_44;
        unsigned int length_left_eye = x_39 - x_36;
        unsigned int length_right_eye = x_45 - x_42;
        //眼睛上下距离
        float height_left_eye = (height_left_eye1 + height_left_eye2)/2;
        float height_right_eye = (height_right_eye1 + height_right_eye2)/2;
        float length_eye = (length_left_eye + length_right_eye)/2;

        if (height_left_eye == 0)  //当眼睛闭合的时候，距离可能检测为0，宽高比出错
            height_left_eye = 1;
        if (height_right_eye == 0)  //当眼睛闭合的时候，距离可能检测为0，宽高比出错
            height_right_eye = 1;
        //眼睛宽高比
        float ar_eye;
        ar_eye = (height_left_eye + height_right_eye) / (2 * length_eye);
        mnum[1] = ar_eye;

        //在屏幕上显示眼睛的高度及宽高比
        //cout << "mou:" <<mnum[0] << endl;
        //cout << "eye:" <<mnum[1] << endl;
        if (mnum[1] < EYE_AR_THRESH)
        {
            e_counter += 1;
        }
        else
        {
            // 如果连续2次都小于阈值，则表示进行了一次眨眼活动
            if(e_counter >= EYE_AR_CONSEC_FRAMES)
            {
                e_total += 1;
                cout<<"e_total:"<<e_total<<endl;
                cui->label_3->setText("Eye_Blink：" + QString::number(e_total,10));
                // 重置眼帧计数器
                e_counter = 0;

            }
        }

        if (mnum[0] > MAR_THRESH)
        {

            m_counter += 1;
            //Rollmouth += 1
        }
        else
        {
            if (m_counter >= MOUTH_AR_CONSEC_FRAMES)
            {

                m_total += 1;
                cout<<"m_total:"<<m_total<<endl;
                 cui->label_4->setText("Mou_Blink：" + QString::number(m_total,10));
                // 重置嘴帧计数器
                m_counter = 0;
            }
        }

        //if(waitKey(1) == 'q') return ;
        //imshow("Dlib标记", frame);
   }
}


void Eyelike::findEyes(cv::Mat frame, cv::Rect face, Ui::MainWindow *cui) {
  //cv::Mat faceROI = frame(face);
  //cv::Mat debugFace = faceROI;
    dlib68(frame, cui);
    cout<<"2"<<endl;
//----------------------------------------

    Mat frame_gray,faceROI;
    cvtColor(frame,frame_gray,CV_BGR2GRAY);
    //截取人脸ROI
    cv::Mat faceROI_rgb = frame(face);
    cvtColor(faceROI_rgb,faceROI,CV_BGR2GRAY);
    cv::Mat debugFace = faceROI_rgb;

  if (kSmoothFaceImage) {
    double sigma = kSmoothFaceFactor * face.width;
    GaussianBlur( faceROI, faceROI, cv::Size( 0, 0 ), sigma);
  }
  //-- Find eye regions and draw them
  int eye_region_width = face.width * (kEyePercentWidth/100.0);
  int eye_region_height = face.width * (kEyePercentHeight/100.0);
  int eye_region_top = face.height * (kEyePercentTop/100.0);
  cv::Rect leftEyeRegion(face.width*(kEyePercentSide/100.0),
                         eye_region_top,eye_region_width,eye_region_height);
  cv::Rect rightEyeRegion(face.width - eye_region_width - face.width*(kEyePercentSide/100.0),
                          eye_region_top,eye_region_width,eye_region_height);

  //-- Find Eye Centers
  cv::Point leftPupil = findEyeCenter(faceROI,leftEyeRegion,"Left Eye");
  cv::Point rightPupil = findEyeCenter(faceROI,rightEyeRegion,"Right Eye");
  // get corner regions
  cv::Rect leftRightCornerRegion(leftEyeRegion);
  leftRightCornerRegion.width -= leftPupil.x;
  leftRightCornerRegion.x += leftPupil.x;
  leftRightCornerRegion.height /= 2;
  leftRightCornerRegion.y += leftRightCornerRegion.height / 2;
  cv::Rect leftLeftCornerRegion(leftEyeRegion);
  leftLeftCornerRegion.width = leftPupil.x;
  leftLeftCornerRegion.height /= 2;
  leftLeftCornerRegion.y += leftLeftCornerRegion.height / 2;
  cv::Rect rightLeftCornerRegion(rightEyeRegion);
  rightLeftCornerRegion.width = rightPupil.x;
  rightLeftCornerRegion.height /= 2;
  rightLeftCornerRegion.y += rightLeftCornerRegion.height / 2;
  cv::Rect rightRightCornerRegion(rightEyeRegion);
  rightRightCornerRegion.width -= rightPupil.x;
  rightRightCornerRegion.x += rightPupil.x;
  rightRightCornerRegion.height /= 2;
  rightRightCornerRegion.y += rightRightCornerRegion.height / 2;
  cv::rectangle(debugFace,leftRightCornerRegion,200);
  cv::rectangle(debugFace,leftLeftCornerRegion,200);
  cv::rectangle(debugFace,rightLeftCornerRegion,200);
  cv::rectangle(debugFace,rightRightCornerRegion,200);
  // change eye centers to face coordinates
  rightPupil.x += rightEyeRegion.x;
  rightPupil.y += rightEyeRegion.y;
  leftPupil.x += leftEyeRegion.x;
  leftPupil.y += leftEyeRegion.y;
  // draw eye centers
  circle(debugFace, rightPupil, 3, 1234);
  circle(debugFace, leftPupil, 3, 1234);

  //-- Find Eye Corners
  if (kEnableEyeCorner) {
    cv::Point2f leftRightCorner = findEyeCorner(faceROI(leftRightCornerRegion), true, false);
    leftRightCorner.x += leftRightCornerRegion.x;
    leftRightCorner.y += leftRightCornerRegion.y;
    cv::Point2f leftLeftCorner = findEyeCorner(faceROI(leftLeftCornerRegion), true, true);
    leftLeftCorner.x += leftLeftCornerRegion.x;
    leftLeftCorner.y += leftLeftCornerRegion.y;
    cv::Point2f rightLeftCorner = findEyeCorner(faceROI(rightLeftCornerRegion), false, true);
    rightLeftCorner.x += rightLeftCornerRegion.x;
    rightLeftCorner.y += rightLeftCornerRegion.y;
    cv::Point2f rightRightCorner = findEyeCorner(faceROI(rightRightCornerRegion), false, false);
    rightRightCorner.x += rightRightCornerRegion.x;
    rightRightCorner.y += rightRightCornerRegion.y;
    circle(debugFace, leftRightCorner, 3, 200);
    circle(debugFace, leftLeftCorner, 3, 200);
    circle(debugFace, rightLeftCorner, 3, 200);
    circle(debugFace, rightRightCorner, 3, 200);
  }

  //imshow(face_window_name, faceROI_rgb);
  Mat temp;
  cvtColor(faceROI_rgb, temp, CV_BGR2RGB);//only RGB of Qt
  QImage srcQImage = QImage((uchar*)(temp.data), temp.cols, temp.rows, temp.cols * temp.channels(), QImage::Format_RGB888);
  srcQImage = srcQImage.scaled(cui->label_11->size(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
  cui->label_11->setPixmap(QPixmap::fromImage(srcQImage));
  cui->label_11->show();
    cout<<"1"<<endl;

}


cv::Mat Eyelike::findSkin (cv::Mat &frame) {
  cv::Mat input;
  cv::Mat output = cv::Mat(frame.rows,frame.cols, CV_8U);

  cvtColor(frame, input, CV_BGR2YCrCb);

  for (int y = 0; y < input.rows; ++y) {
    const cv::Vec3b *Mr = input.ptr<cv::Vec3b>(y);
//    uchar *Or = output.ptr<uchar>(y);
    cv::Vec3b *Or = frame.ptr<cv::Vec3b>(y);
    for (int x = 0; x < input.cols; ++x) {
      cv::Vec3b ycrcb = Mr[x];
//      Or[x] = (skinCrCbHist.at<uchar>(ycrcb[1], ycrcb[2]) > 0) ? 255 : 0;
      if(skinCrCbHist.at<uchar>(ycrcb[1], ycrcb[2]) == 0) {
        Or[x] = cv::Vec3b(0,0,0);
      }
    }
  }
  return output;
}

void Eyelike::detectAndDisplay( cv::Mat frame, Ui::MainWindow *cui) {
    std::vector<cv::Rect> faces;
    std::vector<cv::Mat> rgbChannels(3);
    cv::split(frame, rgbChannels);
    cv::Mat frame_gray = rgbChannels[2];

    //-- Detect faces
  face_cascade.detectMultiScale(frame_gray, faces, 1.1, 2, 0|cv::CASCADE_SCALE_IMAGE|cv::CASCADE_FIND_BIGGEST_OBJECT, cv::Size(150, 150) );
  for(unsigned int i = 0; i < faces.size(); i++ )
  {
    cv::rectangle(debugImage, faces[i], 1234);
  }
  //-- Show what you got
  if (faces.size() > 0)
  {

    //cv::merge(rgbChannels, frame_gray);
    findEyes(frame, faces[0], cui);
  }

}






//--------------------------FindeyeCenter-------------------------
cv::Point Eyelike::unscalePoint(cv::Point p, cv::Rect origSize)
{
  float ratio = (((float)kFastEyeWidth)/origSize.width);
  int x = round(p.x / ratio);
  int y = round(p.y / ratio);
  return cv::Point(x,y);
}

void Eyelike::scaleToFastSize(const cv::Mat &src,cv::Mat &dst)
{
  cv::resize(src, dst, cv::Size(kFastEyeWidth,(((float)kFastEyeWidth)/src.cols) * src.rows));
}

cv::Mat Eyelike::computeMatXGradient(const cv::Mat &mat) {
  cv::Mat out(mat.rows,mat.cols,CV_64F);

  for (int y = 0; y < mat.rows; ++y) {
    const uchar *Mr = mat.ptr<uchar>(y);
    double *Or = out.ptr<double>(y);

    Or[0] = Mr[1] - Mr[0];
    for (int x = 1; x < mat.cols - 1; ++x) {
      Or[x] = (Mr[x+1] - Mr[x-1])/2.0;
    }
    Or[mat.cols-1] = Mr[mat.cols-1] - Mr[mat.cols-2];
  }

  return out;
}


void Eyelike::testPossibleCentersFormula(int x, int y, const cv::Mat &weight,double gx, double gy, cv::Mat &out) {
  // for all possible centers
  for (int cy = 0; cy < out.rows; ++cy) {
    double *Or = out.ptr<double>(cy);
    const unsigned char *Wr = weight.ptr<unsigned char>(cy);
    for (int cx = 0; cx < out.cols; ++cx) {
      if (x == cx && y == cy) {
        continue;
      }
      // create a vector from the possible center to the gradient origin
      double dx = x - cx;
      double dy = y - cy;
      // normalize d
      double magnitude = sqrt((dx * dx) + (dy * dy));
      dx = dx / magnitude;
      dy = dy / magnitude;
      double dotProduct = dx*gx + dy*gy;
      dotProduct = std::max(0.0,dotProduct);
      // square and multiply by the weight
      if (kEnableWeight) {
        Or[cx] += dotProduct * dotProduct * (Wr[cx]/kWeightDivisor);
      } else {
        Or[cx] += dotProduct * dotProduct;
      }
    }
  }
}

cv::Point Eyelike::findEyeCenter(cv::Mat face, cv::Rect eye, std::string debugWindow) {
  cv::Mat eyeROIUnscaled = face(eye);
  cv::Mat eyeROI;
  scaleToFastSize(eyeROIUnscaled, eyeROI);
  // draw eye region
  //cv::rectangle(face,eye,1234);
  cv::rectangle(face,eye,1234);
  //-- Find the gradient
  cv::Mat gradientX = computeMatXGradient(eyeROI);
  cv::Mat gradientY = computeMatXGradient(eyeROI.t()).t();
  //-- Normalize and threshold the gradient
  // compute all the magnitudes
  cv::Mat mags = matrixMagnitude(gradientX, gradientY);
  //compute the threshold
  double gradientThresh = computeDynamicThreshold(mags, kGradientThreshold);
  //double gradientThresh = kGradientThreshold;
  //double gradientThresh = 0;
  //normalize
  for (int y = 0; y < eyeROI.rows; ++y) {
    double *Xr = gradientX.ptr<double>(y), *Yr = gradientY.ptr<double>(y);
    const double *Mr = mags.ptr<double>(y);
    for (int x = 0; x < eyeROI.cols; ++x) {
      double gX = Xr[x], gY = Yr[x];
      double magnitude = Mr[x];
      if (magnitude > gradientThresh) {
        Xr[x] = gX/magnitude;
        Yr[x] = gY/magnitude;
      } else {
        Xr[x] = 0.0;
        Yr[x] = 0.0;
      }
    }
  }
  //imshow(debugWindow,gradientX);
  //-- Create a blurred and inverted image for weighting
  cv::Mat weight;
  GaussianBlur( eyeROI, weight, cv::Size( kWeightBlurSize, kWeightBlurSize ), 0, 0 );
  for (int y = 0; y < weight.rows; ++y) {
    unsigned char *row = weight.ptr<unsigned char>(y);
    for (int x = 0; x < weight.cols; ++x) {
      row[x] = (255 - row[x]);
    }
  }
  //imshow(debugWindow,weight);
  //-- Run the algorithm!
  cv::Mat outSum = cv::Mat::zeros(eyeROI.rows,eyeROI.cols,CV_64F);
  // for each possible gradient location
  // Note: these loops are reversed from the way the paper does them
  // it evaluates every possible center for each gradient location instead of
  // every possible gradient location for every center.
  //printf("Eye Size: %ix%i\n",outSum.cols,outSum.rows);
  for (int y = 0; y < weight.rows; ++y) {
    const double *Xr = gradientX.ptr<double>(y), *Yr = gradientY.ptr<double>(y);
    for (int x = 0; x < weight.cols; ++x) {
      double gX = Xr[x], gY = Yr[x];
      if (gX == 0.0 && gY == 0.0) {
        continue;
      }
      testPossibleCentersFormula(x, y, weight, gX, gY, outSum);
    }
  }
  // scale all the values down, basically averaging them
  double numGradients = (weight.rows*weight.cols);
  cv::Mat out;
  outSum.convertTo(out, CV_32F,1.0/numGradients);
  //imshow(debugWindow,out);
  //-- Find the maximum point
  cv::Point maxP;
  double maxVal;
  cv::minMaxLoc(out, NULL,&maxVal,NULL,&maxP);
  //-- Flood fill the edges
  if(kEnablePostProcess) {
    cv::Mat floodClone;
    //double floodThresh = computeDynamicThreshold(out, 1.5);
    double floodThresh = maxVal * kPostProcessThreshold;
    cv::threshold(out, floodClone, floodThresh, 0.0f, cv::THRESH_TOZERO);
    if(kPlotVectorField) {
      //plotVecField(gradientX, gradientY, floodClone);
      imwrite("eyeFrame.png",eyeROIUnscaled);
    }
    cv::Mat mask = floodKillEdges(floodClone);
    //imshow(debugWindow + " Mask",mask);
    //imshow(debugWindow,out);
    // redo max
    cv::minMaxLoc(out, NULL,&maxVal,NULL,&maxP,mask);
  }
  return unscalePoint(maxP,eye);
}

bool Eyelike::floodShouldPushPoint(const cv::Point &np, const cv::Mat &mat) {
  return inMat(np, mat.rows, mat.cols);
}

cv::Mat Eyelike::floodKillEdges(cv::Mat &mat) {
  cv::rectangle(mat,cv::Rect(0,0,mat.cols,mat.rows),255);

  cv::Mat mask(mat.rows, mat.cols, CV_8U, 255);
  std::queue<cv::Point> toDo;
  toDo.push(cv::Point(0,0));
  while (!toDo.empty()) {
    cv::Point p = toDo.front();
    toDo.pop();
    if (mat.at<float>(p) == 0.0f) {
      continue;
    }
    // add in every direction
    cv::Point np(p.x + 1, p.y); // right
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
    np.x = p.x - 1; np.y = p.y; // left
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
    np.x = p.x; np.y = p.y + 1; // down
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
    np.x = p.x; np.y = p.y - 1; // up
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
    // kill it
    mat.at<float>(p) = 0.0f;
    mask.at<uchar>(p) = 0;
  }
  return mask;
}

cv::Mat *leftCornerKernel;
cv::Mat *rightCornerKernel;

// not constant because stupid opencv type signatures
float kEyeCornerKernel[4][6] = {
  {-1,-1,-1, 1, 1, 1},
  {-1,-1,-1,-1, 1, 1},
  {-1,-1,-1,-1, 0, 3},
  { 1, 1, 1, 1, 1, 1},
};

void Eyelike::createCornerKernels()
{
  rightCornerKernel = new cv::Mat(4,6,CV_32F,kEyeCornerKernel);
  leftCornerKernel = new cv::Mat(4,6,CV_32F);
  // flip horizontally
  cv::flip(*rightCornerKernel, *leftCornerKernel, 1);
}

void Eyelike::releaseCornerKernels() {
  delete leftCornerKernel;
  delete rightCornerKernel;
}

// TODO implement these
cv::Mat Eyelike::eyeCornerMap(const cv::Mat &region, bool left, bool left2) {
  cv::Mat cornerMap;

  cv::Size sizeRegion = region.size();
  cv::Range colRange(sizeRegion.width / 4, sizeRegion.width * 3 / 4);
  cv::Range rowRange(sizeRegion.height / 4, sizeRegion.height * 3 / 4);

  cv::Mat miRegion(region, rowRange, colRange);

  cv::filter2D(miRegion, cornerMap, CV_32F,
               (left && !left2) || (!left && !left2) ? *leftCornerKernel : *rightCornerKernel);

  return cornerMap;
}

cv::Point2f Eyelike::findEyeCorner(cv::Mat region, bool left, bool left2) {
  cv::Mat cornerMap = eyeCornerMap(region, left, left2);

  cv::Point maxP;
  cv::minMaxLoc(cornerMap,NULL,NULL,NULL,&maxP);

  cv::Point2f maxP2;
  maxP2 = findSubpixelEyeCorner(cornerMap, maxP);


  return maxP2;
}

cv::Point2f Eyelike::findSubpixelEyeCorner(cv::Mat region, cv::Point maxP) {

  cv::Size sizeRegion = region.size();

  cv::Mat cornerMap(sizeRegion.height * 10, sizeRegion.width * 10, CV_32F);

  cv::resize(region, cornerMap, cornerMap.size(), 0, 0, cv::INTER_CUBIC);

  cv::Point maxP2;
  cv::minMaxLoc(cornerMap, NULL,NULL,NULL,&maxP2);

  return cv::Point2f(sizeRegion.width / 2 + maxP2.x / 10,
                     sizeRegion.height / 2 + maxP2.y / 10);
}

bool Eyelike::rectInImage(cv::Rect rect, cv::Mat image) {
  return rect.x > 0 && rect.y > 0 && rect.x+rect.width < image.cols &&
  rect.y+rect.height < image.rows;
}

bool Eyelike::inMat(cv::Point p,int rows,int cols) {
  return p.x >= 0 && p.x < cols && p.y >= 0 && p.y < rows;
}

cv::Mat Eyelike::matrixMagnitude(const cv::Mat &matX, const cv::Mat &matY) {
  cv::Mat mags(matX.rows,matX.cols,CV_64F);
  for (int y = 0; y < matX.rows; ++y) {
    const double *Xr = matX.ptr<double>(y), *Yr = matY.ptr<double>(y);
    double *Mr = mags.ptr<double>(y);
    for (int x = 0; x < matX.cols; ++x) {
      double gX = Xr[x], gY = Yr[x];
      double magnitude = sqrt((gX * gX) + (gY * gY));
      Mr[x] = magnitude;
    }
  }
  return mags;
}

double Eyelike::computeDynamicThreshold(const cv::Mat &mat, double stdDevFactor) {
  cv::Scalar stdMagnGrad, meanMagnGrad;
  cv::meanStdDev(mat, meanMagnGrad, stdMagnGrad);
  double stdDev = stdMagnGrad[0] / sqrt(mat.rows*mat.cols);
  return stdDevFactor * stdDev + meanMagnGrad[0];
}

mainwindow.h

#ifndef MAINWINDOW_H
#define MAINWINDOW_H

#include "Cam.h"
#include "eyelike.h"
#include "nThread.h"
#include <dlib/opencv.h>
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing/render_face_detections.h>
#include <dlib/image_processing.h>
#include <dlib/gui_widgets.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/imgproc/types_c.h>
#include <fstream>
#include <sstream>
#include <opencv2/dnn.hpp>      // 深度学习模块
#include <string>
#include <QDebug>
#include <QMainWindow>
#include <QString>
#include <cmath>
//#include <stdlib.h>
#include <error.h>

using namespace std;

QT_BEGIN_NAMESPACE
namespace Ui { class MainWindow; }
QT_END_NAMESPACE
/*
// 自定义配置结构
struct Configuration
{
    public:
    float confThreshold; // Confidence threshold,置信度*分类分数后的阈值
    float nmsThreshold;  // Non-maximum suppression threshold,iou阈值
    float objThreshold;  //Object Confidence threshold,置信度阈值
    string modelpath;   // 模型路径
};*/


class MainWindow : public QMainWindow
{
    Q_OBJECT

public:
    MainWindow(QWidget *parent = nullptr);
    //void detfatigue2(cv::Mat &myframe, cv::VideoCapture &cap, float (&mnum)[2], Ui::MainWindow &ui);
    ~MainWindow();

private slots:
    void on_Open_clicked();

public:
    QTimer *timer;
    Ui::MainWindow *ui;
    //cv::VideoCapture capture;
    //cv::Mat mframe;
    //Cam mcam;
    Eyelike meye;


private slots:
    void on_stop_clicked();
};


#endif // MAINWINDOW_H

mainwindow.cpp

#include "mainwindow.h"
#include "ui_mainwindow.h"
#include "eyelike.h"
#include<QTimer>
#include<dlib/image_processing/frontal_face_detector.h>
#include<dlib/gui_widgets.h>
#include<dlib/image_io.h>
#include<dlib/image_processing.h>

using namespace dlib;
using namespace std;
using namespace cv;

extern VideoCapture mycap;
extern Mat myframe;
extern frontal_face_detector detector;
extern shape_predictor pos_modle;


MainWindow::MainWindow(QWidget *parent)
    : QMainWindow(parent)
    , ui(new Ui::MainWindow)
{
    ui->setupUi(this);
   //timer = new QTimer(this);
   //connect(timer, SIGNAL(timeout()), this, SLOT(importFrame()));

}

MainWindow::~MainWindow()
{
    delete ui;
}

void MainWindow::on_Open_clicked()
{
    ui->textBrowser->setText("Camera is Opening!");
    ui->textBrowser->moveCursor(QTextCursor::End);

    mycap.open(0);
    if(!mycap.isOpened())
    {
            printf("cannot open camera!\n");
    }

    while(1)
    {
         meye.detfatigue(myframe, mycap, ui);
    }


}

void MainWindow::on_stop_clicked()
{
   mycap.release();
}

/*
void mydetect(Mat &myframe)
{
    //string img_path = "./images/bus.jpg";
    string model1_path = "/opt/QTproject/Dms/weights/best.onnx";
    string model_path = "./weights/best.onnx";

    Yolov5 test;
    Net net;
    if (test.readModel(net, model_path, false))
    {
            cout << "read net ok!" << endl;
    }
    else
    {
            return ;
    }


   std:: vector<Scalar> color;
   srand(time(0));
   for (int i = 0; i < 80; i++)
   {
            int b = std::rand() % 256;
            int g = std::rand() % 256;
            int r = std::rand() % 256;
            color.push_back(Scalar(b, g, r));
   }
   std::vector<Output> result;
   //Mat img = imread(img_path);

   if (test.Detect(myframe, net, result) == true)
   {
            cout<<"hereeee"<<endl;
            test.drawPred(myframe, result, color);

   }
   else
   {
            cout << "Detect Failed!"<<endl;
   }

        //system("pause");
        return ;
}
*/

main.cpp

#include "mainwindow.h"
#include "nThread.h"
#include <QApplication>
using namespace std;
using namespace cv;

int main(int argc, char *argv[])
{
    nThread *SThread = new nThread();
    SThread->start();

    QApplication a(argc, argv);
    MainWindow w;
    w.show();

    return a.exec();
}