QT+Dlib

MainWindow.h

#ifndef MAINWINDOW_H
#define MAINWINDOW_H
#include "nThread.h"
#include <QMainWindow>
#include <QTimer>
#include <opencv.hpp>

using namespace cv;

//VideoCapture mycap;
//Mat myframe;

QT_BEGIN_NAMESPACE
namespace Ui { class MainWindow; }
QT_END_NAMESPACE

class MainWindow : public QMainWindow
{
    Q_OBJECT

public:
    MainWindow(QWidget *parent = nullptr);
    QImage MattoQt(cv::Mat src);
    ~MainWindow();

private slots:
    void on_pushButton_clicked();
    void readframe();
    void mshow();

private:
    Ui::MainWindow *ui;
    QTimer *timer;
    //Mat myframe;
    //VideoCapture mycap;
    nThread0 *Thread0;
    nThread1 *Thread1;
    nThread2 *Thread2;
};
#endif // MAINWINDOW_H
.

MainWindow.cpp

#include "mainwindow.h"
#include "ui_mainwindow.h"
#include "dlib.h"
#include "nThread.h"
#include <opencv2/imgproc/types_c.h>
#include <dlib/opencv.h>
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing/render_face_detections.h>
#include <dlib/image_processing.h>
#include <dlib/gui_widgets.h>
#include <QTime>
using namespace dlib;
using namespace cv;

VideoCapture mycap;
Mat myframe;
frontal_face_detector detector;
shape_predictor pos_modle;


extern float mnum[2];
extern int e_total, m_total;
extern int e_counter, m_counter;
extern float EYE_AR_THRESH, MOU_AR_THRESH;
extern float EYE_AR_CONSEC_FRAMES, MOUTH_AR_CONSEC_FRAMES;

MainWindow::MainWindow(QWidget *parent)
    : QMainWindow(parent)
    , ui(new Ui::MainWindow), Thread0(new nThread0), Thread1(new nThread1()), Thread2(new nThread2())
{
    ui->setupUi(this);
    timer = new QTimer(this);
    connect(timer, SIGNAL(timeout()), this, SLOT(readframe()));
    connect(Thread1, SIGNAL(finished()), this, SLOT(mshow()));

    Thread0->start();
}

void delay(int msec)
{
    QTime dieTime = QTime::currentTime().addMSecs(msec);
    while( QTime::currentTime() < dieTime )
        QCoreApplication::processEvents(QEventLoop::AllEvents, 100);
}

MainWindow::~MainWindow()
{
    if (Thread0) {
          Thread0->quit();
          Thread0->wait();
          delete Thread0;
      }

      if (Thread1) {
          Thread1->quit();
          Thread1->wait();
          delete Thread1;
      }

      if (Thread2) {
          Thread2->quit();
          Thread2->wait();
          delete Thread2;
      }
    delete ui;
}

QImage MainWindow::MattoQt(Mat frame)
{
    Mat temp;
    cvtColor(frame, temp, CV_BGR2RGB);
    QImage src = QImage((uchar*)(temp.data), temp.cols, temp.rows, temp.cols * temp.channels(), QImage::Format_RGB888);
    src = src.scaled(ui->label_11->size(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
    return src;
}

void MainWindow::readframe()
{
    mycap.read(myframe);
    Thread1->cap1 = mycap;
    Thread1->frame1 = myframe;
    Thread1->start();
    Thread1->wait();

    Thread2->cap2 = mycap;
    Thread2->frame2 = myframe;
    Thread2->start();
    Thread2->wait();
    //delay(500);
    //detfatigue(myframe, ui);
    //mshow();
    QImage img = MattoQt(myframe);
    ui->label_11->setPixmap(QPixmap::fromImage(img));
}

void MainWindow::on_pushButton_clicked()
{
    mycap.open(0);
    //detector = get_frontal_face_detector();
    //deserialize("/opt/QTproject/mDlib/shape_predictor_68_face_landmarks.dat") >> pos_modle;
    timer->start(100);

}

void MainWindow::mshow()
{
    cout<<"2"<<endl;
    //在屏幕上显示眼睛的高度及宽高比
    //cout << "mou:" <<mnum[0] << endl;
    //cout << "eye:" <<mnum[1] << endl;
    if (mnum[1] < EYE_AR_THRESH)
    {
        e_counter += 1;
    }
    else
    {
        // 如果连续2次都小于阈值，则表示进行了一次眨眼活动
        if(e_counter >= EYE_AR_CONSEC_FRAMES)
        {
            e_total += 1;
            cout<<"e_total:"<<e_total<<endl;
            ui->label_3->setText("Eye_Blink：" + QString::number(e_total,10));
            // 重置眼帧计数器
            e_counter = 0;

        }
    }

    if (mnum[0] > MOU_AR_THRESH)
    {

        m_counter += 1;
        //Rollmouth += 1
    }
    else
    {
        if (m_counter >= MOUTH_AR_CONSEC_FRAMES)
        {

            m_total += 1;
            cout<<"m_total:"<<m_total<<endl;
             ui->label_4->setText("Mou_Blink：" + QString::number(m_total,10));
            // 重置嘴帧计数器
            m_counter = 0;
        }
    }

}

mThread.h

#ifndef NTHREAD_H
#define NTHREAD_H
//#include "dlib.h"
#include <dlib/opencv.h>
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing/render_face_detections.h>
#include <dlib/image_processing.h>
#include <dlib/gui_widgets.h>
#include <opencv2/opencv.hpp>
#include <QThread>
#include <opencv2/imgproc/types_c.h>

using namespace dlib;
using namespace cv;

class nThread0 : public QThread
{
public:
    nThread0();
    void run();
};

class nThread1 : public QThread
{
public:
    nThread1();
    void run() ;
signals:
    void finished();
public:
    VideoCapture cap1;
    Mat frame1;
};

class nThread2 : public QThread
{
public:
    nThread2();
    void run() ;
    void detfatigue();
    void detectAndDisplay();
    cv::Mat findSkin (cv::Mat &frame);
    void findEyes(cv::Mat frame, cv::Rect face);

    //Find eye center
    cv::Mat floodKillEdges(cv::Mat &mat);
    cv::Point unscalePoint(cv::Point p, cv::Rect origSize);
    void scaleToFastSize(const cv::Mat &src,cv::Mat &dst);
    cv::Mat computeMatXGradient(const cv::Mat &mat);
    void testPossibleCentersFormula(int x, int y, const cv::Mat &weight,double gx, double gy, cv::Mat &out);
    cv::Point findEyeCenter(cv::Mat face, cv::Rect eye, std::string debugWindow);
    bool floodShouldPushPoint(const cv::Point &np, const cv::Mat &mat);

    //Find eye corner
    void createCornerKernels();
    void releaseCornerKernels();
    cv::Mat eyeCornerMap(const cv::Mat &region, bool left, bool left2);
    cv::Point2f findEyeCorner(cv::Mat region,bool left, bool left2);
    cv::Point2f findSubpixelEyeCorner(cv::Mat region, cv::Point maxP);

    //Tools
    bool rectInImage(cv::Rect rect, cv::Mat image);
    bool inMat(cv::Point p,int rows,int cols);
    cv::Mat matrixMagnitude(const cv::Mat &matX, const cv::Mat &matY);
    double computeDynamicThreshold(const cv::Mat &mat, double stdDevFactor);


signals:
    void finished2();
public:
    VideoCapture cap2;
    Mat frame2;
    nThread1 *sThread;
};


#endif // NTHREAD_H

mThread.cpp

#include "nThread.h"
#include "dlib.h"
#include <QMutex>

using namespace dlib;
using namespace cv;

extern frontal_face_detector detector;
extern shape_predictor pos_modle;
extern float mnum[2];
extern Mat myframe;
extern VideoCapture mycap;

#if CV_MAJOR_VERSION >= 4
#define CV_WINDOW_NORMAL                cv::WINDOW_NORMAL
#define CV_BGR2YCrCb                    cv::COLOR_BGR2YCrCb
#define CV_HAAR_SCALE_IMAGE             cv::CASCADE_SCALE_IMAGE
#define CV_HAAR_FIND_BIGGEST_OBJECT     cv::CASCADE_FIND_BIGGEST_OBJECT
#endif

#define kEyeLeft true
#define kEyeRight false

const bool kPlotVectorField = false;

// Size constants
const int kEyePercentTop = 25;
const int kEyePercentSide = 13;
const int kEyePercentHeight = 30;
const int kEyePercentWidth = 35;

// Preprocessing
const bool kSmoothFaceImage = false;
const float kSmoothFaceFactor = 0.005;

// Algorithm Parameters
const int kFastEyeWidth = 50;
const int kWeightBlurSize = 5;
const bool kEnableWeight = true;
const float kWeightDivisor = 1.0;
const double kGradientThreshold = 50.0;

// Postprocessing
const bool kEnablePostProcess = true;
const float kPostProcessThreshold = 0.97;

// Eye Corner
const bool kEnableEyeCorner = false;

cv::String face_cascade_name = "/opt/QTproject/mDlib/haarcascade_frontalface_alt.xml";
cv::CascadeClassifier face_cascade;
std::string main_window_name = "Capture - Face detection";
std::string face_window_name = "Capture - Face";
cv::RNG rng(12345);
cv::Mat debugImage;
cv::Mat skinCrCbHist = cv::Mat::zeros(cv::Size(256, 256), CV_8UC1);

nThread0::nThread0()
{

}

void nThread0::run()
{
    QMutex mymutex;
    mymutex.lock();
    detector = get_frontal_face_detector();
    deserialize("/opt/QTproject/mDlib/shape_predictor_68_face_landmarks.dat") >> pos_modle;
    mymutex.unlock();
}

nThread1::nThread1()
{

}

void nThread1::run()
{
    QMutex mutex;
    mutex.lock();
    //cap1.open(0);
    Mat frame;
    //cap1.read(frame1);
    dlib::cv_image<bgr_pixel> cimg(frame1);
    std::vector<dlib::rectangle> faces;
    faces = detector(cimg);
    std::vector<full_object_detection> shapes;
    unsigned int faceNumber = faces.size();   //获取容器中向量的个数即人脸的个数
    for (unsigned int i = 0; i < faceNumber; i++)
    {
        shapes.push_back(pos_modle(cimg, faces[i]));
    }
    for (int i = 36; i <= 68; i++)
    {       //用来画特征值的点
            cv::circle(frame1, cvPoint(shapes[0].part(i).x(), shapes[0].part(i).y()), 1, cv::Scalar(0, 0, 255), -1);
    }

    //mouth points
    unsigned int y_50 = shapes[0].part(50).y();
    unsigned int y_52 = shapes[0].part(52).y();
    unsigned int y_56 = shapes[0].part(56).y();
    unsigned int y_58 = shapes[0].part(58).y();
    unsigned int x_48 = shapes[0].part(48).x();
    unsigned int x_54 = shapes[0].part(54).x();

    int height_mou1 = y_58 - y_50;
    int height_mou2 = y_56 - y_52;
    int length_mou = x_54 - x_48;
    float height_mou = (height_mou1 + height_mou2) / 2;        //眼睛上下距离
    if (height_mou == 0)  //当mouth闭合的时候，距离可能检测为0，宽高比出错
        height_mou = 1;

    float ar_mou = (height_mou / length_mou);
    mnum[0] = ar_mou;

    //Left Eye Points
    unsigned int x_36 = shapes[0].part(36).x();
    unsigned int y_37 = shapes[0].part(37).y();
    unsigned int y_38 = shapes[0].part(38).y();
    unsigned int x_39 = shapes[0].part(39).x();
    unsigned int y_40 = shapes[0].part(40).y();
    unsigned int y_41 = shapes[0].part(41).y();

    //Right Eye Points
    unsigned int x_42 = shapes[0].part(42).x();
    unsigned int y_43 = shapes[0].part(43).y();
    unsigned int y_44 = shapes[0].part(44).y();
    unsigned int x_45 = shapes[0].part(45).x();
    unsigned int y_46 = shapes[0].part(46).y();
    unsigned int y_47 = shapes[0].part(47).y();

    unsigned int height_left_eye1 = y_41 - y_37;            //37到41的纵向距离
    unsigned int height_left_eye2 = y_40 - y_38;            //38到40的纵向距离
    unsigned int height_right_eye1 = y_47 - y_43;            //
    unsigned int height_right_eye2 = y_46 - y_44;
    unsigned int length_left_eye = x_39 - x_36;
    unsigned int length_right_eye = x_45 - x_42;
    //眼睛上下距离
    float height_left_eye = (height_left_eye1 + height_left_eye2)/2;
    float height_right_eye = (height_right_eye1 + height_right_eye2)/2;
    float length_eye = (length_left_eye + length_right_eye)/2;

    if (height_left_eye == 0)  //当眼睛闭合的时候，距离可能检测为0，宽高比出错
        height_left_eye = 1;
    if (height_right_eye == 0)  //当眼睛闭合的时候，距离可能检测为0，宽高比出错
        height_right_eye = 1;
    //眼睛宽高比
    float ar_eye;
    ar_eye = (height_left_eye + height_right_eye) / (2 * length_eye);
    mnum[1] = ar_eye;

    mutex.unlock();
    emit finished();
    //cout<<"1"<<endl;
}

void nThread1::finished()
{

}

nThread2::nThread2()
{

}

void nThread2::run()
{
    QMutex mutex;
    mutex.lock();
    nThread2::detfatigue();
    cout<<"3"<<endl;
    mutex.unlock();
}

void nThread2::detfatigue()
{
    //sThread->start();
    //sThread->wait();
    cout<<"4"<<endl;
    if( !face_cascade.load( face_cascade_name ) )
               printf("--(!)Error loading face cascade, please change face_cascade_name in source code.\n");

         createCornerKernels();
         ellipse(skinCrCbHist, cv::Point(113, 155), cv::Size(23, 15),
                  43.0, 0.0, 360.0, cv::Scalar(255, 255, 255), -1);

          if(cap2.isOpened() )
          {
            //while( true )
            //{
              //cap2.read(frame2);
              // mirror it
              cv::flip(frame2, frame2, 1);
              frame2.copyTo(debugImage);
              // Apply the classifier to the frame
              if( !frame2.empty() )
              {
                detectAndDisplay();
              }
              else
              {
                printf(" --(!) No captured frame -- Break!");
                //break;
              }
              //imshow(main_window_name, eye.frame);
              //imshow(main_window_name,debugImage);

              //int c = cv::waitKey(10);
              //if( (char)c == 'q' ) { break; }
              //if( (char)c == 'f' ) {
              //imwrite("frame.png",frame2);
              //}


          }

          releaseCornerKernels();

}

void nThread2::findEyes(cv::Mat frame, cv::Rect face)
{
    Mat frame_gray,faceROI;
    cvtColor(frame,frame_gray,CV_BGR2GRAY);
    //截取人脸ROI
    cv::Mat faceROI_rgb = frame(face);
    cvtColor(faceROI_rgb,faceROI,CV_BGR2GRAY);
    cv::Mat debugFace = faceROI_rgb;

     if (kSmoothFaceImage) {
       double sigma = kSmoothFaceFactor * face.width;
       GaussianBlur( faceROI, faceROI, cv::Size( 0, 0 ), sigma);
     }
     //-- Find eye regions and draw them
     int eye_region_width = face.width * (kEyePercentWidth/100.0);
     int eye_region_height = face.width * (kEyePercentHeight/100.0);
     int eye_region_top = face.height * (kEyePercentTop/100.0);
     cv::Rect leftEyeRegion(face.width*(kEyePercentSide/100.0),eye_region_top,eye_region_width,eye_region_height);
     cv::Rect rightEyeRegion(face.width - eye_region_width - face.width*(kEyePercentSide/100.0),eye_region_top,eye_region_width,eye_region_height);
     //-- Find Eye Centers
     cv::Point leftPupil = findEyeCenter(faceROI,leftEyeRegion,"Left Eye");
     cv::Point rightPupil = findEyeCenter(faceROI,rightEyeRegion,"Right Eye");
     // get corner regions
     cv::Rect leftRightCornerRegion(leftEyeRegion);
     leftRightCornerRegion.width -= leftPupil.x;
     leftRightCornerRegion.x += leftPupil.x;
     leftRightCornerRegion.height /= 2;
     leftRightCornerRegion.y += leftRightCornerRegion.height / 2;
     cv::Rect leftLeftCornerRegion(leftEyeRegion);
     leftLeftCornerRegion.width = leftPupil.x;
     leftLeftCornerRegion.height /= 2;
     leftLeftCornerRegion.y += leftLeftCornerRegion.height / 2;
     cv::Rect rightLeftCornerRegion(rightEyeRegion);
     rightLeftCornerRegion.width = rightPupil.x;
     rightLeftCornerRegion.height /= 2;
     rightLeftCornerRegion.y += rightLeftCornerRegion.height / 2;
     cv::Rect rightRightCornerRegion(rightEyeRegion);
     rightRightCornerRegion.width -= rightPupil.x;
     rightRightCornerRegion.x += rightPupil.x;
     rightRightCornerRegion.height /= 2;
     rightRightCornerRegion.y += rightRightCornerRegion.height / 2;
     cv::rectangle(debugFace,leftRightCornerRegion,200);
     cv::rectangle(debugFace,leftLeftCornerRegion,200);
     cv::rectangle(debugFace,rightLeftCornerRegion,200);
     cv::rectangle(debugFace,rightRightCornerRegion,200);
     // change eye centers to face coordinates
     rightPupil.x += rightEyeRegion.x;
     rightPupil.y += rightEyeRegion.y;
     leftPupil.x += leftEyeRegion.x;
     leftPupil.y += leftEyeRegion.y;
     // draw eye centers
     circle(debugFace, rightPupil, 3, 1234);
     circle(debugFace, leftPupil, 3, 1234);
     //-- Find Eye Corners
       if (kEnableEyeCorner) {
         cv::Point2f leftRightCorner = findEyeCorner(faceROI(leftRightCornerRegion), true, false);
         leftRightCorner.x += leftRightCornerRegion.x;
         leftRightCorner.y += leftRightCornerRegion.y;
         cv::Point2f leftLeftCorner = findEyeCorner(faceROI(leftLeftCornerRegion), true, true);
         leftLeftCorner.x += leftLeftCornerRegion.x;
         leftLeftCorner.y += leftLeftCornerRegion.y;
         cv::Point2f rightLeftCorner = findEyeCorner(faceROI(rightLeftCornerRegion), false, true);
         rightLeftCorner.x += rightLeftCornerRegion.x;
         rightLeftCorner.y += rightLeftCornerRegion.y;
         cv::Point2f rightRightCorner = findEyeCorner(faceROI(rightRightCornerRegion), false, false);
         rightRightCorner.x += rightRightCornerRegion.x;
         rightRightCorner.y += rightRightCornerRegion.y;
         circle(debugFace, leftRightCorner, 3, 200);
         circle(debugFace, leftLeftCorner, 3, 200);
         circle(debugFace, rightLeftCorner, 3, 200);
         circle(debugFace, rightRightCorner, 3, 200);
       }
}

cv::Mat nThread2::findSkin (cv::Mat &frame) {
  cv::Mat input;
  cv::Mat output = cv::Mat(frame.rows,frame.cols, CV_8U);

  cvtColor(frame, input, CV_BGR2YCrCb);

  for (int y = 0; y < input.rows; ++y) {
    const cv::Vec3b *Mr = input.ptr<cv::Vec3b>(y);
//    uchar *Or = output.ptr<uchar>(y);
    cv::Vec3b *Or = frame.ptr<cv::Vec3b>(y);
    for (int x = 0; x < input.cols; ++x) {
      cv::Vec3b ycrcb = Mr[x];
//      Or[x] = (skinCrCbHist.at<uchar>(ycrcb[1], ycrcb[2]) > 0) ? 255 : 0;
      if(skinCrCbHist.at<uchar>(ycrcb[1], ycrcb[2]) == 0) {
        Or[x] = cv::Vec3b(0,0,0);
      }
    }
  }
  return output;
}

void nThread2::detectAndDisplay() {
    std::vector<cv::Rect> faces;
    std::vector<cv::Mat> rgbChannels(3);
    cv::split(frame2, rgbChannels);
    cv::Mat frame_gray = rgbChannels[2];

    //-- Detect faces
  face_cascade.detectMultiScale(frame_gray, faces, 1.1, 2, 0|cv::CASCADE_SCALE_IMAGE|cv::CASCADE_FIND_BIGGEST_OBJECT, cv::Size(150, 150) );
  for(unsigned int i = 0; i < faces.size(); i++ )
  {
    cv::rectangle(debugImage, faces[i], 1234);
  }
  //-- Show what you got
  if (faces.size() > 0)
  {

    //cv::merge(rgbChannels, frame_gray);
    findEyes(frame2, faces[0]);
  }

}

cv::Point nThread2::unscalePoint(cv::Point p, cv::Rect origSize)
{
  float ratio = (((float)kFastEyeWidth)/origSize.width);
  int x = round(p.x / ratio);
  int y = round(p.y / ratio);
  return cv::Point(x,y);
}

void nThread2::scaleToFastSize(const cv::Mat &src,cv::Mat &dst)
{
  cv::resize(src, dst, cv::Size(kFastEyeWidth,(((float)kFastEyeWidth)/src.cols) * src.rows));
}

cv::Mat nThread2::computeMatXGradient(const cv::Mat &mat) {
  cv::Mat out(mat.rows,mat.cols,CV_64F);

  for (int y = 0; y < mat.rows; ++y) {
    const uchar *Mr = mat.ptr<uchar>(y);
    double *Or = out.ptr<double>(y);

    Or[0] = Mr[1] - Mr[0];
    for (int x = 1; x < mat.cols - 1; ++x) {
      Or[x] = (Mr[x+1] - Mr[x-1])/2.0;
    }
    Or[mat.cols-1] = Mr[mat.cols-1] - Mr[mat.cols-2];
  }

  return out;
}

void nThread2::testPossibleCentersFormula(int x, int y, const cv::Mat &weight,double gx, double gy, cv::Mat &out) {
  // for all possible centers
  for (int cy = 0; cy < out.rows; ++cy) {
    double *Or = out.ptr<double>(cy);
    const unsigned char *Wr = weight.ptr<unsigned char>(cy);
    for (int cx = 0; cx < out.cols; ++cx) {
      if (x == cx && y == cy) {
        continue;
      }
      // create a vector from the possible center to the gradient origin
      double dx = x - cx;
      double dy = y - cy;
      // normalize d
      double magnitude = sqrt((dx * dx) + (dy * dy));
      dx = dx / magnitude;
      dy = dy / magnitude;
      double dotProduct = dx*gx + dy*gy;
      dotProduct = std::max(0.0,dotProduct);
      // square and multiply by the weight
      if (kEnableWeight) {
        Or[cx] += dotProduct * dotProduct * (Wr[cx]/kWeightDivisor);
      } else {
              Or[cx] += dotProduct * dotProduct;
      }
    }
  }
}

cv::Point nThread2::findEyeCenter(cv::Mat face, cv::Rect eye, std::string debugWindow) {
  cv::Mat eyeROIUnscaled = face(eye);
  cv::Mat eyeROI;
  scaleToFastSize(eyeROIUnscaled, eyeROI);
  // draw eye region
  //cv::rectangle(face,eye,1234);
  cv::rectangle(face,eye,1234);
  //-- Find the gradient
  cv::Mat gradientX = computeMatXGradient(eyeROI);
  cv::Mat gradientY = computeMatXGradient(eyeROI.t()).t();
  //-- Normalize and threshold the gradient
  // compute all the magnitudes
  cv::Mat mags = matrixMagnitude(gradientX, gradientY);
  //compute the threshold
  double gradientThresh = computeDynamicThreshold(mags, kGradientThreshold);
  //double gradientThresh = kGradientThreshold;
  //double gradientThresh = 0;
  //normalize
  for (int y = 0; y < eyeROI.rows; ++y) {
    double *Xr = gradientX.ptr<double>(y), *Yr = gradientY.ptr<double>(y);
    const double *Mr = mags.ptr<double>(y);
    for (int x = 0; x < eyeROI.cols; ++x) {
        double gX = Xr[x], gY = Yr[x];
              double magnitude = Mr[x];
              if (magnitude > gradientThresh) {
                Xr[x] = gX/magnitude;
                Yr[x] = gY/magnitude;
              } else {
                Xr[x] = 0.0;
                Yr[x] = 0.0;
              }
            }
          }
          //imshow(debugWindow,gradientX);
          //-- Create a blurred and inverted image for weighting
          cv::Mat weight;
          GaussianBlur( eyeROI, weight, cv::Size( kWeightBlurSize, kWeightBlurSize ), 0, 0 );
          for (int y = 0; y < weight.rows; ++y) {
            unsigned char *row = weight.ptr<unsigned char>(y);
            for (int x = 0; x < weight.cols; ++x) {
              row[x] = (255 - row[x]);
            }
          }
          //-- Run the algorithm!
            cv::Mat outSum = cv::Mat::zeros(eyeROI.rows,eyeROI.cols,CV_64F);
            // for each possible gradient location
            // Note: these loops are reversed from the way the paper does them
            // it evaluates every possible center for each gradient location instead of
            // every possible gradient location for every center.
            //printf("Eye Size: %ix%i\n",outSum.cols,outSum.rows);
            for (int y = 0; y < weight.rows; ++y) {
              const double *Xr = gradientX.ptr<double>(y), *Yr = gradientY.ptr<double>(y);
              for (int x = 0; x < weight.cols; ++x) {
                double gX = Xr[x], gY = Yr[x];
                if (gX == 0.0 && gY == 0.0) {
                  continue;
                }
                testPossibleCentersFormula(x, y, weight, gX, gY, outSum);
              }
            }
            // scale all the values down, basically averaging them
            double numGradients = (weight.rows*weight.cols);
            cv::Mat out;
            outSum.convertTo(out, CV_32F,1.0/numGradients);
              //imshow(debugWindow,out);
              //-- Find the maximum point
              cv::Point maxP;
              double maxVal;
              cv::minMaxLoc(out, NULL,&maxVal,NULL,&maxP);
              //-- Flood fill the edges
              if(kEnablePostProcess) {
                cv::Mat floodClone;
                //double floodThresh = computeDynamicThreshold(out, 1.5);
                double floodThresh = maxVal * kPostProcessThreshold;
                cv::threshold(out, floodClone, floodThresh, 0.0f, cv::THRESH_TOZERO);
                if(kPlotVectorField) {
                  //plotVecField(gradientX, gradientY, floodClone);
                  imwrite("eyeFrame.png",eyeROIUnscaled);
                }
                cv::Mat mask = floodKillEdges(floodClone);
                //imshow(debugWindow + " Mask",mask);
                //imshow(debugWindow,out);
                // redo max
                cv::minMaxLoc(out, NULL,&maxVal,NULL,&maxP,mask);
              }
              return unscalePoint(maxP,eye);
}

bool nThread2::floodShouldPushPoint(const cv::Point &np, const cv::Mat &mat) {
  return inMat(np, mat.rows, mat.cols);
}

cv::Mat nThread2::floodKillEdges(cv::Mat &mat) {
  cv::rectangle(mat,cv::Rect(0,0,mat.cols,mat.rows),255);

  cv::Mat mask(mat.rows, mat.cols, CV_8U, 255);
  std::queue<cv::Point> toDo;
  toDo.push(cv::Point(0,0));
  while (!toDo.empty()) {
    cv::Point p = toDo.front();
    toDo.pop();
    if (mat.at<float>(p) == 0.0f) {
      continue;
    }
    // add in every direction
    cv::Point np(p.x + 1, p.y); // right
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
    np.x = p.x - 1; np.y = p.y; // left
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
    np.x = p.x; np.y = p.y + 1; // down
    if (floodShouldPushPoint(np, mat)) toDo.push(np);
        np.x = p.x; np.y = p.y + 1; // down
        if (floodShouldPushPoint(np, mat)) toDo.push(np);
        np.x = p.x; np.y = p.y - 1; // up
        if (floodShouldPushPoint(np, mat)) toDo.push(np);
        // kill it
        mat.at<float>(p) = 0.0f;
        mask.at<uchar>(p) = 0;
      }
      return mask;
}

cv::Mat *leftCornerKernel;
cv::Mat *rightCornerKernel;

// not constant because stupid opencv type signatures
float kEyeCornerKernel[4][6] = {
  {-1,-1,-1, 1, 1, 1},
  {-1,-1,-1,-1, 1, 1},
  {-1,-1,-1,-1, 0, 3},
  { 1, 1, 1, 1, 1, 1},
};

void nThread2::createCornerKernels()
{
  rightCornerKernel = new cv::Mat(4,6,CV_32F,kEyeCornerKernel);
  leftCornerKernel = new cv::Mat(4,6,CV_32F);
  // flip horizontally
  cv::flip(*rightCornerKernel, *leftCornerKernel, 1);
}

void nThread2::releaseCornerKernels() {
  delete leftCornerKernel;
  delete rightCornerKernel;
}

// TODO implement these
cv::Mat nThread2::eyeCornerMap(const cv::Mat &region, bool left, bool left2) {
  cv::Mat cornerMap;

  cv::Size sizeRegion = region.size();
  cv::Range colRange(sizeRegion.width / 4, sizeRegion.width * 3 / 4);
  cv::Range rowRange(sizeRegion.height / 4, sizeRegion.height * 3 / 4);

  cv::Mat miRegion(region, rowRange, colRange);

  cv::filter2D(miRegion, cornerMap, CV_32F,
               (left && !left2) || (!left && !left2) ? *leftCornerKernel : *rightCornerKernel);

  return cornerMap;
}

cv::Point2f nThread2::findEyeCorner(cv::Mat region, bool left, bool left2) {
  cv::Mat cornerMap = eyeCornerMap(region, left, left2);

  cv::Point maxP;
  cv::minMaxLoc(cornerMap,NULL,NULL,NULL,&maxP);

  cv::Point2f maxP2;
  maxP2 = findSubpixelEyeCorner(cornerMap, maxP);


  return maxP2;
}

cv::Point2f nThread2::findSubpixelEyeCorner(cv::Mat region, cv::Point maxP) {

  cv::Size sizeRegion = region.size();

  cv::Mat cornerMap(sizeRegion.height * 10, sizeRegion.width * 10, CV_32F);

  cv::resize(region, cornerMap, cornerMap.size(), 0, 0, cv::INTER_CUBIC);

  cv::Point maxP2;
  cv::minMaxLoc(cornerMap, NULL,NULL,NULL,&maxP2);

  return cv::Point2f(sizeRegion.width / 2 + maxP2.x / 10,
                     sizeRegion.height / 2 + maxP2.y / 10);
}

bool nThread2::rectInImage(cv::Rect rect, cv::Mat image) {
  return rect.x > 0 && rect.y > 0 && rect.x+rect.width < image.cols && rect.y+rect.height < image.rows;
}

bool nThread2::inMat(cv::Point p,int rows,int cols) {
  return p.x >= 0 && p.x < cols && p.y >= 0 && p.y < rows;
}

cv::Mat nThread2::matrixMagnitude(const cv::Mat &matX, const cv::Mat &matY) {
  cv::Mat mags(matX.rows,matX.cols,CV_64F);
  for (int y = 0; y < matX.rows; ++y) {
    const double *Xr = matX.ptr<double>(y), *Yr = matY.ptr<double>(y);
    double *Mr = mags.ptr<double>(y);
    for (int x = 0; x < matX.cols; ++x) {
      double gX = Xr[x], gY = Yr[x];
      double magnitude = sqrt((gX * gX) + (gY * gY));
      Mr[x] = magnitude;
    }
  }
  return mags;
}

double nThread2::computeDynamicThreshold(const cv::Mat &mat, double stdDevFactor) {
  cv::Scalar stdMagnGrad, meanMagnGrad;
  cv::meanStdDev(mat, meanMagnGrad, stdMagnGrad);
  double stdDev = stdMagnGrad[0] / sqrt(mat.rows*mat.cols);
  return stdDevFactor * stdDev + meanMagnGrad[0];
}