eye closure detection using flandmark toolbox

eye_closure_detection.h

#ifndef _EYE_CLOSURE_DETECTION_H
#define _EYE_CLOSURE_DETECTION_H

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <vector>
#include <iostream>

using namespace cv;
using namespace std;

extern void HorizonProjection(const Mat& src, Mat& dst);
extern void VerticalProjection(const Mat& src, Mat& dst);
extern void Diff(const Mat& src, Mat& dst);
extern void FindTopTwoPeakPos(const Mat& src, int& top1_pos, int& top2_pos);
extern Mat DrawProjectionMat(const Mat& projection);
extern Mat VPF_Horizon(const Mat& src, const Mat& IPF);
#endif //_EYE_CLOSURE_DETECTION_H

eye_closure_detection.cpp

#include "eye_closure_detection.h"

#define _DEBUG_INFO

void HorizonProjection(const Mat& src, Mat& dst)
{
	// accept only char type matrices
	CV_Assert(src.depth() != sizeof(uchar));

	dst.create(src.rows, 1, CV_32F);

	int i, j;
	const uchar* p;
	float* p_dst;
	for(i = 0; i < src.rows; i++){
		p = src.ptr<uchar>(i);
		p_dst = dst.ptr<float>(i);
		p_dst[0] = 0;
		for(j = 0; j < src.cols; j++){
			p_dst[0] += p[j];
		}
		p_dst[0] /= src.cols;
	}
}

void VerticalProjection(const Mat& src, Mat& dst)
{
	// accept only char type matrices
	CV_Assert(src.depth() != sizeof(uchar));

	dst.create(1, src.cols, CV_32F);

	int i, j;
	const uchar* p;
	float* p_dst = dst.ptr<float>(0);
	for(j = 0; j < src.cols; j++){
		p_dst[j] = 0;
		for(i = 0; i < src.rows; i++){
			p = src.ptr<uchar>(i);
			p_dst[j] += p[j];
		}
		p_dst[j] /= src.rows;
	}
}

void Diff(const Mat& src, Mat& dst)
{
	CV_Assert(src.type() == CV_32F);
	dst.create(src.size(), src.type());

	int total_cnt = src.total();
	dst.at<float>(0) = 0;
	for(int i = 1; i < total_cnt; i++){
		dst.at<float>(i) = abs(src.at<float>(i) - src.at<float>(i-1));
	}
}


void FindTopTwoPeakPos(const Mat& src, int& top1_pos, int& top2_pos)
{
	CV_Assert(src.type() == CV_32F);

	vector<float> vec_val;
	vector<int> vec_pos;

	int total_cnt = src.total();
	for(int i = 1; i < total_cnt-1; i++){
		if(src.at<float>(i) > src.at<float>(i-1)
			&& src.at<float>(i) > src.at<float>(i+1)){
				vec_val.push_back(src.at<float>(i));
				vec_pos.push_back(i);
		}
	}


	float top1 = -1, top2 = -1;
	int vec_pos1 = 0, vec_pos2 = 0;
	for(int i = 0; i < vec_val.size(); i++){
		if(top1 < vec_val[i]){
			top1 = vec_val[i];
			vec_pos1 = i;
		}

#ifdef _DEBUG_INFO
		printf("val(%.1f), pos(%d)\n", vec_val[i], vec_pos[i]);
#endif	//_DEBUG_INFO

	}

	for(int i = 0; i < vec_val.size(); i++){
		if(top2 < vec_val[i] && vec_pos1 != i){
			top2 = vec_val[i];
			vec_pos2 = i;
		}
	}

	top1_pos = vec_pos[vec_pos1];
	top2_pos = vec_pos[vec_pos2];
}


Mat DrawProjectionMat(const Mat& projection)
{
	CV_Assert(projection.type() == CV_32F);
	
	int total_cnt = projection.total();
	const int BIN_WIDTH = 10;
	const int BIN_HEIGHT = 256;

	Mat histImage( BIN_HEIGHT, BIN_WIDTH*total_cnt, CV_8UC3, Scalar( 0,0,0) );
	for(int i = 1; i < total_cnt; i++){
		line( histImage, Point( BIN_WIDTH*(i-1), BIN_HEIGHT - cvRound(projection.at<float>(i-1)) ) ,
			Point( BIN_WIDTH*(i), BIN_HEIGHT - cvRound(projection.at<float>(i)) ),
			Scalar( 255, 0, 0), 2, 8, 0  );
	}

	return histImage;
	/// Display
	/*namedWindow("calcHist Demo", CV_WINDOW_AUTOSIZE );
	imshow("calcHist Demo", histImage );
	waitKey(0);*/
}

Mat VPF_Horizon(const Mat& src, const Mat& IPF)
{
	// accept only char type matrices
	CV_Assert(src.depth() != sizeof(uchar));

	Mat VPF;
	VPF.create(src.rows, 1, CV_32F);

	int i, j;
	const uchar* p;
	float* p_vpf;
	const float* p_ipf;
	for(i = 0; i < src.rows; i++){
		p = src.ptr<uchar>(i);
		p_vpf = VPF.ptr<float>(i);
		p_ipf = IPF.ptr<float>(i);
		p_vpf[0] = 0;
		for(j = 0; j < src.cols; j++){
			p_vpf[0] += (p[j]-p_ipf[0])*(p[j]-p_ipf[0]);
			//p_vpf[0] += (p[j]-p_ipf[0]);
		}
		p_vpf[0] /= src.cols;
	}

	return VPF;
}

example1.cpp

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2012 Michal Uricar
 * Copyright (C) 2012 Michal Uricar
 */

//#include <cv.h>
//#include <cvaux.h>
//#include <highgui.h>
#include <opencv2/core/core.hpp>
#include "opencv2/imgproc/imgproc.hpp"
#include <opencv2/highgui/highgui.hpp>
#include "opencv2/objdetect/objdetect.hpp"
using namespace cv;

#include <cstring>
#include <cmath>

#include "../libflandmark/flandmark_detector.h"
#include "eye_closure_detection.h"
//#define _DEBUG_INFO

int detectFaceInImage(IplImage *orig, IplImage* input, CvHaarClassifierCascade* cascade, FLANDMARK_Model *model, int *bbox, double *landmarks)
{
	int ret = 0;

    // Smallest face size.
    CvSize minFeatureSize = cvSize(40, 40);
    int flags =  CV_HAAR_DO_CANNY_PRUNING;
    // How detailed should the search be.
    float search_scale_factor = 1.1f;
    CvMemStorage* storage;
    CvSeq* rects;
    int nFaces;

    storage = cvCreateMemStorage(0);
    cvClearMemStorage(storage);

    // Detect all the faces in the greyscale image.
    rects = cvHaarDetectObjects(input, cascade, storage, search_scale_factor, 2, flags, minFeatureSize);
    nFaces = rects->total;

    double t = (double)cvGetTickCount();
    for (int iface = 0; iface < (rects ? nFaces : 0); ++iface)
    {
        CvRect *r = (CvRect*)cvGetSeqElem(rects, iface);
        
        bbox[0] = r->x;
        bbox[1] = r->y;
        bbox[2] = r->x + r->width;
        bbox[3] = r->y + r->height;
        
        ret = flandmark_detect(input, bbox, model, landmarks);

        // display landmarks
        cvRectangle(orig, cvPoint(bbox[0], bbox[1]), cvPoint(bbox[2], bbox[3]), CV_RGB(255,0,0) );
        cvRectangle(orig, cvPoint(model->bb[0], model->bb[1]), cvPoint(model->bb[2], model->bb[3]), CV_RGB(0,0,255) );
        cvCircle(orig, cvPoint((int)landmarks[0], (int)landmarks[1]), 3, CV_RGB(0, 0,255), CV_FILLED);
        for (int i = 2; i < 2*model->data.options.M; i += 2)
        {
            cvCircle(orig, cvPoint(int(landmarks[i]), int(landmarks[i+1])), 3, CV_RGB(255,0,0), CV_FILLED);

        }
    }
    t = (double)cvGetTickCount() - t;
    int ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );

    if (nFaces > 0)
    {
        printf("Faces detected: %d; Detection of facial landmark on all faces took %d ms\n", nFaces, ms);
    } else {
        printf("NO Face\n");
		ret = -1;
    }
    
    cvReleaseMemStorage(&storage);

	return ret;
}


void extractEyes(Mat image, double* landmarks, Mat& left_eye, Mat& right_eye)
{
	/*
	 *    5   1    2   6	
	 *
	 *
	 *          0/7
	 *
	 *
	 *       3       4
	 *
	 */
	//Mat g_left_eye;
	//Mat g_right_eye;

	Point pt_left_up;
	Point pt_right_down;

	Point left_eye_center((landmarks[5*2]+landmarks[1*2])/2, (landmarks[5*2+1]+landmarks[1*2+1])/2);
	Point right_eye_center((landmarks[2*2]+landmarks[6*2])/2, (landmarks[2*2+1]+landmarks[6*2+1])/2);
	float eye_dist = right_eye_center.x - left_eye_center.x;
	const float EYE_WIDTH_RATIO = 0.8;
	const float EYE_HEIGHT_RATIO = 0.4;


	// left eye
	pt_left_up = Point(left_eye_center.x - eye_dist*EYE_WIDTH_RATIO/2, left_eye_center.y - eye_dist*EYE_HEIGHT_RATIO/2);
	pt_right_down = Point(left_eye_center.x + eye_dist*EYE_WIDTH_RATIO/2, left_eye_center.y + eye_dist*EYE_HEIGHT_RATIO/2);
	if(pt_left_up.x < 0)
		pt_left_up.x = 0;
	if(pt_left_up.y < 0)
		pt_left_up.y = 0;
	if(pt_right_down.x >= image.cols)
		pt_right_down.x = image.cols-1;
	if(pt_right_down.y >= image.rows)
		pt_right_down.y = image.rows-1;

#ifdef _DEBUG_INFO
	printf("pt_left_up(%d,%d), pt_right_down(%d,%d)\n", pt_left_up.x, pt_left_up.y,
		pt_right_down.x, pt_right_down.y);
#endif // _DEBUG_INFO

	left_eye = image(Rect(pt_left_up, pt_right_down));		


#ifdef _DEBUG_INFO
	// show left eye area
	rectangle(image, Rect(pt_left_up, pt_right_down), CV_RGB(0,255,0), 1);
#endif // _DEBUG_INFO

	// right eye
	pt_left_up = Point(right_eye_center.x - eye_dist*EYE_WIDTH_RATIO/2, right_eye_center.y - eye_dist*EYE_HEIGHT_RATIO/2);
	pt_right_down = Point(right_eye_center.x + eye_dist*EYE_WIDTH_RATIO/2, right_eye_center.y + eye_dist*EYE_HEIGHT_RATIO/2);
	if(pt_left_up.x < 0)
		pt_left_up.x = 0;
	if(pt_left_up.y < 0)
		pt_left_up.y = 0;
	if(pt_right_down.x >= image.cols)
		pt_right_down.x = image.cols-1;
	if(pt_right_down.y >= image.rows)
		pt_right_down.y = image.rows-1;
	right_eye = image(Rect(pt_left_up, pt_right_down));

#ifdef _DEBUG_INFO
	// show right eye area
	rectangle(image, Rect(pt_left_up, pt_right_down), CV_RGB(0,255,0), 1);
	imshow("image", image);
	waitKey(0);
#endif // _DEBUG_INFO
}


int main( int argc, char** argv ) 
{
    char flandmark_window[] = "flandmark_example1";
    double t;
    int ms;
    
    if (argc < 2)
    {
      fprintf(stderr, "Usage: flandmark_1 <path_to_input_image> [<path_to_output_image>]\n");
      exit(1);
    }
    
    cvNamedWindow(flandmark_window, CV_WINDOW_AUTOSIZE);
    
    // Haar Cascade file, used for Face Detection.
    char faceCascadeFilename[] = "haarcascade_frontalface_alt.xml";
    // Load the HaarCascade classifier for face detection.
    CvHaarClassifierCascade* faceCascade;
    faceCascade = (CvHaarClassifierCascade*)cvLoad(faceCascadeFilename, 0, 0, 0);
    if( !faceCascade )
    {
        printf("Couldnt load Face detector '%s'\n", faceCascadeFilename);
        exit(1);
    }

     // ------------- begin flandmark load model
    t = (double)cvGetTickCount();
    FLANDMARK_Model * model = flandmark_init("flandmark_model.dat");

    if (model == 0)
    {
        printf("Structure model wasn't created. Corrupted file flandmark_model.dat?\n");
        exit(1);
    }

    t = (double)cvGetTickCount() - t;
    ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );
    printf("Structure model loaded in %d ms.\n", ms);
    // ------------- end flandmark load model
    
    // input image
    IplImage *frame = cvLoadImage(argv[1]);
    if (frame == NULL)
    {
      fprintf(stderr, "Cannot open image %s. Exiting...\n", argv[1]);
      exit(1);
    }
    // convert image to grayscale
    IplImage *frame_bw = cvCreateImage(cvSize(frame->width, frame->height), IPL_DEPTH_8U, 1);
    cvConvertImage(frame, frame_bw);
    
    int *bbox = (int*)malloc(4*sizeof(int));
    double *landmarks = (double*)malloc(2*model->data.options.M*sizeof(double));
    int ret = detectFaceInImage(frame, frame_bw, faceCascade, model, bbox, landmarks);
    
    cvShowImage(flandmark_window, frame);
    cvWaitKey(0);
    
	if(0 != ret){
		printf("Landmark not detected!\n");
		return -1;
	}

    if (argc == 3)
    {
      printf("Saving image to file %s...\n", argv[2]);
      cvSaveImage(argv[2], frame);
    }

	// extract eyes
	Mat image_gray(frame_bw);
	Mat left_eye, right_eye;
	extractEyes(image_gray, landmarks, left_eye, right_eye);

	// eye closure detection
	Mat hp_l, hp_r, hp_l_diff;
	HorizonProjection(left_eye, hp_l);
	Diff(hp_l, hp_l_diff);
	Mat hist_projection, hist_projection_diff;
	hist_projection = DrawProjectionMat(hp_l);
	hist_projection_diff = DrawProjectionMat(hp_l_diff);

	/// Display  
 //   namedWindow("hist_projection", CV_WINDOW_AUTOSIZE );  
 //   imshow("hist_projection", hist_projection );  
	//namedWindow("hist_projection_diff", CV_WINDOW_AUTOSIZE );  
 //   imshow("hist_projection_diff", hist_projection_diff );  
 //   waitKey(0); 

	//int top1, top2;
	//FindTopTwoPeakPos(hp_l_diff, top1, top2);
	//line(left_eye, Point(0, top1), Point(left_eye.cols-1, top1), Scalar( 255, 0, 0), 1, 8, 0);
	//line(left_eye, Point(0, top2), Point(left_eye.cols-1, top2), Scalar( 255, 0, 0), 1, 8, 0);
	//cout << top1 << " " << top2 << endl;


	Mat vpf = VPF_Horizon(left_eye, hp_l);
	Mat vpf_diff;
	Diff(vpf, vpf_diff);
	Mat vpf_hist, vpf_diff_hist;
	vpf_hist = DrawProjectionMat(vpf);
	vpf_diff_hist = DrawProjectionMat(vpf_diff);

	cout << hp_l << endl;
	cout << vpf.size() << endl;
	cout << vpf << endl;
	/// Display  
    namedWindow("vpf_hist", CV_WINDOW_AUTOSIZE );  
    imshow("vpf_hist", vpf_hist );  
	namedWindow("vpf_diff_hist", CV_WINDOW_AUTOSIZE );  
    imshow("vpf_diff_hist", vpf_diff_hist );  
    waitKey(0); 

	int top1, top2;
	FindTopTwoPeakPos(vpf_diff, top1, top2);
	line(left_eye, Point(0, top1), Point(left_eye.cols-1, top1), Scalar( 255, 0, 0), 1, 8, 0);
	line(left_eye, Point(0, top2), Point(left_eye.cols-1, top2), Scalar( 255, 0, 0), 1, 8, 0);
	cout << top1 << " " << top2 << endl;


	/// Display  
    namedWindow("left_eye", CV_WINDOW_AUTOSIZE );  
    imshow("left_eye", left_eye);  
    waitKey(0);  
    
    // cleanup
    free(bbox);
    free(landmarks);
    cvDestroyWindow(flandmark_window);
    cvReleaseImage(&frame);
    cvReleaseImage(&frame_bw);
    cvReleaseHaarClassifierCascade(&faceCascade);
    flandmark_free(model);
}