原著:
《Patch-based Probabilistic Image Quality Assessment for Face Selection and Improved Video-based Face 》
这篇论文主要思想是将人脸分成许多8×8的小block,作者认为每个block代表脸上不同部分,分别提取block的前三个交流分量,然后在feret的fa标准人脸上进行统计出均值和协方差,然后依据多维的正态分布计算出测试图相应block的概率,然后以所有block的概率和作为打分标准。但是实现出来的效果并不好,实际研究发现在标准人脸的block中的dct分量并没有符合正态分布,绝大多数的方差相差都特别大。理论上来说,如果人脸对齐的都非常准确,人脸大小和形状也差不多,这样每个block代表的人脸区域也差不多,这时候的dct应该才会有统计特性。
1 前期处理及提取特征向量部分
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc.hpp>
using namespace cv;
namespace ImageUtil {
/*
* 《Patch-based Probabilistic Image Quality Assessment for Face Selection and Improved Video-based Face Recognition》
*/
cv::Mat getfeaturevector(cv::Mat& img){
cv::Mat m;
cv::resize(img,m,cv::Size(64,64));
cv::Mat logimg(m.rows,m.cols,CV_64FC1);
for(int i=0;i!=m.rows;i++){
uchar* pr=m.ptr<uchar>(i);
double* pr_log=logimg.ptr<double>(i);
for(int j=0;j!=m.cols;j++){
pr_log[j]=std::log(pr[j]+1);
}
}
cv::Mat patchimg(8,8,CV_64FC1);
cv::Mat dctimg(8,8,CV_64FC1);
cv::Mat featureMat((m.rows-7)*(m.cols-7),3,CV_64FC1);
int index=0;
for(int r=0;r!=m.rows-7;r++){
for(int c=0;c!=m.cols-7;c++){
Mat mean,stddev;
cv::meanStdDev(logimg(cv::Rect(r,c,8,8)),mean,stddev);
for(int i=0;i!=8;i++)//normalised
for(int j=0;j!=8;j++){
double v=logimg.at<double>(r+i,c+j);
patchimg.at<double>(i,j)=(v-mean.at<double>(0,0))/stddev.at<double>(0,0);
}
cv::dct(patchimg,dctimg);//dct变换
featureMat.at<double>(index,0)=dctimg.at<double>(0,1);
featureMat.at<double>(index,1)=dctimg.at<double>(1,0);
featureMat.at<double>(index,2)=dctimg.at<double>(1,1);
index++;
}
}
std::cout<<featureMat.rows<<std::endl;
return featureMat;
}
cv::Mat getNormMean();
std::vector<cv::Mat> getNormCor();
float getImageQuality_patchbased(cv::Mat& featureMat){
cv::Mat mean_n=getNormMean();
std::vector<cv::Mat> cor_vec_n=getNormCor();
std::cout<<"cor vec size:"<<cor_vec_n.size()<<std::endl;
double score=0.0f;
for(int i=0;i<3249;i++){
// if(i<100){
// std::cout<<"mean "<<i<<":"<<mean_n.row(i)<<std::endl;
// std::cout<<"cor "<<i<<":"<<cor_vec_n[i]<<std::endl;
// }
cv::Mat x_u= featureMat.row(i)-mean_n.row(i);
cv::Mat x_u_t;
cv::transpose(x_u,x_u_t);
cv::Mat cor_inv=cor_vec_n[i].inv();
cv::Mat t1=-0.5*(x_u)*cor_inv*x_u_t;
std::cout<<"t1 size:"<<t1.size()<<std::endl;
double det=cv::determinant(cor_vec_n[i]);
if(det==0)
{
std::cout<<"error ,det =0";
}
float p=std::exp(t1.at<double>(0,0))/std::pow(2*3.1415926,1.5)/std::sqrt(det);
std::cout<<"p:"<<p<<std::endl;
score+=std::log(p);
}
std::cout<<"score:"<<score<<std::endl;
return score;
}
cv::Mat getNormMean(){
cv::FileStorage fs("./wong.xml",FileStorage::READ);
cv::Mat size;
fs["mean_size"]>>size;
cv::Mat means(size.at<ushort>(0,0),3,CV_64FC1);
for(int i=0;i<size.at<ushort>(0,0);i++){
cv::Mat rowMat;
std::stringstream ss;
ss<<"means"<<i;
fs[ss.str()]>>rowMat;
rowMat.copyTo(means.row(i));
}
fs.release();
return means;//MEAN_NORMAL;
}
std::vector<cv::Mat> getNormCor(){
std::vector<cv::Mat> vec;
cv::FileStorage fs("./wong.xml",FileStorage::READ);
cv::Mat size;
fs["cor_size"]>>size;
for(int i=0;i<size.at<ushort>(0,0);i++){
cv::Mat corMat;
std::stringstream ss;
ss<<"cors"<<i;
fs[ss.str()]>>corMat;
vec.push_back(corMat);
}
fs.release();
return vec;
}
}
2 获取协方差矩阵及均值部分
// ----------------------------------------------------------------------------
#include <iostream>
#include <dlib/image_io.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_processing.h>
#include <opencv/cv.h>
#include <opencv2/highgui.hpp>
#include "imagequality.hpp"
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/opencv/cv_image.h>
#include <dlib/opencv/to_open_cv.h>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem.hpp>
namespace fs=boost::filesystem;
using namespace std;
using namespace dlib;
using namespace cv;
int main(int argc, char *argv[])
{
dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
shape_predictor sp;
deserialize("11.dat") >> sp;
fs::path imagefolder(argv[1]);
if(!fs::exists(imagefolder) || !fs::is_directory(imagefolder))
{
std::cout<<"can't find the folder or the path is not a folder";
return 0;
}
std::vector<cv::Mat> feat_train;
for(fs::directory_entry& x:fs::directory_iterator(imagefolder))
{
std::cout<<"开始提取:"<<x.path().native()<<std::endl;
Mat image1 = imread(x.path().native(), IMREAD_COLOR);
dlib::cv_image<dlib::bgr_pixel> d_img(image1);
std::vector<dlib::rectangle> faces=detector(d_img);
if(faces.size()>0){
full_object_detection shape = sp(d_img, faces[0]);
chip_details chipdetail=get_face_chip_details(shape,150,0);
matrix<bgr_pixel> face_chip;
extract_image_chip(d_img, chipdetail, face_chip);
cv::Mat face_chip_gray;
cv::cvtColor(dlib::toMat(face_chip),face_chip_gray,CV_BGR2GRAY);
cv::Mat featMats=ImageUtil::getfeaturevector(face_chip_gray);
feat_train.push_back(featMats);
}
}
std::vector<cv::Mat> train_cor;
std::vector<cv::Mat> train_mean;
std::cout<<feat_train[0].rows<<std::endl;
std::cout<<feat_train.size()<<std::endl;
for(int i=0;i<feat_train[0].rows;i++){
cv::Mat x_all(feat_train.size(),3,CV_32FC1);
for(int j=0;j<feat_train.size();j++){
feat_train[j].row(i).copyTo(x_all.row(j));
}
cv::Mat covar,means;
cv::calcCovarMatrix(x_all,covar,means,CV_COVAR_NORMAL|CV_COVAR_ROWS);
train_cor.push_back(covar);
train_mean.push_back(means);
//std::cout<<means<<std::endl;
}
cv::FileStorage fs("./wong.xml",cv::FileStorage::WRITE);
fs<<"mean_size"<<cv::Mat(1,1,CV_16UC1,cv::Scalar(train_mean.size()));
for(int i=0;i<train_mean.size();i++){
std::stringstream ss;
ss<<"means"<<i;
fs<<ss.str()<<train_mean[i];
}
fs<<"cor_size"<<cv::Mat(1,1,CV_16UC1,cv::Scalar(train_cor.size()));
for(int i=0;i<train_cor.size();i++){
std::stringstream ss;
ss<<"cors"<<i;
fs<<ss.str()<<train_cor[i];
}
fs.release();
waitKey(0);
return 0;
}
3,人脸评分部分
// ----------------------------------------------------------------------------
#include <iostream>
#include <dlib/image_io.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_processing.h>
#include <opencv/cv.h>
#include <opencv2/highgui.hpp>
#include "imagequality.hpp"
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/opencv/cv_image.h>
#include <dlib/opencv/to_open_cv.h>
using namespace std;
using namespace dlib;
using namespace cv;
int main(int argc, char *argv[])
{
Mat image1 = imread(argv[1], IMREAD_COLOR);
if(!image1.data)
{
printf( "No image data \n" );
return -1;
}
namedWindow( "Display Image", CV_WINDOW_AUTOSIZE );
imshow( "Display Image", image1);
dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
shape_predictor sp;
deserialize("11.dat") >> sp;
dlib::cv_image<dlib::bgr_pixel> d_img(image1);
std::vector<dlib::rectangle> faces=detector(d_img);
if(faces.size()>0){
full_object_detection shape = sp(d_img, faces[0]);
chip_details chipdetail=get_face_chip_details(shape,150,0);
matrix<bgr_pixel> face_chip;
extract_image_chip(d_img, chipdetail, face_chip);
cv::Mat face_chip_gray;
cv::cvtColor(dlib::toMat(face_chip),face_chip_gray,CV_BGR2GRAY);
cv::resize(face_chip_gray,face_chip_gray,cv::Size(64,64));
cv::Mat fe= ImageUtil::getfeaturevector(face_chip_gray);
ImageUtil::getImageQuality_patchbased(fe);
}
waitKey(0);
return 0;
}