基于频率分量的概率分布人脸质量评价

原著：

《Patch-based Probabilistic Image Quality Assessment for Face Selection and Improved Video-based Face 》

这篇论文主要思想是将人脸分成许多8×8的小block，作者认为每个block代表脸上不同部分，分别提取block的前三个交流分量，然后在feret的fa标准人脸上进行统计出均值和协方差，然后依据多维的正态分布计算出测试图相应block的概率，然后以所有block的概率和作为打分标准。但是实现出来的效果并不好，实际研究发现在标准人脸的block中的dct分量并没有符合正态分布，绝大多数的方差相差都特别大。理论上来说，如果人脸对齐的都非常准确，人脸大小和形状也差不多，这样每个block代表的人脸区域也差不多，这时候的dct应该才会有统计特性。

1 前期处理及提取特征向量部分

#include <opencv2/opencv.hpp>
#include <opencv2/imgproc.hpp>

using namespace cv;
namespace ImageUtil {

/*
 * 《Patch-based Probabilistic Image Quality Assessment for Face Selection and Improved Video-based Face Recognition》
 */
cv::Mat getfeaturevector(cv::Mat& img){
    cv::Mat m;
    cv::resize(img,m,cv::Size(64,64));
    cv::Mat logimg(m.rows,m.cols,CV_64FC1);
    for(int i=0;i!=m.rows;i++){
        uchar* pr=m.ptr<uchar>(i);
        double* pr_log=logimg.ptr<double>(i);
        for(int j=0;j!=m.cols;j++){
            pr_log[j]=std::log(pr[j]+1);
        }
    }
    cv::Mat patchimg(8,8,CV_64FC1);
    cv::Mat dctimg(8,8,CV_64FC1);
    cv::Mat featureMat((m.rows-7)*(m.cols-7),3,CV_64FC1);
    int index=0;
    for(int r=0;r!=m.rows-7;r++){
        for(int c=0;c!=m.cols-7;c++){
            Mat mean,stddev;
            cv::meanStdDev(logimg(cv::Rect(r,c,8,8)),mean,stddev);
            for(int i=0;i!=8;i++)//normalised
                for(int j=0;j!=8;j++){
                    double v=logimg.at<double>(r+i,c+j);
                    patchimg.at<double>(i,j)=(v-mean.at<double>(0,0))/stddev.at<double>(0,0);
                }
            cv::dct(patchimg,dctimg);//dct变换
            featureMat.at<double>(index,0)=dctimg.at<double>(0,1);
            featureMat.at<double>(index,1)=dctimg.at<double>(1,0);
            featureMat.at<double>(index,2)=dctimg.at<double>(1,1);
            index++;
        }
    }
    std::cout<<featureMat.rows<<std::endl;
    return featureMat;
}
cv::Mat getNormMean();
std::vector<cv::Mat> getNormCor();

float getImageQuality_patchbased(cv::Mat& featureMat){
    cv::Mat mean_n=getNormMean();
    std::vector<cv::Mat> cor_vec_n=getNormCor();
    std::cout<<"cor vec size:"<<cor_vec_n.size()<<std::endl;
    double score=0.0f;
    for(int i=0;i<3249;i++){
//        if(i<100){
//            std::cout<<"mean "<<i<<":"<<mean_n.row(i)<<std::endl;
//            std::cout<<"cor "<<i<<":"<<cor_vec_n[i]<<std::endl;
//        }
        cv::Mat x_u= featureMat.row(i)-mean_n.row(i);
        cv::Mat x_u_t;
        cv::transpose(x_u,x_u_t);
        cv::Mat cor_inv=cor_vec_n[i].inv();
        cv::Mat t1=-0.5*(x_u)*cor_inv*x_u_t;
        std::cout<<"t1 size:"<<t1.size()<<std::endl;
        double det=cv::determinant(cor_vec_n[i]);
        if(det==0)
        {
            std::cout<<"error ,det =0";
        }
        float p=std::exp(t1.at<double>(0,0))/std::pow(2*3.1415926,1.5)/std::sqrt(det);
        std::cout<<"p:"<<p<<std::endl;
        score+=std::log(p);
    }
    std::cout<<"score:"<<score<<std::endl;
    return score;

}
cv::Mat getNormMean(){
    cv::FileStorage fs("./wong.xml",FileStorage::READ);
    cv::Mat size;
    fs["mean_size"]>>size;
    cv::Mat means(size.at<ushort>(0,0),3,CV_64FC1);
    for(int i=0;i<size.at<ushort>(0,0);i++){
        cv::Mat rowMat;
        std::stringstream ss;
        ss<<"means"<<i;
        fs[ss.str()]>>rowMat;
        rowMat.copyTo(means.row(i));
    }
    fs.release();
    return means;//MEAN_NORMAL;
}
std::vector<cv::Mat> getNormCor(){
    std::vector<cv::Mat> vec;
    cv::FileStorage fs("./wong.xml",FileStorage::READ);
    cv::Mat size;
    fs["cor_size"]>>size;
    for(int i=0;i<size.at<ushort>(0,0);i++){
        cv::Mat corMat;
        std::stringstream ss;
        ss<<"cors"<<i;
        fs[ss.str()]>>corMat;
        vec.push_back(corMat);
    }
    fs.release();
    return vec;
}
}

2 获取协方差矩阵及均值部分

//  ----------------------------------------------------------------------------



#include <iostream>
#include <dlib/image_io.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_processing.h>
#include <opencv/cv.h>
#include <opencv2/highgui.hpp>
#include "imagequality.hpp"
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/opencv/cv_image.h>
#include <dlib/opencv/to_open_cv.h>

#include <boost/filesystem/path.hpp>
#include <boost/filesystem.hpp>

namespace fs=boost::filesystem;

using namespace std;
using namespace dlib;
using namespace cv;



int main(int argc, char *argv[])
{
    dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
    shape_predictor sp;
    deserialize("11.dat") >> sp;

    fs::path imagefolder(argv[1]);
    if(!fs::exists(imagefolder) || !fs::is_directory(imagefolder))
    {
        std::cout<<"can't find the folder or the path is not a folder";
        return 0;
    }
    std::vector<cv::Mat> feat_train;
    for(fs::directory_entry& x:fs::directory_iterator(imagefolder))
    {
        std::cout<<"开始提取："<<x.path().native()<<std::endl;
        Mat image1 = imread(x.path().native(), IMREAD_COLOR);
        dlib::cv_image<dlib::bgr_pixel> d_img(image1);
        std::vector<dlib::rectangle> faces=detector(d_img);
        if(faces.size()>0){
            full_object_detection shape = sp(d_img, faces[0]);
            chip_details chipdetail=get_face_chip_details(shape,150,0);

            matrix<bgr_pixel> face_chip;
            extract_image_chip(d_img, chipdetail, face_chip);

            cv::Mat face_chip_gray;
            cv::cvtColor(dlib::toMat(face_chip),face_chip_gray,CV_BGR2GRAY);
            cv::Mat featMats=ImageUtil::getfeaturevector(face_chip_gray);

            feat_train.push_back(featMats);
        }
    }

    std::vector<cv::Mat> train_cor;
    std::vector<cv::Mat> train_mean;
    std::cout<<feat_train[0].rows<<std::endl;
    std::cout<<feat_train.size()<<std::endl;
    for(int i=0;i<feat_train[0].rows;i++){
        cv::Mat x_all(feat_train.size(),3,CV_32FC1);
        for(int j=0;j<feat_train.size();j++){
            feat_train[j].row(i).copyTo(x_all.row(j));
        }
        cv::Mat covar,means;
        cv::calcCovarMatrix(x_all,covar,means,CV_COVAR_NORMAL|CV_COVAR_ROWS);
        train_cor.push_back(covar);
        train_mean.push_back(means);
        //std::cout<<means<<std::endl;
    }
    cv::FileStorage fs("./wong.xml",cv::FileStorage::WRITE);
    fs<<"mean_size"<<cv::Mat(1,1,CV_16UC1,cv::Scalar(train_mean.size()));
    for(int i=0;i<train_mean.size();i++){
        std::stringstream ss;
        ss<<"means"<<i;
        fs<<ss.str()<<train_mean[i];
    }
    fs<<"cor_size"<<cv::Mat(1,1,CV_16UC1,cv::Scalar(train_cor.size()));
    for(int i=0;i<train_cor.size();i++){
        std::stringstream ss;
        ss<<"cors"<<i;
        fs<<ss.str()<<train_cor[i];
    }
    fs.release();
    waitKey(0);

    return 0;

}

3，人脸评分部分

//  ----------------------------------------------------------------------------
#include <iostream>
#include <dlib/image_io.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_processing.h>
#include <opencv/cv.h>
#include <opencv2/highgui.hpp>
#include "imagequality.hpp"
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/opencv/cv_image.h>
#include <dlib/opencv/to_open_cv.h>

using namespace std;
using namespace dlib;
using namespace cv;

int main(int argc, char *argv[])
{
    Mat image1 = imread(argv[1], IMREAD_COLOR);
    if(!image1.data)
    {
        printf( "No image data \n" );
        return -1;
    }
    namedWindow( "Display Image", CV_WINDOW_AUTOSIZE );
    imshow( "Display Image", image1);
    dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
    shape_predictor sp;
    deserialize("11.dat") >> sp;

    dlib::cv_image<dlib::bgr_pixel> d_img(image1);
    std::vector<dlib::rectangle> faces=detector(d_img);
    if(faces.size()>0){
        full_object_detection shape = sp(d_img, faces[0]);
        chip_details chipdetail=get_face_chip_details(shape,150,0);

        matrix<bgr_pixel> face_chip;
        extract_image_chip(d_img, chipdetail, face_chip);

        cv::Mat face_chip_gray;
        cv::cvtColor(dlib::toMat(face_chip),face_chip_gray,CV_BGR2GRAY);

        cv::resize(face_chip_gray,face_chip_gray,cv::Size(64,64));
        cv::Mat fe= ImageUtil::getfeaturevector(face_chip_gray);
        ImageUtil::getImageQuality_patchbased(fe);
    }
    waitKey(0);

    return 0;
}