caffe detect model检测模型的性能指标测试 ---ubuntu系统

对于检测模型，我们不能用简单的准确率、精度等概念评价。这里介绍FDDB提供的一套检测的评价方法（ROC），流程主要如下：

1.下载FDDB数据库 http://tamaraberg.com/faceDataset/originalPics.tar.gz

解压后得到多个目录下的图片

2.下载对应图片的标注信息 http://vis-www.cs.umass.edu/fddb/FDDB-folds.tgz

解压后得到多个记录图片路径的txt和对应的图片标注信息（其中标注的gt是以椭圆标注的，即）

写个脚本或者简易代码将所有FDDB-fold -%d.txt 整合到一个文件  allfilepath.txt;

所有FDDB-fold-%d-ellipseList.txt整合到一个文件annotFile.txt

在FDDB-folds 文件夹下创建.pl文件，将下列代码粘入

my $listfile = "allfilepath";
my $annotFile = "annotFile.txt";
foreach my $fi (1..10){
  $foldFile = sprintf("FDDB-fold-%02d.txt",$fi);
  system("cat $foldFile >> $listfile ");
  $foldFile = sprintf("FDDB-fold-%02d-ellipseList.txt", $fi);
  system("cat $foldFile >> $annotFile");
}

 

执行 perl ××.pl你可以在该目录下得到allfile.txt annotFile.txt

3.接下来我们要获得的是我们的模型在FDDB数据集上的测试结果

我用的是opencv导入模型并进行批量测试的，代码如下：

#include <iostream>

#include <cstdlib>

#include <stdio.h>

#include <opencv2/opencv.hpp>

#include <opencv2/dnn.hpp>

#include <opencv2/dnn/shape_utils.hpp>

#include "opencv2/core/ocl.hpp"

#include <fstream>

#include <string>

using namespace cv;

using namespace cv::dnn;

using namespace std;

const size_t inWidth = 300;

const size_t inHeight = 300;

const double inScaleFactor = 1;

const int widthDelta = 0;

const int heightDelta = 0;

const Scalar meanVal(104.0, 177.0, 123.0);//104 177 123

void help(){

cout << "./FaceDetect [OPTIONS]" << endl;

cout << " -h : print usage" << endl;

cout << " -b basedir : base dir of images to be tested" << endl;

cout << " -f filename : file with detections " << endl;

cout << " -o outfile : output the predicts of images in the format" << endl;

cout << " : picname" << endl;

cout << " : FaceCount" << endl;

cout << " : <face1>x y w h 1" << endl;

cout << " : <face2>x y w h 1" << endl;

}


int main(int argc, char* argv[])

{

if(argc==1||argc>4){

help();

return 0;

}

float min_confidence = 0.0;

String modelConfiguration = "./face_detector/deploy.prototxt";//模型文件～～～

String modelBinary = "./face_detector/res10_300x300_ssd_iter_100000.caffemodel"; //模型参数～～～

dnn::Net net = readNetFromCaffe(modelConfiguration, modelBinary);

if (net.empty())

{

cerr << "Can't load network by using the following files: " << endl;

cerr << "prototxt: " << modelConfiguration << endl;

cerr << "caffemodel: " << modelBinary << endl;

cerr << "Models are available here:" << endl;

cerr << "<OPENCV_SRC_DIR>/samples/dnn/face_detector" << endl;

cerr << "or here:" << endl;

cerr << "https://github.com/opencv/opencv/tree/master/samples/dnn/face_detector" << endl;

exit(-1);

}

net.setPreferableTarget(DNN_TARGET_OPENCL);

    /*检测人脸*/

    

int num_gpu_devices = cv::cuda::getCudaEnabledDeviceCount();


/*open the test list file*/

string basedir = argv[1];

string filename = argv[2];

string DetectOutFile =argv[3];

cout<<"basedir: "<<basedir<<endl;

cout<<"inputfilename "<<filename<<endl;

cout<<"outputfilename "<<DetectOutFile<<endl;

ifstream testfile(filename.c_str());

if(!testfile.is_open()){

cout<<"check out the input filename"<<endl;

return 0;

}

ofstream Outfile(DetectOutFile.c_str());

vector<Rect> LostRect;

vector<int> LostTimes;

cout<<"processing.....\n";

for (;;)

{

    

Mat frame;

string picname;

getline(testfile,picname);

string picpath = basedir+picname+".jpg";

cout<<"proc..."<<picname<<endl;

frame = imread(picpath);


        int FaceCount=0;

if (frame.empty())

{ cout<<"done..."<<endl;

waitKey();

break;

}

if (frame.channels() == 4)

cvtColor(frame, frame, COLOR_BGRA2BGR);

//! [Prepare blob]

Mat inputBlob =blobFromImage(frame, inScaleFactor,

Size(inWidth, inHeight), meanVal, false, false); //Convert Mat to batch of images

//! [Prepare blob]

net.setInput(inputBlob, "data"); //set the network input

Mat detection = net.forward("detection_out"); //compute output

vector<double> layersTimings;

Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());

        ostringstream ss;

float confidenceThreshold = min_confidence;

vector<Rect> NowFaceRect;

vector<float> nowconf;

for (int i = 0; i < detectionMat.rows; i++)

{

float confidence = detectionMat.at<float>(i, 2); //判断为人脸的概率

if (confidence > confidenceThreshold)

{

    

int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frame.cols);

int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frame.rows);

int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frame.cols);

int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frame.rows);

                

if(xLeftBottom>xRightTop){

int temp = xLeftBottom;

xLeftBottom = xRightTop;

xRightTop = temp;

}

if(yLeftBottom>yRightTop){

int temp = yLeftBottom;

yLeftBottom = yRightTop;

yRightTop = temp;

}

if(xLeftBottom>frame.cols)continue;

if(yRightTop<0)continue;

xLeftBottom = xLeftBottom - widthDelta;

yLeftBottom = yLeftBottom - heightDelta;

xRightTop = xRightTop +widthDelta;

yRightTop = yRightTop+heightDelta;

xLeftBottom=xLeftBottom<0?2:xLeftBottom;

yLeftBottom=yLeftBottom<0?2:yLeftBottom;

xRightTop =xRightTop>frame.cols?frame.cols-2:xRightTop ;

yRightTop =yRightTop >frame.rows?frame.rows-2:yRightTop;

Rect object((int)xLeftBottom, (int)yLeftBottom,

(int)(xRightTop - xLeftBottom),

(int)(yRightTop - yLeftBottom));

FaceCount++;


rectangle(frame, object, Scalar(0, 255, 0),2);

NowFaceRect.push_back(object);

nowconf.push_back(confidence);

}

}

Outfile<<picname<<endl;

Outfile << FaceCount<<endl;

for(int n=0;n<FaceCount;n++){

Outfile<<NowFaceRect[n].x<<" "<<NowFaceRect[n].y<<" "<<NowFaceRect[n].width<<" "<<NowFaceRect[n].height<<" "<<nowconf[n]<<endl;//

}

        

}

return 0;

}

 

建议将代码复制新建的cpp文件中，并命名FaceDetectDemo为传入的三个参数分别为图片的基文件路径 、输入的图片路径txt、输出的测试标注txt。为方便起见，可以新建一个.sh,将下列代码拷入，并参照修改对应参数。。。

./FaceDetectDemo \
/home/xah/Documents/FaceDection/FDDB_evaluate/originalPics/ \
/home/xah/Documents/FaceDection/FDDB_evaluate/FDDB-folds/allfilepath.txt \
/home/xah/Documents/FaceDection/FDDB_evaluate/FDDB-folds/predictDetections.txt

 

4.执行玩上述步骤，你已经得到了标准的测试图片库、所有图片的路径txt、所有的groundtruth txt、图片的测试结果txt

接下来就要生成FDDB的可执行文件了，源码可以在这上面下载http://vis-www.cs.umass.edu/fddb/evaluation.tgz

解压后，修改Makefile，

INCS = -I/usr/local/include/opencv

LIBS = -L/usr/local/lib -lopencv_core -lopencv_imgproc -lopencv_highgui
       -lopencv_ml -lopencv_video -lopencv_features2d -lopencv_calib3d 
       -lopencv_objdetect -lopencv_contrib -lopencv_legacy

如果你用的是opencv3.2之后的版本，那就在LIBS中将 lopencv_contrib 和lopencv_legacy 移除后加上 lopencv_imgcodecs

确保make之后没问题，那么就要开始运行文件了。这里同样提供了一个.sh文件，参数如下

./evaluate \
-a /home/xah/Documents/FaceDection/FDDB_evaluate/FDDB-folds/annotFile.txt \
-d /home/xah/Documents/FaceDection/FDDB_evaluate/FDDB-folds/predictDetections.txt \
-f 0 \
-i /home/xah/Documents/FaceDection/FDDB_evaluate/originalPics/ \
-l /home/xah/Documents/FaceDection/FDDB_evaluate/FDDB-folds/allfilepath.txt \
-r /home/xah/Documents/FaceDection/FDDB_evaluate/evaluation/Roc/DetDir/ \

 

其中 -a FDDB的标注信息文件

        -d 用自己模型生成的预测文件

        -f 0为矩形框 1为椭圆 2为像素

        -i 图片的基准路径

       -l 图片路径集合

       -r 生成的ROC.txt的路径

运行后可在-r参数所设置的路径中找到ContROC.txt和DiscROC.txt。

5.确保你已经安装gunplot

如果没有可以执行 gunplot sudo apt-get install gnuplot-x11

6.下载compareROC.tar.gz http://vis-www.cs.umass.edu/fddb/results.html

解压后将步骤4生成的2个文件拷贝到该目录下rocCurves的文件夹中，然后修改contROC.p

7.执行gunplot contROC.p可以在目录下得到对应的png文件。为了方便观察本模型的性能，建议注释一部分其他的模型的曲线，得到的结果如下图所示（我画的是文件中提供的所有的模型性能曲线，加上我自己的3个，所以很多，你们可以自行删减）