访问网址查看 人脸对齐的发展
https://max.book118.com/html/2017/1111/139651668.shtm
1.准备数据集:UMD 把 21个关键点 提取出5个。
2.Loss 函数使用 absloss ,如何添加 caffe absloss 查看我的其他博文。
C:\WorkSpace\SoftWare\caffe\convert_multilabel_imageset.exe --shuffle D:\UMD\result_img_96\ D:\UMD\result_img_96\new_shutter_96_5_train.txt E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\train_data_npd E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\train_label_npd 15
C:\WorkSpace\SoftWare\caffe\convert_multilabel_imageset.exe D:\UMD\result_img_96\ D:\UMD\result_img_96\shutter_96_5_val.txt E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\test_data_npd E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\test_label_npd 15
pause
# The train/test net protocol buffer definition
net: "train_5_op_abs.prototxt"
# test_iter specifies how many forward passes the test should carry out.
# In the case of MNIST, we have test batch size 100 and 100 test iterations,
# covering the full 10,000 testing images.
test_iter: 48
# Carry out testing every 500 training iterations.
test_interval: 1000
# The base learning rate, momentum and the weight decay of the network.
base_lr: 0.0004
#base_lr:0.0002
#加快梯度下降的速度
momentum: 0.95
#momentum2: 0.999
weight_decay: 0.0001
# The learning rate policy
lr_policy: "inv"
gamma: 0.0001
power: 0.75
# Display every 100 iterations
display: 1000
# The maximum number of iterations
max_iter: 300000
# snapshot intermediate results
snapshot: 10000
snapshot_prefix: "save_path_abs/"
# solver mode: CPU or GPU
solver_mode: GPU
#type:"Adam"
name: "Faceregressions"
layer {
name: "landmark_5"
type: "Data"
top: "data"
include {
phase: TRAIN
}
transform_param {
mean_value: 104
mean_value: 117
mean_value: 124
scale: 0.0078125
}
data_param {
source: "lmdb/train_data_npd"
batch_size: 64
backend: LMDB
}
}
layer {
name: "landmark_5"
type: "Data"
top: "label"
include {
phase: TRAIN
}
data_param {
source: "lmdb/train_label_npd"
batch_size: 64
backend: LMDB
}
}
layer {
name: "landmark_5"
type: "Data"
top: "data"
include {
phase: TEST
}
#transform_param {
# scale: 0.00390625
#}
transform_param {
mean_value: 104
mean_value: 117
mean_value: 124
scale: 0.0078125
}
data_param {
source: "lmdb/test_data_npd"
batch_size: 64
backend: LMDB
}
}
layer {
name: "landmark_5"
type: "Data"
top: "label"
include {
phase: TEST
}
data_param {
source: "lmdb/test_label_npd"
batch_size: 64
backend: LMDB
}
}
layer {
name: "conv1"
type: "Convolution"
bottom: "data"
top: "conv1"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 32
kernel_size: 3
pad:1
stride: 2
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu1"
type: "ReLU"
bottom: "conv1"
top: "conv1"
}
layer {
name: "conv2"
type: "Convolution"
bottom: "conv1"
top: "conv2"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 64
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu2"
type: "ReLU"
bottom: "conv2"
top: "conv2"
}
layer {
name: "conv3"
type: "Convolution"
bottom: "conv2"
top: "conv3"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu3"
type: "ReLU"
bottom: "conv3"
top: "conv3"
}
layer {
name: "conv4"
type: "Convolution"
bottom: "conv3"
top: "conv4"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu4"
type: "ReLU"
bottom: "conv4"
top: "conv4"
}
layer {
name: "conv5"
type: "Convolution"
bottom: "conv4"
top: "conv5"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu5"
type: "ReLU"
bottom: "conv5"
top: "conv5"
}
layer {
name: "conv6"
type: "Convolution"
bottom: "conv5"
top: "conv6"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu6"
type: "ReLU"
bottom: "conv6"
top: "conv6"
}
layer {
name: "ip1"
type: "InnerProduct"
bottom: "conv6"
top: "ip1"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
inner_product_param {
num_output: 256
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu7"
type: "ReLU"
bottom: "ip1"
top: "ip1"
}
layer {
name: "ip2"
type: "InnerProduct"
bottom: "ip1"
top: "ip2"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
inner_product_param {
num_output: 10
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "slice"
type: "Slice"
bottom: "label"
top: "label1"
top: "label2"
slice_param {
axis: 1
slice_point: 10
}
}
# 5个 visible 信息暂时没用 label2
layer {
name: "accuracy"
type: "AbsoluteLoss"
bottom: "ip2"
bottom: "label1"
top: "accuracy"
include {
phase: TEST
}
}
layer {
name: "loss"
type: "AbsoluteLoss"
bottom: "ip2"
bottom: "label1"
top: "loss"
}
name: "Faceregressions"
layer
{
name: "data"
type: "MemoryData"
top: "data"
top: "label"
memory_data_param
{
batch_size: 1
channels: 3
height: 96
width: 96
}
transform_param {
mean_value: 104
mean_value: 117
mean_value: 124
scale: 0.0078125
}
}
layer {
name: "conv1"
type: "Convolution"
bottom: "data"
top: "conv1"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 32
kernel_size: 3
pad:1
stride: 2
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu1"
type: "ReLU"
bottom: "conv1"
top: "conv1"
}
layer {
name: "conv2"
type: "Convolution"
bottom: "conv1"
top: "conv2"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 64
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu2"
type: "ReLU"
bottom: "conv2"
top: "conv2"
}
layer {
name: "conv3"
type: "Convolution"
bottom: "conv2"
top: "conv3"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu3"
type: "ReLU"
bottom: "conv3"
top: "conv3"
}
layer {
name: "conv4"
type: "Convolution"
bottom: "conv3"
top: "conv4"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu4"
type: "ReLU"
bottom: "conv4"
top: "conv4"
}
layer {
name: "conv5"
type: "Convolution"
bottom: "conv4"
top: "conv5"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu5"
type: "ReLU"
bottom: "conv5"
top: "conv5"
}
layer {
name: "conv6"
type: "Convolution"
bottom: "conv5"
top: "conv6"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
convolution_param {
num_output: 128
kernel_size: 3
stride: 2
pad:1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu6"
type: "ReLU"
bottom: "conv6"
top: "conv6"
}
layer {
name: "ip1"
type: "InnerProduct"
bottom: "conv6"
top: "ip1"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
inner_product_param {
num_output: 256
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu7"
type: "ReLU"
bottom: "ip1"
top: "ip1"
}
layer {
name: "ip2"
type: "InnerProduct"
bottom: "ip1"
top: "ip2"
param {
lr_mult: 1
}
param {
lr_mult: 2
}
inner_product_param {
num_output: 10
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
#include "Util.h"
#include <string>
#include <vector>
#include <fstream>
#include <sstream> //使用stringstream需要引入这个头文件
#include <iostream>
#include <stdio.h>
#include <io.h>
#include <opencv2/opencv.hpp>
#include <opencv2/xfeatures2d.hpp>
#include "../../../../SoftWare/libsvm-3.22/svm.h"
using namespace std;
using namespace glasssix;
using namespace cv;
const int LEFT_EYE = 0;
const int RIGHT_EYE = 9;
const int NOSE = 34;
const int LEFT_MOUTH = 46;
const int RIGHT_MOUTH = 47;
const int ALL = 95;
template <class Type>
Type stringToNum(const string& str)
{
istringstream iss(str);
Type num;
iss >> num;
return num;
}
void getFiles(string path, vector<string>& files, vector<string>& filenames, const string & tail)
{
//文件句柄
intptr_t hFile = 0;
//文件信息
struct _finddata_t fileinfo;
string p;
if ((hFile = _findfirst(p.assign(path).append("\\*").c_str(), &fileinfo)) != -1)
{
do
{
//如果是目录,迭代之
//如果不是,加入列表
if ((fileinfo.attrib & _A_SUBDIR))
{
if (strcmp(fileinfo.name, ".") != 0 && strcmp(fileinfo.name, "..") != 0)
{
getFiles(p.assign(path).append("\\").append(fileinfo.name), files, filenames, tail);
}
}
else
{
string filename = fileinfo.name;
size_t found = filename.find(tail);
if (found != string::npos)
{
files.push_back(p.assign(path).append("\\").append(filename));
filenames.push_back(filename);
}
}
} while (_findnext(hFile, &fileinfo) == 0);
_findclose(hFile);
}
}
void splitString(const string& s, vector<string>& v, const string& c)
{
string::size_type pos1, pos2;
pos2 = s.find(c);
pos1 = 0;
while (string::npos != pos2)
{
v.push_back(s.substr(pos1, pos2 - pos1));
pos1 = pos2 + c.size();
pos2 = s.find(c, pos1);
}
if (pos1 != s.length()) {
v.push_back(s.substr(pos1));
}
}
void parseData_5(string buf, LandMark& mark)
{
vector<string> result1;
splitString(buf, result1, "\t");
mark.fileName = result1[0];
vector<string> result2;
splitString(result1[1], result2, ",");
mark.points[0].x = stringToNum<double>(result2[2 * LEFT_EYE]);
mark.points[0].y = stringToNum<double>(result2[2 * LEFT_EYE + 1]);
mark.points[1].x = stringToNum<double>(result2[2 * RIGHT_EYE]);
mark.points[1].y = stringToNum<double>(result2[2 * RIGHT_EYE + 1]);
mark.points[2].x = stringToNum<double>(result2[2 * NOSE]);
mark.points[2].y = stringToNum<double>(result2[2 * NOSE + 1]);
mark.points[3].x = stringToNum<double>(result2[2 * LEFT_MOUTH]);
mark.points[3].y = stringToNum<double>(result2[2 * LEFT_MOUTH + 1]);
mark.points[4].x = stringToNum<double>(result2[2 * RIGHT_MOUTH]);
mark.points[4].y = stringToNum<double>(result2[2 * RIGHT_MOUTH + 1]);
}
void parseKey_26_Data(string buf, LandMark& mark)
{
vector<string> result1;
splitString(buf, result1, "\t");
mark.fileName = result1[0];
vector<string> result2;
splitString(result1[1], result2, ",");
mark.points[0].x = stringToNum<double>(result2[2 * LEFT_EYE]);
mark.points[0].y = stringToNum<double>(result2[2 * LEFT_EYE + 1]);
mark.points[1].x = stringToNum<double>(result2[2 * RIGHT_EYE]);
mark.points[1].y = stringToNum<double>(result2[2 * RIGHT_EYE + 1]);
mark.points[2].x = stringToNum<double>(result2[2 * NOSE]);
mark.points[2].y = stringToNum<double>(result2[2 * NOSE + 1]);
mark.points[3].x = stringToNum<double>(result2[2 * LEFT_MOUTH]);
mark.points[3].y = stringToNum<double>(result2[2 * LEFT_MOUTH + 1]);
mark.points[4].x = stringToNum<double>(result2[2 * RIGHT_MOUTH]);
mark.points[4].y = stringToNum<double>(result2[2 * RIGHT_MOUTH + 1]);
int j = 5;
for (int i = 74; i < ALL; i++)
{
if (i >= 74 && i < 95) {
mark.points[j].x = stringToNum<double>(result2[2 * i]);
mark.points[j].y = stringToNum<double>(result2[2 * i + 1]);
j++;
}
}
}
void parseKey_68_Data(string buf, LandMark& mark)
{
vector<string> result1;
splitString(buf, result1, "\t");
mark.fileName = result1[0];
vector<string> result2;
splitString(result1[1], result2, ",");
int j = 0;
//add left eye 18个
for (int i = 0; i < 37; i++)
{
if ((i >= 0 && i < 9) || (i >= 18 && i < 26) || (i == 36)) {
mark.points[j].x = stringToNum<double>(result2[2 * i]);
mark.points[j].y = stringToNum<double>(result2[2 * i + 1]);
j++;
}
}
//add right eye 18个
for (int i = 9; i < 38; i++)
{
if ((i >= 9 && i < 18) || (i >= 26 && i < 34) || (i == 37)) {
mark.points[j].x = stringToNum<double>(result2[2 * i]);
mark.points[j].y = stringToNum<double>(result2[2 * i + 1]);
j++;
}
}
//add nose eye 10个
for (int i = 34; i < 46; i++)
{
if ((i >= 34 && i < 36) || (i >= 38 && i < 46)) {
mark.points[j].x = stringToNum<double>(result2[2 * i]);
mark.points[j].y = stringToNum<double>(result2[2 * i + 1]);
j++;
}
}
//add mouth eye 22个
for (int i = 46; i < 68; i++)
{
if (i >= 46 && i < 68) {
mark.points[j].x = stringToNum<double>(result2[2 * i]);
mark.points[j].y = stringToNum<double>(result2[2 * i + 1]);
j++;
}
}
}
void parseData(string buf, LandMark& mark, int num_mark)
{
vector<string> result1;
splitString(buf, result1, " ");
mark.fileName = result1[0];
for (int i = 0; i < num_mark; i++)
{
mark.points[i].x = stringToNum<double>(result1[2 * i + 1]);
mark.points[i].y = stringToNum<double>(result1[2 * (i + 1)]);
}
}
void parseData(string buf, LandMark& mark)
{
vector<string> result1;
splitString(buf, result1, "\t");
mark.fileName = result1[0];
vector<string> result2;
splitString(result1[1], result2, ",");
for (int i = 0; i < LANDMARK_NUM; i++)
{
mark.points[i].x = stringToNum<double>(result2[2 * i]);
mark.points[i].y = stringToNum<double>(result2[2 * i + 1]);
}
}
void DataPrepareUtil::clearFileData(string filePath)
{
if (filePath == "")
{
return;
}
ofstream in;
in.open(filePath, ios::trunc);
in.close();
}
void DataPrepareUtil::readallFile(std::string dir, vector<std::string> & files, vector<string>& filenames, string const & tail)
{
getFiles(dir, files, filenames, tail);
}
int DataPrepareUtil::getpts(string filename, LandMark & landmark)
{
if (filename != "")
{
vector<string> result;
splitString(filename, result, ".");
string file = result[0] + ".pts";
ifstream fileA(file);
if (!fileA)
{
cout << "没有找到需要读取的 " << file << " 请将文件放到指定位置再次运行本程序。" << endl;
return -1;
}
//landmark.fileName = filename;
for (int i = 0; !fileA.eof(); i++)
{
string buf;
getline(fileA, buf, '\n');
if (buf == "")
{
//cout << "buf is empty." << endl;
continue;
}
if (i >= 3 && i < 71)
{
vector<string> point;
splitString(buf, point, " ");
landmark.points[i - 3].x = stringToNum<float>(point[0]);
landmark.points[i - 3].y = stringToNum<float>(point[1]);
}
}
fileA.close();
return 0;
}
else
{
cout << "file name is null." << endl;
return -1;
}
}
void DataPrepareUtil::readLabelData(string filePath, std::vector<Label> &data, int count)
{
ifstream fileA(filePath);
if (!fileA)
{
cout << "没有找到需要读取的 " << filePath << " 请将文件放到指定位置再次运行本程序。" << endl << " 按任意键以退出";
return;
}
for (int i = 0; !fileA.eof() && (i < count); i++)
{
Label mark;
string buf;
getline(fileA, buf, '\n');
if (buf == "")
{
cout << "buf is empty." << endl;
continue;
}
vector<string> resu;
splitString(buf, resu, " ");
mark.fileName = resu[0];
mark.label = resu[1];
data.push_back(mark);
}
fileA.close();
}
void parsePointVisibleData(string buf, LandMark& mark, int landmark)
{
vector<string> result1;
splitString(buf, result1, " ");
mark.fileName = result1[0];
for (int i = 0; i < landmark * 2; i++)
{
if (i < landmark)
{
mark.points[i].x = stringToNum<double>(result1[2 * i + 1]);
mark.points[i].y = stringToNum<double>(result1[2 * i + 2]);
}
else {
mark.visiable[i - landmark] = stringToNum<float>(result1[landmark + i + 1]);
}
}
}
vector<LandMark> DataPrepareUtil::readStageOneData(string filePath, int numMark, int count)
{
vector<LandMark> result;
ifstream fileA(filePath);
if (!fileA)
{
cout << "没有找到需要读取的 " << filePath << " 请将文件放到指定位置再次运行本程序。" << endl << " 按任意键以退出";
return result;
}
for (int i = 0; !fileA.eof() && (i < count); i++)
{
LandMark mark;
string buf;
getline(fileA, buf, '\n');
if (buf == "")
{
cout << "buf is empty." << endl;
continue;
}
parseData(buf, mark, numMark);
result.push_back(mark);
}
fileA.close();
return result;
}
vector<LandMark> DataPrepareUtil::readPointVisibleData(string filePath, int numMark, int count)
{
vector<LandMark> result;
ifstream fileA(filePath);
if (!fileA)
{
cout << "没有找到需要读取的 " << filePath << " 请将文件放到指定位置再次运行本程序。" << endl << " 按任意键以退出";
return result;
}
for (int i = 0; !fileA.eof() && (i < count); i++)
{
LandMark mark;
string buf;
getline(fileA, buf, '\n');
if (buf == "")
{
cout << "buf is empty." << endl;
continue;
}
parsePointVisibleData(buf, mark, numMark);
result.push_back(mark);
}
fileA.close();
return result;
}
void DataPrepareUtil::writeDatatoFile(std::string filePath, vector<Label> & data)
{
if (filePath == "" || data.size() == 0)
{
return;
}
ofstream in;
in.open(filePath, ios::app); //ios::trunc
int length = data.size();
for (int i = 0; i < length; i++)
{
string dataline = "H:\\CASIA\\CASIA-WebFace\\" + data[i].fileName;
dataline.append(" ");
dataline.append(data[i].label);
in << dataline << "\n";
}
in.close();
}
void DataPrepareUtil::writeDatatoFile(std::string filePath, vector<LandMark> & data, int landmark)
{
if (filePath == "" || data.size() == 0)
{
return;
}
ofstream in;
in.open(filePath, ios::app);
int length = data.size();
for (int i = 0; i < length; i++)
{
string dataline = data[i].fileName;
dataline.append(" ");
for (int j = 0; j < landmark; j++)
{
dataline.append(to_string(data[i].points[j].x));
dataline.append(" ");
dataline.append(to_string(data[i].points[j].y));
if (j != (landmark - 1))
{
dataline.append(" ");
}
//cout.precision(20);
//cout << "double x:" << data[i].points[j].x << endl;
//cout.precision(20); // 设置输出精度
//cout << "to_string():" << doubleToString(data[i].points[j].x)<<endl;
}
in << dataline << "\n";
}
in.close();
}
void DataPrepareUtil::writeDatatoFileX_Y(std::string filePath, vector<LandMark> & data, int landmark)
{
if (filePath == "" || data.size() == 0)
{
return;
}
ofstream in;
in.open(filePath, ios::app);
int length = data.size();
for (int i = 0; i < length; i++)
{
string dataline = data[i].fileName;
dataline.append(" ");
for (int j = 0; j < landmark; j++)
{
dataline.append(to_string(data[i].points[j].x));
dataline.append(" ");
}
for (int j = 0; j < landmark; j++)
{
dataline.append(to_string(data[i].points[j].y));
if (j != (landmark - 1))
{
dataline.append(" ");
}
}
in << dataline << "\n";
}
in.close();
}
void DataPrepareUtil::writeDatatoFile(std::string filePath, vector<LandMark> & data, int landmark, int start, int end)
{
if (filePath == "" || data.size() == 0)
{
return;
}
ofstream in;
in.open(filePath, ios::app);
int length = data.size();
if (start < 0 || start > length || end < start || end > length)
{
cout << "start or end is error." << endl;
}
for (int i = start; i < end; i++)
{
string dataline = data[i].fileName;
dataline.append(" ");
for (int j = 0; j < landmark; j++)
{
dataline.append(to_string(data[i].points[j].x));
dataline.append(" ");
dataline.append(to_string(data[i].points[j].y));
if (j != (landmark - 1))
{
dataline.append(" ");
}
//cout.precision(20);
//cout << "double x:" << data[i].points[j].x << endl;
//cout.precision(20); // 设置输出精度
//cout << "to_string():" << doubleToString(data[i].points[j].x)<<endl;
}
in << dataline << "\n";
}
in.close();
}
void sift_feature(Point &point, Mat & desc, Mat &img)
{
//Mat src;
//cvtColor(img, src, CV_BGR2GRAY);
if (img.data == NULL)
{
cout << "No exist" << endl;
return;
}
Ptr<Feature2D> sift = xfeatures2d::SIFT::create(1);
std::vector<KeyPoint> keypointsa;
keypointsa.clear();
KeyPoint keyp;
keyp.pt.x = point.x;
keyp.pt.y = point.y;
keyp.size = 16;
keypointsa.push_back(keyp);
//sift->detectAndCompute(src, mask, keypointsa, a);//得到特征点和特征点描述
//drawKeypoints(src, keypointsa, src, Scalar(0,0,255));//画出特征点
sift->detectAndCompute(img, Mat(), keypointsa, desc, true);
//drawKeypoints(img, keypointsa, img, Scalar(0, 0, 255));
/*imshow("src", img);
waitKey(0);*/
}
int DataPrepareUtil::getImageHogFeature(Mat &img, vector<float> & descriptors)
{
if (img.data == NULL)
{
cout << "No exist" << endl;
return -1;
}
HOGDescriptor *hog = new HOGDescriptor(Size(8, 8), Size(8, 8), Size(8, 8), Size(4, 4), 9); //Size(4,4) cell 大小
hog->compute(img, descriptors, Size(1, 1), Size(0, 0));
return 0;
}
void check_xy(int &x, int &y, int width, int height, int stride)
{
if (x < 0)
{
x = 0;
}
if (y < 0)
{
y = 0;
}
if (x > width - stride)
{
x = width - stride;
}
if (y > height - stride)
{
y = height - stride;
}
}
void DataPrepareUtil::PCA_Reduce(cv::Mat & input_train, cv::Mat & input_test, cv::Mat & output_train, cv::Mat & output_test, Mat & mean, Mat & eigenvectors)
{
cout << "start pca" << endl;
double t0 = (double)cvGetTickCount();
PCA pca(input_train, Mat(), PCA::DATA_AS_ROW, 1700);
cout << "end pca" << endl;
double t1 = (double)cvGetTickCount();
cout << "cost time is: " << ((t1 - t0) / ((double)cvGetTickFrequency() * 1000 * 1000)) << "s" << endl;
//cout << pca.eigenvalues << endl;
//cout << pca.eigenvectors << endl;
output_train = pca.project(input_train);
output_test = pca.project(input_test);
cout << " point size :" << output_train.rows << " " << output_train.cols << endl;
//imwrite("D:\\face\\face_img_96\\img\\feature_sift\\pca\\mean.jpg", pca.mean);
//imwrite("D:\\face\\face_img_96\\img\\feature_sift\\pca\\engv.jpg", pca.eigenvectors);
mean = pca.mean;
eigenvectors = pca.eigenvectors;
}
void DataPrepareUtil::getData(std::string base, std::vector<LandMark> & really, vector<LandMark> & predict, cv::Mat & train_data, cv::Mat & train_label, cv::Mat & test_data, cv::Mat & test_label, int train_num, int test_num, int flage)
{
int length = predict.size();
cout << "train image is :" << length << endl;
int n = 0;
int featureNum = 0;
int cols = 68 * 2;
for (int i = 0; i < length; i++)
{
Mat img = imread(base + predict[i].fileName, CV_LOAD_IMAGE_COLOR);
std::vector<float> descriptors;
if (flage == HOG_FEATURE)
{
for (int j = 0; j < 68; j++)
{
std::vector<float> descriptor;
Rect rect;
rect.x = predict[i].points[j].x;
rect.y = predict[i].points[j].y;
rect.x = rect.x - 4;
rect.y = rect.y - 4;
check_xy(rect.x, rect.y, 96, 96, 8);
rect.width = 8;
rect.height = 8;
Mat roi = img(rect);
getImageHogFeature(roi, descriptor);
int le = descriptor.size();
for (int k = 0; k < le; k++)
{
descriptors.push_back(descriptor[k]);
}
}
}
else if (flage == SIFT_FEATURE)
{
for (int j = 0; j < LANDMARK_NUM; j++)
{
Mat desc;
sift_feature(Point(predict[i].points[j].x, predict[i].points[j].y), desc, img);
for (int k = 0; k < desc.cols; k++)
{
descriptors.push_back(desc.at<float>(0, k));
}
//cout << " value :" << desc << endl;
//system("PAUSE");
}
}
if (i < train_num)
{
if (i == 0)
{
featureNum = descriptors.size();
cout << "featureNum is:" << featureNum << endl;
train_label = Mat::zeros(train_num, cols, CV_32FC1); //注意其中训练和自动训练的接口,还有labelMat一定要用CV_32SC1的类型
train_data = Mat::zeros(train_num, descriptors.size(), CV_32FC1);
}
for (int j = 0; j < cols / 2; j++)
{
train_label.at<float>(i, 2 * j) = really[i].points[j].x - predict[i].points[j].x;
train_label.at<float>(i, 2 * j + 1) = really[i].points[j].y - predict[i].points[j].y;
//cout << train_label.at<int>(i, 2 * j) << endl;
}
n = 0;
for (std::vector<float>::iterator iter = descriptors.begin(); iter != descriptors.end(); iter++)
{
train_data.at<float>(i, n) = *iter;
n++;
}
}
else
{
if (i == train_num)
{
featureNum = descriptors.size();
cout << "test featureNum is:" << featureNum << endl;
test_label = Mat::zeros(test_num, cols, CV_32FC1);
test_data = Mat::zeros(test_num, descriptors.size(), CV_32FC1);
}
for (int j = 0; j < cols / 2; j++)
{
test_label.at<float>(i - train_num, 2 * j) = really[i - train_num].points[j].x - predict[i - train_num].points[j].x;
test_label.at<float>(i - train_num, 2 * j + 1) = really[i - train_num].points[j].y - predict[i - train_num].points[j].y;
}
n = 0;
for (std::vector<float>::iterator iter = descriptors.begin(); iter != descriptors.end(); iter++)
{
test_data.at<float>(i - train_num, n) = *iter;
n++;
}
}
}
}
void DataPrepareUtil::getPatchData(std::string base, std::vector<LandMark> & really, vector<LandMark> & predict, std::vector<cv::Mat> & train_data, cv::Mat & train_label, std::vector<cv::Mat> & test_data, cv::Mat & test_label, int train_num, int test_num)
{
int length = predict.size();
cout << "train image is :" << length << endl;
int n = 0;
int featureNum = 0;
int cols = 68 * 2;
for (int i = 0; i < length; i++)
{
Mat img = imread(base + predict[i].fileName, CV_LOAD_IMAGE_COLOR);
Mat patch[68];
vector<Rect> rect;
for (int j = 0; j < 68; j++)
{
Rect re;
re.x = predict[i].points[j].x;
re.y = predict[i].points[j].y;
re.x = re.x - 8;
re.y = re.y - 8;
check_xy(re.x, re.y, 96, 96, 16);
re.width = 16;
re.height = 16;
Mat roi = img(re);
patch[j] = roi;
rect.push_back(re);
}
if (i < train_num)
{
if (i == 0)
{
train_label = Mat::zeros(train_num, cols, CV_32FC1); //注意其中训练和自动训练的接口,还有labelMat一定要用CV_32SC1的类型
}
for (int j = 0; j < cols / 2; j++)
{
train_label.at<float>(i, 2 * j) = really[i].points[j].x - rect[j].x;//predict[i].points[j].x;
train_label.at<float>(i, 2 * j + 1) = really[i].points[j].y - rect[j].y;//predict[i].points[j].y;
//cout << train_label.at<int>(i, 2 * j) << endl;
}
/*Mat patch1, patch2;
for (int i = 0; i < 17; i++)
{
for (int j = 0; j < 3; j++)
{
if (j == 0)
{
hconcat(patch[i][j], patch[i][j + 1], patch1);
}
else {
hconcat(patch1, patch[i][j + 1], patch1);
}
}
if (i == 0)
{
patch2 = patch1;
}
else if (i > 0)
{
vconcat(patch1, patch2, patch2);
}
}*/
Mat dst;
merge(patch, 68, dst);
train_data.push_back(dst);
}
else
{
if (i == train_num)
{
test_label = Mat::zeros(test_num, cols, CV_32FC1); //注意其中训练和自动训练的接口,还有labelMat一定要用CV_32SC1的类型
}
for (int j = 0; j < cols / 2; j++)
{
test_label.at<float>(i - train_num, 2 * j) = really[i - train_num].points[j].x - predict[i - train_num].points[j].x;
test_label.at<float>(i - train_num, 2 * j + 1) = really[i - train_num].points[j].y - predict[i - train_num].points[j].y;
}
Mat dst;
merge(patch, 68, dst);
test_data.push_back(dst);
}
}
}
void DataPrepareUtil::switch68to5landmark(std::vector<LandMark> & data_68, vector<LandMark> & data_5)
{
int four_patch[] = { 36,42, 42,48, 30, 48, 54 };
int length = data_68.size();
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = data_68[i].fileName;
for (int j = 0; j < LANDMARK_NUM; j++)
{
float x = 0, y = 0;
if (j >= four_patch[0] && j < four_patch[1])
{
landmark.points[0].x += data_68[i].points[j].x;
landmark.points[0].y += data_68[i].points[j].y;
}
if (j >= four_patch[2] && j < four_patch[3])
{
landmark.points[1].x += data_68[i].points[j].x;
landmark.points[1].y += data_68[i].points[j].y;
}
if (j == four_patch[4])
{
landmark.points[2].x += data_68[i].points[j].x;
landmark.points[2].y += data_68[i].points[j].y;
}
if (j == four_patch[5])
{
landmark.points[3].x += data_68[i].points[j].x;
landmark.points[3].y += data_68[i].points[j].y;
}
if (j == four_patch[6])
{
landmark.points[4].x += data_68[i].points[j].x;
landmark.points[4].y += data_68[i].points[j].y;
}
}
for (int k = 0; k < 2; k++)
{
landmark.points[k].x = landmark.points[k].x / 6.0;
landmark.points[k].y = landmark.points[k].y / 6.0;
}
data_5.push_back(landmark);
}
}
void DataPrepareUtil::switchSizePoint(vector<LandMark> & data, int mark_num, cv::Size input_size, cv::Size out_size)
{
int length = data.size();
for (int i = 0; i < length; i++)
{
for (int j = 0; j < mark_num; j++)
{
data[i].points[j].x = data[i].points[j].x * ((float)out_size.width / input_size.width);
data[i].points[j].y = data[i].points[j].y * ((float)out_size.height / input_size.height);
}
}
}
void DataPrepareUtil::writePointVisibletoFile(std::string filePath, vector<LandMark> & data, int landmark)
{
if (filePath == "" || data.size() == 0)
{
cout << " filePath == "" || data.size() == 0 " << endl;
return;
}
ofstream in;
in.open(filePath, ios::app);
int length = data.size();
for (int i = 0; i < length; i++)
{
string dataline = data[i].fileName;
dataline.append(" ");
for (int j = 0; j < landmark; j++)
{
dataline.append(to_string(data[i].points[j].x));
dataline.append(" ");
dataline.append(to_string(data[i].points[j].y));
//if (j != (landmark - 1))
//{
dataline.append(" ");
//}
}
for (int j = 0; j < landmark; j++)
{
dataline.append(to_string(data[i].visiable[j]< 0.5 ? 0 : 1));
if (j != (landmark - 1))
{
dataline.append(" ");
}
}
in << dataline << "\n";
}
in.close();
}
svm_parameter param;
void init_param()
{
param.svm_type = EPSILON_SVR;
param.kernel_type = RBF;
param.degree = 3;
param.gamma = 0.01;
param.coef0 = 0;
param.nu = 0.5;
param.cache_size = 1000;
param.C = 20;
param.eps = 1e-6;
param.shrinking = 1;
param.probability = 0;
param.nr_weight = 0;
param.weight_label = NULL;
param.weight = NULL;
}
//void DataPrepareUtil::libSVM_Train(cv::Mat & data, cv::Mat &label, string save_path)
//{
// init_param();
// int rows = data.rows;
// int cols = data.cols;
//
// svm_problem prob;
// prob.l = rows;
//
// svm_node *x_space = new svm_node[(cols + 1)*prob.l];//样本特征存储空间
// prob.x = new svm_node *[prob.l]; //每一个X指向一个样本
// cout << "size :" << sizeof(x_space) << endl;
// prob.y = new double[prob.l];
//
// //libsvm train data prepare.
// for (int i = 0; i < rows; i++)
// {
// for (int j = 0; j < cols + 1; j++)
// {
// if (j == cols)
// {
// x_space[i*(cols + 1) + j].index = -1;
// prob.x[i] = &x_space[i * (cols + 1)];
// prob.y[i] = label.at<uchar>(i, 0);
// break;
// }
// x_space[i*(cols + 1) + j].index = j + 1;
// x_space[i*(cols + 1) + j].value = data.at<float>(i, j);
// }
// }
// cout << "start train svm." << endl;
// svm_model *model = svm_train(&prob, ¶m);
//
// cout << "save model" << endl;
// svm_save_model(save_path.c_str(), model);
// cout << "done!" << endl;
//
// delete[] x_space;
// delete[] prob.x;
// delete[] prob.y;
//}
//void DataPrepareUtil::libSVM_Predict(std::string mode, cv::Mat & data, cv::Mat &label)
//{
// svm_model* model = svm_load_model(mode.c_str());
// int test_cols = data.cols;
// int test_rows = data.rows;
// svm_node *test_space = new svm_node[test_cols + 1];
// //svm_problem prob_test;
// //libsvm test data prepare.
// int error = 0;
// double t0 = (double)cvGetTickCount();
// for (int i = 0; i < test_rows; i++)
// {
// for (int j = 0; j < test_cols + 1; j++)
// {
// if (j == test_cols)
// {
// test_space[j].index = -1;
// break;
// }
// test_space[j].index = j + 1;
// test_space[j].value = data.at<float>(i, j);
// }
// int d = svm_predict(model, test_space);
// if (d != label.at<int>(i, 0))
// {
// cout << "predict is :" << d << " really is :" << label.at<int>(i, 0) << endl;
// error++;
// }
// }
// double t1 = (double)cvGetTickCount();
// cout << "average time is: " << ((t1 - t0) / ((double)cvGetTickFrequency() * 1000 * 1000))*1000.0 / test_rows << "ms" << endl;
// cout << "acurcy is :" << (float)(test_rows - error) / test_rows << endl;
// delete[] test_space;
//}
#include <iostream>
#include "../CascadRegression/MCLC.h"
#include <opencv2/opencv.hpp>
#include "../CascadRegression/util/Util.h"
#include <io.h>
#include <direct.h>
#include <time.h>
using namespace caffe;
using namespace cv;
using namespace glasssix;
using namespace std;
#define DEVICE 0
#define RESIZE 96
#define IMAGESIZE 96
#define NEED_LANDMARK 5
double predict(Mat &img, MCLC & mclc, int net_id, LandMark & landmark, int mark_num)
{
double time = 0;
double t0 = (double)cvGetTickCount();
vector<Mat> imgdata;
imgdata.push_back(img);
unordered_map<std::string, DataBlob> result = mclc.Forward(imgdata, net_id);
double t1 = (double)cvGetTickCount();
time = (t1 - t0) / ((double)cvGetTickFrequency() * 1000);
//cout << "time is:" << time << " ms" << endl;
for (unordered_map<string, DataBlob>::iterator iter = result.begin(); iter != result.end(); iter++) {
string key = iter->first;
DataBlob result1 = iter->second;
vector<int> mt = result1.size;
if (result1.name == "ip2")
{
for (int i = 0; i < mark_num; i++)
{
float x = *(result1.data);
float y = *(++result1.data);
landmark.points[i].x = x;
landmark.points[i].y = y;
(result1.data)++;
}
}
else if (result1.name == "ip3")
{
for (int i = 0; i < mark_num; i++)
{
landmark.visiable[i] = *(result1.data);
(result1.data)++;
}
}
}
return time;
}
void showError(vector<LandMark> &label, vector<LandMark> & predictRe, int mark_num)
{
int size = label.size();
float ave_error = 0, eye_error = 0;
cout << label.size() << " " << predictRe.size() << endl;
for (int i = 0; i < size; i++)
{
float sum = 0, sum_eye = 0;
for (int j = 0; j < mark_num; j++)
{
if (j == 0)
{
sum_eye += pow((label[i].points[j].x - predictRe[i].points[j].x), 2) + pow((label[i].points[j].y - predictRe[i].points[j].y), 2);
}
sum += pow((label[i].points[j].x - predictRe[i].points[j].x), 2) + pow((label[i].points[j].y - predictRe[i].points[j].y), 2);
}
eye_error += sum_eye;
float distance = sqrt(pow((label[i].points[0].x - label[i].points[1].x), 2) + pow((label[i].points[0].y - label[i].points[1].y), 2));
ave_error += 1 / distance * sqrt(sum);
}
cout << "eye-error is:" << eye_error / size << endl;
cout << "all error is:" << ave_error / size << endl;
}
void showVisibleError(vector<LandMark> &label, vector<LandMark> & predictRe, int mark_num)
{
int size = label.size();
float error1 = 0, error2 = 0;
for (int i = 0; i < size; i++)
{
for (int j = 0; j < mark_num; j++)
{
if (label[i].visiable[j] == 0)
{
error2++;
if (predictRe[i].visiable[j] > 0.5)
{
error1++;
}
}
}
}
cout << "visiable error is:" << error1 / error2 << endl;
}
void switchPointSize(DataPrepareUtil &dpu)
{
//switch point size
string shutter_train = "D:\\face_new\\landmark_5\\shutter_train.txt";
string shutter_test = "D:\\face_new\\landmark_5\\shutter_test.txt";
string shutter_val = "D:\\face_new\\landmark_5\\shutter_val.txt";
vector<LandMark> data_96, data_96_test, data_96_val;
data_96 = dpu.readStageOneData(shutter_train, NEED_LANDMARK);
data_96_test = dpu.readStageOneData(shutter_test, NEED_LANDMARK);
data_96_val = dpu.readStageOneData(shutter_val, NEED_LANDMARK);
dpu.switchSizePoint(data_96, NEED_LANDMARK, Size(96, 96), Size(128, 128));
dpu.switchSizePoint(data_96_test, NEED_LANDMARK, Size(96, 96), Size(128, 128));
dpu.switchSizePoint(data_96_val, NEED_LANDMARK, Size(96, 96), Size(128, 128));
/*for (int i = 0; i < 10; i++)
{
Mat img = imread("D:\\face_new\\landmark_5\\result_img_128\\" + data_96[i].fileName);
for (int j = 0; j < NEED_LANDMARK; j++)
{
circle(img, Point(data_96[i].points[j]), 2, Scalar(255, 0, 0));
}
imshow("img", img);
waitKey(0);
}*/
dpu.writeDatatoFile("D:\\face_new\\landmark_5\\result_img_128\\shutter_train.txt", data_96, NEED_LANDMARK);
dpu.writeDatatoFile("D:\\face_new\\landmark_5\\result_img_128\\shutter_test.txt", data_96_test, NEED_LANDMARK);
dpu.writeDatatoFile("D:\\face_new\\landmark_5\\result_img_128\\shutter_val.txt", data_96_val, NEED_LANDMARK);
}
void Test300WError(DataPrepareUtil &dpu, MCLC &mclc, int id_68)
{
// 300W 错误
string img_300w = "D:\\face\\face_alignment\\300W_resize\\";
string text_300w = "D:\\face\\face_alignment\\300W_resize\\result_point_96_68.txt";
double time1 = 0;
vector<LandMark> data_68;
data_68 = dpu.readStageOneData(text_300w, LANDMARK_NUM);
vector<LandMark> data_5;
vector<LandMark> pre_data_5;
dpu.switch68to5landmark(data_68, data_5);
int size = data_5.size();
for (int j = 0; j < size; j++)
{
LandMark landmark;
landmark.fileName = data_5[j].fileName;
Mat img = imread(img_300w + data_5[j].fileName);
time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);
pre_data_5.push_back(landmark);
}
cout << "300W test result" << endl;
showError(data_5, pre_data_5, NEED_LANDMARK);
cout << "average time is :" << time1 / size << endl;
//dpu.writeDatatoFile("D:\\face\\face_alignment\\300W_resize\\result_point_96_5.txt", data_5, NEED_LANDMARK);
}
void L2LossPre(vector<LandMark> &preRe)
{
string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_96_5\\";
string prototext = base + "test_5_op.prototxt";
string mode_path = base + "save_path\\_iter_150000.caffemodel";
string img_path = "D:\\face_new\\landmark_5\\result_img_96\\";
string text_label = img_path + "shutter_test.txt";
MCLC mclc;
int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
DataPrepareUtil dpu;
vector<LandMark> data;
data = dpu.readStageOneData(text_label, NEED_LANDMARK);
cout << "data length is:" << data.size() << endl;
int length = data.size();
double time1 = 0;
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = data[i].fileName;
Mat img = imread(img_path + data[i].fileName);
time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);
preRe.push_back(landmark);
//for (int j = 0; j < NEED_LANDMARK; j++)
//{
// circle(img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
//}
//imshow("img", img);
//waitKey(0);
}
cout << "average time1 is:" << time1 / length << " ms" << endl;
showError(data, preRe, NEED_LANDMARK);
}
void absLossPre(string img_path, string text_label, MCLC &mclc, vector<LandMark> &preRe, DataPrepareUtil &dpu, vector<LandMark> & data)
{
string base = "E:\\work\\face_alignment\\lanmark_5\\model\\umd_96_5\\";
string prototext = base + "test_5_op_change_abs.prototxt";
string mode_path = base + "save_path_abs\\_iter_200000.caffemodel";
int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
data = dpu.readPointVisibleData(text_label, NEED_LANDMARK);
cout << "data length is:" << data.size() << endl;
int length = data.size();
double time1 = 0;
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = data[i].fileName;
Mat img = imread(img_path + data[i].fileName);
time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);
preRe.push_back(landmark);
}
cout << "average time1 is:" << time1 / length << " ms" << endl;
showError(data, preRe, NEED_LANDMARK);
}
void pointVisibleLossPre(string img_path, string text_label, MCLC &mclc, vector<LandMark> &preRe, DataPrepareUtil &dpu, vector<LandMark> & data)
{
string base = "E:\\work\\face_alignment\\lanmark_5\\model\\umd_128_5\\";
string prototext = base + "test_5_op_loss.prototxt";
string mode_path = base + "save_path1\\_iter_200000.caffemodel";
int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
data = dpu.readPointVisibleData(text_label, NEED_LANDMARK);
cout << "data length is:" << data.size() << endl;
int length = data.size();
double time1 = 0;
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = data[i].fileName;
Mat img = imread(img_path + data[i].fileName);
time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);
preRe.push_back(landmark);
//for (int j = 0; j < NEED_LANDMARK; j++)
//{
// Scalar scalar;
// if (landmark.visiable[j] < 0.5)
// {
// scalar = Scalar(255, 0, 0);
// }
// else {
// scalar = Scalar(0, 0, 255);
// }
// circle(img, Point(landmark.points[j]), 2, scalar, -1);
//}
//imshow("img", img);
//waitKey(0);
}
cout << "average time1 is:" << time1 / length << " ms" << endl;
showError(data, preRe, NEED_LANDMARK);
}
void smoothLossPre(string img_path, string text_label, MCLC &mclc, vector<LandMark> &preRe, DataPrepareUtil &dpu, vector<LandMark> & data)
{
string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_96_5\\";
string prototext = base + "test_5_op_loss.prototxt";
string mode_path = base + "save_path_smooth\\_iter_200000.caffemodel";
int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
data = dpu.readStageOneData(text_label, NEED_LANDMARK);
cout << "data length is:" << data.size() << endl;
int length = data.size();
double time1 = 0;
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = data[i].fileName;
Mat img = imread(img_path + data[i].fileName);
time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);
preRe.push_back(landmark);
}
cout << "average time1 is:" << time1 / length << " ms" << endl;
showError(data, preRe, NEED_LANDMARK);
}
void size64Pre()
{
string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_64_5\\";
string prototext = base + "test_5_op.prototxt";
string mode_path = base + "save_path\\_iter_150000.caffemodel";
string img_path = "D:\\face_new\\landmark_5\\result_img_64\\";
string text_label = img_path + "shutter_test.txt";
MCLC mclc;
int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
DataPrepareUtil dpu;
vector<LandMark> data;
data = dpu.readStageOneData(text_label, NEED_LANDMARK);
cout << "data length is:" << data.size() << endl;
int length = data.size();
vector<LandMark> preRe;
double time1 = 0;
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = data[i].fileName;
Mat img = imread(img_path + data[i].fileName);
time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);
preRe.push_back(landmark);
//for (int j = 0; j < NEED_LANDMARK; j++)
//{
// circle(img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
//}
//imshow("img", img);
//waitKey(0);
}
cout << "average time1 is:" << time1 / length << " ms" << endl;
showError(data, preRe, NEED_LANDMARK);
}
//size: brow, eye, nose, mouth, face
const float four_size[] = { 24,24, 28,28, 48,32 };
string mark_4_img_dir[] = { "eye\\", "nose\\", "mouth\\" };
string mark_4_img_train[] = { "eye_train.txt", "nose_train.txt", "mouth_train.txt" };
string mark_4_img_test[] = { "eye_test.txt", "nose_test.txt", "mouth_test.txt" };
void getRect(LandMark landmark, int flag, vector<Rect> &rects, int cols, int rows)
{
Rect rect, rect1;
Point p1;
if (flag == 0)
{
p1 = landmark.points[0];
}
else if (flag == 1)
{
p1 = landmark.points[2];
}
else if (flag == 2)
{
p1 = landmark.points[3];
}
float x, y, width, height;
rect.width = four_size[2 * flag];
rect.height = four_size[2 * flag + 1];
x = p1.x;
y = p1.y;
rect.x = min(max(x - four_size[2 * flag] * 0.5f, 0.0f), (float)(IMAGESIZE - four_size[2 * flag]));
rect.y = min(max(y - four_size[2 * flag + 1] * 0.5f, 0.0f), (float)(IMAGESIZE - four_size[2 * flag + 1]));
rects.push_back(rect);
}
void resetRect(Rect & rect, int cols, int rows)
{
if (rect.x < 0)
{
rect.x = 0;
}
if (rect.y < 0)
{
rect.y = 0;
}
if (rect.x + rect.width > cols)
{
rect.x = cols - rect.width;
}
if (rect.y + rect.height > rows)
{
rect.y = rows - rect.height;
}
}
void addRect(vector<Rect> & rects)
{
srand((unsigned)time(NULL));
int dir[] = { 1,1, 1,-1, 1,0, -1,1, -1,-1, -1,0, 0,1, 0,-1 };
if (rects.size() == 1)
{
for (int i = 0; i < 8; i++)
{
Rect r(rects[0]);
r.x += dir[2 * i] * (rand() % 4 + 1);
r.y += dir[2 * i + 1] * (rand() % 4 + 1);
resetRect(r, IMAGESIZE, IMAGESIZE);
rects.push_back(r);
}
}
}
void createPatchFile(string base, vector<LandMark> &data, vector<LandMark> &preRe, DataPrepareUtil &dpu)
{
const int path_size = 1;
vector<LandMark> landmark_four[path_size][2];
int istd = 0;
for (int j = 0; j < path_size; j++)
{
dpu.clearFileData(base + mark_4_img_dir[j] + mark_4_img_train[j]);
dpu.clearFileData(base + mark_4_img_dir[j] + mark_4_img_test[j]);
}
int length = preRe.size();
for (int i = 0; i < length; i++)
{
if (i >= 12000)
{
istd = 1;
}
for (int j = 0; j < path_size; j++)
{
Mat img = imread(base + preRe[i].fileName);
vector<Rect> rects;
getRect(preRe[i], j, rects, img.cols, img.rows);
//cout << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
addRect(rects);
for (int n = 0; n < rects.size(); n++)
{
Mat roi = img(rects[n]);
string dir = base + mark_4_img_dir[j];
if (_access(dir.c_str(), 0) == -1)
{
_mkdir(dir.c_str());
}
imwrite(dir + to_string(n) + "_" + preRe[i].fileName, roi);
//坐标转换
LandMark mark;
mark.fileName = to_string(n) + "_" + preRe[i].fileName;
float offsetX = (data[i].points[0].x - rects[n].x) * four_size[2 * j] / rects[n].width;
float offsetY = (data[i].points[0].y - rects[n].y) * four_size[2 * j + 1] / rects[n].height;
mark.points[0].x = offsetX;
mark.points[0].y = offsetY;
landmark_four[j][istd].push_back(mark);
}
}
if (landmark_four[0][0].size() > 1000)
{
for (int j = 0; j < path_size; j++)
{
dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][0], 1);
(landmark_four[j][0]).clear();
}
}
if (landmark_four[0][1].size() > 1000)
{
for (int j = 0; j < path_size; j++)
{
dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_test[j], landmark_four[j][1], 1);
(landmark_four[j][1]).clear();
}
}
}
for (int j = 0; j < path_size; j++)
{
dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][0], 1);
增加验证集数据 数据平衡
//dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][1], mark_num[j]);
//dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][1], mark_num[j]);
//dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][1], mark_num[j]);
dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_test[j], landmark_four[j][1], 1);
}
}
void stageTwoPredict(string img_path, MCLC &mclc, vector<LandMark> &label, vector<LandMark> &preRe)
{
string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_stage_two\\eye\\";
string prototext_eye = base + "test_eye_op.prototxt";
string mode_path_eye = base + "save_path\\_iter_150000.caffemodel";
int id_eye = mclc.AddNet(prototext_eye, mode_path_eye, DEVICE);
double time_four = 0;
int length = preRe.size();
for (int i = 0; i < length; i++)
{
LandMark landmark;
landmark.fileName = preRe[i].fileName;
Mat img = imread(img_path + preRe[i].fileName);
vector<Rect> rects;
getRect(preRe[i], 0, rects, img.cols, img.rows);
Mat roi = img(rects[0]);
time_four += predict(roi, mclc, id_eye, landmark, 1);
preRe[i].points[0].x = landmark.points[0].x * (rects[0].width / four_size[0]) + rects[0].x;
preRe[i].points[0].y = landmark.points[0].y * (rects[0].height / four_size[1]) + rects[0].y;
}
showError(label, preRe, NEED_LANDMARK);
cout << "stage two ........................................................." << endl;
cout << "average time is :" << time_four / length << " ms" << endl;
}
void showMeanShape()
{
string shape_text = "D:\\face_new\\landmark_5\\result_img_128\\landmark_128_5.txt";
vector<LandMark>data;
DataPrepareUtil dpu;
data = dpu.readStageOneData(shape_text, NEED_LANDMARK);
cout << "data length is:" << data.size() << endl;
LandMark landmark;
int length = data.size();
for (int i = 0; i < length; i++)
{
for (int j = 0; j < NEED_LANDMARK; j++)
{
landmark.points[j].x += data[i].points[j].x;
landmark.points[j].y += data[i].points[j].y;
}
}
for (int j = 0; j < NEED_LANDMARK; j++)
{
landmark.points[j].x = landmark.points[j].x / length;
landmark.points[j].y = landmark.points[j].y / length;
cout << "shape j:" << j << " ( " << landmark.points[j].x << " , " << landmark.points[j].y << " )" << endl;
}
}
int main(int argc, const char** argv) {
//L2LossPre();
//AbsLossPre();
//size64Pre();
DataPrepareUtil dpu;
vector<LandMark> preRe;
vector<LandMark> data;
string img_path = "D:\\UMD\\result_img_96\\";
string text_label = img_path + "shutter_96_5_test.txt";
MCLC mclc;
//showMeanShape();
absLossPre(img_path, text_label, mclc, preRe, dpu, data);
//smoothLossPre(img_path, text_label, mclc, preRe, dpu, data);
//string base = "D:\\face_new\\landmark_5\\result_img_96\\";
//pointVisibleLossPre(img_path, text_label, mclc, preRe, dpu, data);
//showVisibleError(data, preRe, NEED_LANDMARK);
std::system("PAUSE");
return 0;
}
Prepare data
#include "../data300w/util/Util.h"
#include "cunpd.hpp"
#include <iostream>
#include <opencv2/opencv.hpp>
#include <vector>
#include <direct.h>
#include <io.h>
using namespace std;
using namespace cv;
using namespace glasssix;
#define ZOOM 1.0
#define RESIZE 96
#define NEED_LANDMARK 5
#define CURRENT_LANDMARK 21
string search_base = "H:\\UMD\\";
string search_dir_[] = { search_base + "umdfaces_batch1", search_base + "umdfaces_batch2", search_base + "umdfaces_batch3"};
string text_file[] = { search_base +"umdfaces_batch1\\umdfaces_batch1_ultraface.csv", search_base + "umdfaces_batch2\\umdfaces_batch2_ultraface.csv", search_base + "umdfaces_batch3\\umdfaces_batch3_ultraface.csv" };
string text_pre[] = { "batch1_", "batch2_", "batch3_"};
string tail_[] = { ".jpg", ".jpg" , ".jpg" };
string base = search_base + "landmark_5\\";
string result_img = base + "result_img4_" + to_string(RESIZE) + "\\";
string result_txt = base + "landmark4_" + to_string(RESIZE) + "_5.txt";
const int five_point[5] = { 7, 10, 14, 17, 19};
int main11() {
// prepare data code.
cunpd *pd = new cunpd();
vector<LandMark> result_five_point;
int model_id = pd->AddNpdModel(0);
int faceNum = 0;
DataPrepareUtil dpUtil;
dpUtil.clearFileData(result_txt);
string base_dir = base;
if (_access(base_dir.c_str(), 0) == -1)
{
_mkdir(base_dir.c_str());
}
long count = 0;
for (int temp = 0; temp < 3; temp++)
{
long countNum = 0;
vector<LandMark> result_point;
dpUtil.readFileData(text_file[temp], result_point, CURRENT_LANDMARK);
for (vector<LandMark>::iterator itor = result_point.begin(); itor != result_point.end();)
{
if (count % 500 == 0)
{
cout << "prepare count is: " << count << " current img path is :" << result_point[0].fileName << endl;
}
count++;
Mat img;
int widths = 0, heights = 0;
Mat dstImg;
try {
string img_path = search_dir_[temp] + "\\" + result_point[0].fileName;
if (_access(img_path.c_str(), 0) == -1)
{
cout << "coun't found filename" << search_dir_[temp] + result_point[0].fileName << endl;
result_point.erase(itor);
continue;
}
img = imread(search_dir_[temp] + "\\" + result_point[0].fileName);
widths = img.cols;
heights = img.rows;
dstImg.create(heights, widths, CV_8UC1);
cvtColor(img, dstImg, CV_BGR2GRAY);
}
catch (Exception e)
{
cout << "exception:" << " current file:" << result_point[0].fileName << endl;
system("PAUSE");
}
//for (int k = 0; k < CURRENT_LANDMARK; k++)
//{
// circle(img, Point(result_point[0].points[k]), 2, Scalar(255, 0, 0));
// putText(img, to_string(k), Point(Point(result_point[0].points[k])), 0.3, 0.3, Scalar(255, 0, 0));
//}
//imshow("img", img);
//waitKey(0);
vector<FaceInfomation> face = pd->detect(dstImg, model_id, 48);
LandMark landmark;
int length = face.size();
if (length == 0)
{
cout << "not found face ." << endl;
}
for (int j = 0; j < length; j++)
{
if (face[j].score > 15)
{
Rect rect = face[j].rect;
if (result_point[0].points[0].x > rect.x && result_point[0].points[0].x < rect.x + rect.width
&& result_point[0].points[0].y > rect.y && result_point[0].points[0].y < rect.y + rect.height
&&result_point[0].points[12].x > rect.x && result_point[0].points[12].x < rect.x + rect.width
&& result_point[0].points[12].y > rect.y && result_point[0].points[12].y < rect.y + rect.height
&&result_point[0].points[16].x > rect.x && result_point[0].points[16].x < rect.x + rect.width
&& result_point[0].points[16].y > rect.y && result_point[0].points[16].y < rect.y + rect.height
&&result_point[0].points[20].x > rect.x && result_point[0].points[20].x < rect.x + rect.width
&& result_point[0].points[20].y > rect.y && result_point[0].points[20].y < rect.y + rect.height
&& (abs(result_point[0].points[7].y - result_point[0].points[17].y) > (rect.height / 6.0)))
{
int rect_w = rect.width;
int rect_h = rect.height;
rect.width = rect_w * ZOOM;
rect.height = rect_h * ZOOM;
rect.x = max(rect.x - (ZOOM - 1.0) * rect_w / 2.0, 0.0);
rect.y = max(rect.y - (ZOOM - 1.0) * rect_h / 2.0, 0.0);
if (rect.x + rect.width > widths)
{
rect.width = widths - rect.x;
//rect.height = rect_h * rect.width / rect_w;
}
if (rect.y + rect.height > heights)
{
rect.height = heights - rect.y;
//rect.width = rect_w * rect.height / rect_h;
}
Mat roi_face, result_face;
roi_face = img(rect);
result_face.create(RESIZE, RESIZE, CV_8UC3);
cv::resize(roi_face, result_face, Size(RESIZE, RESIZE));
string dir = result_img;
if (_access(dir.c_str(), 0) == -1)
{
_mkdir(dir.c_str());
}
imwrite(dir + text_pre[temp] + to_string(countNum) + ".png", result_face);
landmark.fileName = text_pre[temp] + to_string(countNum) + ".png";
countNum++;
//坐标转换
//double t0 = (double)cvGetTickCount();
//#pragma omp parallel for num_threads(4)
for (int k = 0; k < NEED_LANDMARK; k++)
{
landmark.visible[k] = result_point[0].visible[five_point[k]];
if (result_point[0].points[five_point[k]].x == -1 || result_point[0].points[five_point[k]].y == -1) {
cout << "this points is NULL" << endl;
continue;
}
landmark.points[k].x = ((float)RESIZE / rect.width) * (result_point[0].points[five_point[k]].x - rect.x);
landmark.points[k].y = ((float)RESIZE / rect.height) * (result_point[0].points[five_point[k]].y - rect.y);
}
faceNum++;
result_five_point.push_back(landmark);
if (result_five_point.size() >= 1000)
{
dpUtil.writePointVisibletoFile(result_txt, result_five_point, NEED_LANDMARK);
result_five_point.clear();
}
//for (int k = 0; k < 5; k++)
//{
// circle(result_face, Point(landmark.points[k]), 2, Scalar(255, 0, 0));
//}
//imshow("img", result_face);
//waitKey(0);
break;
}
}
}
result_point.erase(itor);
}
}
if (result_five_point.size() > 0)
{
dpUtil.writePointVisibletoFile(result_txt, result_five_point, NEED_LANDMARK);
result_five_point.clear();
}
std::system("PAUSE");
delete pd;
return 0;
}
Argument data
#include <iostream>
#include <opencv2/opencv.hpp>
#include "../data300w/util/Util.h"
#include "cunpd.hpp"
#include <ctime>
#include <io.h>
#include <direct.h>
#include <thread>
#include <condition_variable>
#include <mutex>
#include <Windows.h>
using namespace std;
using namespace cv;
using namespace glasssix;
#define ZOOM_ 1.0
#define SIZE 96
extern void splitString(const string& s, vector<string>& v, const string& c);
template <class Type>
Type stringToNum(const string& str)
{
istringstream iss(str);
Type num;
iss >> num;
return num;
}
void writeHistoFile(std::string filePath, vector<int> & densi_data)
{
if (filePath == "" || densi_data.size() == 0)
{
return;
}
ofstream in;
in.open(filePath, ios::app); //ios::trunc
int length = densi_data.size();
for (int i = 0; i < length; i++)
{
string dataline = to_string(densi_data[i]);
in << dataline << "\n";
}
in.close();
}
float getHorizontal(LandMark &landmark)
{
float tan_theta = (landmark.points[1].y - landmark.points[0].y) / (landmark.points[1].x - landmark.points[0].x);
float theta = atan(tan_theta);
return theta * 180 / 3.1415926;
}
void showHistogram(vector<float> & hor_data)
{
int densi[60] = { 0 };
int length = hor_data.size();
for (int i = 0; i < length; i++)
{
if (floor((hor_data[i] + 30)) >= 0 && floor((hor_data[i] + 30)) < 60)
{
densi[(int)floor((hor_data[i] + 30))]++;
}
if (floor((hor_data[i] + 30)) < 0)
{
densi[0]++;
}
else if (floor((hor_data[i] + 30)) >= 60)
{
densi[60]++;
}
}
string density_text = "D:\\UMD\\density_text.txt";
vector<int>density_data(densi, densi + 60);
writeHistoFile(density_text, density_data);
Mat histImg;
histImg.create(1000, 1600, CV_8UC3);
histImg.setTo(0);
int offset = 10;
for (int i = 0; i < 60; i++)
{
double tmpCount = densi[i];
rectangle(histImg, Point2f(offset + i * 25, 1000), Point2f(offset + i * 25, 1000 - tmpCount / 15.0), Scalar::all(255), -1); //画出直方图
putText(histImg, to_string(i - 29), Point2f(offset + i * 25 + 3, 1000 - 3), 0.3, 0.3, Scalar(0, 0, 255));
Point2f pt0;
pt0.x = offset + i * 25;
pt0.y = 1000 - densi[i] / 15.0;
Point2f pt1;
pt1.x = offset + (i + 1) * 25;
pt1.y = 1000 - densi[i + 1] / 15.0;
line(histImg, pt0, pt1, Scalar(255, 0, 0), 1); //连接直方图的顶点
}
imshow("hist", histImg);
waitKey(0);
}
void getDatahor(string file1, vector<float> & hor_data)
{
int mark_num = 5;
DataPrepareUtil dpu;
vector<LandMark> data;
dpu.readFileData(file1, data, mark_num);
int length = data.size();
for (int i = 0; i < length; i++)
{
float hor = getHorizontal(data[i]);
hor_data.emplace_back(hor);
}
}
void rotation(float theta, Mat &img, Mat &dst, Size img_size, LandMark &landmark, int mark_num)
{
//rotation
Mat mat = img;
Point2f center(img_size.width / 2, img_size.height / 2);
double angle = theta;
Mat rot = getRotationMatrix2D(center, angle, 1);
Rect bbox = RotatedRect(center, mat.size(), angle).boundingRect();
cv::warpAffine(mat, dst, rot, bbox.size());
for (int j = 0; j < mark_num; j++)
{
float theta = -3.1415926 / (180 / angle);
float x1 = landmark.points[j].x - rot.at<double>(1, 2);
float y1 = landmark.points[j].y - rot.at<double>(0, 2);
landmark.points[j].x = x1 * cos(theta) - y1 * sin(theta);
landmark.points[j].y = x1 * sin(theta) + y1 * cos(theta);
//circle(dst, Point(x, y), 2, Scalar(255, 0, 0));
}
//cv::imshow("dst", dst);
//cv::waitKey(0);
}
void augment_data(string img_path, string img_text, string result_path, string result_text)
{
DataPrepareUtil dpu;
int mark_num = 5;
srand((unsigned)time(NULL));
vector<LandMark> data;
dpu.readFileData(img_text, data, mark_num);
vector<LandMark> rotation_data;
vector<float> hor_data;
getDatahor(img_text, hor_data);
int length = hor_data.size();
for (int i = 0; i < length; i++)
{
if (hor_data[i] > 0 && hor_data[i] < 3)
{
Mat dst;
Mat img = imread(img_path + data[i].fileName);
LandMark landmark(data[i]);
rotation(25, img, dst, Size(96, 96), landmark, mark_num);
rotation_data.push_back(landmark);
}
}
}
bool getFaceRect(cunpd &pd, int model_id, Mat &dstImg, LandMark & landmark, Rect & rect)
{
const int widths = dstImg.cols;
const int heights = dstImg.rows;
vector<FaceInfomation> face = pd.detect(dstImg, model_id, 48);
int length = face.size();
if (length == 0)
{
cout << "not found face ." << endl;
}
for (int j = 0; j < length; j++)
{
if (face[j].score > 15)
{
rect = face[j].rect;
if (landmark.points[0].x > rect.x && landmark.points[0].x < rect.x + rect.width
&& landmark.points[0].y > rect.y && landmark.points[0].y < rect.y + rect.height
&&landmark.points[12].x > rect.x && landmark.points[12].x < rect.x + rect.width
&& landmark.points[12].y > rect.y && landmark.points[12].y < rect.y + rect.height
&&landmark.points[16].x > rect.x && landmark.points[16].x < rect.x + rect.width
&& landmark.points[16].y > rect.y && landmark.points[16].y < rect.y + rect.height
&&landmark.points[20].x > rect.x && landmark.points[20].x < rect.x + rect.width
&& landmark.points[20].y > rect.y && landmark.points[20].y < rect.y + rect.height
&& (abs(landmark.points[7].y - landmark.points[17].y) > (rect.height / 6.0)))
{
int rect_w = rect.width;
int rect_h = rect.height;
rect.width = rect_w * ZOOM_;
rect.height = rect_h * ZOOM_;
rect.x = max(rect.x - (ZOOM_ - 1.0) * rect_w / 2.0, 0.0);
rect.y = max(rect.y - (ZOOM_ - 1.0) * rect_h / 2.0, 0.0);
if (rect.x + rect.width > widths)
{
rect.width = widths - rect.x;
}
if (rect.y + rect.height > heights)
{
rect.height = heights - rect.y;
}
return true;
}
}
}
return false;
}
void getoffsetRect(Rect & rect, vector<Rect> & all_rect, int cols, int rows, int max_offset)
{
srand((unsigned)time(NULL));
Rect rect0(rect), rect1(rect);
int offsetx = rand() % max_offset + 1;
int offsety = rand() % max_offset + 1;
if (rect.x > offsetx && rect.y > offsety)
{
rect0.x = rect.x - offsetx;
rect0.y = rect.y - offsety;
}
offsetx = rand() % max_offset + 1;
offsety = rand() % max_offset + 1;
if (rect.x + rect.width + offsetx < cols && rect.y + rect.height + offsety < rows)
{
rect1.x = rect.x + offsetx;
rect1.y = rect.y + offsety;
}
all_rect.push_back(rect0);
all_rect.push_back(rect1);
}
#define NEED_LANDMARK 5
#define CURRENT_LANDMARK 21
const int five_points[5] = { 7, 10, 14, 17, 19 };
string search_base = "H:\\UMD\\";
string search_dir_[] = { search_base + "umdfaces_batch1", search_base + "umdfaces_batch2", search_base + "umdfaces_batch3" };
string text_file[] = { search_base + "umdfaces_batch1\\umdfaces_batch1_ultraface.csv", search_base + "umdfaces_batch2\\umdfaces_batch2_ultraface.csv", search_base + "umdfaces_batch3\\umdfaces_batch3_ultraface.csv" };
string text_pre[] = { "batch1_aug_", "batch2_aug_", "batch3_aug_" };
string tail_[] = { ".jpg", ".jpg" , ".jpg" };
string base = search_base + "landmark_5\\augment_img\\";
string result_img = base + "result_img_" + to_string(SIZE) + "\\";
string result_txt = base + "landmark_" + to_string(SIZE) + "_5.txt";
const int theta_offset = 5;
const int theta_max = 20;
vector<LandMark> rotation_point;
int countNum = 0;
bool ready = false;
std::mutex mtx_syn;
std::condition_variable cv_syn;
void roll_yaw_pitch_data(LandMark result_mark, int temp, cunpd &pd, int model_id, DataPrepareUtil &dpu)
{
float roll = result_mark.direct[2];
string img_path = search_dir_[temp] + "\\" + result_mark.fileName;
if (_access(img_path.c_str(), 0) == -1)
{
cout << "coun't found filename" << img_path << endl;
return;
}
Mat img = imread(img_path);
Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
cvtColor(img, dstImg, CV_BGR2GRAY);
//yaw 增强 pitch 增强
for (int j = 0; j < 2; j++)
{
if (result_mark.direct[j] > -theta_offset && result_mark.direct[j] < theta_offset)
{
Rect rect;
LandMark landmark(result_mark);
bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
if (success)
{
vector<Rect> all_rect;
getoffsetRect(rect, all_rect, img.cols, img.rows, 4);
for (int i = 0; i < 2; i++)
{
LandMark dst_landmark;
//vector<string> filenames;
//splitString(landmark.fileName, filenames, "/");
//string filename = filenames[filenames.size()-1];
std::unique_lock<std::mutex> lck(mtx_syn);
dst_landmark.fileName = text_pre[temp] + to_string(countNum++) + ".png";
lck.unlock();
//cout << img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
Mat roi_face = img(all_rect[i]);
cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
//坐标转换
for (int k = 0; k < 5; k++)
{
dst_landmark.visible[k] = landmark.visible[five_points[k]];
dst_landmark.points[k].x = ((float)SIZE / all_rect[i].width) * (landmark.points[five_points[k]].x - all_rect[i].x);
dst_landmark.points[k].y = ((float)SIZE / all_rect[i].height) * (landmark.points[five_points[k]].y - all_rect[i].y);
}
imwrite(result_img + dst_landmark.fileName, roi_face);
std::unique_lock<std::mutex> lck1(mtx_syn);
rotation_point.push_back(dst_landmark);
lck1.unlock();
}
}
}
}
// roll 增强
if (roll > -theta_offset && roll < theta_offset)
{
for (int i = -1; i < 2; i = i + 2)
{
Mat rotat_img;
LandMark landmark(result_mark);
int theta = (rand() % theta_max + theta_offset) * i;
rotation(theta, img, rotat_img, Size(SIZE, SIZE), landmark, CURRENT_LANDMARK);
Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
cvtColor(rotat_img, dstImg, CV_BGR2GRAY);
//for (int j = 0; j < CURRENT_LANDMARK; j++)
//{
// circle(rotat_img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
//}
//imshow("img", rotat_img);
//waitKey(0);
LandMark dst_landmark;
//vector<string> filenames;
//splitString(landmark.fileName, filenames, "/");
//string filename = filenames[filenames.size()-1];
std::unique_lock<std::mutex> lck(mtx_syn);
dst_landmark.fileName = text_pre[temp] + to_string(countNum++) + ".png";
lck.unlock();
Rect rect;
bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
if (success)
{
//cout << rotat_img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
Mat roi_face = rotat_img(rect);
cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
//坐标转换
for (int k = 0; k < 5; k++)
{
dst_landmark.visible[k] = landmark.visible[five_points[k]];
dst_landmark.points[k].x = ((float)SIZE / rect.width) * (landmark.points[five_points[k]].x - rect.x);
dst_landmark.points[k].y = ((float)SIZE / rect.height) * (landmark.points[five_points[k]].y - rect.y);
}
imwrite(result_img + dst_landmark.fileName, roi_face);
std::unique_lock<std::mutex> lck(mtx_syn);
rotation_point.push_back(dst_landmark);
if (rotation_point.size() > 500)
{
dpu.writePointVisibletoFile(result_txt, rotation_point, NEED_LANDMARK);
rotation_point.clear();
}
if (countNum % 500 == 0)
{
cout << "prepare data:" << countNum << endl;
}
lck.unlock();
}
}
}
}
vector<LandMark> result_point;
void deal_thread(int temp, int model_id, DataPrepareUtil &dpu, cunpd &pd)
{
while (true)
{
std::unique_lock<std::mutex> lck(mtx_syn);
while (!ready) {
cv_syn.wait(lck);
}
//
auto itor = result_point.begin();
auto itor2 = result_point.end();
if (itor == itor2)
{
break;
}
LandMark landmark(result_point[0]);
result_point.erase(itor);
// cout << "landmark.fileName is:"<<landmark.fileName<< "thread id"<< this_thread::get_id()<< endl;
lck.unlock();
roll_yaw_pitch_data(landmark, temp, pd, model_id, dpu);
}
}
void go()
{
std::unique_lock<std::mutex> lck(mtx_syn);
ready = true;
cv_syn.notify_all();
}
int main()
{
cunpd pd;
int model_id = pd.AddNpdModel(0);
// 查看数据分布
//string file1 = "D:\\UMD\\landmark_128_5.txt";
//vector<float> hor_data;
//getDatahor(file1, hor_data);
//showHistogram(hor_data);
/*string img_path = "D:\\UMD\\result_img_96\\";
string result_path = "D:\\UMD\\arguement_data\\";
string img_text = img_path + "shutter_96_5_train.txt";
string result_text = result_path + "augment_96_5_train.txt";
augment_data(img_path, img_text, result_path, result_text);*/
string base_dir = base;
if (_access(base_dir.c_str(), 0) == -1)
{
_mkdir(base_dir.c_str());
}
string dir = result_img;
if (_access(dir.c_str(), 0) == -1)
{
_mkdir(dir.c_str());
}
srand((unsigned)time(NULL));
DataPrepareUtil dpUtil;
dpUtil.clearFileData(result_txt);
long count = 0;
vector<LandMark> rotation_point;
for (int temp = 0; temp < 3; temp++)
{
long countNum = 0;
//vector<LandMark> result_point;
dpUtil.readFileData(text_file[temp], result_point, CURRENT_LANDMARK);
std::thread threads[4];
for (int i = 0; i < 4; i++)
{
threads[i] = std::thread(deal_thread, temp, model_id, dpUtil, pd);
//threads[i] = std::thread(threadA, i, result_point, temp, model_id, dpUtil, pd);
}
cout << "temp start:" << temp << endl;
go();
for (auto &th : threads) {
th.join();
}
cout << "temp end:" << temp << endl;
//for (vector<LandMark>::iterator itor = result_point.begin(); itor != result_point.end();)
//{
// if (count % 500 == 0)
// {
// cout << "prepare count is: " << count << " current img path is :" << result_point[0].fileName << endl;
// }
// count++;
/*
float yaw = result_point[0].direct[0];
float pitch = result_point[0].direct[1];
float roll = result_point[0].direct[2];
string img_path = search_dir_[temp] + "\\" + result_point[0].fileName;
if (_access(img_path.c_str(), 0) == -1)
{
cout << "coun't found filename" << img_path << endl;
result_point.erase(itor);
continue;
}
Mat img = imread(img_path);
Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
cvtColor(img, dstImg, CV_BGR2GRAY);
//yaw 增强 pitch 增强
for (int j = 0; j < 2; j++)
{
if (result_point[0].direct[j] > -theta_offset && result_point[0].direct[j] < theta_offset)
{
Rect rect;
LandMark landmark(result_point[0]);
bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
if (success)
{
vector<Rect> all_rect;
getoffsetRect(rect, all_rect, img.cols, img.rows, 4);
for (int i = 0; i < 2; i++)
{
LandMark dst_landmark;
//vector<string> filenames;
//splitString(landmark.fileName, filenames, "/");
//string filename = filenames[filenames.size()-1];
dst_landmark.fileName = text_pre[temp] + to_string(countNum) + ".png";
//cout << img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
Mat roi_face = img(all_rect[i]);
cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
//坐标转换
for (int k = 0; k < 5; k++)
{
dst_landmark.visible[k] = landmark.visible[five_points[k]];
dst_landmark.points[k].x = ((float)SIZE / all_rect[i].width) * (landmark.points[five_points[k]].x - all_rect[i].x);
dst_landmark.points[k].y = ((float)SIZE / all_rect[i].height) * (landmark.points[five_points[k]].y - all_rect[i].y);
}
imwrite(result_img + dst_landmark.fileName, roi_face);
rotation_point.push_back(dst_landmark);
countNum++;
}
}
}
}
// roll 增强
if (roll > -theta_offset && roll < theta_offset)
{
for (int i = -1; i < 2; i = i + 2)
{
Mat rotat_img;
LandMark landmark(result_point[0]);
int theta = (rand() % theta_max + theta_offset) * i;
rotation(theta, img, rotat_img, Size(SIZE, SIZE), landmark, CURRENT_LANDMARK);
Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
cvtColor(rotat_img, dstImg, CV_BGR2GRAY);
//for (int j = 0; j < CURRENT_LANDMARK; j++)
//{
// circle(rotat_img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
//}
//imshow("img", rotat_img);
//waitKey(0);
LandMark dst_landmark;
//vector<string> filenames;
//splitString(landmark.fileName, filenames, "/");
//string filename = filenames[filenames.size()-1];
dst_landmark.fileName = text_pre[temp] + to_string(countNum) + ".png";
Rect rect;
bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
if (success)
{
//cout << rotat_img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
Mat roi_face = rotat_img(rect);
cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
//坐标转换
for (int k = 0; k < 5; k++)
{
dst_landmark.visible[k] = landmark.visible[five_points[k]];
dst_landmark.points[k].x = ((float)SIZE / rect.width) * (landmark.points[five_points[k]].x - rect.x);
dst_landmark.points[k].y = ((float)SIZE / rect.height) * (landmark.points[five_points[k]].y - rect.y);
}
imwrite(result_img + dst_landmark.fileName, roi_face);
rotation_point.push_back(dst_landmark);
countNum++;
}
}
}
*/
//}
}
if (rotation_point.size() > 0)
{
dpUtil.writePointVisibletoFile(result_txt, rotation_point, NEED_LANDMARK);
rotation_point.clear();
}
system("PAUSE");
return 0;
}