caffe 版本 yolo 过程记录

【注】后续再写可训练版本的yolo caffe版
【caffe训练版本已经更新caffe-yolo训练】
YOLO 框的获取

 x = image.cols*(result[idx++]+j)/7;//i表示gride的位置
 y = image.rows*(result[idx++]+i)/7;//j表示gride的位置
 w = image.cols*result[idx]*result[idx++];
 h = image.rows*result[idx]*result[idx];

一、yolo 网络 weight 准备

yolo网络里面含有权重的层分别为卷积层和全链接层，因此，我对yolo里面将每一个含有权重的层进行了存储权重的操作，并将它们分别存储在txt中，下面的程序是loadweights函数实现的是将txt中的权重赋值给net。
下面涉及到的大部分文件都在百度盘中（链接：http://pan.baidu.com/s/1hs8XCW0 密码：8rcu）

二、yolo网络prototxt的准备

这里只是展示一下yolo-tiny-deploy.prototxt。 yolo-deploy.prototxt可以参见百度盘

name: "yolotiny"
input: "data"
input_dim: 1
input_dim: 3
input_dim: 448
input_dim: 448
layer{
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  convolution_param{
    num_output: 16
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky1"
  type: "Leaky"
  bottom: "conv1"
  top: "Leaky1"
}
layer{
  name: "pool1"
  type: "Pooling"
  bottom: "Leaky1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer{
  name: "conv2"
  type: "Convolution"
  bottom: "pool1"
  top: "conv2"
  convolution_param{
    num_output: 32
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky2"
  type: "Leaky"
  bottom: "conv2"
  top: "Leaky2"
}
layer{
  name: "pool2"
  type: "Pooling"
  bottom: "Leaky2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer{
  name: "conv3"
  type: "Convolution"
  bottom: "pool2"
  top: "conv3"
  convolution_param{
    num_output: 64
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky3"
  type: "Leaky"
  bottom: "conv3"
  top: "Leaky3"
}
layer{
  name: "pool3"
  type: "Pooling"
  bottom: "Leaky3"
  top: "pool3"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer{
  name: "conv4"
  type: "Convolution"
  bottom: "pool3"
  top: "conv4"
  convolution_param{
    num_output: 128
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky4"
  type: "Leaky"
  bottom: "conv4"
  top: "Leaky4"
}
layer{
  name: "pool4"
  type: "Pooling"
  bottom: "Leaky4"
  top: "pool4"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer{
  name: "conv5"
  type: "Convolution"
  bottom: "pool4"
  top: "conv5"
  convolution_param{
    num_output: 256
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky5"
  type: "Leaky"
  bottom: "conv5"
  top: "Leaky5"
}
layer{
  name: "pool5"
  type: "Pooling"
  bottom: "Leaky5"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer{
  name: "conv6"
  type: "Convolution"
  bottom: "pool5"
  top: "conv6"
  convolution_param{
    num_output: 512
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky6"
  type: "Leaky"
  bottom: "conv6"
  top: "Leaky6"
}
layer{
  name: "pool6"
  type: "Pooling"
  bottom: "Leaky6"
  top: "pool6"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer{
  name: "conv7"
  type: "Convolution"
  bottom: "pool6"
  top: "conv7"
  convolution_param{
    num_output: 1024
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky7"
  type: "Leaky"
  bottom: "conv7"
  top: "Leaky7"
}
layer{
  name: "conv8"
  type: "Convolution"
  bottom: "Leaky7"
  top: "conv8"
  convolution_param{
    num_output: 1024
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky8"
  type: "Leaky"
  bottom: "conv8"
  top: "Leaky8"
}
layer{
  name: "conv9"
  type: "Convolution"
  bottom: "Leaky8"
  top: "conv9"
  convolution_param{
    num_output: 1024
    kernel_size: 3
    stride: 1
    pad: 1
  }
}
layer{
  name: "Leaky9"
  type: "Leaky"
  bottom: "conv9"
  top: "Leaky9"
}
layer {
  name: "connect1"
  type: "InnerProduct"
  bottom: "Leaky9"
  top: "connect1"
  inner_product_param {
    num_output: 256
  }
}
layer {
  name: "connect2"
  type: "InnerProduct"
  bottom: "connect1"
  top: "connect2"
  inner_product_param {
    num_output: 4096
  }
}
layer{
  name: "Leaky10"
  type: "Leaky"
  bottom: "connect2"
  top: "Leaky10"
}
layer {
  name: "connect3"
  type: "InnerProduct"
  bottom: "Leaky10"
  top: "connect3"
  inner_product_param {
    num_output: 1470
  }
}

三、添加层（Leaky层）

由于caffe没有自带Leaky层，因此需要添加以下层。
leaky_layer.hpp

template <typename Dtype>
class LeakyLayer : public NeuronLayer<Dtype>{
public:
    explicit LeakyLayer(const LayerParameter& param)
       : NeuronLayer<Dtype>(param) {}
    virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
          const vector<Blob<Dtype>*>& top);
    virtual inline const char* type() const { return "Leaky"; }
    virtual inline int ExactNumBottomBlobs() const { return 1; }
    virtual inline int ExactNumTopBlobs() const { return 1; }
protected:
      virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
          const vector<Blob<Dtype>*>& top);
      virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
          const vector<Blob<Dtype>*>& top);
      virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
          const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom){};//暂时还没实现
      virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
          const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom){};//暂时还没实现
};

leaky_layer.cpp

//add by yang
template <typename Dtype>
void LeakyLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
          const vector<Blob<Dtype>*>& top) {
    NeuronLayer<Dtype>::LayerSetUp(bottom, top);
      CHECK_NE(top[0], bottom[0]) << this->type() << " Layer does not "
        "allow in-place computation.";
}

template <typename Dtype>
void LeakyLayer<Dtype>::Forward_cpu(
        const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
      const int count = top[0]->count();
      Dtype* top_data = top[0]->mutable_cpu_data();
      const Dtype* bottom_data = bottom[0]->cpu_data();
      for(int i = 0; i < count; ++i){
          if(bottom_data[i] > 0)
              top_data[i] = bottom_data[i];
          else
              top_data[i] = 0.1*bottom_data[i];
          //top_data[i] = bottom_data[i] > 0？ bottom_data[i]: (Dtype(0.1)*bottom_data[i]);
      }
}
//add by yang
#ifdef CPU_ONLY
STUB_GPU(LeakyLayer);
#endif
INSTANTIATE_CLASS(LeakyLayer);
REGISTER_LAYER_CLASS(Leaky);

leaky_layer.cu

//add by yang
template <typename Dtype>
__global__ void LeakyForward(const int n, const Dtype* in, Dtype* out){
    CUDA_KERNEL_LOOP(index, n){
        out[index] = in[index] > 0 ? in[index] : in[index]*0.1;
    }
}

template <typename Dtype>
void LeakyLayer<Dtype>::Forward_gpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  const int count = bottom[0]->count();
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* top_data = top[0]->mutable_gpu_data();
  LeakyForward<Dtype><<<CAFFE_GET_BLOCKS(count), CAFFE_CUDA_NUM_THREADS>>>(
          count, bottom_data, top_data);
  CUDA_POST_KERNEL_CHECK;
}
INSTANTIATE_LAYER_GPU_FUNCS(LeakyLayer);

四、使用

将第六步的代码复制到caffe根目录下面的tools文件夹，make通过之后。

argv[1]网络的prototxt
argv[2]权重文件(如果编译时候使用的是loadweight，argv[2]对应的是txt文件夹，如果使用的是CopyTrainedLayersFromBinaryProto，argv[2]对应的是caffemodel)
argv[3]图片文件

五、结果展示

六、代码

#include <string>
#include <vector>

#include "boost/algorithm/string.hpp"
#include "google/protobuf/text_format.h"
#include <opencv2/opencv.hpp>

#include "caffe/blob.hpp"
#include "caffe/layer.hpp"
#include "caffe/common.hpp"
#include "caffe/net.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
#include "caffe/util/format.hpp"
#include "caffe/util/io.hpp"
#include <stdio.h>
#include <malloc.h>
#include <fstream>
#include <boost/progress.hpp>


#include <boost/math/special_functions/next.hpp>
#include <boost/random.hpp>

#include <limits>

#include "caffe/common.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/util/rng.hpp"


//#include "caffe/util/math_functions.hpp"

using caffe::Blob;
using caffe::Caffe;
using caffe::Datum;
using caffe::Net;
using caffe::Layer;
using std::string;
namespace db = caffe::db;

template<typename Dtype>
Dtype lap(Dtype x1_min,Dtype x1_max,Dtype x2_min,Dtype x2_max){
    if(x1_min < x2_min){
        if(x1_max < x2_min){
            return 0;
        }else{
            if(x1_max > x2_min){
                if(x1_max < x2_max){
                    return x1_max - x2_min;
                }else{
                    return x2_max - x2_min;
                }
            }else{
                return 0;
            }
        }
    }else{
        if(x1_min < x2_max){
            if(x1_max < x2_max)
                return x1_max-x1_min;
            else
                return x2_max-x1_min;
        }else{
            return 0;
        }
    }
}

template int lap(int x1_min, int x1_max, int x2_min, int x2_max);
template float lap(float x1_min, float x1_max, float x2_min, float x2_max);

void loadweights(boost::shared_ptr<Net<float> >& net,char* argv);
void loaddata(boost::shared_ptr<Net<float> >& net, std::string image_path);
void getbox(std::vector<float> result,float* pro_obj,int* idx_class,std::vector<std::vector<int> >& bboxs,float thresh,cv::Mat image);
//int lap(int x1_min,int x1_max,int x2_min,int x2_max);
int main(int argc, char** argv){
  //boost::progress_timer t;
  char *labelname[] = {"aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"};
  Caffe::set_mode(Caffe::GPU);
  boost::shared_ptr<Net<float> > net(new Net<float>(argv[1], caffe::TEST));
  //loadweights(net,argv[2]);//这行代码是还没caffemodel的时候使用的。
  net->CopyTrainedLayersFromBinaryProto(argv[2]);//有caffemodel的时候就可以使用这行代码，跟上面一行互补使用。
  loaddata(net,std::string(argv[3]));
  //std::cout<<"load weights and data 's time = "<< t.elapsed() <<std::endl;
  net->Forward();
  //std::cout<<"to process finish time = "<< t.elapsed() <<std::endl;
  Blob<float>* output_layer = net->output_blobs()[0];
  const float* begin = output_layer->cpu_data();
  const float* end = begin + output_layer->channels();
  std::vector<float> result(begin, end);
  //接下来就是生成框。
  std::vector<std::vector<int> > bboxs;
  float pro_obj[49][2];
  int idx_class[49];
  cv::Mat image = cv::imread(argv[3]);
  float overlap;
  float overlap_thresh = 0.4;
  //cv::resize(image,image,cv::Size(448,448));
  getbox(result, &pro_obj[0][0],idx_class,bboxs,0.2,image);
  std::vector<bool> mark(bboxs.size(),true);
  for(int i = 0; i < bboxs.size(); ++i){
      for(int j = i+1; j < bboxs.size(); ++j){
          int overlap_x = lap(bboxs[i][0],bboxs[i][2],bboxs[j][0],bboxs[j][2]);
          int overlap_y = lap(bboxs[i][1],bboxs[i][3],bboxs[j][1],bboxs[j][3]);
          overlap = (overlap_x*overlap_y)*1.0/((bboxs[i][0]-bboxs[i][2])*(bboxs[i][1]-bboxs[i][3])+(bboxs[j][0]-bboxs[j][2])*(bboxs[j][1]-bboxs[j][3])-(overlap_x*overlap_y));
          if(overlap > overlap_thresh){
              if(bboxs[i][4] > bboxs[j][4]){
                  mark[j] = false;
              }else{
                  mark[i] = false;
              }
          }
      }
  }
  for(int i = 0; i < bboxs.size();++i){
      if(mark[i]){
          cv::Point point1(bboxs[i][1],bboxs[i][2]);
          cv::Point point2(bboxs[i][3],bboxs[i][4]);
          cv::rectangle(image, cv::Rect(point1,point2),cv::Scalar(0,bboxs[i][0]/20.0*225,255),bboxs[i][5]/8);
          char ch[100];
          sprintf(ch,"%s %.2f",labelname[bboxs[i][0]-1], bboxs[i][5]*1.0/100);
          std::string temp(ch);
          cv::putText(image,temp,point1,CV_FONT_HERSHEY_COMPLEX,0.4,cv::Scalar(255,255,255));
      }
  }
  //输出结果，画框。。
  cv::imshow("yolo",image);

  std::string save_name(argv[3]);
  save_name = save_name.substr(0, save_name.find('.'));
  save_name = save_name + "_yolo_detect.jpg";
  std::cout << save_name << std::endl;
  cv::imwrite(save_name, image);

  cv::waitKey(0);
  //下面这三行注释的代码的作用，当你只有txt存储的weight的时候，需要将weight转化为caffemodel的时候就使用这三行代码。
  //caffe::NetParameter net_param;
 // net->ToProto(&net_param,false);
  //WriteProtoToBinaryFile(net_param, "/home/yang/yolo2caffe/yolo.caffemodel");
  return 1;
}
void loadweights(boost::shared_ptr<Net<float> >& net,char* argv){
  char txt_name[200];
  strcat(txt_name,argv);
  char path[200];
  const std::vector<boost::shared_ptr<Layer<float> > > layers = net->layers();
  int convolution_n = 0;
  int connect_n = 0;
  FILE* fp;
  char* name = (char*)malloc(sizeof(char)*100);
  boost::shared_ptr<Layer<float> > layer;
  std::vector<boost::shared_ptr<Blob<float> > > blobs;
  for(int i = 0; i < layers.size(); ++i){
    layer = layers[i];
    blobs = layer->blobs();
    if(layer->type() == std::string("Convolution")){
        ++convolution_n;
        std::cout << "convolution" << convolution_n <<std::endl;
        sprintf(path,"%s/convolution%d.txt",argv,convolution_n);
        //std::cout << path << std::endl;
        //sprintf(name,"/home/yang/yolo2caffe/yolo/yolo_convolution%d.txt",convolution_n);
        fp = fopen(path,"r");
        fread(blobs[1]->mutable_cpu_data(), sizeof(float), blobs[1]->count(), fp);
        fread(blobs[0]->mutable_cpu_data(), sizeof(float), blobs[0]->count(), fp);
    }
    else{
      if(layer->type() == std::string("InnerProduct")){
        ++connect_n;
        std::cout << "Connect" << connect_n <<std::endl;
        sprintf(path,"%s/connect%d.txt",argv,connect_n);
        //std::cout << path << std::endl;
        fp = fopen(path,"r");
        fread(blobs[1]->mutable_cpu_data(), sizeof(float), blobs[1]->count(), fp);
        fread(blobs[0]->mutable_cpu_data(), sizeof(float), blobs[0]->count(), fp);
      }
    }
  }
  if(fp != NULL)
    fclose(fp);
  delete []name;
}
void loaddata(boost::shared_ptr<Net<float> >& net, std::string image_path){
  Blob<float>* input_layer = net->input_blobs()[0];
  int width, height;
  width = input_layer->width();
  height = input_layer->height();
  int size = width*height;
  cv::Mat image = cv::imread(image_path,-1);
  cv::Mat image_resized;
  cv::resize(image, image_resized, cv::Size(height, width));
  float* input_data = input_layer->mutable_cpu_data();
  int temp,idx;
  for(int i = 0; i < height; ++i){
    uchar* pdata = image_resized.ptr<uchar>(i);
    for(int j = 0; j < width; ++j){
      temp = 3*j;
      idx = i*width+j;
      input_data[idx] = (pdata[temp+2]/127.5)-1;
      input_data[idx+size] = (pdata[temp+1]/127.5)-1;
      input_data[idx+2*size] = (pdata[temp+0]/127.5)-1;
    }
  }
  //cv::imshow("image",image_resized);
}
void getbox(std::vector<float> result,float* pro_obj,int* idx_class,std::vector<std::vector<int> >& bboxs,float thresh,cv::Mat image){
  float pro_class[49];
  int idx;
  float max_idx;
  float max;
  for(int i = 0; i < 7; ++i){
    for(int j = 0; j < 7;++j){
      max = 0;
      max_idx = 0;
      idx = 20*(i*7+j);
      for(int k = 0; k < 20; ++k){
        if (result[idx+k] > max){
          max = result[idx+k];
          max_idx = k+1;
        }
      }
      idx_class[i*7+j] = max_idx;
      pro_class[i*7+j] = max;
      pro_obj[(i*7+j)*2] = max*result[7*7*20+(i*7+j)*2];
      pro_obj[(i*7+j)*2+1] = max*result[7*7*20+(i*7+j)*2+1];
    }
  }
  std::vector<int> bbox;
  int x_min,x_max,y_min,y_max;
  float x,y,w,h;
  for(int i = 0; i < 7;++i){
    for(int j = 0; j < 7;++j){
      for(int k = 0; k < 2; ++k){
          if(pro_obj[(i*7+j)*2 + k] > thresh){
              //std::cout << "(" << i << "," << j << "," << k << ")" << " prob="<<pro_obj[(i*7+j)*2 + k] << " class="<<idx_class[i*7+j]<<std::endl;
              idx = 49*20 + 49*2 + ((i*7+j)*2+k)*4;
              x = image.cols*(result[idx++]+j)/7;
              y = image.rows*(result[idx++]+i)/7;
              w = image.cols*result[idx]*result[idx++];
              h = image.rows*result[idx]*result[idx];
              //std::cout << x <<" "<< y << " " << w <<" "<< h <<std::endl;
              x_min = x - w/2;
              y_min = y - h/2;
              x_max = x + w/2;
              y_max = y + h/2;
              bbox.clear();
              bbox.push_back(idx_class[i*7+j]);
              bbox.push_back(x_min);
              bbox.push_back(y_min);
              bbox.push_back(x_max);
              bbox.push_back(y_max);
              bbox.push_back(int(pro_obj[(i*7+j)*2 + k]*100));
              bboxs.push_back(bbox);
          }
      }
    }
  }
}