添加caffe layers

caffe 的C++ 接口对于train 非常的方便，但是对于perdict 而言就不太友好，我们只能看到loss 。但是对于multilabel 问题，比如输出的perdict 和label 是一副图片的话，用c++ 就不太方便，所以我们要在caffe layers 中 添加一些自己的layer 来实现可视化。

在一开始添加的时候，会遇到各种各样的bug，这个mark 下以备不时之需。

比如说，我要添加一个visual layer，把perdict 的图片输出出来。
在caffe/include/caffe/layers/ 下添加visual_layer.hpp
在 caffe/src/caffe/layers/ 下添加visual_layer.cpp
修改 caffe/src/proto 的caffe.proto 下修改相应的属性。

message LayerParameter {
...
optional VisualParameter visual_param = 1000; 这里添加的是caffe 的type，数字在message 上有题型下一个可用的数字是多少，这个因人而异
}

....
添加多一个message
message VisualParameter {
  optional string save_folder = 1;
  optional uint32 save_interval = 2 [default = 2];
  optional float scale = 3 [default = 1];
}

visual_layer.hpp

#ifndef CAFFE_VISUAL_LAYER_HPP_
#define CAFFE_VISUAL_LAYER_HPP_

#include <stdio.h>
#include <vector>

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/highgui/highgui_c.h>
#include <opencv2/imgproc/imgproc.hpp>

#include "caffe/blob.hpp"
#include "caffe/layer.hpp"
#include "caffe/proto/caffe.pb.h"

#include "caffe/layers/loss_layer.hpp"

namespace caffe {


template <typename Dtype>
class VisualLayer : public Layer<Dtype> {
 public:

  explicit VisualLayer(const LayerParameter& param)
      : Layer<Dtype>(param) {}
  virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top);
  virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
      //vector<int> top_shape(0); 
      //top[0]->Reshape(top_shape);
  };

  virtual inline const char* type() const { return "Visual"; }
  virtual inline int ExactNumBottomBlobs() const { return 3; }

  virtual inline int MinTopBlobs() const { return 0; }
  virtual inline int MaxTopBlos() const { return 0; }

 protected:

  virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top);

  /// @brief Not implemented -- VisualLayer 不需要回传梯度.
  virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
    for (int i = 0; i < propagate_down.size(); ++i) {
      if (propagate_down[i]) { NOT_IMPLEMENTED; }
    }
  }

 private:
  cv::Mat BlobToColorImage(const Blob<Dtype>* blob, const int n, const int height, const int width, const Dtype img_scale);
  cv::Mat BlobToGreyImage(const Blob<Dtype>* blob, const int n, const int height, const int width, const Dtype img_scale);
  void BlobToFourChannelImage(const Blob<Dtype>* blob, const int n, const int height, const int width, const Dtype img_scale);

  int iteration_num;
  int save_interval;
  string save_path;
  float scale;
};

}  // namespace caffe

#endif  // CAFFE_VISUAL_LAYER_HPP_

visual_layer.cpp

#include <functional>
#include <utility>
#include <vector>

#include "caffe/layers/visual_layer.hpp"
#include "caffe/util/math_functions.hpp"

namespace caffe {

template <typename Dtype>
void VisualLayer<Dtype>::LayerSetUp(
  const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {

  iteration_num = 0;
  save_interval = this->layer_param_.visual_param().save_interval();
  save_path = this->layer_param_.visual_param().save_folder();
  scale = this->layer_param_.visual_param().scale();

  CHECK_GT(save_interval, 0);
}



template <typename Dtype>
void VisualLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
    const vector<Blob<Dtype>*>& top) {
  ++iteration_num;
  if(iteration_num % save_interval != 0)
    return;

  char name_buffer[100];
  cv::Mat rgb_img;
  cv::Mat label_img;
  cv::Mat prediction_img;
  vector<cv::Mat> data_img;
  for(int img_id = 0; img_id < bottom[0]->num(); ++img_id) {
    //BlobToFourChannelImage(bottom[0], img_id, bottom[0]->height(), bottom[0]->width(), scale);
    label_img      = BlobToGreyImage(bottom[1], img_id, bottom[1]->height(), bottom[1]->width(), scale);
    prediction_img = BlobToGreyImage(bottom[2], img_id, bottom[2]->height(), bottom[2]->width(), scale);
    //LOG(INFO) << "pred_rows: " << prediction_img.rows;
    //LOG(INFO) << "pred_cols: " << prediction_img.cols;
   //channel LOG(INFO) << "pred_chan: " << prediction_img.channels;
    //label_img = BlobToGreyImage(bottom[1], img_id, scale);
    //prediction_img = BlobToGreyImage(bottom[2], img_id, scale);


    sprintf(name_buffer, "%d_%d_label.jpg", iteration_num, img_id);
    cv::imwrite(save_path + string(name_buffer), label_img);

    sprintf(name_buffer, "%d_%d_prediction.jpg", iteration_num, img_id);
    cv::imwrite(save_path + string(name_buffer), prediction_img);
  }
}

template <typename Dtype>
void VisualLayer<Dtype>::BlobToFourChannelImage(const Blob<Dtype>* blob, const int n,
  const int height, const int width, const Dtype img_scale) {
  CHECK_GE(blob->channels(), 4) << "Only Support FourChannel Images";
  const Dtype* cpu_data = blob->cpu_data();
  int offset = n * blob->channels() * height * width;

  cv::Mat img1(height, width, CV_8UC3);
  cv::Mat img2(height, width, CV_8UC1);
  for (int c = 0; c < 4; ++c) {
    for (int h = 0; h < img1.rows; ++h) {
      for (int w = 0; w < img1.cols; ++w) {
        Dtype value = cpu_data[offset];
        //if(value < 0) value = 0;
        //if(value > 255) value = 255;
        if (c < 3) {
          img1.at<cv::Vec3b>(h, w)[c] = (int)cv::saturate_cast<uchar>(value);
          //LOG(INFO) << "cv::saturate_cast: " << (int)cv::saturate_cast<uchar>(value);
        }
        else img2.at<uchar>(h,w) = cv::saturate_cast<uchar>(value);

        ++offset;
      }
    }
  }
  //cv::imshow("color data", img1);
  //cv::waitKey(10000);
  //cv::imshow("gray data", img2);
  //cv::waitKey(10000);
}



template <typename Dtype>
cv::Mat VisualLayer<Dtype>::BlobToColorImage(const Blob<Dtype>* blob, const int n, 
  const int height, const int width, const Dtype img_scale) {
  CHECK_GE(blob->channels(), 3) << "Only Support Color images";

  const Dtype* cpu_data = blob->cpu_data();
  int offset = n * blob->channels() * height * width;

  cv::Mat img(height, width, CV_8UC3);
  for (int c = 0; c < 3; ++c) {
    for (int h = 0; h < img.rows; ++h) {
      for (int w = 0; w < img.cols; ++w) {
        Dtype value = cpu_data[offset] * img_scale;
        if(value < 0) value = 0;
        if(value > 255) value = 255;

        img.at<cv::Vec3b>(h, w)[c] = cv::saturate_cast<uchar>(value);

        ++offset;
      }
    }
  }

  return img;
}

template <typename Dtype>
cv::Mat VisualLayer<Dtype>::BlobToGreyImage(const Blob<Dtype>* blob, const int n, 
  const int height, const int width, const Dtype img_scale) {
  CHECK_GE(blob->channels(), 1) << "Only Support grey images";

 const Dtype* cpu_data = blob->cpu_data();
 int offset = n * height * width;

  cv::Mat img(height, width, CV_8UC1);
  for (int h = 0; h < img.rows; ++h) {
    for (int w = 0; w < img.cols; ++w) {
      //Dtype value = blob->data_at(n, 0, h, w) * img_scale;
      Dtype value = cpu_data[offset] * img_scale;
      if(value < 0) value = 0;
      if(value > 255) value = 255;

      img.at<uchar>(h, w) = cv::saturate_cast<uchar>(value);
      ++offset;
    }
  }
  /*cv::resize(img, img, cv::Size(128,128));
  cv::namedWindow( "Gray image", CV_WINDOW_NORMAL);
  cv::imshow("Gray image", img);
  cv::waitKey(1);*/
  //cv::destroyWindow("Gray image");
  return img;
}

INSTANTIATE_CLASS(VisualLayer);
REGISTER_LAYER_CLASS(Visual);

}  // namespace caffe

再写好之后，回到caffe 根目录，
make
要是编译通过了就说明添加成功了。然后就可以在train.prototxt 添加这个新的layer了。