Segnet 学习

Segnet C++ Class

1. 在caffe-segnet的源码中，github上已经更新了其C++接口和例程；

@ubuntu:~/mumu/SegNet/caffe-segnet/examples/SegNet_with_C++$ ls
test_segmentation.cpp

新建一个文件夹mu_SegnetClass:
文件架构如图：

工程文件夹下的CMakeLists.txt文件：

PROJECT(mu_SegnetClass)
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
SET(CMAKE_BUILD_TYPE Debug)
# 设置编译选项,允许c++11标准、O3优化、多线程。match选项可避免一些cpu上的问题
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -march=native -O3 -pthread")
find_package( OpenCV REQUIRED )
find_package( Caffe REQUIRED )
include_directories(${Caffe_INCLUDE_DIRS})
add_definitions(${Caffe_DEFINITIONS})
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
#二进制文件输出到bin
SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/lib)
#库输出到lib
add_subdirectory(${PROJECT_SOURCE_DIR}/src/)

源文件目录下的CMakeLists.txt

set(build_libraries
        ${Caffe_LIBRARIES}
        ${OpenCV_LIBS}
        ${BOOST_LIBRARIES}
        )
add_executable(mu_SegnetClass
        test_segmentation.cpp
        )
target_link_libraries(mu_SegnetClass
        ${build_libraries}
        )

终端命令行键入：

@ubuntu:~/mumu/SegNet$ ./mu_SegnetClass/bin/mu_SegnetClass Example_Models/segnet_sun.prototxt segnet_sun.caffemodel image/0001231.jpg Scripts/sun.png
---------- Semantic Segmentation for image/0001231.jpg ----------
Processing time = 1223 ms
output_blob(n,c,h,w) = 1, 1, 360, 480
//此处输出语义分割后的图片//本机GPU实在太差，速率慢成狗

test_segmentation.cpp源码

#define USE_OPENCV 1
#include <caffe/caffe.hpp>
#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#endif  // USE_OPENCV
#include <algorithm>
#include <iosfwd>
#include <memory>
#include <string>
#include <utility>
#include <vector>
//#include <chrono> //Just for time measurement. This library requires compiler and library support for the ISO C++ 2011 standard. This support is currently experimental in Caffe, and must be enabled with the -std=c++11 or -std=gnu++11 compiler options.
#ifdef USE_OPENCV
using namespace caffe;  // NOLINT(build/namespaces)
using std::string;

class Classifier {
 public:
  Classifier(const string& model_file, //模型文件 segnet_sun.prototxt
             const string& trained_file); //训练文件 segnet_sun.caffemodel

  void Predict(const cv::Mat& img, string LUT_file);
 private:
  void SetMean(const string& mean_file);
  void WrapInputLayer(std::vector<cv::Mat>* input_channels);
  void Preprocess(const cv::Mat& img,
                  std::vector<cv::Mat>* input_channels);
  void Visualization(Blob<float>* output_layer, string LUT_file);
 private:
  shared_ptr<Net<float> > net_;
  cv::Size input_geometry_;
  int num_channels_;
};

Classifier::Classifier(const string& model_file,
                       const string& trained_file) {

  Caffe::set_mode(Caffe::GPU);
  /* Load the network. */
  net_.reset(new Net<float>(model_file, TEST));
  net_->CopyTrainedLayersFrom(trained_file);

  CHECK_EQ(net_->num_inputs(), 1) << "Network should have exactly one input.";
  CHECK_EQ(net_->num_outputs(), 1) << "Network should have exactly one output.";

  Blob<float>* input_layer = net_->input_blobs()[0];
  num_channels_ = input_layer->channels();
  CHECK(num_channels_ == 3 || num_channels_ == 1)
    << "Input layer should have 1 or 3 channels.";
  input_geometry_ = cv::Size(input_layer->width(), input_layer->height());
}

void Classifier::Predict(const cv::Mat& img, string LUT_file) {
  Blob<float>* input_layer = net_->input_blobs()[0];
  input_layer->Reshape(1, num_channels_,
                       input_geometry_.height, input_geometry_.width);
  /* Forward dimension change to all layers. */
  net_->Reshape();

  std::vector<cv::Mat> input_channels;
  WrapInputLayer(&input_channels);

  Preprocess(img, &input_channels);

  struct timeval time;
  gettimeofday(&time, NULL); // Start Time
  long totalTime = (time.tv_sec * 1000) + (time.tv_usec / 1000);
  //std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); //Just for time measurement

  net_->Forward();

  gettimeofday(&time, NULL);  //END-TIME
  totalTime = (((time.tv_sec * 1000) + (time.tv_usec / 1000)) - totalTime);
  std::cout << "Processing time = " << totalTime << " ms" << std::endl;

  //std::chrono::steady_clock::time_point end= std::chrono::steady_clock::now();
  //std::cout << "Processing time = " << (std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count())/1000000.0 << " sec" <<std::endl; //Just for time measurement

  /* Copy the output layer to a std::vector */
  Blob<float>* output_layer = net_->output_blobs()[0];

  Visualization(output_layer, LUT_file);
}

void Classifier::Visualization(Blob<float>* output_layer, string LUT_file) {

  std::cout << "output_blob(n,c,h,w) = " << output_layer->num() << ", " << output_layer->channels() << ", "
              << output_layer->height() << ", " << output_layer->width() << std::endl;

  cv::Mat merged_output_image = cv::Mat(output_layer->height(), output_layer->width(), CV_32F, const_cast<float *>(output_layer->cpu_data()));
  //merged_output_image = merged_output_image/255.0;

  merged_output_image.convertTo(merged_output_image, CV_8U);
  cv::cvtColor(merged_output_image.clone(), merged_output_image, CV_GRAY2BGR);
  cv::Mat label_colours = cv::imread(LUT_file,1);
  cv::Mat output_image;
  LUT(merged_output_image, label_colours, output_image);

  cv::imshow( "Display window", output_image);
  cv::waitKey(0);
}

/* Wrap the input layer of the network in separate cv::Mat objects
 * (one per channel). This way we save one memcpy operation and we
 * don't need to rely on cudaMemcpy2D. The last preprocessing
 * operation will write the separate channels directly to the input
 * layer. */
void Classifier::WrapInputLayer(std::vector<cv::Mat>* input_channels) {
  Blob<float>* input_layer = net_->input_blobs()[0];

  int width = input_layer->width();
  int height = input_layer->height();
  float* input_data = input_layer->mutable_cpu_data();
  for (int i = 0; i < input_layer->channels(); ++i) {
    cv::Mat channel(height, width, CV_32FC1, input_data);
    input_channels->push_back(channel);
    input_data += width * height;
  }
}

void Classifier::Preprocess(const cv::Mat& img,
                            std::vector<cv::Mat>* input_channels) {
  /* Convert the input image to the input image format of the network. */
  cv::Mat sample;
  if (img.channels() == 3 && num_channels_ == 1)
    cv::cvtColor(img, sample, cv::COLOR_BGR2GRAY);
  else if (img.channels() == 4 && num_channels_ == 1)
    cv::cvtColor(img, sample, cv::COLOR_BGRA2GRAY);
  else if (img.channels() == 4 && num_channels_ == 3)
    cv::cvtColor(img, sample, cv::COLOR_BGRA2BGR);
  else if (img.channels() == 1 && num_channels_ == 3)
    cv::cvtColor(img, sample, cv::COLOR_GRAY2BGR);
  else
    sample = img;

  cv::Mat sample_resized;
  if (sample.size() != input_geometry_)
    cv::resize(sample, sample_resized, input_geometry_);
  else
    sample_resized = sample;

  cv::Mat sample_float;
  if (num_channels_ == 3)
    sample_resized.convertTo(sample_float, CV_32FC3);
  else
    sample_resized.convertTo(sample_float, CV_32FC1);

  /* This operation will write the separate BGR planes directly to the
   * input layer of the network because it is wrapped by the cv::Mat
   * objects in input_channels. */
  cv::split(sample_float, *input_channels);

  CHECK(reinterpret_cast<float*>(input_channels->at(0).data)
        == net_->input_blobs()[0]->cpu_data())
    << "Input channels are not wrapping the input layer of the network.";
}

int main(int argc, char** argv) {
  if (argc != 5) {
    std::cerr << "Usage: " << argv[0]
              << " \ndeploy.prototxt \nnetwork.caffemodel"
              << " \nimg.jpg" << " \ncamvid12.png (for example: /SegNet-Tutorial/Scripts/camvid12.png)" << std::endl;
    return 1;
  }

  ::google::InitGoogleLogging(argv[0]);

  string model_file   = argv[1];
  string trained_file = argv[2]; //for visualization

  Classifier classifier(model_file, trained_file);

  string file = argv[3];
  string LUT_file = argv[4];

  std::cout << "---------- Semantic Segmentation for "
            << file << " ----------" << std::endl;

  cv::Mat img = cv::imread(file, 1);
  CHECK(!img.empty()) << "Unable to decode image " << file;
  cv::Mat prediction;

  classifier.Predict(img, LUT_file);
}
#else
int main(int argc, char** argv) {
  LOG(FATAL) << "This example requires OpenCV; compile with USE_OPENCV.";
}
#endif  // USE_OPENCV

Bug记录—数组越界

此处主要的原因是char num[3];
当图片的数据超过100的时候，数组越界，改为char num[5];即可

---------- Semantic Segmentation for ../image/image_3/99.png ----------
Processing time = 886 ms
output_blob(n,c,h,w) = 1, 1, 360, 480
*** buffer overflow detected ***: ../bin/./mu_SegnetClass terminated
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x7329f)[0x7f1eb114529f]
/lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x5c)[0x7f1eb11e083c]
/lib/x86_64-linux-gnu/libc.so.6(+0x10d710)[0x7f1eb11df710]
/lib/x86_64-linux-gnu/libc.so.6(+0x10cc19)[0x7f1eb11dec19]
/lib/x86_64-linux-gnu/libc.so.6(_IO_default_xsputn+0xbc)[0x7f1eb114d61c]
/lib/x86_64-linux-gnu/libc.so.6(_IO_vfprintf+0x161e)[0x7f1eb111d25e]
/lib/x86_64-linux-gnu/libc.so.6(__vsprintf_chk+0x84)[0x7f1eb11deca4]
/lib/x86_64-linux-gnu/libc.so.6(__sprintf_chk+0x7d)[0x7f1eb11debfd]
../bin/./mu_SegnetClass(main+0x114)[0x403874]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5)[0x7f1eb10f3f45]
../bin/./mu_SegnetClass[0x404534]
======= Memory map: ========
00400000-00409000 r-xp 00000000 08:13 6688007                            /home/relaybot/mumu/SegNet/mu_SegnetClass/bin/mu_SegnetClass
00609000-0060a000 r--p 00009000 08:13 6688007                            /home/relaybot/mumu/SegNet/mu_SegnetClass/bin/mu_SegnetClass
0060a000-0060b000 rw-p 0000a000 08:13 6688007                            /home/relaybot/mumu/SegNet/mu_SegnetClass/bin/mu_SegnetClass
00d9b000-0fff9000 rw-p 00000000 00:00 0                                  [heap]
10000000-10001000 rw-s 00000000 00:06 533                                /dev/nvidia0

遇到opencv的resize函数错误

在网上查资料以为是opencv2中的imgwarp.c文件的函数断言有问题，然后又换成opencv3，其实是cv::imread得到的图片的路径出了问题，也就是没有读到正确的图片数据，才会出错显示。

OpenCV Error: Assertion failed (ssize.width > 0 && ssize.height > 0) in resize,
file /home/relaybot/mumu/opencv-3.3.1/modules/imgproc/src/resize.cpp, line 3289
terminate called after throwing an instance of 'cv::Exception'
  what():

错误

/usr/bin/ld: cannot find -l./segnet.h collect2: error: ld returned 1 exit 

此处是因为CMakeLists.txt中的错误：

add_library( rgbd_tutor_1
     segnet.cpp)
target_link_libraries( rgbd_tutor_1
    ${OpenCV_LIBRARIES}
        ./segnet.h)

改成：

add_library( rgbd_tutor_1
     segnet.cpp segnet.h)
target_link_libraries( rgbd_tutor_1
    ${OpenCV_LIBRARIES}
)