(caffe)python 3.6编译libboost_python3.so并编译cpu-caffe

写这篇主要是使用fast-reid中torch转caffe模型，通过run_inference.sh调用caffe python3.6接口失败，花了太多时间，所以记录以下。这里感谢Hwijune，一起讨论fast-reid pytorch转caffe的问题。已在多台设备上编译成功多次。

0.环境

python3.6
numpy
boost_1_68_0

1.下载boost

boost_1_68_0.tar.gz

wget https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz

2.解压文件

tar xzvf boost_1_68_0.tar.gz      
cd boost_1_68_0/  

3.安装附加依赖库

apt-get update  
sudo apt-get install git
sudo apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev
libhdf5-serial-dev protobuf-compiler
sudo apt-get install --no-install-recommends libboost-all-dev
sudo apt-get install libatlas-base-dev
sudo apt-get install python-dev
sudo apt-get insall libgflags-dev libgoogle-glog-dev liblmdb-dev

4.安装boost

sh ./bootstrap.sh --with-libraries=all --with-toolset=gcc

 

./b2 include="/usr/include/python3.6m/"  

 可以通过whereis命令来查看python3.6m路径。 

./b2 install  

 

 5.创建软链接

cd /usr/local/lib  
ln -s libboost_python-py36.so libboost_python3.so  
ln -s libboost_python-py36.a libboost_python3.a  

cp /usr/local/lib/libboost_python36.a  /usr/lib/x86_64-linux-gnu/libboost_python_python36.a
cp /usr/local/lib/libboost_python36.so.1.68.0  /usr/lib/x86_64-linux-gnu/libboost_python3.so

   将部分依赖的链接复制到/usr/lib/x86_64-linux-gnu/下，就不需要export了。

cp /usr/local/lib/libboost_python36.so.1.68.0 /usr/lib/x86_64-linux-gnu/
cp /usr/local/lib/libboost_system.so.1.68.0 /usr/lib/x86_64-linux-gnu/
cp /usr/local/lib/libboost_filesystem.so.1.68.0 /usr/lib/x86_64-linux-gnu/
cp /usr/local/lib/libboost_thread.so.1.68.0 /usr/lib/x86_64-linux-gnu/

  6.安装cpu版本caffe

apt-get -y install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler
apt-get -y install --no-install-recommends libboost-all-dev
apt-get -y install libopenblas-dev liblapack-dev libatlas-base-dev
apt-get -y install libgflags-dev libgoogle-glog-dev liblmdb-dev
apt-get -y install git cmake build-essential

git clone https://github.com/BVLC/caffe.git
chmod 777 -R caffe
cd caffe
cp Makefile.config.example Makefile.config
vim Makefile.config

CPU_ONLY := 1
WITH_PYTHON_LAYER := 1
# Uncomment to use Python 3 (default is Python 2)
PYTHON_LIBRARIES := boost_python3 python3.6m
PYTHON_INCLUDE := /usr/include/python3.6m \
                /usr/lib/python3.6/dist-packages/numpy/core/include

vim Makefile
PYTHON_LIBRARIES := boost_python python2.7
to
PYTHON_LIBRARIES := boost_python3 python3.6m

除此之外，还要修改三个文件，参考此处：https://blog.csdn.net/u014090429/article/details/89960239：

（1）caffe-master/include/caffe/layers/pooling_layer.hpp

line 54-55:
bool ceil_mode_;   //添加的类成员变量
// PoolingParameter_RoundMode round_mode_;

 

make all -j8  #-j8根据自己电脑配置决定
make test -j8
make runtest -j8 #测试
# export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/boost_1_68_0/lib:$LD_LIBRARY_PATH

 （2）caffe-master/src/caffe/layers/pooling_layer.cpp

line 38-39:
ceil_mode_ = pool_param.ceil_mode();      //添加的代码，主要作用是从参数文件中获取ceil_mode_的参数数值。
//round_mode_ = pool_param.round_mode();

line 92-121:
// 添加的代码-----------------------------------
  if (ceil_mode_) {
    pooled_height_ = static_cast<int>(ceil(static_cast<float>(
         height_ + 2 * pad_h_ - kernel_h_) / stride_h_)) + 1;
    pooled_width_ = static_cast<int>(ceil(static_cast<float>(
         width_ + 2 * pad_w_ - kernel_w_) / stride_w_)) + 1;
  } else {
    pooled_height_ = static_cast<int>(floor(static_cast<float>(
         height_ + 2 * pad_h_ - kernel_h_) / stride_h_)) + 1;
    pooled_width_ = static_cast<int>(floor(static_cast<float>(
         width_ + 2 * pad_w_ - kernel_w_) / stride_w_)) + 1;
  }
  // ------------------------------------------------------
  
  /*switch (round_mode_) {
  case PoolingParameter_RoundMode_CEIL:
    pooled_height_ = static_cast<int>(ceil(static_cast<float>(
        height_ + 2 * pad_h_ - kernel_h_) / stride_h_)) + 1;
    pooled_width_ = static_cast<int>(ceil(static_cast<float>(
        width_ + 2 * pad_w_ - kernel_w_) / stride_w_)) + 1;
    break;
  case PoolingParameter_RoundMode_FLOOR:
    pooled_height_ = static_cast<int>(floor(static_cast<float>(
        height_ + 2 * pad_h_ - kernel_h_) / stride_h_)) + 1;
    pooled_width_ = static_cast<int>(floor(static_cast<float>(
        width_ + 2 * pad_w_ - kernel_w_) / stride_w_)) + 1;
    break;
  default:
    LOG(FATAL) << "Unknown rounding mode.";
  }*/

（3）/caffe-master/src/caffe/proto/caffe.proto

  optional Engine engine = 11 [default = DEFAULT];
  // If global_pooling then it will pool over the size of the bottom by doing
  // kernel_h = bottom->height and kernel_w = bottom->width
  optional bool global_pooling = 12 [default = false];
  // How to calculate the output size - using ceil (default) or floor rounding.
  //enum RoundMode {
 //   CEIL = 0;
  //  FLOOR = 1;
  //}
  //optional RoundMode round_mode = 13 [default = CEIL];
  optional bool ceil_mode = 13 [default = true];

 给一个最后成功的显示： 

fast-reid成功运行run_inference.sh

7.出现过的问题

7.1. .build_release/lib/libcaffe.so: undefined reference to `boost::system::detail::generic_category_ncx()'
collect2: error: ld returned 1 exit status

原因：boost编译不完整

解决：

重新安装boost，编译完整：

sh ./bootstrap.sh --with-libraries=all --with-toolset=gcc 

 7.2 make runtest 导入库libboost_filesystem.so.1.68.0失败

make runtest -j8
make: brew: Command not found
.build_release/tools/caffe
.build_release/tools/caffe: error while loading shared libraries: libboost_filesystem.so.1.68.0: cannot open shared object file: No such file or directory
Makefile:542: recipe for target 'runtest' failed
make: *** [runtest] Error 127

解决：

通过whereis找到对应的库，然后将其复制到/usr/lib/x86_64-linux-gnu/中去：

cp /usr/local/lib/libboost_filesystem.so.1.68.0 /usr/lib/x86_64-linux-gnu/

7.3 Error parsing text-format caffe.NetParameter: 60:14: Message type "caffe.PoolingParameter" has no field named "ceil_mode".

使用https://github.com/dujianyi/caffe.git

但是后面又有dilation，所以还是只能用BVLC的。修改在文中，不赘述。

7.4 undefined symbol: _ZN5boost6python6detail11init_moduleER11PyModuleDefPFvvE

修改Makefile

vim Makefile
PYTHON_LIBRARIES := boost_python python2.7
to
PYTHON_LIBRARIES := boost_python3 python3.6m

 7.5 Error parsing text-format caffe.NetParameter: 22:13: Message type "caffe.ConvolutionParameter" has no field named "dilation".

修改在文中，不赘述。

参考

1.Ubuntu下安装和编译boost库

2.Ubuntu16.04 python3.6 caffe安装教程

3.caffe跑densenet的错误：Message type "caffe.PoolingParameter" has no field named "ceil_mode".

4.Add parameter for pooling layer to specify "ceil" or "floor" #3057

5.Ubuntu 16 安装 Boost 1.68

本来第二篇是主要参考的，但是由于这里面的一个（--with-libraries=python），boost安装不完整，所以降级。。。