踩坑绵绵无绝期,完美总结版
初步整理了一下,不想继续整理了,先这样吧
折腾了5天,我是垃圾
此版适合:
-
没有root权限的普通孩子
-
对linux系统不熟悉,里面可能有做的多余的地方
经验
-
问题一般都是版本问题跟路径问题
-
有奇怪的问题就卸掉重装
-
caffe重新cmake之前最好删掉cmakecache.txt和cmakefiles文件夹
安装依赖项
安装protobuf 2.6.1(caffe要安这个版本)
第一次安的3.15.6,不行
#无法下载,手动下载
下载地址:https://github.com/google/googletest/tree/release-1.5.0
参考:http://www.likecs.com/show-129322.html
#如果permission denied ,修改文件权限:chmod +x autogen.sh
#prefix是指定安装目录,没有root权限的可以随意设置一个,如:$HOME/bin等,
#之后记得修改 ~/.bashrc 文件
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/wangsuwei/bin/protocbuf_2_6_1/lib
export PATH=$PATH:/home/wangsuwei/bin/protocbuf_2_6_1/bin
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/home/wangsuwei/bin/protocbuf_2_6_1/lib/pkgconfig/
-
进入文件夹 cd ./protobuf
-
缺少google test库:(用1.5.0的可以),把下载的googletest库放到:/opt/RDB/protobuf-2.6.1,改名为gtest(先删除之前老的),
-
./autogen.sh
-
./configure --prefix=$HOME/bin/protocbuf_2_6_1
-
make
-
make install
-
变量设置
查看版本信息的方法:进入bin目录,有protoc文件。./protoc --version
安装boost1_67_0
第一次安了1_75,说版本高,后来又安了1_45,说版本低
include:/home/wangsuwei/program/boost_1_67_0 \
lib:/home/wangsuwei/program/boost_1_67_0/stage/lib \
报错 permission denied chmod +x bootstrap.sh
报错build.sh permission denied
chmod +x ./tools/build/src/engine/build.sh
-
下载解压
-
进入目录 cd ~/source/
-
注意如果有多个python,设置python版本。怎么设置见问题合集,搜索boost,一定要看
-
./bootstrap.sh -with-libraries=all --with-toolset=gcc --with-python=/usr/bin/python2.7 --prefix=$HOME/bin/boost_1_67_0
-
./b2
-
./b2 install --prefix=$HOME/bin/boost_1_67_0
-
进入boost安装目录下/lib,创建软连接指向文件位置,ln -s libboost_python27.so libboost_python.so
安装gflags,lunwind, glog,
需要先安装gflags,unwind,再安装glog。
gflags
#cmake 的时候要指定shared_libs=on,否则之后会很多关于share的error。
-
下载解压:手动下载,丰衣足食https://codeload.github.com/gflags/gflags/zip/master
-
进入目录
-
mkdir build && cd build
-
#ccmake .. 修改DBUILD_SHARED_LIBS=ON,和安装路径!!!!
-
cmake -DBUILD_SHARED_LIBS=ON -DGFLAGS_NAMESPACE=google -G "Unix Makefiles” ../
-
make
-
make install
参考:
https://cmake.org/cmake/help/v3.10/envvar/LDFLAGS.html
https://blog.csdn.net/whatday/article/details/88410313
unwind(glog需要unwind)
-
CFLAGS=-fPIC
-
chmod +x autogen.sh
-
./autogen.sh --prefix=$HOME/bin --enable-shared
-
make CFLAGS=-fPIC
-
make CFLAGS=-fPIC install
glog
-
下载解压:手动下载,丰衣足食https://github.com/google/glog
-
进入目录
-
mkdir build && cd build
-
#ccmake .. 修改DBUILD_SHARED_LIBS=ON,安装路径,设置unwind路径(3个)。t进入高级模式往下翻就行!!!!
-
cmake
-
make
-
make install
安装leveldb ,lbdb,hdf5,openblas,snappy
这些的安装跟前面大同小异
snappy(openblas需要它)
下载链接: https://codeload.github.com/google/snappy/zip/refs/heads/master
openblas
git clone git://github.com/xianyi/OpenBLAS #https://codechina.csdn.net/mirrors/xianyi/openblas
-
make USE_MASS=0#因为没有安装mass
-
make install PREFIX=/home/wangsuwei/bin
LMDB
-
git clone https://github.com/LMDB/lmdb
-
cd ./lmdb-mdb.master/libraries/liblmdb
-
make
-
make PREFIX=/home/wangsuwei/bin install
#??怎么还是往usr/local写?
换下面的做法:
-
修改makefile文件,prefix = /home/wangsuwei/bin
-
make
-
make install
LevelDB
-
git clone https://github.com/google/leveldb
-
下载googletest和benchmark,放到thirdparty文件夹中,并修改名字,去掉master
https://github.com/google/benchmark
https://github.com/google/googletest
-
mkdir build && cd build
-
export CXXFLAGS="-fPIC"
-
ccmake .. #修改安装路径!!!!
-
cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build .
-
make
-
make install
HDF5
这是个大坑,下载了多个版本都不行,最后使用了anaconda带的hdf。
git https://support.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.10.4.tar #https://www.hdfgroup.org/downloads/hdf5/source-code/
-
chmod +x configure
-
./configure --prefix=/home/wangsuwei/bin/hdf_1_8
-
make
-
make install
anaconda虽然解决了这个问题,但是会带来新的坑,就是又多了一个python。
根据后面出现的问题,觉得protobuf的find_package的处理可能有帮助,但是心好累,能用就好了。。。
配置caffe
下载caffe,并解压
cd caffe
cp Makefile.config.example Makefile.config
修改配置
修改的文件,
makefile.config
BLAS := open
-
opencv是3版本的,取消注释 OPENCV_VERSION=3
-
修改cuda,python目录
-
设置BLAS方式和include,lib的路径
-
如果是用 的anaconda的python,设置其目录和python_include 目录
-
修改 INCLUDE_DIRS 和LIBRARY_DIRS,将之前安装的依赖包的 include 和lib文件路径,写进去。
cmakelist
找到下面添加 -std=c++11
if(UNIX OR APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -std=c++11")
endif()
cmake/Modules中的find文件的路径
-
moudles中findGFlags,FindGlog,FindLMDB,FindLevelDB
-
cmake中ProtoBuf的find_package的替换
-
cmake中Dependencies.cmake中opencv相关变量的设置,不想研究了,直接写成绝对路径了,指定opencv版本
cmake,里面可能有没用的,最后能用就行
cmake . $caffe_root -G "Unix Makefiles" \
-DBLAS=open \
-DGLOG_ROOT_DIR=/home/wangsuwei/bin \
-DGFLAGS_ROOT_DIR=/home/wangsuwei/bin \
-DLMDB_DIR=/home/wangsuwei/bin \
-DLEVELDB_ROOT=/home/wangsuwei/bin \
-DSNAPPY_ROOT_DIR=/home/wangsuwei/bin \
-DBOOST_ROOT=/home/wangsuwei/bin/boost_1_67_0 \
-DProtobuf_DIR=/home/wangsuwei/bin/protocbuf_2_6_1
make -j
make pycaffe
export PYTHONPATH=$(pwd)/python:$PYTHONPATH
参考:https://blog.csdn.net/y_ear/article/details/84622383
备用
#进入boost安装目录下/lib,创建软连接指向文件位置,ln -s libboost_python27.so libboost_python.so
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/wangsuwei/bin/protocbuf_2_6_1/lib
export PATH=$PATH:/home/wangsuwei/bin/protocbuf_2_6_1/bin
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/home/wangsuwei/bin/protocbuf_2_6_1/lib/pkgconfig/
export PATH=$PATH:/home/wangsuwei/bin
出现过的问题:
问题1 找不到propoc
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/wangsuwei/bin/lib
export PATH=$PATH:/home/wangsuwei/bin/bin
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/home/wangsuwei/bin/lib/pkgconfig/
问题2:编译选项问题
https://blog.csdn.net/m_buddy/article/details/88085425
/usr/include/c++/5/bits/c++0x_warning.h:32:2: error: #error This file requires compiler and library support for the ISO C++ 2011 standard. This support must be enabled with the -std=c++11 or -std=gnu++11 compiler options.
/usr/include/c++/4.9/bits/c++0x_warning.h:32:2: error: #error This file requires compiler and library support for the ISO C++ 2011 standard. This support is currently experimental, and must be enabled with the -std=c++11 or -std=gnu++11 compiler options.
#error This file requires compiler and library support for the
/usr/local/include/google/protobuf/arena.h:324:36: note: template argument deduction/substitution failed:
Makefile:600: recipe for target ‘.build_release/src/caffe/proto/caffe.pb.o’ failed
make: *** [.build_release/src/caffe/proto/caffe.pb.o] Error 1
解决方案
打开Makefile文件,给CXXFLAGS、NVCCFLAGS、LINKFLAGS都使用-std=c++11标准编译:
CXXFLAGS += -pthread -fPIC $(COMMON_FLAGS) $(WARNINGS)
改为:
CXXFLAGS += -pthread -fPIC $(COMMON_FLAGS) $(WARNINGS) -std=c++11
NVCCFLAGS += -D_FORCE_INLINES -ccbin=$(CXX) -Xcompiler -fPIC $(COMMON_FLAGS)
改为:
NVCCFLAGS += -D_FORCE_INLINES -ccbin=$(CXX) -Xcompiler -fPIC $(COMMON_FLAGS) -std=c++11make
LINKFLAGS += -pthread -fPIC $(COMMON_FLAGS) $(WARNINGS)
改为:
LINKFLAGS += -pthread -fPIC $(COMMON_FLAGS) $(WARNINGS) -std=c++11
问题3 levelDB 找不到
在生成动态库时,常常习惯性的加上fPIC选项,fPIC有什么作用和意义,加不加有什么区别,这里做下小结:fPIC的全称是 Position Independent Code, 用于生成位置无关代码。什么是位置无关代码,个人理解是代码无绝对跳转,跳转都为相对跳转。
解决方案
重新安装leveldb
make前加入 export CXXFLAGS="-fPIC"
问题4 找不到lsnappy,找不到libboost_python
安装snappy库 https://codeload.github.com/google/snappy/zip/refs/heads/master
cblas,lapack 修改makefile文件:BLAS := open
boost 创建软连接指向文件位置,ln -s libboost_python27.so libboost_python.so
注意:
一开始设置BLAS := open就好,后来不行了,在cmake后加选项 -DBLAS=open
问题5,6,7,8本质上是一类问题
问题5 cmake 找不到gflag,glog
原因 参考https://blog.csdn.net/10km/article/details/72967656
这个问题的根本原因在于代码作者对cmake的find_library的查找机制没有全搞明白
简单说就是调用find_library时,会以从一组路径中顺序查找指定的库,这有一个优先序问题,如果没有指定了NO_DEFAULT_PATH,则会先查找默认的系统库路径
如果指定了NO_DEFAULT_PATH,则只查找PATHS提供的路径,不会查找系统库路径。因为所以原始代码中没有加NO_DEFAULT_PATH导致每次只能找到系统路径下的库。
如何保证PATHS指定的路径优先被搜索呢?
解决的办法cmake官网也给出来了,就是调用两次find_library,第一次加NO_DEFAULT_PATH,第二次则不加。
方法
cmake .. $caffe_root -G "Unix Makefiles" \
-DGLOG_ROOT_DIR=/home/wangsuwei/bin \
-DGFLAGS_ROOT_DIR=/home/wangsuwei/bin
修改FindGlog.cmake,FindGFlags.cmake
注意添加/include; NO_DEFAULT_PATH
问题6 找不到 protobuf(这个解决方案不行,后面还会出问题,略过,也不是完全错误,而是做了一半,后面有类似问题,采用了两种解决方案)
在问题12中得到解决
参考:
https://blog.csdn.net/fb_941219/article/details/88526157
https://zhuanlan.zhihu.com/p/50829542
方法:
-
在caffe/cmake中找到ProtoBUf.cmake。
-
在ProtoBUf.cmake找到中加find_package( Protobuf REQUIRED),前面加入:
set(Protobuf_DIR /home/wangsuwei/bin/)
set(Protobuf_INCLUDE_DIR /home/wangsuwei/bin/include)
set(Protobuf_LIBRARIES /home/wangsuwei/bin/lib)
图示:
问题7 找不到HDF5,找到版本1,12,0!!
#下载了1.8,1.10,1.12都不行
#最后使用anaconda自带的hdf5,
set(HDF5_DIR /home/wangsuwei/bin/) 没用
set(HDF5_INCLUDE_DIRS /home/wangsuwei/bin/include)
set(HDF5_LIBRARIES /home/wangsuwei/bin/lib)
加到path里 没用
export HDF5_DIR=/home/wangsuwei/bin 没用
猜测是版本不匹配
export HDF5_DISABLE_VERSION_CHECK=1 没用
问题8找不到LMDB
cmake/moudles/FindLMDB中
find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS "$ENV{LMDB_DIR}/include")
find_library(LMDB_LIBRARIES NAMES lmdb PATHS "$ENV{LMDB_DIR}/lib" )
改为
find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS ${LMDB_DIR}/include NO_DEFAULT_PATH)
find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS ${LMDB_DIR}/include)
find_library(LMDB_LIBRARIES NAMES lmdb PATHS ${LMDB_DIR}/lib NO_DEFAULT_PATH)
find_library(LMDB_LIBRARIES NAMES lmdb PATHS ${LMDB_DIR}/lib )
LevelDB
find_library(LevelDB_LIBRARY NAMES leveldb
PATHS /usr/lib ${LEVELDB_ROOT}/lib
DOC "Path to leveldb library." NO_DEFAULT_PATH)
find_library(LevelDB_LIBRARY NAMES leveldb
PATHS /usr/lib ${LEVELDB_ROOT}/lib
DOC "Path to leveldb library.")
export LD_LIBRARY_PATH=/home/wangsuwei/bin/lib:$LD_LIBRARY_PATH #不行
export C_INCLUDE_PATH=/home/wangsuwei/bin/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/home/wangsuwei/bin/include:$CPLUS_INCLUDE_PATH
问题9:protobuf requires c++11
在Makefile或者CMakeLists.txt中增加 -std=c++11
if(UNIX OR APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -std=c++11")
endif()
https://blog.csdn.net/sdlypyzq/article/details/108791239
指定编译器为gcc:没用
export CC=/usr/bin/gcc
export CXX=/usr/bin/g++
问题10 boost版本
下了三个版本,1.75太新,1.46太旧,1.67可以
问题解析:版本太新
确定cmake对应的版本https://cmake.org/cmake/help/v3.12/module/FindBoost.html
下载地址 :https://dl.bintray.com/boostorg/release/1.75.0/source/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/wangsuwei/bin/lib
问题11 cmake问题
found thefollowing Boost libraries
could not find boost
python interface is disabled or not all required dependencide found.Builing without it.
d
参考:https://www.jianshu.com/p/17a5fdbfa8a9
在\tools\build\example下的user-config.jam中最后一行指定python
# Configure specific Python version.
using python : 2.7 : /usr/bin : /usr/include/python2.7 : /usr/lib/python2.7 ;
-
查看boost中的python是不是 libboost_python27.dylib ,我的是 libboost_python38.dylib 得到,就悲剧了。那就重新安装boost
-
指定使用的python版本,重新安装
-
.\bootstrap 中添加参数:--with-python="python/bin/python27"
-
安装时出现icu的问题,没管。。。。心累
-
别忘了ln -s 那一行
问题12
make -j 错误 undefined reference to `google::protobuf:
之前忽略掉了protobuf的warning,还是得解决
原因:protobuf是自己安装的,使用find_package是找不到的,需要自己找路径
参考:https://www.jianshu.com/p/2946b0e5c45b
修改ProtoBuf.cmake
注释掉#find_package( Protobuf REQUIRED)
替换为:
include(FindPackageHandleStandardArgs)
set(Protobuf_DIR /home/wangsuwei/bin/protocbuf_2_6_1)
set(PROTOBUF_PROTOC_EXECUTABLE /home/wangsuwei/bin/protocbuf_2_6_1/bin/protoc)
#set(Protobuf_INCLUDE_DIR /home/wangsuwei/bin/protocbuf_2_6_1/include)
#set(Protobuf_LIBRARIES /home/wangsuwei/bin/protocbuf_2_6_1/lib)
find_path(Protobuf_INCLUDE_DIR
google/protobuf/service.h
PATHS ${Protobuf_DIR}/include NO_DEFAULT_PATH
)
find_path(Protobuf_INCLUDE_DIR
google/protobuf/service.h
PATHS ${Protobuf_DIR}/include
)
find_library(Protobuf_LIBRARIES NAMES libprotobuf.so PATHS ${Protobuf_DIR}/lib NO_DEFAULT_PATH)
find_library(Protobuf_LIBRARIES NAMES libprotobuf.so PATHS ${Protobuf_DIR}/lib )
find_library(Protobuf_LITE_LIBRARIES NAMES libprotobuf-lite.so PATHS ${Protobuf_DIR}/lib NO_DEFAULT_PATH)
find_library(Protobuf_LITE_LIBRARIES NAMES libprotobuf-lite.so PATHS ${Protobuf_DIR}/lib)
find_library(Protobuf_PROTOC_LIBRARIES NAMES libprotoc.so PATHS ${Protobuf_DIR}/lib NO_DEFAULT_PATH)
find_library(Protobuf_PROTOC_LIBRARIES NAMES libprotoc.so PATHS ${Protobuf_DIR}/lib )
find_package_handle_standard_args(PROTOBUF DEFAULT_MSG Protobuf_INCLUDE_DIR Protobuf_LIBRARIES Protobuf_LITE_LIBRARIES Protobuf_PROTOC_LIBRARIES)
if(PROTOBUF_FOUND)
set(PROTOBUF_DIR ${Protobuf_DIR})
set(PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIR})
set(PROTOBUF_LIBRARIES ${Protobuf_LIBRARIES})
set(PROTOBUF_LITE_LIBRARIES ${Protobuf_LITE_LIBRARIES})
set(PROTOBUF_PROTOC_LIBRARIES ${Protobuf_PROTOC_LIBRARIES})
#message(STATUS "Found gflags (include: ${GFLAGS_INCLUDE_DIR}, library: ${GFLAGS_LIBRARY})")
mark_as_advanced(PROTOBUF_LITE_LIBRARIES PROTOBUF_PROTOC_LIBRARIES
PROTOBUF_LIBRARIES Protobuf_INCLUDE_DIR PROTOBUF_DIR PROTOBUF_PROTOC_EXECUTABLE)
endif()
问题13 opencv找不到部分路径
make -j报错
猜测是因为opencv是我自己装的
dependencies强制改:
set(OpenCV_DIR "/home/wangsuwei/program/opencv3")
set(OpenCV_INCLUDE_DIRS "/home/wangsuwei/program/opencv3/include")
set(OpenCV_LIBS "/home/wangsuwei/program/opencv3/libopencv_core.so /home/wangsuwei/program/opencv3/libopencv_highgui.so /home/wangsuwei/program/opencv3/libopencv_imgproc.so /home/wangsuwei/program/opencv3/libopencv_imgcodecs.so")
https://blog.csdn.net/haluoluo211/article/details/80559341
https://www.jianshu.com/p/39fc5e548310
问题14:undefined reference to 'ULx86_64_step
Glog links to libunwind atomically. You can remove libunwind to avoid the dependency. Or you can try to install libunwind-devel.
安装unwind
-
CFLAGS=-fPIC
-
chmod +x autogen.sh
-
./autogen.sh --prefix=$HOME/bin --enable-shared
-
make CFLAGS=-fPIC
-
make CFLAGS=-fPIC install
-
重新安装 glog