龙芯3a5000,指令集架构从mips64变更为loongarch64,这事和搞java spring cloud微服务注册的nacos有什么关系呢?有!nacos内部使用了原生c++库rocksdb,这个原生c++库没有在龙芯下的动态库.so文件,说白了就是nacos使用rocksdb的低版本在龙芯3a5000下面起不来!下面就是研究自行编译rocksdb动态库的过程。
多说一句,最新的nacos版本,比如2022年的版本,它里面已经移除了对rocksdb的依赖,以下内容对于新版本没有意义。
1.rocksdb下载
github有源码,自己去clone。
2.编译
需要先安装一些编译依赖项:
apt install libsnappy-dev libgflags-dev libgzstd-dev liblz4-dev
rocksdb源码目录内的CMakelist.txt里面有这么一段话
看上去三个命令就可以编译成功,要是真这么简单我也没必要写这篇博客了。
修改这个CMakelist.txt文件,在第49行后边增加2行配置
if(POLICY CMP0042)
cmake_policy(SET CMP0042 NEW)
endif()
#在这后面加
set(CMAKE_VERBOSE_MAKEFILE ON)
set(CMAKE_BUILD_TYPE,"Release")
继续往下找,大概70多行的位置有几个压缩算法开关,把它们打开,变成下面这样:
option(WITH_SNAPPY "build with SNAPPY" ON)
option(WITH_LZ4 "build with lz4" ON)
option(WITH_ZLIB "build with zlib" ON)
option(WITH_ZSTD "build with zstd" ON)
mkdir build
cd build
cmake ..
make -j 4
这是正常的编译过程,不正常的往下看:
cmake ..以后loongnix系统下编译提示:
-- The CXX compiler identification is GNU 8.3.0
-- The C compiler identification is GNU 8.3.0
-- The ASM compiler identification is GNU
-- Found assembler: /usr/bin/cc
-- Check for working CXX compiler: /usr/bin/c++
-- Check for working CXX compiler: /usr/bin/c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Check for working C compiler: /usr/bin/cc
-- Check for working C compiler: /usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Could NOT find gflags (missing: gflags_DIR)
CMake Error at /usr/share/cmake-3.13/Modules/FindPackageHandleStandardArgs.cmake:137 (message):
Could NOT find gflags (missing: GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIR)
Call Stack (most recent call first):
/usr/share/cmake-3.13/Modules/FindPackageHandleStandardArgs.cmake:378 (_FPHSA_FAILURE_MESSAGE)
cmake/modules/Findgflags.cmake:15 (find_package_handle_standard_args)
CMakeLists.txt:125 (find_package)
找不到gflags库, 这个在开头说过,需要安装编译依赖就没问题了。
再次 make -j 4 后开始报这样的错误:
[ 0%] Building CXX object CMakeFiles/rocksdb.dir/db/column_family.cc.o
[ 0%] Building CXX object CMakeFiles/rocksdb-shared.dir/db/c.cc.o
/tmp/cchrbuvK.s: Assembler messages:
/tmp/cchrbuvK.s:6359: 错误:no match insn: vld $vr0,$r14,0
/tmp/cchrbuvK.s:6361: 错误:no match insn: vst $vr0,$r14,0
/tmp/cchrbuvK.s:7033: 错误:no match insn: xvld $xr0,$r13,480
/tmp/cchrbuvK.s:7034: 错误:no match insn: xvst $xr0,$r12,480
/tmp/cchrbuvK.s:7036: 错误:no match insn: vld $vr0,$r13,0
/tmp/cchrbuvK.s:7038: 错误:no match insn: vst $vr0,$r12,0
/tmp/cchrbuvK.s:7042: 错误:no match insn: xvld $xr0,$r14,0
/tmp/cchrbuvK.s:7044: 错误:no match insn: xvst $xr0,$r14,0
/tmp/cchrbuvK.s:7046: 错误:no match insn: vld $vr0,$r13,0
/tmp/cchrbuvK.s:7048: 错误:no match insn: vst $vr0,$r12,0
/tmp/cchrbuvK.s:44167: 错误:no match insn: xvld $xr0,$r12,0
/tmp/cchrbuvK.s:44168: 错误:no match insn: xvst $xr0,$r22,-64
/tmp/cchrbuvK.s:44579: 错误:no match insn: vld $vr0,$r13,0
/tmp/cchrbuvK.s:44581: 错误:no match insn: vst $vr0,$r14,0
/tmp/ccgLkxGO.s: Assembler messages:
/tmp/ccgLkxGO.s:6112: 错误:no match insn: vld $vr0,$r14,0
/tmp/ccgLkxGO.s:6114: 错误:no match insn: vst $vr0,$r14,0
/tmp/ccgLkxGO.s:6769: 错误:no match insn: xvld $xr0,$r13,480
/tmp/ccgLkxGO.s:6770: 错误:no match insn: xvst $xr0,$r12,480
/tmp/ccgLkxGO.s:6772: 错误:no match insn: vld $vr0,$r13,0
/tmp/ccgLkxGO.s:6774: 错误:no match insn: vst $vr0,$r12,0
/tmp/ccgLkxGO.s:6778: 错误:no match insn: xvld $xr0,$r14,0
/tmp/ccgLkxGO.s:6780: 错误:no match insn: xvst $xr0,$r14,0
/tmp/ccgLkxGO.s:6782: 错误:no match insn: vld $vr0,$r13,0
/tmp/ccgLkxGO.s:6784: 错误:no match insn: vst $vr0,$r12,0
/tmp/ccgLkxGO.s:43111: 错误:no match insn: xvld $xr0,$r12,0
/tmp/ccgLkxGO.s:43112: 错误:no match insn: xvst $xr0,$r22,-64
/tmp/ccgLkxGO.s:43498: 错误:no match insn: vld $vr0,$r13,0
/tmp/ccgLkxGO.s:43500: 错误:no match insn: vst $vr0,$r14,0
/tmp/cchrbuvK.s:111212: 错误:no match insn: vld $vr0,$r14,0
/tmp/cchrbuvK.s:111214: 错误:no match insn: vst $vr0,$r14,0
/tmp/cchrbuvK.s:111392: 错误:no match insn: xvld $xr0,$r13,480
/tmp/cchrbuvK.s:111393: 错误:no match insn: xvst $xr0,$r12,480
/tmp/cchrbuvK.s:111395: 错误:no match insn: vld $vr0,$r13,0
/tmp/cchrbuvK.s:111397: 错误:no match insn: vst $vr0,$r12,0
/tmp/cchrbuvK.s:111401: 错误:no match insn: xvld $xr0,$r14,0
/tmp/cchrbuvK.s:111403: 错误:no match insn: xvst $xr0,$r14,0
/tmp/cchrbuvK.s:111405: 错误:no match insn: vld $vr0,$r13,0
/tmp/cchrbuvK.s:111407: 错误:no match insn: vst $vr0,$r12,0
/tmp/ccgLkxGO.s:107953: 错误:no match insn: vld $vr0,$r14,0
/tmp/ccgLkxGO.s:107955: 错误:no match insn: vst $vr0,$r14,0
/tmp/ccgLkxGO.s:108135: 错误:no match insn: xvld $xr0,$r13,480
/tmp/ccgLkxGO.s:108136: 错误:no match insn: xvst $xr0,$r12,480
/tmp/ccgLkxGO.s:108138: 错误:no match insn: vld $vr0,$r13,0
/tmp/ccgLkxGO.s:108140: 错误:no match insn: vst $vr0,$r12,0
/tmp/ccgLkxGO.s:108144: 错误:no match insn: xvld $xr0,$r14,0
/tmp/ccgLkxGO.s:108146: 错误:no match insn: xvst $xr0,$r14,0
/tmp/ccgLkxGO.s:108148: 错误:no match insn: vld $vr0,$r13,0
/tmp/ccgLkxGO.s:108150: 错误:no match insn: vst $vr0,$r12,0
/tmp/ccSJJqAl.s: Assembler messages:
/tmp/ccSJJqAl.s:71788: 错误:no match insn: vld $vr0,$r12,0
/tmp/ccSJJqAl.s:71789: 错误:no match insn: vst $vr0,$r22,-64
make[2]: *** [CMakeFiles/rocksdb.dir/build.make:362:CMakeFiles/rocksdb.dir/db/c.cc.o] 错误 1
make[2]: *** 正在等待未完成的任务....
查了一通,这些vld ,vst都是arm cpu的指令啊,难道是源码里面有写死的汇编代码?
grep -r 'vld' ,grep -r 'vst' 看了看源码文件里面没有这样的汇编指令啊,那就是编译器受到什么东西指挥按照目标cpu是arm 去编译了?
修改CMakeList.txt,增加一行 set(CMAKE_VERBOSE_MAKEFILEON ON) 让它打印详细一点的信息,仔细观察后发现有个 -march=native ,修改CMakeList.txt内容,大概在第282行左右
else()
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
else()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x" AND NOT HAS_S390X_MARCH_NATIVE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") #修改这里
endif()
endif()
endif()
将-march=native改为loongarch64,如下:
else()
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
else()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x" AND NOT HAS_S390X_MARCH_NATIVE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64 -Os -Wall -s") #改成这样
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Os -Wall -s") #再加一行这个
endif()
endif()
endif()
再次
cd build
cmake ..
make clean
make -j 4
大部分文件都可以正常编译了,但是又报出来新的错误:
/phx/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/../portability/toku_time.h:158:2: error: #error No timer implementation for this platform
#error No timer implementation for this platform
^~~~~
/phx/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/../portability/toku_time.h: In function ‘tokutime_t toku_time_now()’:
/phx/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/../portability/toku_time.h:160:1: error: no return statement in function returning non-void [-Werror=return-type]
}
这是源码没有适配龙芯平台啊,改改它试试
vi /utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h
在上面那个#error No timer implementation for this platform那里开始改
参考这篇文件:GCC 为龙芯 CPU的预定义宏_c++_swordholder_InfoQ写作社区
增加以下内容:
#elif defined(__loongarch__)
struct timeval tv;
gettimeofday(&tv,NULL);
return tv.tv_sec*(uint64_t)1000000+tv.tv_usec;
改完以后再次接着make,编译成功。