彻底解决低版本boost在新版CUDA下编译出错的问题

解决marian在Ubuntu上的编译问题：

在安装了boost 1.58 - 1.65左右的版本时，编译会报错：

In file included from /usr/local/cuda-10.2/include/cuda_runtime.h:115:0,
                 from <command-line>:0:
/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^

这个问题坑在没有给出引用/usr/local/cuda-10.2/include/cuda_runtime.h的文件，止步于：

from <command-line>:0:

第一步：

使用make VERBOSE=1 重新编译，得到下面信息：

/usr/local/cuda-10.2/bin/nvcc -M -D__CUDACC__ marian/src/tensors/gpu/tensor_operators.cu -o marian/build/src/CMakeFiles/marian_cuda.dir/tensors/gpu/marian_cuda_generated_tensor_operators.cu.o.NVCC-depend -m64 -DBLAS_FOUND=1 -DMKL_FOUND=1 -DCUDA_FOUND -DUSE_NCCL --default-stream per-thread -O3 -g --use_fast_math -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_70,code=compute_70 -ccbin /usr/bin/cc -std=c++11 -Xcompiler -fPIC -Xcompiler -Wno-unused-result -Xcompiler -Wno-deprecated -Xcompiler -Wno-pragmas -Xcompiler -Wno-unused-value -Xcompiler -Werror -Xcompiler -msse2 -Xcompiler -msse3 -Xcompiler -msse4.1 -Xcompiler -msse4.2 -Xcompiler -mavx -Xcompiler -mavx2 -DNVCC -I/usr/local/cuda-10.2/include -I/opt/intel/compilers_and_libraries_2019.1.144/linux/mkl/include -Imarian/src -Imarian/src/. -Imarian/src/3rd_party -Imarian/src/3rd_party/SQLiteCpp/include -Imarian/src/3rd_party/sentencepiece -Imarian/src/3rd_party/fbgemm/include -Imarian/build/local/include
In file included from /usr/local/cuda-10.2/include/cuda_runtime.h:115:0,
                 from <command-line>:0:
/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^
/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^
CMake Error at marian_cuda_generated_tensor_operators.cu.o.Release.cmake:215 (message):
  Error generating
  marian/build/src/CMakeFiles/marian_cuda.dir/tensors/gpu/./marian_cuda_generated_tensor_operators.cu.o

第二步：

在nvcc后加上 -v 选项，重新执行上面的nvcc编译指令，得到下面输出：

"/usr/bin"/cc -std=c++11 -D__CUDA_ARCH__=600 -E -x c++  -DCUDA_DOUBLE_MATH_FUNCTIONS -D__CUDACC__ -D__NVCC__  -fPIC -Wno-unused-result -Wno-deprecated -Wno-pragmas -Wno-unused-value -Werror -msse2 -msse3 -msse4.1 -msse4.2 -mavx -mavx2 -O3 -I"/usr/local/cuda-10.2/include" -I"/opt/intel/compilers_and_libraries_2019.1.144/linux/mkl/include" -I"marian/src" -I"marian/src/." -I"marian/src/3rd_party" -I"marian/src/3rd_party/SQLiteCpp/include" -I"marian/src/3rd_party/sentencepiece" -I"marian/src/3rd_party/fbgemm/include" -I"marian/build/local/include" "-I/usr/local/cuda-10.2/bin/../targets/x86_64-linux/include"    -D "__CUDACC__" -D "BLAS_FOUND=1" -D "MKL_FOUND=1" -D "CUDA_FOUND" -D "USE_NCCL" -D "NVCC" -D__CUDACC_VER_MAJOR__=10 -D__CUDACC_VER_MINOR__=2 -D__CUDACC_VER_BUILD__=89 -DCUDA_API_PER_THREAD_DEFAULT_STREAM=1 -include "cuda_runtime.h" -m64 -g -gdwarf-2 "marian/src/tensors/gpu/tensor_operators.cu" -o "/tmp/tmpxft_00004b67_00000000-7_tensor_operators.compute_60.cpp1.ii" 

第三步：

继续在cc后加上 -v选项，重新执行上面的cc编译指令，得到下面输出：

/usr/libexec/gcc/x86_64-unknown-linux-gnu/5.4.0/cc1plus -E -quiet -v -I /usr/local/cuda-10.2/include -I /opt/intel/compilers_and_libraries_2019.1.144/linux/mkl/include -I marian/src -I marian/src/. -I marian/src/3rd_party -I marian/src/3rd_party/SQLiteCpp/include -I marian/src/3rd_party/sentencepiece -I marian/src/3rd_party/fbgemm/include -I marian/build/local/include -I /usr/local/cuda-10.2/bin/../targets/x86_64-linux/include -imultiarch x86_64-linux-gnu -D_GNU_SOURCE -D __CUDA_ARCH__=600 -D CUDA_DOUBLE_MATH_FUNCTIONS -D __CUDACC__ -D __NVCC__ -D __CUDACC__ -D BLAS_FOUND=1 -D MKL_FOUND=1 -D CUDA_FOUND -D USE_NCCL -D NVCC -D __CUDACC_VER_MAJOR__=10 -D __CUDACC_VER_MINOR__=2 -D __CUDACC_VER_BUILD__=89 -D CUDA_API_PER_THREAD_DEFAULT_STREAM=1 -include cuda_runtime.h marian/src/tensors/gpu/tensor_operators.cu -o /tmp/tmpxft_00004b67_00000000-7_tensor_operators.compute_60.cpp1.ii -msse2 -msse3 -msse4.1 -msse4.2 -mavx -mavx2 -m64 -mtune=generic -march=x86-64 -std=c++11 -Wno-unused-result -Wno-deprecated -Wno-pragmas -Wno-unused-value -Werror -fPIC -g -gdwarf-2 -fworking-directory -O3

…

In file included from /usr/local/cuda-10.2/include/cuda_runtime.h:115:0,
                 from <command-line>:0:
/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^
/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^

第四步：

由于cc1plus已经加上了-v参数，最后的输出也并未比最开始多出有用信息，故只能使用strace来打印出所有系统调用。在上面命令前加上strace，并重新执行，得到下面信息：

…
open("/usr/lib/gcc/x86_64-unknown-linux-gnu/5.4.0/../../../../include/c++/5.4.0/backward/boost/config/compiler/nvcc.hpp", O_RDONLY|O_NOCTTY) = -1 ENOENT (No such file or directory)
lstat("/usr", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/lib", {st_mode=S_IFDIR|0755, st_size=12288, ...}) = 0
lstat("/usr/lib/gcc", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/lib/gcc/x86_64-unknown-linux-gnu", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/lib/gcc/x86_64-unknown-linux-gnu/5.4.0", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/lib/gcc/x86_64-unknown-linux-gnu/5.4.0/include", {st_mode=S_IFDIR|0775, st_size=4096, ...}) = 0
lstat("/usr/lib/gcc/x86_64-unknown-linux-gnu/5.4.0/include/boost", 0x7fffc8a511d0) = -1 ENOENT (No such file or directory)
open("/usr/lib/gcc/x86_64-unknown-linux-gnu/5.4.0/include/boost/config/compiler/nvcc.hpp", O_RDONLY|O_NOCTTY) = -1 ENOENT (No such file or directory)
lstat("/usr", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/local", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/local/include", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/local/include/boost", {st_mode=S_IFDIR|0755, st_size=12288, ...}) = 0
lstat("/usr/local/include/boost/config", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/local/include/boost/config/compiler", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/usr/local/include/boost/config/compiler/nvcc.hpp", {st_mode=S_IFREG|0644, st_size=1246, ...}) = 0
open("/usr/local/include/boost/config/compiler/nvcc.hpp", O_RDONLY|O_NOCTTY) = 5
fstat(5, {st_mode=S_IFREG|0644, st_size=1246, ...}) = 0
read(5, "//  (C) Copyright Eric Jourdanne"..., 1246) = 1246
close(5)                                = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale-langpack/en_US.UTF-8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale-langpack/en_US.utf8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale-langpack/en_US/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale-langpack/en.UTF-8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale-langpack/en.utf8/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale-langpack/en/LC_MESSAGES/cpplib.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/cuda-10.2/include/crt/common_functions.h", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0644, st_size=11839, ...}) = 0
read(5, "/*\n * Copyright 1993-2018 NVIDIA"..., 4096) = 4096
read(5, "tin__ void*   __cdecl memset(voi"..., 4096) = 4096
read(5, "  __THROW; \n#elif defined(_WIN32"..., 8192) = 3647
read(5, "", 4096)                       = 0
write(2, "In file included from \33[01m\33[K/u"..., 602In file included from /usr/local/cuda-10.2/include/cuda_runtime.h:115:0,
                 from <command-line>:0:
/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^
) = 602
write(2, "\33[01m\33[K/usr/local/cuda-10.2/inc"..., 461/usr/local/cuda-10.2/include/crt/common_functions.h:74:24: error: token ""__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."" is not valid in preprocessor expressions
 #define __CUDACC_VER__ "__CUDACC_VER__ is no longer supported.  Use __CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__, and __CUDACC_VER_BUILD__ instead."
                        ^
) = 461

可以看到在出错前，open的最后一个文件是/usr/local/include/boost/config/compiler/nvcc.hpp，因此可以推断问题和boost。
继续向上回溯，可以找到出错前最后打开的最后一个工程内的文件是marian/src/3rd_party/CLI/Optional.hpp
这个文件中使用了__has_include这个C++11新提供的宏，根据文件是否存在来决定是否include，来实现选择性include。
可能是这个新特性导致编译器无法输出引用/usr/local/cuda-10.2/include/cuda_runtime.h的文件。

解决方案：

由于定位到boost，很容易就能发现是/usr/local/include/boost/config/compiler/nvcc.hpp中使用了__CUDACC_VER__。
这个问题有两中方式解决：
一是升级boost库版本，升级到1.65以上。
二是在CMakeList.txt中CUDA_NVCC_FLAGS中加上-DCLI11_BOOST_OPTIONAL=0，禁止使用boost option