一、spconv 编译及使用
1、安装准备
目前spconv库 2.0 版本以上没有生成c++库文件,需要下载 1.0 版本库,如下为 spconv 库下载命令:
git clone -b v1.2.1 https://github.com/traveller59/spconv.git --recurse-submodules
在编译 spconv 之前需要安装 pytorch 库,可自行搜索安装方式.
2、修改编译脚本
打开 setup.py 脚本,找到 build_extension
函数,找到 cuda_flags
这个参数,在下方添加如下代码:
cuda_flags += ["-gencode", "arch=compute_52,code=sm_52",
"-gencode", "arch=compute_60,code=sm_60",
"-gencode", "arch=compute_61,code=sm_61",
"-gencode", "arch=compute_70,code=sm_70",
"-gencode", "arch=compute_75,code=sm_75",
"-gencode", "arch=compute_80,code=sm_80",
"-gencode", "arch=compute_86,code=sm_86",
"-gencode", "arch=compute_86,code=compute_86"]
打开 CMakeLists.txt ,在其内添加如下代码:
set(CMAKE_VERBOSE_MAKEFILE ON)
3、编译安装
执行编译命令:
python setup.py bdist_wheel
编译完成后生成的动态库文件在 build 文件夹下以 lib 开头的文件夹内,开发时主要使用的是 libcuhash.so
及 libspconv.so 这两个文件,可以将这两个文件拷贝到 /usr/local/lib
目录下。头文件在项目根目录的 include 文件夹下,可以将其内的所有文件放到 /usr/local/include
目录。
4、编译出现的问题及解决办法
编译过程中出现,找不到 <THC/THCNumerics.cuh> 头文件
找到 include/spconv/reordering.cu.h
头文件,删除第 18 行的 #include <THC/THCNumerics.cuh>
代码,之后重新编译即可。
二、使用 libtorch 编译
1、下载 libtorch 库
如果没有安装 python 版本的 torch 库,也可以直接使用 c++ 版本的 libtorch 进行编译。
下载地址:https://pytorch.org/get-started/locally/
官网上提供了两个版本(Pre-cxx11 ABI版本和cxx11 ABI版本),Pre-cxx11 ABI
是 c++11 之前的 ABI,cxx11 ABI
是 c++11 的 ABI,根据自己的需求下载使用。
2、修改编译脚本
- 找到第 9 行
import torch
代码进行注释
...
7 from pathlib import Path
8
9 # import torch
10 from setuptools import Extension, find_packages, setup
11 from setuptools.command.build_ext import build_ext
...
- 修改第16行
LIBTORCH_ROOT
值为解压后的 libtorch 库路径
16 LIBTORCH_ROOT = "/opt/libtorch"
- 修改
PYTORCH_VERSION
为libtorch库版本,注释 22、24、25 行代码
22 #remove_device = re.search(r"(\+|\.)(dev|cu|cpu)", torch.__version__)
23 PYTORCH_VERSION = "1.13.1"
24 #if remove_device is not None:
25 # PYTORCH_VERSION = torch.__version__[:remove_device.start()]
- 找到 58、59、60 行代码进行注释,并将余下代码与上方代码对齐
...
52 cmake_args = [# '-G "Visual Studio 15 2017 Win64"',
53 '-DCMAKE_PREFIX_PATH={}'.format(LIBTORCH_ROOT),
54 '-DPYBIND11_PYTHON_VERSION={}'.format(PYTHON_VERSION),
55 '-DSPCONV_BuildTests=OFF',
56 '-DPYTORCH_VERSION={}'.format(PYTORCH_VERSION_NUMBER)
57 ] # -arch=sm_61
58 # if not torch.cuda.is_available() and SPCONV_FORCE_BUILD_CUDA is None:
59 # cmake_args += ['-DSPCONV_BuildCUDA=OFF']
60 # else:
61 cuda_flags = ["\"--expt-relaxed-constexpr\""]
62 # must add following flags to use at::Half
63 # but will remove raw half operators.
64 cuda_flags += ["-D__CUDA_NO_HALF_OPERATORS__", "-D__CUDA_NO_HALF_CONVERSIONS__"]
65 # cuda_flags += ["-D__CUDA_NO_HALF2_OPERATORS__"]
66 cmake_args += ['-DCMAKE_CUDA_FLAGS=' + " ".join(cuda_flags)]
67 cuda_flags += ["-gencode", "arch=compute_52,code=sm_52",
...
- 找到 111行
setup_requires = ['torch>=1.3.0'],
进行注释
...
103 packages = find_packages(exclude=('tools', 'tools.*'))
104 setup(
105 name='spconv',
106 version='1.2.1',
107 author='Yan Yan',
108 author_email='scrin@foxmail.com',
109 description='spatial sparse convolution for pytorch',
110 long_description='',
111 # setup_requires = ['torch>=1.3.0'],
112 packages=packages,
113 package_dir = {'spconv': 'spconv'},
114 ext_modules=[CMakeExtension('spconv', library_dirs=[])],
115 cmdclass=dict(build_ext=CMakeBuild),
116 zip_safe=False,
117 )
修改完后的 setup.py 内容如下:
import os
import platform
import re
import subprocess
import sys
from distutils.version import LooseVersion
from pathlib import Path
# import torch
from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext
# if 'LIBTORCH_ROOT' not in os.environ:
# raise ValueError("You must set LIBTORCH_ROOT to your torch c++ library.")
LIBTORCH_ROOT = "/opt/libtorch"
SPCONV_FORCE_BUILD_CUDA = os.getenv("SPCONV_FORCE_BUILD_CUDA")
PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
#remove_device = re.search(r"(\+|\.)(dev|cu|cpu)", torch.__version__)
PYTORCH_VERSION = "1.13.1"
#if remove_device is not None:
# PYTORCH_VERSION = torch.__version__[:remove_device.start()]
PYTORCH_VERSION = list(map(int, PYTORCH_VERSION.split(".")))
PYTORCH_VERSION_NUMBER = PYTORCH_VERSION[0] * 10000 + PYTORCH_VERSION[1] * 100 + PYTORCH_VERSION[2]
class CMakeExtension(Extension):
def __init__(self, name, sourcedir='', library_dirs=[]):
Extension.__init__(self, name, sources=[], library_dirs=library_dirs)
self.sourcedir = os.path.abspath(sourcedir)
class CMakeBuild(build_ext):
def run(self):
try:
out = subprocess.check_output(['cmake', '--version'])
except OSError:
raise RuntimeError("CMake must be installed to build the following extensions: " +
", ".join(e.name for e in self.extensions))
if platform.system() == "Windows":
cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
if cmake_version < '3.13.0':
raise RuntimeError("CMake >= 3.13.0 is required on Windows")
for ext in self.extensions:
self.build_extension(ext)
def build_extension(self, ext):
extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
cmake_args = [# '-G "Visual Studio 15 2017 Win64"',
'-DCMAKE_PREFIX_PATH={}'.format(LIBTORCH_ROOT),
'-DPYBIND11_PYTHON_VERSION={}'.format(PYTHON_VERSION),
'-DSPCONV_BuildTests=OFF',
'-DPYTORCH_VERSION={}'.format(PYTORCH_VERSION_NUMBER)
] # -arch=sm_61
# if not torch.cuda.is_available() and SPCONV_FORCE_BUILD_CUDA is None:
# cmake_args += ['-DSPCONV_BuildCUDA=OFF']
# else:
cuda_flags = ["\"--expt-relaxed-constexpr\""]
# must add following flags to use at::Half
# but will remove raw half operators.
cuda_flags += ["-D__CUDA_NO_HALF_OPERATORS__", "-D__CUDA_NO_HALF_CONVERSIONS__"]
# cuda_flags += ["-D__CUDA_NO_HALF2_OPERATORS__"]
cmake_args += ['-DCMAKE_CUDA_FLAGS=' + " ".join(cuda_flags)]
cuda_flags += ["-gencode", "arch=compute_52,code=sm_52",
"-gencode", "arch=compute_60,code=sm_60",
"-gencode", "arch=compute_61,code=sm_61",
"-gencode", "arch=compute_70,code=sm_70",
"-gencode", "arch=compute_75,code=sm_75",
"-gencode", "arch=compute_80,code=sm_80",
"-gencode", "arch=compute_86,code=sm_86",
"-gencode", "arch=compute_86,code=compute_86"]
cfg = 'Debug' if self.debug else 'Release'
assert cfg == "Release", "pytorch ops don't support debug build."
build_args = ['--config', cfg]
print(cfg)
if platform.system() == "Windows":
cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), str(Path(extdir) / "spconv"))]
# cmake_args += ['-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), str(Path(extdir) / "spconv"))]
cmake_args += ['-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), str(Path(extdir) / "spconv"))]
cmake_args += ["-DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=TRUE"]
if sys.maxsize > 2**32:
cmake_args += ['-A', 'x64']
build_args += ['--', '/m']
else:
cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(str(Path(extdir) / "spconv"))]
cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
build_args += ['--', '-j4']
env = os.environ.copy()
env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''),
self.distribution.get_version())
if not os.path.exists(self.build_temp):
os.makedirs(self.build_temp)
print("|||||CMAKE ARGS|||||", cmake_args)
subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)
subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp)
packages = find_packages(exclude=('tools', 'tools.*'))
setup(
name='spconv',
version='1.2.1',
author='Yan Yan',
author_email='scrin@foxmail.com',
description='spatial sparse convolution for pytorch',
long_description='',
# setup_requires = ['torch>=1.3.0'],
packages=packages,
package_dir = {'spconv': 'spconv'},
ext_modules=[CMakeExtension('spconv', library_dirs=[])],
cmdclass=dict(build_ext=CMakeBuild),
zip_safe=False,
)
脚本修改完成后即可按照上述步骤进行编译了。