使用setup.py
主要是使用distutils进行构建,打包,发布,安装
setup当中核心是extension, 这是一个list, 里面参数在使用不同的编译选项的时候对应不同的编译dict, 解引用之后增加到extension当中作为编译选项。
单纯的使用gcc进行编译,使用python setup.py build_ext --inplace
其中标准命令build_ext 代表build C/C++ extensions(compile/link to build directory)
其他可以使用 --help进行查看
import os
import os.path
# from distutils.core import setup
# from distutils.extension import Extension
from setuptools import setup, find_packages, Extension
from Cython.Build import cythonize
# setup compilation environment
dir_path = os.path.dirname(os.path.realpath(__file__))
parrots_path = os.path.join(dir_path, '..')
parrots_include = os.path.join(parrots_path, 'include')
def get_version_fromlib():
from ctypes import cdll, c_int, byref
ld_path = os.path.join(parrots_path, 'lib', 'libparrots.so')
lib = cdll.LoadLibrary(ld_path)
major, minor, patch = c_int(), c_int(), c_int()
lib.parrots_getVersion(byref(major), byref(minor), byref(patch))
return '{}.{}.{}'.format(major.value, minor.value, patch.value)
def get_version_fromfile():
config_path = os.path.join(parrots_include, 'parrots', 'config.h')
with open(config_path) as f:
count = 0
major, minor, patch = '*' * 3
for l in f.readlines():
if 'PARROTS_VAR_MAJOR' in l:
major = l.strip().split(' ')[2]
count += 1
if 'PARROTS_VAR_MINOR' in l:
minor = l.strip().split(' ')[2]
count += 1
if 'PARROTS_VAR_PATCH' in l:
patch = l.strip().split(' ')[2]
count += 1
if count >= 3:
break
assert count == 3, 'cannot get version info from config.h'
return '{}.{}.{}'.format(major, minor, patch)
def get_version():
try:
return get_version_fromlib()
except Exception as e:
import warnings
warnings.warn('fail to get version from lib, reason is {}, '
'try to find it from head file'.format(str(e)))
try:
return get_version_fromfile()
except Exception as e:
warnings.warn('fail to get version info, error: {}'.format(str(e)))
return '0.0.0'
version_suffix = 'b1'
version = '{}{}'.format(get_version(), version_suffix)
inc_dirs = [
parrots_include
]
pybind_inc_dirs = inc_dirs + [
os.path.join(parrots_path, '3rdparty', 'pybind11', 'include'),
os.path.join(parrots_path, '3rdparty', 'clue', 'include'),
]
lib_dirs = [
os.path.join(parrots_path, 'lib')
]
# extensions
ext_args = dict(
include_dirs=inc_dirs,
library_dirs=lib_dirs,
language='c++',
extra_compile_args=['-Wno-unused-function', '-Wno-write-strings'],
extra_link_args=["-Wl,-rpath=$ORIGIN"] + ["-Wl,-rpath={}".format(d)
for d in lib_dirs],
libraries=['parrots']
)
pybind_compile_args = [
'-DVERSION_INFO="%s"' % version,
'-std=c++11',
]
pybind_ext_args = dict(
include_dirs=pybind_inc_dirs,
library_dirs=lib_dirs,
language='c++',
extra_compile_args=pybind_compile_args,
extra_link_args=["-Wl,-rpath=$ORIGIN"] + ["-Wl,-rpath={}".format(d)
for d in lib_dirs],
libraries=['parrots']
)
extensions = [
Extension('parrots.cbase', ['parrots/cbase.pyx'], **ext_args),
Extension('parrots.cdarray', ['parrots/cdarray.pyx'], **ext_args),
Extension('parrots.cdevice', ['parrots/cdevice.pyx'], **ext_args),
Extension('parrots.clogging', ['parrots/clogging.pyx'], **ext_args),
Extension('parrots.config', ['parrots/config.pyx'], **ext_args),
Extension('parrots.ir', ['parrots/ir.pyx'], **ext_args),
Extension('parrots.ssdata', ['parrots/ssdata.pyx'], **ext_args),
Extension('parrots.ccomm', ['parrots/ccomm.pyx'], **ext_args),
]
extensions = cythonize(extensions, include_path=['.'], nthreads=4)
pybind_extensions = [
Extension('parrots.pybind', ['../src/pybind.cpp'], **pybind_ext_args),
]
setup_requires = [
'Cython',
'flake8',
'pytest-runner',
]
install_requires = [
'colorlog',
'numpy',
'six',
]
tests_require = [
'scipy',
'pytest',
'pytest-cov',
'pytest-ordering',
'pytest-timeout',
]
scripts = [
]
def package_files(directory):
paths = []
for (path, directories, filenames) in os.walk(directory, followlinks=True):
for filename in filenames:
paths.append(os.path.join('..', path, filename))
return paths
package_data = [
'*.so',
'compute/*.so',
# '*.so.*',
]
# package_data += package_files('parrots/include')
if __name__ == '__main__':
setup(
name='parrots',
version=version,
ext_modules=extensions + pybind_extensions,
packages=find_packages(),
package_data={'parrots': package_data},
# include_package_data=True,
scripts=scripts,
setup_requires=setup_requires,
install_requires=install_requires,
tests_require=tests_require,
zip_safe=False
)
2.编写makefile
一些变量可以直接赋值,本质上执行一组命令。
PYTHON ?= python3
all:
$(PYTHON) setup.py build_ext --inplace
bash linklib.sh
pkg_pyc:
$(PYTHON) setup.py build_ext --inplace
bash linklib.sh
python setup.py bdist_egg --exclude-source-files
lint:
$(PYTHON) setup.py flake8
flake8 --config=.flake8.cython
format:
yapf -i -r parrots torch torchvision tests test_torch
clean:
find parrots -type l -exec unlink {} \;
find parrots -name "*.so" -delete
find parrots torch torchvision -name "*.pyc" -delete
find parrots -name "*.cpp" -exec grep -l "Generated by Cython\|failed Cython" {} \; -delete
test:
$(PYTHON) setup.py -q test --addopts "-q ./tests/"
$(PYTHON) -m pytest -q ./test_torch/
test-v:
$(PYTHON) setup.py test --addopts "-v ./tests/"
$(PYTHON) -m pytest -v ./test_torch/
torch.test:
$(PYTHON) setup.py test --addopts "-v ./test_torch/"
parrots_cov = ./tests --cov=parrots/ --cov-report=
torch_cov = ./test_torch --cov=parrots/ --cov=torch/ --cov=torchvision/ --cov-append
coverage:
PARROTS_COVERAGE_TEST=ON $(PYTHON) setup.py -q test --addopts "-q $(parrots_cov)"
$(PYTHON) -m pytest -q $(torch_cov)
coverage-v:
PARROTS_COVERAGE_TEST=ON $(PYTHON) setup.py test --addopts "-v $(parrots_cov)"
$(PYTHON) -m pytest -v $(torch_cov)
coverage-vv:
PARROTS_COVERAGE_TEST=ON $(PYTHON) setup.py test --addopts "-v $(parrots_cov)"
$(PYTHON) -m pytest -v $(torch_cov)
coverage html
compute.lint:
if [ -d "../src/compute/python" ];then flake8 ../src/compute/python;else echo >&2 "'parrots2/src/compute/python/' directory doesn't exist." && exit 1;fi
compute.test:
if [ -d "test_compute" ];then $(PYTHON) setup.py test --addopts "-v ./test_compute/";else echo >&2 "'python/test_compute/' doesn't exist" && exit 1;fi
compute.clean:
if [ ! -d "parrots/compute" ];then echo >&2 "'parrots/compute/' doesn't exist." && exit 1;fi
find parrots/compute -name "*.so" -delete
find parrots/compute -name "*.pyc" -delete
find parrots/compute -type l -exec unlink {} \;
make相关命令: