python构建_python构建过程

使用setup.py

主要是使用distutils进行构建，打包，发布，安装

setup当中核心是extension, 这是一个list, 里面参数在使用不同的编译选项的时候对应不同的编译dict, 解引用之后增加到extension当中作为编译选项。

单纯的使用gcc进行编译，使用python setup.py build_ext --inplace

其中标准命令build_ext 代表build C/C++ extensions(compile/link to build directory)

其他可以使用 --help进行查看

import os

import os.path

# from distutils.core import setup

# from distutils.extension import Extension

from setuptools import setup, find_packages, Extension

from Cython.Build import cythonize

# setup compilation environment

dir_path = os.path.dirname(os.path.realpath(__file__))

parrots_path = os.path.join(dir_path, '..')

parrots_include = os.path.join(parrots_path, 'include')

def get_version_fromlib():

from ctypes import cdll, c_int, byref

ld_path = os.path.join(parrots_path, 'lib', 'libparrots.so')

lib = cdll.LoadLibrary(ld_path)

major, minor, patch = c_int(), c_int(), c_int()

lib.parrots_getVersion(byref(major), byref(minor), byref(patch))

return '{}.{}.{}'.format(major.value, minor.value, patch.value)

def get_version_fromfile():

config_path = os.path.join(parrots_include, 'parrots', 'config.h')

with open(config_path) as f:

count = 0

major, minor, patch = '*' * 3

for l in f.readlines():

if 'PARROTS_VAR_MAJOR' in l:

major = l.strip().split(' ')[2]

count += 1

if 'PARROTS_VAR_MINOR' in l:

minor = l.strip().split(' ')[2]

count += 1

if 'PARROTS_VAR_PATCH' in l:

patch = l.strip().split(' ')[2]

count += 1

if count >= 3:

break

assert count == 3, 'cannot get version info from config.h'

return '{}.{}.{}'.format(major, minor, patch)

def get_version():

try:

return get_version_fromlib()

except Exception as e:

import warnings

warnings.warn('fail to get version from lib, reason is {}, '

'try to find it from head file'.format(str(e)))

try:

return get_version_fromfile()

except Exception as e:

warnings.warn('fail to get version info, error: {}'.format(str(e)))

return '0.0.0'

version_suffix = 'b1'

version = '{}{}'.format(get_version(), version_suffix)

inc_dirs = [

parrots_include

]

pybind_inc_dirs = inc_dirs + [

os.path.join(parrots_path, '3rdparty', 'pybind11', 'include'),

os.path.join(parrots_path, '3rdparty', 'clue', 'include'),

]

lib_dirs = [

os.path.join(parrots_path, 'lib')

]

# extensions

ext_args = dict(

include_dirs=inc_dirs,

library_dirs=lib_dirs,

language='c++',

extra_compile_args=['-Wno-unused-function', '-Wno-write-strings'],

extra_link_args=["-Wl,-rpath=$ORIGIN"] + ["-Wl,-rpath={}".format(d)

for d in lib_dirs],

libraries=['parrots']

)

pybind_compile_args = [

'-DVERSION_INFO="%s"' % version,

'-std=c++11',

]

pybind_ext_args = dict(

include_dirs=pybind_inc_dirs,

library_dirs=lib_dirs,

language='c++',

extra_compile_args=pybind_compile_args,

extra_link_args=["-Wl,-rpath=$ORIGIN"] + ["-Wl,-rpath={}".format(d)

for d in lib_dirs],

libraries=['parrots']

)

extensions = [

Extension('parrots.cbase', ['parrots/cbase.pyx'], **ext_args),

Extension('parrots.cdarray', ['parrots/cdarray.pyx'], **ext_args),

Extension('parrots.cdevice', ['parrots/cdevice.pyx'], **ext_args),

Extension('parrots.clogging', ['parrots/clogging.pyx'], **ext_args),

Extension('parrots.config', ['parrots/config.pyx'], **ext_args),

Extension('parrots.ir', ['parrots/ir.pyx'], **ext_args),

Extension('parrots.ssdata', ['parrots/ssdata.pyx'], **ext_args),

Extension('parrots.ccomm', ['parrots/ccomm.pyx'], **ext_args),

]

extensions = cythonize(extensions, include_path=['.'], nthreads=4)

pybind_extensions = [

Extension('parrots.pybind', ['../src/pybind.cpp'], **pybind_ext_args),

]

setup_requires = [

'Cython',

'flake8',

'pytest-runner',

]

install_requires = [

'colorlog',

'numpy',

'six',

]

tests_require = [

'scipy',

'pytest',

'pytest-cov',

'pytest-ordering',

'pytest-timeout',

]

scripts = [

]

def package_files(directory):

paths = []

for (path, directories, filenames) in os.walk(directory, followlinks=True):

for filename in filenames:

paths.append(os.path.join('..', path, filename))

return paths

package_data = [

'*.so',

'compute/*.so',

# '*.so.*',

]

# package_data += package_files('parrots/include')

if __name__ == '__main__':

setup(

name='parrots',

version=version,

ext_modules=extensions + pybind_extensions,

packages=find_packages(),

package_data={'parrots': package_data},

# include_package_data=True,

scripts=scripts,

setup_requires=setup_requires,

install_requires=install_requires,

tests_require=tests_require,

zip_safe=False

)

2.编写makefile

一些变量可以直接赋值，本质上执行一组命令。

PYTHON ?= python3

all:

$(PYTHON) setup.py build_ext --inplace

bash linklib.sh

pkg_pyc:

$(PYTHON) setup.py build_ext --inplace

bash linklib.sh

python setup.py bdist_egg --exclude-source-files

lint:

$(PYTHON) setup.py flake8

flake8 --config=.flake8.cython

format:

yapf -i -r parrots torch torchvision tests test_torch

clean:

find parrots -type l -exec unlink {} \;

find parrots -name "*.so" -delete

find parrots torch torchvision -name "*.pyc" -delete

find parrots -name "*.cpp" -exec grep -l "Generated by Cython\|failed Cython" {} \; -delete

test:

$(PYTHON) setup.py -q test --addopts "-q ./tests/"

$(PYTHON) -m pytest -q ./test_torch/

test-v:

$(PYTHON) setup.py test --addopts "-v ./tests/"

$(PYTHON) -m pytest -v ./test_torch/

torch.test:

$(PYTHON) setup.py test --addopts "-v ./test_torch/"

parrots_cov = ./tests --cov=parrots/ --cov-report=

torch_cov = ./test_torch --cov=parrots/ --cov=torch/ --cov=torchvision/ --cov-append

coverage:

PARROTS_COVERAGE_TEST=ON $(PYTHON) setup.py -q test --addopts "-q $(parrots_cov)"

$(PYTHON) -m pytest -q $(torch_cov)

coverage-v:

PARROTS_COVERAGE_TEST=ON $(PYTHON) setup.py test --addopts "-v $(parrots_cov)"

$(PYTHON) -m pytest -v $(torch_cov)

coverage-vv:

PARROTS_COVERAGE_TEST=ON $(PYTHON) setup.py test --addopts "-v $(parrots_cov)"

$(PYTHON) -m pytest -v $(torch_cov)

coverage html

compute.lint:

if [ -d "../src/compute/python" ];then flake8 ../src/compute/python;else echo >&2 "'parrots2/src/compute/python/' directory doesn't exist." && exit 1;fi

compute.test:

if [ -d "test_compute" ];then $(PYTHON) setup.py test --addopts "-v ./test_compute/";else echo >&2 "'python/test_compute/' doesn't exist" && exit 1;fi

compute.clean:

if [ ! -d "parrots/compute" ];then echo >&2 "'parrots/compute/' doesn't exist." && exit 1;fi

find parrots/compute -name "*.so" -delete

find parrots/compute -name "*.pyc" -delete

find parrots/compute -type l -exec unlink {} \;

make相关命令：