python3 交叉编译环境搭建

说明

1. 以python-3.6.8为例
2. 在ubuntu x86_64环境编译在arm aarch64环境上运行的python代码
3. ubuntu x86_64已经安装python
4. 编译目标的python版本与编译机(ubuntu) 的版本须一致
5. 参考：链接: Crossenv

步骤

1. 安装交叉编译工具，因为要编译运行在aarch64环境的python代码，所以要在ubuntu环境安装aarch64-linux-gnu编译工具

sudo apt-get install gcc-7-aarch64-linux-gnu  # 版本根据需要去选择，这里使用7

2. 安装依赖包

sudo apt-get install zlib1g-dev openssl libffi-dev 

3. 下载python源码，解压

tar -xzf Python-3.6.8.tgz
cd Python-3.6.8

4. 执行命令：

# --host: 生成的目标环境的host_gnu_type
# --build：当前编译环境的host_gnu_type
# 查看ubuntu host_gnu_type: python3 -m sysconfig | grep HOST_GNU_TYPE
./configure --prefix=/home/fkx/python/Python-3.6.8 \
            --host=aarch64-linux-gnu \
            --build=x86_64-linux-gnu \
            --without-ensurepip \
            CC=aarch64-linux-gnu-gcc-7 \
            ac_cv_buggy_getaddrinfo=no \
            ac_cv_file__dev_ptmx=yes \
            ac_cv_file__dev_ptc=no
make
make install

5. 安装crossenv

pip3 install crossenv

6. 生成python虚拟环境

cd ..  # 离开python源码目录
python3 -m crossenv /home/fkx/python/Python-3.6.8/bin/python3 cross_venv  # 在当前目录下生成虚拟环境cross_venv
souce ./cross_venv/bin/activate  # 使用虚拟环境, 退出虚拟环境命令：deactivate
build-pip install --upgrade pip
build-pip install cython -i https://mirrors.aliyun.com/pypi/simple/

7. 编译

# 编译使用python脚本的方式编译：
python3 compile.py

"""
compile.py
"""
import multiprocessing
import re
import shutil
import os
import time
import compileall
from distutils.core import setup
from distutils.sysconfig import get_config_var
from Cython.Build import cythonize
from Cython.Distutils.extension import Extension


parent_path = "test_compile/"
out_dir = "compile_out/"
# 需要编译为so的目录
compile_so_dirs = ["test_compile/api", "test_compile/common", "test_compile/db", "test_compile/driver", "test_compile/manager"]

ignore_compile_so_dirs = []

# 编译pyc的目录
compile_pyc_dirs = []

# 忽略编译的文件
ignore_compile_pyc_files = []


def is_ignore_dir(dir):
    for ignore in ignore_compile_so_dirs:
        if dir.__contains__(ignore):
            return True
    return False


def copy_file(src_path, dest_path):
    dest_dir = os.path.dirname(dest_path)
    if not os.path.isdir(dest_dir):
        os.makedirs(dest_dir, exist_ok=True)
    shutil.copy(src_path, dest_path)


def move_pyc_to_out_dir(file_dir):
    for dirpath, foldernames, filenames in os.walk(file_dir):
        for f in filenames:
            if f.endswith(".pyc"):
                f_splits = f.split(".")
                f_splits.pop(len(f_splits) - 2)
                real_file_name = ".".join(f_splits)
                if os.path.basename(dirpath) == "__pycache__":
                    if os.path.join(os.path.dirname(dirpath), real_file_name.rstrip("c")) in ignore_compile_pyc_files:
                        continue
                    dst_path = os.path.join(dirpath.replace(parent_path, out_dir), real_file_name)
                else:
                    dst_path = os.path.join(dirpath.replace(parent_path, out_dir), real_file_name)
                copy_file(os.path.join(dirpath, f), dst_path)
                os.remove(dst_path.rstrip("c"))


def compile_so(extension):
    setup(ext_modules=cythonize(extension),
          script_args=["build_ext", "--inplace"])
    file_path = extension.sources[0]
    file_name = os.path.basename(file_path)
    compile_file_name = file_name.rsplit(".", 1)[0] + get_config_var('EXT_SUFFIX')
    compile_file_path = os.path.join(os.path.dirname(file_path), compile_file_name)
    dst_path = file_path.replace(parent_path, out_dir).replace(".py", ".so")
    # copy so文件到指定目录
    copy_file(compile_file_path, dst_path)
    # 删除原py文件
    os.remove(os.path.join(os.path.dirname(dst_path), file_name))


def do_compile_so():
    extensions = []
    for compile_dir in compile_so_dirs:
        for dirpath, foldernames, filenames in os.walk(compile_dir):
            # 顶级目录下是main.py，只需编译为pyc即可
            if dirpath == parent_path or dirpath == parent_path.rstrip("/"):
                continue
            if is_ignore_dir(dirpath):
                continue
            for filename in filter(lambda x: re.match(r'.*[.]py$', x), filenames):
                file_path = os.path.join(dirpath, filename)
                extensions.append(
                    Extension(file_path[:-3].replace('/', '.'),
                              [file_path],
                              extra_compile_args=["-Os", "-g0"],
                              extra_link_args=["-Wl,--strip-all"]))
    cythonize(extensions, nthreads=8, language_level=3)
    pool = multiprocessing.Pool(processes=8)
    pool.map(compile_so, extensions)
    pool.close()
    pool.join()
    return extensions


def compile_pyc():
    # 编译pyc文件
    for compile_pyc_dir in compile_pyc_dirs:
        compileall.compile_dir(compile_pyc_dir, maxlevels=0, force=True, quiet=1)
        move_pyc_to_out_dir(compile_pyc_dir)


if __name__ == '__main__':
    start_time = time.time()
    shutil.rmtree(out_dir, ignore_errors=True)
    shutil.copytree(parent_path, out_dir)
    do_compile_so()
    compile_pyc()
    # 清除源代码，替换为编译后代码
    shutil.rmtree(parent_path, ignore_errors=True)
    os.rename(out_dir.rstrip("/"), parent_path.rstrip("/"))
    print("compile complete! time:", time.time() - start_time, 's')