python项目py代码编译为so，保持原目录结构

参考：https://blog.csdn.net/REAL_liudebai/article/details/125126432

我的项目代码结构：

deploy一般应用部署的的相关代码，包含配置、测试、入口main、版本文件等
code的所有要编译的py文件目录，编译为so的文件都放在这里

要安装cython：pip install cython

编写setup.py文件

setup是编译的入口代码，执行指令编译：python setup.py build_ext --inplace

import sys
from distutils.core import setup
from Cython.Build import cythonize
from buildso import BuildSo, CustomBuildExt

bso = BuildSo()

setup(
    name = "",
    # packages = get_packages(),
    ext_modules = cythonize(bso.get_ext_modules()),
    # 自定义的CustomBuildExt
    cmdclass={
        'build_ext': CustomBuildExt
    }
)

编写buildso.py文件

功能包括组织编译哪些文件，编译后打包，清理临时文件等。

1. get_ext_modules – 获取要编译的文件，并以Extension类别的给到setup的ext_modules。
2. CustomBuildExt的run – 编译、打包的整个过程。

import os
import datetime
import json
import shutil
import tarfile
from distutils.extension import Extension
from distutils.command.clean import clean
from Cython.Distutils import build_ext

##################################################################################
# 需要编译的文件夹绝对路径
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
BASE_DIR = os.path.join(CUR_DIR, "deploy")

# 项目根目录下不用（能）转译的py文件（夹）名，用于启动的入口脚本文件一定要加进来
ignore_files = ['main.py', '__init__.py']
ignore_dirs = ['__pycache__', 'test']

# 项目子目录下不用（能）转译的'py文件（夹）名
ignore_names = ['__init__.py']

# 不需要原样复制到编译文件夹的文件或者文件夹
ignore_move = ['__pycache__']

# 是否将编译打包到指定文件夹内 (True)，还是和源文件在同一目录下(False)，默认True
ispackage = True
package_name = "my_service"     # 打包文件夹名 (package = True 时有效)
##################################################################################

def make_targz(source_dir, output_filename, arcname):
    with tarfile.open(output_filename, "w:gz") as tar:
        tar.add(source_dir, arcname=arcname)

def translate_dir(path):
    """
    编译需要的py文件
    """
    icnt = 0
    complie_files = []
    for root, dirs, files in os.walk(path):
        dir_name = os.path.basename(root)
        if dir_name in ignore_dirs:
            continue
        for file_name in files:
            if file_name in ignore_names:
                continue
            if file_name.startswith('__') or file_name.startswith('.') or file_name.startswith('build'):
                continue
            if file_name in ignore_files:
                continue
            fpath = os.path.join(root, file_name)
            if not fpath.endswith('.py') and not fpath.endswith('.pyx'):
                continue
            if fpath.endswith('__init__.py') or fpath.endswith('__init__.pyx'):
                continue
            icnt += 1
            print(f"要编译文件{icnt}: {fpath}")
            complie_files.append(fpath)
    return complie_files

class BuildSo():
    def __init__(self) -> None:
        self.complie_files = translate_dir(BASE_DIR)

    def get_ext_modules(self):
        """
        扩展模块
        """
        py_files = self.complie_files
        extensions = []
        for fpath in py_files:
            ext_name = os.path.splitext(fpath)[0]
            ext_name = ext_name.replace(os.path.dirname(BASE_DIR), "")     # 要保留basename
            ext_name = ext_name.replace(os.path.sep, '.').strip(".")        # 在build上的路径

            extension = Extension(ext_name, [fpath], include_dirs=['.'])
            extension.cython_directives = {'language_level': "3"}
            extensions.append(extension)    # 扩展
        return extensions


# 继承Cython的build_ext类
class CustomBuildExt(build_ext):
    def run(self):
        print("编译run ...")
        build_ext.run(self)  # 编译

        print("移动工程需要的文件：so, __init__.py, configs, test, main.py")
        print("清理py对应的.c, .so, so重新命令")
        self.build_package() # 打包

        # print(f"清理bluid: {self.distribution}")
        # self.clean_build(self.distribution)

    def clean_build(self, distribution):
        clean_command = clean(distribution)
        clean_command.all = True
        clean_command.finalize_options()
        clean_command.run()
 
    def build_package(self):
        """
        构建打包
        """
        try:
            if not ispackage:
                return
            # package_path = os.path.join(CUR_DIR, "outputs", package_name + f"""_{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}""")  # 打包文件夹路径
            with open(os.path.join(CUR_DIR, "deploy/version"), "r") as f:
                version = json.load(f)
            # 打包文件夹路径
            package_path = os.path.join(CUR_DIR, f"""outputs/{package_name}_{version["version"]}""") # + f"""_{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}""")
            os.makedirs(package_path, exist_ok=True)
            # main.py
            shutil.copyfile(os.path.join(BASE_DIR, "main.py"), os.path.join(package_path, "main.py"))
            shutil.copyfile(os.path.join(BASE_DIR, "version"), os.path.join(package_path, "version"))
            shutil.copy(os.path.join(CUR_DIR, "scripts/run_deploy_prod.sh"), os.path.join(CUR_DIR, f"{package_path}/run_deploy_prod.sh"))
            # configs、test、tcwp
            self._copy_congigs(package_path)
            self._copy_test(package_path)
            self._copy_code(package_path)
            # 打包
            output_filename=f"""{package_path}.tar.gz"""
            print(f"make_targz--->> {output_filename}")
            make_targz(source_dir=package_path, output_filename=output_filename, arcname=package_name)
        except Exception as e:
            print(f"Exception: {e}")
    
    def _copy_congigs(self, package_path):
        """
        拷贝配置文件（所有配置文件）
        """
        src_dir = os.path.join(BASE_DIR, "configs")
        dst_dir = os.path.join(package_path, "configs")
        for root, _, files in os.walk(src_dir):
            dst_path = root.replace(src_dir, dst_dir)
            os.makedirs(dst_path, exist_ok=True)
            for fname in files:
                shutil.copyfile( os.path.join(root, fname), os.path.join(dst_path, fname))
 
    def _copy_test(self, package_path):
        """
        拷贝测试文件（test目录下的py文件）
        """
        new_test_path = os.path.join(package_path, "test")
        os.makedirs(new_test_path, exist_ok=True)

        old_test_path = os.path.join(BASE_DIR, "test")
        for root, dirs, files in os.walk(old_test_path):
            for file_name in files:
                if not file_name.endswith(".py"):
                    continue
                fpath = os.path.join(root, file_name)
                new_path = os.path.join(new_test_path, root.replace(old_test_path, "").strip(os.path.sep))
                os.makedirs(new_path, exist_ok=True)
                shutil.copyfile(fpath, os.path.join(new_path,file_name))

    def _copy_tcwp(self, package_path):
        """
        拷贝tcwp包（编译的文件，例如：so，__init__.py等）
        """
        new_test_path = os.path.join(package_path, "code")
        os.makedirs(new_test_path, exist_ok=True)

        old_tcwp_path = os.path.join(BASE_DIR, "code")
        for root, dirs, files in os.walk(old_tcwp_path):
            dir_name = os.path.basename(root)
            if dir_name in ignore_dirs:
                continue
            for file_name in files:
                # .so, __init__.py
                if file_name.endswith(".so"):
                    fpath = os.path.join(root, file_name)
                    new_path = os.path.join(new_test_path, root.replace(old_tcwp_path, "").strip(os.path.sep))
                    os.makedirs(new_path, exist_ok=True)
                    # copy so
                    file_name = "".join((file_name.split(".")[:-2]))
                    so_file = os.path.join(new_path, file_name + ".so")
                    print(f"mv: {fpath} --> {so_file}")
                    shutil.move(fpath, so_file)
                    # remove
                    c_file = os.path.join(root, file_name + ".c")
                    if os.path.exists(c_file):
                        print(f"rm: {c_file}")
                        os.remove(c_file)
                elif file_name in ['__init__.py', '__init__.pyx']:
                    fpath = os.path.join(root, file_name)
                    new_path = os.path.join(new_test_path, root.replace(old_tcwp_path, "").strip(os.path.sep))
                    os.makedirs(new_path, exist_ok=True)
                    new_fpath = os.path.join(new_path, file_name)
                    print(f"mv: {fpath} --> {new_fpath}")
                    shutil.copyfile(fpath, new_fpath)

if __name__ == '__main__':
    pass

编写build.sh

运行这个脚本编译：

CURRENT_DIR=$(cd `dirname $0`; pwd)
BASE_DIR=$(cd $(dirname $0);pwd)

## 打包成so版本
python ${BASE_DIR}/setup.py build_ext --inplace