Python多进程执行程序

需求: 创建多个进程执行一个可执行文件(e.g. .exe文件)，这些进程可能会有不同的命令参数，执行不同的功能
Solution: 使用Python自带的multiprocessing模块
p.start() 表示进程p开始执行，p.join( ) 表示等待进程p的结束

代码:

import multiprocessing
import subprocess

def run_exe(exe_name,args1='',args2='',args3=''):
    #执行exe文件的代码
    print(exe_name)
    with open(args3,'w')as file:
        subprocess.run([exe_name,args1,args2],stdout=file,stderr=file)
    return 

if __name__=='__main__':
    exe_files =["./app_encoder",
    "./app_encoder",
    "./app_encoder"]
    arguments1 =["--config",
    "--config",
    "--config"]
    arguments2 =["uni_encode_RA_1.cfg",
    "uni_encode_RA_3.cfg",
    "uni_encode_RA_5.cfg"]
    arguments3 =["uni_encode_RA_encoder_output_1.txt",
    "uni_encode_RA_encoder_output_3.txt",
    "uni_encode_RA_encoder_output_5.txt"]
    processes =[]
    for i in range(len(exe_files)):
        exe = exe_files[i]
        args1 = arguments1[i]
        args2 = arguments2[i]
        args3 = arguments3[i]
        p = multiprocessing.Process(target=run_exe,args=(exe,args1,args2,args3))
        processes.append(p)
        p.start()
    for p in processes:
        p.join()

理解:
这段代码会调用3个进程p1,p2,p3.
其中p1是执行可执行文件app_encoder, 命令参数1为–config, 命令参数2为uni_encode_RA_1.cfg(表示命令参数config的取值为uni_encode_RA_1.cfg)，将输出结果输出到文件uni_encode_RA_encoder_output_1.txt中。
p2是执行可执行文件app_encoder, 命令参数1为–config, 命令参数2为uni_encode_RA_2.cfg(表示命令参数config的取值为uni_encode_RA_2.cfg), 将输出结果输出到文件uni_encode_RA_encoder_output_2.txt中。
p3是执行可执行文件app_encoder, 命令参数1为–config, 命令参数2为uni_encode_RA_3.cfg(表示命令参数config的取值为uni_encode_RA_3.cfg), 将输出结果输出到文件uni_encode_RA_encoder_output_3.txt中。
p = multiprocessing.Process(target=run_exe,args=(exe,args1,args2,args3)) 表示创建一个进程，执行run_exe这个函数，这个函数的参数为args这个参数列表
注意: 这里不能像我们平时输入给VS中那样，直接传给程序一个命令参数 --config uni_encode_RA_1.cfg. 必须像这样分开传

另外一种多进程并行执行程序的写法

(待研究)

from __future__ import division
import os
import sys
import math
import rom
import multiprocessing


def encoder(sequence, profile, QP, preset, frames, framerate, width, height, bitdepth, intraperiod, frameskip, slice_idx):
    print(f"Start {sequence}_Qp{QP}...")
    # Check the directory
    encoder_cfg = "cfg/encode_{}.cfg".format(profile)
    if not os.path.exists(encoder_cfg):
        print("No encoder cfg. ({})".format(encoder_cfg))
        return -1

    if profile.lower() == "ra":
        ip = intraperiod
    elif profile.lower() in ["ldp", "ldb"]:
        ip = 0
    elif profile.lower() == "ai":
        ip = 1
    path = rom.seqPath
    # Encoder
    status = os.system(
        ".\\bin\\encoder_app.exe \
        --config {encoder_cfg} \
        -i {path}/{seq_name}.yuv \
        -w {width} \
        -h {height} \
        -z {fr} \
        -p {ip} \
        -f {fram} \
        --skip_frames {frameskip} \
        -d {bitdepth} \
        --internal_bit_depth 10 \
        -q {QP} \
        -s 1 \
        --temporal_filter_strength_0 95 --temporal_filter_period_0 8 --temporal_filter_strength_1 150 --temporal_filter_period_1 16 \
        -o {preset}/{profile}_dat/{seq_name}_Qp{QP}_rap{slice_idx}.bin "
        "> {preset}/{profile}_log/{seq_name}_Qp{QP}_rap{slice_idx}.txt".\
        format(encoder_cfg = encoder_cfg, path = path, seq_name = sequence, fram = frames, slice_idx=slice_idx, frameskip = frameskip, QP = QP, profile = profile, preset = preset, ip = ip, fr = framerate, width = width, height = height, bitdepth = bitdepth
        )
    )
    if status != 0:
        print("Encoder crashed! ({}: Qp{} {})".format(sequence, QP, profile))	
    else:
        #decoder
        status = os.system(
            ".\\bin\\decoder_app.exe "
            " -i {preset}/{profile}_dat/{seq_name}_Qp{QP}_rap{slice_idx}.bin -s "
            ">> {preset}/{profile}_log/{seq_name}_Qp{QP}_rap{slice_idx}.txt". format(
                seq_name = sequence, QP = QP, profile = profile, preset = preset, slice_idx=slice_idx
            )
        )
        if status != 0:
            print("Decoder crashed! ({}: Qp{})".format(sequence, QP))
    print(f"End {sequence}_Qp{QP}")
    return status


def submit():
    if len(sys.argv) < 3:
        print ("Usage: python tasks.py profile(AI/RA/LDP) runName")
    # Global parameters
    #seqs_per_submission = 1
    profile = sys.argv[1]
    runName = sys.argv[2]
    preset = 'AVS'

    # Check some essential directory
    def _check(directory):
        if not os.path.exists(directory):
            os.makedirs(directory)

    _check(preset)
    _check("log")
    _check(preset + "/" + profile + "_dat")
    _check(preset + "/" + profile + "_log")
    
    # Begin to submit the profiler
    pool = multiprocessing.Pool(processes=16)
    for i in range(0, len(rom.sequences)):
        # Write the Sequence profiler
        i_seq   = rom.sequences[i][0]
        for qp in ['27','32','38','45']:
            if profile.lower() == "ra":
                framecount_all=rom.framecount[i_seq]
                ip = rom.intraperiod[i_seq]
                slice_num = int(math.ceil(framecount_all / ip))
                
                for slice_idx in range(slice_num):
                    frameskip = slice_idx * ip                    
                    
                    if (framecount_all - frameskip) > ip:
                        frameEncodeThisTime = ip + 1
                    else:
                        frameEncodeThisTime = (framecount_all - frameskip)

                    # with open("task.sh", 'w') as f:
                    pool.apply_async(func = encoder, args=(i_seq, profile, qp, preset, frameEncodeThisTime, rom.framerate[i_seq],rom.width[i_seq], rom.height[i_seq], rom.bitdepth[i_seq], rom.intraperiod[i_seq], frameskip, slice_idx))
                    # encoder(i_seq, profile, qp, preset, frameEncodeThisTime, rom.framerate[i_seq],rom.width[i_seq], rom.height[i_seq], rom.bitdepth[i_seq], rom.intraperiod[i_seq], frameskip, slice_idx)
                    
                    # Submit the tasks
                    # os.system("sbatch task.sh")
                    # os.remove("task.sh")
    pool.close()
    pool.join()


if __name__ == "__main__":
    submit()