需求: 创建多个进程执行一个可执行文件(e.g. .exe文件),这些进程可能会有不同的命令参数,执行不同的功能
Solution: 使用Python自带的multiprocessing模块
p.start() 表示进程p开始执行,p.join( ) 表示等待进程p的结束
代码:
import multiprocessing
import subprocess
def run_exe(exe_name,args1='',args2='',args3=''):
#执行exe文件的代码
print(exe_name)
with open(args3,'w')as file:
subprocess.run([exe_name,args1,args2],stdout=file,stderr=file)
return
if __name__=='__main__':
exe_files =["./app_encoder",
"./app_encoder",
"./app_encoder"]
arguments1 =["--config",
"--config",
"--config"]
arguments2 =["uni_encode_RA_1.cfg",
"uni_encode_RA_3.cfg",
"uni_encode_RA_5.cfg"]
arguments3 =["uni_encode_RA_encoder_output_1.txt",
"uni_encode_RA_encoder_output_3.txt",
"uni_encode_RA_encoder_output_5.txt"]
processes =[]
for i in range(len(exe_files)):
exe = exe_files[i]
args1 = arguments1[i]
args2 = arguments2[i]
args3 = arguments3[i]
p = multiprocessing.Process(target=run_exe,args=(exe,args1,args2,args3))
processes.append(p)
p.start()
for p in processes:
p.join()
理解:
这段代码会调用3个进程p1,p2,p3.
其中p1是执行可执行文件app_encoder, 命令参数1为–config, 命令参数2为uni_encode_RA_1.cfg(表示命令参数config的取值为uni_encode_RA_1.cfg),将输出结果输出到文件uni_encode_RA_encoder_output_1.txt中。
p2是执行可执行文件app_encoder, 命令参数1为–config, 命令参数2为uni_encode_RA_2.cfg(表示命令参数config的取值为uni_encode_RA_2.cfg), 将输出结果输出到文件uni_encode_RA_encoder_output_2.txt中。
p3是执行可执行文件app_encoder, 命令参数1为–config, 命令参数2为uni_encode_RA_3.cfg(表示命令参数config的取值为uni_encode_RA_3.cfg), 将输出结果输出到文件uni_encode_RA_encoder_output_3.txt中。
p = multiprocessing.Process(target=run_exe,args=(exe,args1,args2,args3)) 表示创建一个进程,执行run_exe这个函数,这个函数的参数为args这个参数列表
注意: 这里不能像我们平时输入给VS中那样,直接传给程序一个命令参数 --config uni_encode_RA_1.cfg. 必须像这样分开传
另外一种多进程并行执行程序的写法
(待研究)
from __future__ import division
import os
import sys
import math
import rom
import multiprocessing
def encoder(sequence, profile, QP, preset, frames, framerate, width, height, bitdepth, intraperiod, frameskip, slice_idx):
print(f"Start {sequence}_Qp{QP}...")
# Check the directory
encoder_cfg = "cfg/encode_{}.cfg".format(profile)
if not os.path.exists(encoder_cfg):
print("No encoder cfg. ({})".format(encoder_cfg))
return -1
if profile.lower() == "ra":
ip = intraperiod
elif profile.lower() in ["ldp", "ldb"]:
ip = 0
elif profile.lower() == "ai":
ip = 1
path = rom.seqPath
# Encoder
status = os.system(
".\\bin\\encoder_app.exe \
--config {encoder_cfg} \
-i {path}/{seq_name}.yuv \
-w {width} \
-h {height} \
-z {fr} \
-p {ip} \
-f {fram} \
--skip_frames {frameskip} \
-d {bitdepth} \
--internal_bit_depth 10 \
-q {QP} \
-s 1 \
--temporal_filter_strength_0 95 --temporal_filter_period_0 8 --temporal_filter_strength_1 150 --temporal_filter_period_1 16 \
-o {preset}/{profile}_dat/{seq_name}_Qp{QP}_rap{slice_idx}.bin "
"> {preset}/{profile}_log/{seq_name}_Qp{QP}_rap{slice_idx}.txt".\
format(encoder_cfg = encoder_cfg, path = path, seq_name = sequence, fram = frames, slice_idx=slice_idx, frameskip = frameskip, QP = QP, profile = profile, preset = preset, ip = ip, fr = framerate, width = width, height = height, bitdepth = bitdepth
)
)
if status != 0:
print("Encoder crashed! ({}: Qp{} {})".format(sequence, QP, profile))
else:
#decoder
status = os.system(
".\\bin\\decoder_app.exe "
" -i {preset}/{profile}_dat/{seq_name}_Qp{QP}_rap{slice_idx}.bin -s "
">> {preset}/{profile}_log/{seq_name}_Qp{QP}_rap{slice_idx}.txt". format(
seq_name = sequence, QP = QP, profile = profile, preset = preset, slice_idx=slice_idx
)
)
if status != 0:
print("Decoder crashed! ({}: Qp{})".format(sequence, QP))
print(f"End {sequence}_Qp{QP}")
return status
def submit():
if len(sys.argv) < 3:
print ("Usage: python tasks.py profile(AI/RA/LDP) runName")
# Global parameters
#seqs_per_submission = 1
profile = sys.argv[1]
runName = sys.argv[2]
preset = 'AVS'
# Check some essential directory
def _check(directory):
if not os.path.exists(directory):
os.makedirs(directory)
_check(preset)
_check("log")
_check(preset + "/" + profile + "_dat")
_check(preset + "/" + profile + "_log")
# Begin to submit the profiler
pool = multiprocessing.Pool(processes=16)
for i in range(0, len(rom.sequences)):
# Write the Sequence profiler
i_seq = rom.sequences[i][0]
for qp in ['27','32','38','45']:
if profile.lower() == "ra":
framecount_all=rom.framecount[i_seq]
ip = rom.intraperiod[i_seq]
slice_num = int(math.ceil(framecount_all / ip))
for slice_idx in range(slice_num):
frameskip = slice_idx * ip
if (framecount_all - frameskip) > ip:
frameEncodeThisTime = ip + 1
else:
frameEncodeThisTime = (framecount_all - frameskip)
# with open("task.sh", 'w') as f:
pool.apply_async(func = encoder, args=(i_seq, profile, qp, preset, frameEncodeThisTime, rom.framerate[i_seq],rom.width[i_seq], rom.height[i_seq], rom.bitdepth[i_seq], rom.intraperiod[i_seq], frameskip, slice_idx))
# encoder(i_seq, profile, qp, preset, frameEncodeThisTime, rom.framerate[i_seq],rom.width[i_seq], rom.height[i_seq], rom.bitdepth[i_seq], rom.intraperiod[i_seq], frameskip, slice_idx)
# Submit the tasks
# os.system("sbatch task.sh")
# os.remove("task.sh")
pool.close()
pool.join()
if __name__ == "__main__":
submit()