多进程系列：计算各子进程时间

多进程系列：计算各子进程时间

了解各个子进程耗时对于负载均衡至关重要，下面展示代码示例和结果示例说明如何计算各子进程时间。

代码示例

import multiprocessing
import time
from pprint import pprint

# 假设以下是五个分类模型函数
def classify_model_1(image_path, queue):
    # 模拟分类操作
    time.sleep(1)
    print("classify_model_1")
    # return f"模型1分类结果: {image_path}"
    queue.put(f"模型1分类结果: {image_path}")


def classify_model_2(image_path, queue):
    # 模拟分类操作
    time.sleep(1)
    print("classify_model_2")
    # return f"模型2分类结果: {image_path}"
    queue.put(f"模型2分类结果: {image_path}")


# 假设以下是五个分割模型函数
def segment_model_1(image_path, queue):
    # 模拟分割操作
    time.sleep(1)
    print("segment_model_1")
    # return f"模型1分割结果: {image_path}"
    queue.put(f"模型1分割结果: {image_path}")


def segment_model_2(image_path, queue):
    # 模拟分割操作
    time.sleep(1)
    print("segment_model_2")
    # return f"模型2分割结果: {image_path}"
    queue.put(f"模型2分割结果: {image_path}")


# 进程执行函数
def process_image(image_path, result_dict, result_key):
    # 创建用于进程间通信的队列
    classify_queue = multiprocessing.Queue()
    segment_queue = multiprocessing.Queue()

    # 创建分类模型和分割模型的进程
    classify_processes = [
        multiprocessing.Process(
            target=classify_model_1, args=(image_path, classify_queue)
        ),
        multiprocessing.Process(
            target=classify_model_2, args=(image_path, classify_queue)
        ),
    ]


    segment_processes = [
        multiprocessing.Process(
            target=segment_model_1, args=(image_path, segment_queue)
        ),
        multiprocessing.Process(
            target=segment_model_2, args=(image_path, segment_queue)
        ),
    ]
    model_processes = classify_processes + segment_processes
    # 启动所有进程并记录开始时间
    start_time_dict = {}
    run_time_dict = {}
    for p in model_processes:
        p.start()
        start_time_dict[p.pid] = time.time()
    print(f"==>> start_time_dict: {start_time_dict}")
    # 等待所有分类模型进程完成并记录结束时间
    for p in classify_processes:
        p.join()
        run_time_dict[p.pid] = str(time.time() - start_time_dict[p.pid])

    # 等待所有分割模型进程完成并记录结束时间
    for p in segment_processes:
        p.join()
        run_time_dict[p.pid] = str(time.time() - start_time_dict[p.pid])
    print(f"==>> run_time_dict: {run_time_dict}")
    # 方法一
    # 按照顺序等待所有进程完成，比较耗时，
    # 适用于确保分类模型在分割模型之前完成
    # for p in model_processes:
    #     p.join()

    # 方法二
    # 分类模型进程和分割模型进程同时运行
    for p in classify_processes:
        p.join()

    # 等待所有分割模型进程完成
    for p in segment_processes:
        p.join()

    # 收集所有分类和分割模型的结果
    classify_results = []
    while not classify_queue.empty():
        classify_results.append(classify_queue.get())

    segment_results = []
    while not segment_queue.empty():
        segment_results.append(segment_queue.get())

    # return classify_results, segment_results
    # 将结果存储在共享字典中
    result_dict[result_key] = (classify_results, segment_results, run_time_dict)

# 主函数
if __name__ == "__main__":
    tic = time.time()
    # 假设我们有一个图像路径列表
    image_paths = [
        "image1.jpg",
        "image2.jpg",
    ]

    # 创建一个Manager来共享结果字典
    manager = multiprocessing.Manager()
    result_dict = manager.dict()
    processes = []
    for i, image_path in enumerate(image_paths):
        p = multiprocessing.Process(target=process_image, args=(image_path, result_dict, i))
        processes.append(p)
        p.start()

    # 等待所有图像处理进程完成
    for p in processes:
        p.join()

    # 输出结果
    for key, value in result_dict.items():
        pprint(value)

    toc = time.time()
    run_time = toc - tic
    print("程序运行时间：", run_time)

结果示例

classify_model_1
classify_model_2
classify_model_1
segment_model_1
segment_model_2
==>> start_time_dict: {203732: 1723085307.806911, 204608: 1723085307.9630837, 202820: 1723085307.9887679, 203004: 1723085308.0356302}
==>> run_time_dict: {203732: '2.909015655517578', 204608: '2.8934247493743896', 202820: '3.10205078125', 203004: '3.1345412731170654'}
classify_model_2
segment_model_1
segment_model_2
==>> start_time_dict: {184112: 1723085308.0356302, 200372: 1723085308.1605997, 204200: 1723085308.207462, 203284: 1723085308.4573998}
==>> run_time_dict: {184112: '3.023956775665283', 200372: '3.478219985961914', 204200: '3.510946035385132', 203284: '3.3078691959381104'}
(['模型1分类结果: image1.jpg', '模型2分类结果: image1.jpg'],
 ['模型1分割结果: image1.jpg', '模型2分割结果: image1.jpg'],
 {202820: '3.10205078125',
  203004: '3.1345412731170654',
  203732: '2.909015655517578',
  204608: '2.8934247493743896'})
(['模型1分类结果: image2.jpg', '模型2分类结果: image2.jpg'],
 ['模型1分割结果: image2.jpg', '模型2分割结果: image2.jpg'],
 {184112: '3.023956775665283',
  200372: '3.478219985961914',
  203284: '3.3078691959381104',
  204200: '3.510946035385132'})
程序运行时间： 5.741686582565308