python多进程测试
多进程测试
通过编写不使用多进程和使用多进程的方式进行对比,查看多进程速度提升效果,和另外个人其它多进程相关实验记录。
1、速度对比测试
分别计算常规方法、concurrent.futures和multiprocessing.Pool所消耗的时间,使用time.sleep(0.01)代替函数中处理逻辑所需时间,进行对比:
若fig_classify函数没有逻辑处理时间,直接返回结果,常规方法所用时间更少,通常是由于多进程构建列表或者字典额外使用了 一段时间 。
import os
from multiprocessing import Pool
from multiprocessing import freeze_support
import concurrent.futures
import time
def fig_classify(file_path, filename):
# 这里是假设的函数实现,你应该替换成实际的处理逻辑
# 例如,根据文件名返回一些处理结果
time.sleep(0.01)
# absolute_dir = os.path.join(file_path, filename)
# return f"Processed {absolute_dir}"
return f"Processed {filename}"
def main():
file_path = r'D:\data\猫狗\kaggle\train' # 确保这是一个有效的目录路径
# 记录开始时间
start_time = time.time()
for file in os.listdir(file_path):
result = fig_classify(file_path,file)
print(result)
# 记录结束时间
end_time = time.time()
# 计算并打印运行时间
elapsed_time = end_time - start_time
print(f"代码运行了 {elapsed_time:.2f} 秒。")
# 记录开始时间
start_time = time.time()
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
results = [executor.submit(fig_classify, file_path, f) for f in os.listdir(file_path)]
for future in concurrent.futures.as_completed(results):
result = future.result()
print(result)
# 记录结束时间
end_time = time.time()
# 计算并打印运行时间
elapsed_time = end_time - start_time
print(f"代码运行了 {elapsed_time:.2f} 秒。")
# 记录开始时间
start_time = time.time()
result_list = []
with Pool(8) as p:
for f in os.listdir(file_path):
result = p.apply_async(fig_classify, args=(file_path,f,))
result_list.append(result)
for result in result_list:
print(result.get()) # 等待并打印结果
# 记录结束时间
end_time = time.time()
# 计算并打印运行时间
elapsed_time = end_time - start_time
print(f"代码运行了 {elapsed_time:.2f} 秒。")
if __name__ == '__main__':
freeze_support()
main()
2、对应关系测试
测试concurrent.futures异步文件和future对应关系.
import os
from multiprocessing import Pool
from multiprocessing import freeze_support
import concurrent.futures
import time
def fig_classify(filename):
# 这里是假设的函数实现,你应该替换成实际的处理逻辑
# 例如,根据文件名返回一些处理结果
time.sleep(0.01)
return f"Processed {filename}"
def main():
file_path = r'D:\data\猫狗\kaggle\train' # 确保这是一个有效的目录路径
# 记录开始时间
start_time = time.time()
for file in os.listdir(file_path):
result = fig_classify(file)
print(result)
# 记录结束时间
end_time = time.time()
# 计算并打印运行时间
elapsed_time = end_time - start_time
print(f"代码运行了 {elapsed_time:.2f} 秒。")
# 记录开始时间
start_time = time.time()
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
results = {executor.submit(fig_classify, f): f for f in os.listdir(file_path)}
for future in concurrent.futures.as_completed(results):
filename = results[future]
try:
result = future.result()
print(f"{filename}: {result}")
except Exception as e:
print(f"Error processing {filename}: {e}")
# 记录结束时间
end_time = time.time()
# 计算并打印运行时间
elapsed_time = end_time - start_time
print(f"代码运行了 {elapsed_time:.2f} 秒。")
# 记录开始时间
start_time = time.time()
result_list = []
with Pool(8) as p:
for f in os.listdir(file_path):
result = p.apply_async(fig_classify, args=(f,))
result_list.append(result)
for result in result_list:
result.get() # 等待并打印结果
# 记录结束时间
end_time = time.time()
# 计算并打印运行时间
elapsed_time = end_time - start_time
print(f"代码运行了 {elapsed_time:.2f} 秒。")
if __name__ == '__main__':
freeze_support()
main()
3、共享变量测试
在多进程中,每个进程都有自己的内存空间,因此它们不能直接共享变量。在这个例子中,ls 是一个全局变量,但在多进程中,每个进程都会创建一个新的 ls 副本,而不是共享同一个列表。所以,当你尝试在多个进程中修改 ls 时,实际上每个进程都在修改自己的副本,而不会影响其他进程的副本。
如果想要在多个进程之间共享数据,可以使用 multiprocessing.Manager。这是一个特殊的对象,它可以创建一个可以在多个进程之间共享的数据结构。以下是如何使用 Manager 来共享一个列表的示例:
from multiprocessing import freeze_support, Manager
import concurrent.futures
import numpy as np
from PIL import Image,ImageOps
def trans(data):
# 初始化一个空字典
result_dict = {}
# 遍历数据
for key, value in data:
# 如果key在字典中不存在,则创建一个新的列表并添加value
if key not in result_dict:
if isinstance(value,np.ndarray):
# 假设img_ori_temp_np是一个numpy数组
img_ori_temp = Image.fromarray(ImageOps.grayscale(value))
result_dict[key] = [img_ori_temp]
else:
result_dict[key] = [value]
# 如果key已经存在,则将value添加到对应的列表中
else:
if isinstance(value,np.ndarray):
# 假设img_ori_temp_np是一个numpy数组
img_ori_temp = Image.fromarray(ImageOps.grayscale(value))
result_dict[key].append(img_ori_temp)
else:
result_dict[key].append(value)
return result_dict
def odd_even(i, shared_ls, shared_dic_for_ls):
shared_ls.append(i+1)
if i%2==1:
cls='odd'
shared_dic_for_ls.append((cls, i))
else:
cls='even'
shared_dic_for_ls.append((cls, i))
return f"Processed {i}"
def main():
with Manager() as manager:
shared_ls = manager.list()
shared_dic_for_ls = manager.list()
with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor:
results = [executor.submit(odd_even, i, shared_ls, shared_dic_for_ls) for i in range(0, 100)]
for future in concurrent.futures.as_completed(results):
result = future.result()
print(result)
print(shared_ls)
result_dic = trans(shared_dic_for_ls)
print(result_dic)
if __name__ == '__main__':
freeze_support()
main()
4、可序列化pickle测试
测试不同变量obj是否可序列化,若不可序列化,使用多进程时不可作为函数的参数
import io
import pickle
import sys
import threading
print(sys.version_info)
lock = threading.RLock()
buffer = io.BytesIO()
pickle.dump(obj, buffer)
参考链接:1、multiprocessing.Manager().dict()使用坑点https://blog.csdn.net/qq_40276813/article/details/124746189
2、multiprocessing.freeze_support()介绍
https://blog.csdn.net/Owen_goodman/article/details/115521388
3、彻底解决python多进程can‘t pickle问题
https://blog.csdn.net/weixin_42787086/article/details/133589904