python开多少进程合适_Python多进程与单进程效率对比

运行环境：Python3 in win10

先生成200个测试文件

# generate.py

i = 0

while i < 200:

o = open("test\\" + str(i) + ".py", "w")

content = str(i)

o.write(content)

o.close()

i += 1

多进程拷贝文件

# multi-pool-copy.py

from multiprocessing import Pool, Manager

import time

import os

import shutil

import random

start = time.time()

def copyFile(file_name, old_folder_name, new_folder_name, q):

time.sleep(random.random())

shutil.copyfile(old_folder_name + '\\' + file_name, new_folder_name + '\\' + file_name,)

q.put(file_name) # put item into the queue

def main():

pool = Pool(5)

q = Manager().Queue()

old_folder_name = input("Please input the folder name you want to copy: ")

new_folder_name = old_folder_name + "-copy"

os.mkdir(new_folder_name)

file_name_list = os.listdir(old_folder_name)

for file in file_name_list:

pool.apply_async(copyFile, args=(file, old_folder_name, new_folder_name, q))

cnt = 0

allLength = len(file_name_list)

while cnt < allLength:

message = q.get()

cnt += 1

print("\rCopying %s, Process Bar is:%d%%" % (message, (cnt / allLength) * 100), end="")

print("Copy Done!")

if __name__ == "__main__":

main()

end = time.time()

print("Time-consuming: %#.2fs" % (end-start))

在使用单进程拷贝文件之前，需要手动删除test-copy文件夹

# single-pool-copy.py

import time

import os

import shutil

import random

start = time.time()

def copyFile(file_name, old_folder_name, new_folder_name):

time.sleep(random.random())

shutil.copyfile(old_folder_name + '\\' + file_name, new_folder_name + '\\' + file_name, )

def main():

old_folder_name = input("Please input the folder name you want to copy: ")

new_folder_name = old_folder_name + "-copy"

os.mkdir(new_folder_name)

file_name_list = os.listdir(old_folder_name)

cnt = 0

allLength = len(file_name_list)

for file in file_name_list:

copyFile(file, old_folder_name, new_folder_name)

cnt += 1

print("\rCopying %s, Process Bar is:%d%%" % (file, (cnt / allLength) * 100), end="")

print("Copy Done!")

if __name__ == "__main__":

main()

end = time.time()

print("Time-consuming: %#.2fs" % (end-start))