python 多进程读取五千万个文件_Python多进程练习（通过多进程、进程间通信等实现60个G的电影拷贝）...

此篇为学习Python多进程这个知识点是的练习Demo,有兴趣的可以看一下。这 里面涉及到的知识点有：进程池(Pool)、进程间通信(Manager().Queue() > 、文件读写等。

# coding:utf-8

import os

from multiprocessing import Pool, Manager

# 定义一个copy文件的方法

def copy_file(file, old_folder, new_folder, queue):

# 读取文件

fr = open(old_folder + "/" + file, "rb")

fw = open(new_folder + "/" + file, "wb")

file_size = (os.path.getsize(old_folder + "/" + file)) / 1024 // 1024

print("大小：%d" % file_size + "M|||" + (old_folder + file))

# 文件内容

while True:

content = fr.read(1024 * 1024)

fw.write(content)

if not content:

break

fr.close()

fw.close()

queue.put(file_size)

# 定义一个主方法来操作进程等

def main():

current_path = "/Users/jackyang/Movies/"

# 输入需要拷贝的文件夹

old_folder_name = input("请输入您要拷贝的文件夹：")

old_folder_name = current_path + old_folder_name

# 目标位置的文件夹

new_folder_name = old_folder_name + "-备份"

# 创建文件夹

if os.path.exists(new_folder_name):

os.removedirs(new_folder_name)

else:

os.mkdir(new_folder_name)

# 获取copy目录下所有的文件名

all_file = os.listdir(old_folder_name)

# 创建进程池

pool = Pool(5)

# 创建队列管理用来进程间通信

queue = Manager().Queue()

for file in all_file:

pool.apply_async(copy_file, args=(file, old_folder_name, new_folder_name, queue))

num = 0

total = len(all_file)

# 计算出所有文件的总大小

sum_size = 0

for path in all_file:

sum_size += (os.path.getsize(old_folder_name + "/" + path)) / 1024 // 1024

print("文件总大小：%dM" % sum_size)

current_progress = 0

while num < total:

num += 1

current_progress += queue.get()

copyProgress = current_progress / sum_size

print("当前的进度是：%.f%%" % (copyProgress * 100))

if __name__ == '__main__':

main()