python多进程读同一个文件_python 多进程读写文件

import time

from multiprocessing import Process, JoinableQueue, cpu_count

import csv

####处理一条数据的方法

def deal_line(line, writer, csv_file):

writer.writerow((line[3], line[1]))

csv_file.flush()#重点,在多进程中写文件需要尽快刷新,否则可能会导致数据丢失

####消费者模型

def consumer(queue, writer, csv_file):

while True:

line = queue.get()

deal_line(line, writer, csv_file)

queue.task_done()

####生产者模型

def producer(queue):

with open('test.txt', 'r') as f:

for line in f:

queue.put(line)

####启动N个生产者N个消费者模型

def main():

with open('t1.csv', 'w+') as csv_file:

writer = csv.writer(csv_file)

queue = JoinableQueue(8)#可限制队列长度

pc = Process(target=producer, args=(queue,))

pc.start()

#多消费者

for _ in range(cpu_count()):