我有一个相当标准的多处理脚本,可以处理一个数据库表中的200万条记录。在我把工作放入worker_队列之前,内存使用量就膨胀到12GB以上并崩溃。有更好的设计方法吗?在import math
import psycopg2
from psycopg2.extras import DictCursor
from multiprocessing import Process, Manager
from config import DB
connection = psycopg2.connect(DB)
cursor = connection.cursor(cursor_factory=DictCursor)
def worker(worker_queue, progress):
for row in iter(worker_queue.get, None):
# Do work
progress.put(1)
if __name__ == "__main__":
total, done = 0, 0
cursor.execute("SELECT * from table")
manager = Manager()
worker_queue = manager.Queue()
progress = manager.Queue()
for row in cursor:
worker_queue.put(row)
total += 1
workers = [Process(target=worker, args=(worker_queue, progress)) for i in range(50)]
for each in workers:
each.start()
for i in iter(progress.get, None):
done += 1
remaining = total - done
if remaining == 0:
print 'Done'
elif ((remaining % (10 ** int(math.log10(remaining)))) == 0):
print str(remaining) + ' remaining'