python psycopg2_Python和psycopg2在大数据下的多处理性能

我有一个相当标准的多处理脚本，可以处理一个数据库表中的200万条记录。在我把工作放入worker_队列之前，内存使用量就膨胀到12GB以上并崩溃。有更好的设计方法吗？在import math

import psycopg2

from psycopg2.extras import DictCursor

from multiprocessing import Process, Manager

from config import DB

connection = psycopg2.connect(DB)

cursor = connection.cursor(cursor_factory=DictCursor)

def worker(worker_queue, progress):

for row in iter(worker_queue.get, None):

# Do work

progress.put(1)

if __name__ == "__main__":

total, done = 0, 0

cursor.execute("SELECT * from table")

manager = Manager()

worker_queue = manager.Queue()

progress = manager.Queue()

for row in cursor:

worker_queue.put(row)

total += 1

workers = [Process(target=worker, args=(worker_queue, progress)) for i in range(50)]

for each in workers:

each.start()

for i in iter(progress.get, None):

done += 1

remaining = total - done

if remaining == 0:

print 'Done'

elif ((remaining % (10 ** int(math.log10(remaining)))) == 0):

print str(remaining) + ' remaining'