python 多进程 内存_Python多进程内存增量

如果找不到泄漏源，可以通过让每个工人只处理有限数量的任务来解决它。一旦它们达到任务限制，就可以让它们退出，并用新的工作进程替换它们。内置的multiprocessing.Pool对象通过^{}关键字参数支持这一点。你可以做类似的事情：import multiprocessing

import threading

class WorkerPool(object):

def __init__(self, workers=multiprocessing.cpu_count(),

maxtasksperchild=None, lock=multiprocessing.Semaphore(1)):

self._lock = multiprocessing.Semaphore(1)

self._max_tasks = maxtasksperchild

self._workers = workers

self._pool = []

self._repopulate_pool()

self._pool_monitor = threading.Thread(self._monitor_pool)

self._pool_monitor.daemon = True

self._pool_monitor.start()

def _monitor_pool(self):

""" This runs in its own thread and monitors the pool. """

while True:

self._maintain_pool()

time.sleep(0.1)

def _maintain_pool(self):

""" If any workers have exited, start a new one in its place. """

if self._join_exited_workers():

self._repopulate_pool()

def _join_exited_workers(self):

""" Find exited workers and join them. """

cleaned = False

for i in reversed(range(len(self._pool))):

worker = self._pool[i]

if worker.exitcode is not None:

# worker exited

worker.join()

cleaned = True

del self._pool[i]

return cleaned

def _repopulate_pool(self):

""" Start new workers if any have exited. """

for i in range(self._workers - len(self._pool)):

w = Worker(self._lock, self._max_tasks)

self._pool.append(w)

w.start()

class Worker(multiprocessing.Process):

def __init__(self, lock, max_tasks):

multiprocesing.Process.__init__(self)

self.lock = lock

self.queue = Redis(..) # this is a redis based queue

self.res_queue = Redis(...)

self.max_tasks = max_tasks

def run():

runs = 0

while self.max_tasks and runs < self.max_tasks:

job = self.queue.get(block=True)

job.results = process(job)

with self.lock:

post_process(self.res_queue, job)

if self.max_tasks:

runs += 1

def main():

pool = WorkerPool(workers=4, maxtasksperchild=1000)

# The program will block here since none of the workers are daemons.

# It's not clear how/when you want to shut things down, but the Pool

# can be enhanced to support that pretty easily.

请注意，上面的池监视代码与multiprocessing.Pool中用于相同目的的代码几乎完全相同。在