关于python的multiprocessing.Manager在将Queue放在Dict中时的异常情况

关于python的multiprocessing.Manager在将Queue放在Dict中时的异常情况

python3在多进程编程时，利用Manager进行进程间通讯。当讲Manager.Queue村妇Dict中时，会引发如下错误:

Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 243, in serve_client
    request = recv()
  File "/usr/lib/python3.8/multiprocessing/connection.py", line 251, in recv
    return _ForkingPickler.loads(buf.getbuffer())
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 959, in RebuildProxy
    return func(token, serializer, incref=incref, **kwds)
TypeError: AutoProxy() got an unexpected keyword argument 'manager_owned'

利用如下补丁可以修复问题：

### multiprocessing Manager 修复及二次开发补丁        ###
### 1. 修改 Manager 的默认进程名为 network            ###
### 2. 修复 Queue 在 Dict 中的异常情况                ###

import setproctitle
import multiprocessing.managers # 这两次引用都很重要
from multiprocessing import Manager # 这两次引用都很重要

def _rename_manager():
    setproctitle.setproctitle("network") # 这里可以自定义Manager进程的名称

def start(self, initializer=_rename_manager, initargs=()):
'
    Spawn a server process for this manager object
'
    if self._state.value != multiprocessing.managers.State.INITIAL:
        if self._state.value == multiprocessing.managers.State.STARTED:
            raise ProcessError("Already started server") # type: ignore
        elif self._state.value == multiprocessing.managers.State.SHUTDOWN:
            raise ProcessError("Manager has shut down") # type: ignore
        else:
            raise ProcessError( # type: ignore
                "Unknown state {!r}".format(self._state.value))

    if initializer is not None and not callable(initializer):
        raise TypeError('initializer must be a callable')

    # pipe over which we will retrieve address of server
    reader, writer = multiprocessing.managers.connection.Pipe(duplex=False)

    # spawn process which runs a server
    self._process = self._ctx.Process(
        target=type(self)._run_server,
        args=(self._registry, self._address, self._authkey,
                self._serializer, writer, initializer, initargs),
        )
    ident = ':'.join(str(i) for i in self._process._identity)
    self._process.name = type(self).__name__  + '-' + ident
    self._process.start()

    # get address of server
    writer.close()
    self._address = reader.recv()
    reader.close()

    # register a finalizer
    self._state.value = multiprocessing.managers.State.STARTED
    self.shutdown = multiprocessing.managers.util.Finalize(
        self, type(self)._finalize_manager,
        args=(self._process, self._address, self._authkey,
                self._state, self._Client),
        exitpriority=0
        )

def AutoProxy(token, serializer, manager=None, authkey=None,
              exposed=None, incref=True, manager_owned=False):
'
    Return an auto-proxy for `token`
'
    _Client = multiprocessing.managers.listener_client[serializer][1]

    if exposed is None:
        conn = _Client(token.address, authkey=authkey)
        try:
            exposed = dispatch(conn, None, 'get_methods', (token,)) # type: ignore
        finally:
            conn.close()

    if authkey is None and manager is not None:
        authkey = manager._authkey
    if authkey is None:
        authkey = multiprocessing.process.current_process().authkey

    ProxyType = multiprocessing.managers.MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
    proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
                      incref=incref, manager_owned=manager_owned)
    proxy._isauto = True
    return proxy

multiprocessing.managers.AutoProxy = AutoProxy
multiprocessing.managers.BaseManager.start = start
### multiprocessing Manager 修复及二次开发补丁        ###

在初始化mg = Manager()的文件顶部，引用如上补丁，即可修复异常。

如果存在多个文件初始化Manager，需要在每个文件都引用如上补丁。