python zeromq rpc介紹

https://github.com/geoffwatts/zmqrpc
傳輸資料格式：bson
模型: 多線程
Client             Server   
------       ------------------------------          
client                        worker(thread)     
             \                /
cleint   --   queue    
             /                \
client                        worker(thread)

server.py

"""
server: Implementing ZMQRPCServer class to export a user class via zmqrpc to ZMQRPC clients, and to arrange queued calls to server threads.
"""

import sys
if sys.version < '2.6':
    sys.exit('ERROR: Sorry, python 2.6 is required for the way this module uses threading.')

import zmq
from bson import BSON   
import threading
import os, sys, traceback
from zmqrpc import ZMQRPCError, ZMQRPCRemoteError


LISTEN=0
CONNECT=1

class ZMQRPCServer(object):
    def _thread(self,context,worker_id,import_class,pid,serverid,counters,methods,target,stype,worker_args):
        """
        Worker thread for zmqrpc server - binds to zmq socket (target) and works ZMQRPCServer import_class.
        Instantiated by work() threading
        Handles BSON in/out, zmq REP to zmq QUEUE or REQ
        """
        socket = self.context.socket(zmq.REP)
        job_count = 0
        if stype == LISTEN:
            socket.bind(target)
        else:
            socket.connect(target)

        if worker_args:
            nuclass = import_class(**worker_args)
        else:
            nuclass = import_class()
        
        while True:
            sockin = socket.recv()
            message = BSON(sockin).decode()
            result = None
            fail = None
            tb = None
            method = str(message['method'])
            args = message.get('args',[])
            if self.export and (not method in self.export):
                tb = "NameError: name '"+method+"' is not exported in ZMQRPC class '"+import_class.__name__+"'"
                socket.send(BSON.encode({'fail':True,'result':None,'runner':None,'traceback':tb}))
                return
                

            # Convert kwargs from unicode strings to 8bit strings
            
            if method == '__threadstatus__':
                x = threading.current_thread()
                socket.send(BSON.encode({'runner':None,'traceback':None,'fail':False,'result':{'id':serverid+':'+str(pid)+':'+str(x.name),'alive':x.is_alive(),'job_count':counters.get(x.name,0),'last_method':methods.get(x.name,''),}}))   
            else:
                try:
                    kwargs = {}
                    for (k,v) in message.get('kwargs',{}).iteritems():
                        kwargs[str(k)] = v

                    job_count+=1
                    counters[threading.currentThread().name] = job_count
                    methods[threading.currentThread().name] = method
                    runner = {'job_count':job_count,'thread':threading.currentThread().name,'method':import_class.__name__+'.'+method,}
                    
                    # Find the method in the module, run it.
                    try:
                        if hasattr(nuclass,method):
                            result = getattr(nuclass,method)(*args,**kwargs)
                            fail = False
                        else:
                            fail = True
                            tb = "NameError: name '"+method+"' is not defined in ZMQRPC class '"+import_class.__name__+"'"
                    except:
                        etype, evalue, etb = sys.exc_info()
                        fail = True
                        tb = "\n".join(traceback.format_exception(etype, evalue, etb))
                    socket.send(BSON.encode({'fail':fail,'result':result,'runner':runner,'traceback':tb}))
                except:
                    etype, evalue, etb = sys.exc_info()
                    fail = True
                    tb = "\n".join(traceback.format_exception(etype, evalue, etb))
                    socket.send(BSON.encode({'fail':fail,'result':None,'runner':None,'traceback':tb}))


    def __init__(self,import_class,export=None):
        """
        Instantiate this class with your class to export via zmqrpc
        """
        self.iclass = import_class
        self.pid = os.getpid()
        self.serverid = os.uname()[1]
        self.context = zmq.Context(1)
        self.export = export

    def work(self,workers=1,target="inproc://workers",stype=CONNECT,worker_args={}):
        """
        Call to spawn serverthreads that will then work forever.
        stype: socket type, either zmqrpc.server.CONNECT or zmqrpc.server.LISTEN
        target: zmq socket (eg: 'tcp://127.0.0.1:5000')
        workers: number of worker threads to spwan
        """
        counters = {}
        methods = {}
        for i in range(0,workers):
            thread = threading.Thread(target=self._thread, name='zmqrpc-'+str(i), args=(self.context,i,self.iclass,self.pid,self.serverid,counters,methods,target,stype,worker_args))
            thread.start()

            
    def queue(self,listen,bind='inproc://workers',thread=False):
        """
        Call to start a zmq queue device to disatch zmqrpc work.
        listen: zmq socket to listen on for CLIENTS (eg: 'tcp://127.0.0.1:5 000')
        target: zmq socket to listen on for worker threads (eg: 'tcp://127.0.0.1:6000')
        workers: number of worker threads to spwan
        """
        def q(listen,worker_target):
            self.workers = self.context.socket(zmq.XREQ)
            self.workers.bind(worker_target);

            self.clients = self.context.socket(zmq.XREP)
            self.clients.bind(listen) 
            zmq.device(zmq.QUEUE, self.clients, self.workers)
        if thread:
            thread = threading.Thread(target=q, name='zmqrpc-queue', args=(listen,bind ))
            thread.start()
        else:
            q(listen,bind)
        

client.py

"""client: client class to export a class to an zmqrpc queue or client."""
import zmq
from bson import BSON   
import os, sys, traceback
import time

from zmqrpc import ZMQRPCError, ZMQRPCRemoteError

class ZMQRPC(object):
    """
    ZMQRPC: client class to export a class to an zmqrpc queue or client.
    """
    def __init__(self,target,timeout=30):
        """
        Instantiate this class with a zmq target (eg 'tcp://127.0.0.1:5000') and a timeout (in seconds) for method calls.
        Then call zmqrpc server exported methods from the class.
        """
        
        self._context = zmq.Context()
        self._zmqsocket = self._context.socket(zmq.REQ)
        # Connect to everything, or just one
        if isinstance(target,list):
            for t in target:
                self._zmqsocket.connect(target)
        else:
            self._zmqsocket.connect(target)
        self._socket = target
        self._pollin = zmq.Poller()
        self._pollin.register(self._zmqsocket,zmq.POLLIN)
        self._pollout = zmq.Poller()
        self._pollout.register(self._zmqsocket,zmq.POLLOUT)
        self._timeout = timeout
        
        self._lastrun = None

    def _dorequest(self,msg,timeout=5):
        """
        _dorequest: Set up a BSON string and send zmq REQ to ZMQRPC target
        """
        # Set up bson message
        bson = BSON.encode(msg)
        
        # Send...
        try:
            self._pollout.poll(timeout=timeout*1000) # Poll for outbound send, then send
            self._zmqsocket.send(bson,flags=zmq.NOBLOCK)
        except:
            raise ZMQRPCError('Request failure')

        # Poll for inbound then rx
        try:        
            for i in range(0,timeout*100):
                if len(self._pollin.poll(timeout=1)) > 0:
                    break
                time.sleep(0.01)
            msg_in = self._zmqsocket.recv(flags=zmq.NOBLOCK)
        
        except:
            raise ZMQRPCError('Response timeout')

        
        if msg_in == None:
            raise ZMQRPCError('No response')
    
        result = BSON(msg_in).decode()
        
        self._lastrun = result.get('runner')
        
        return result
    
    
    def _debug_call(self,name,*args,**kwargs):
        """
        _debug_call: Convenience method to call _dorequest with pre-filled dict with method name, args, kwargs and timeout
        """
        return self._dorequest({'method':name,'args':args,'kwargs':kwargs},timeout=self._timeout)
               
    def __serverstatus__(self,max_nodes=1000):
        """
        __serverstatus__: Slightly hackish method to retreive threadstatus from all listening threads on a zmqrpc queue
        """
        results = {}
        try:
            for r in range(0,max_nodes):
                res = self._dorequest({'method':'__threadstatus__'},timeout=self._timeout)[u'result']
                id = res[u'id']
                if results.has_key(id): break
                del res[u'id']
                results[id] = res
        except:
            raise ZMQRPCError('Error finding server threads')
        return results
            
            
         
    class RPC(object):
        """
        RPC: zmqrpc Remote procedure call class - encapsulates method calls to imported class
        """

        def __init__(self,name,fn,timeout,target):
            self._name = name
            self._fn = fn
            self._timeout = timeout
            self._socket = target
            
        def __call__(self,*args,**kwargs):
            result = self._fn({'method':self._name,'args':args,'kwargs':kwargs},timeout=self._timeout)
            if result['fail']:
                raise ZMQRPCRemoteError(result['traceback']) #+"  RUNNER:"+str(result['runner']))
            else:
                return result['result']
        def __repr__(self):
            return '<zmqrpc method '+self._name+' to zmq socket '+self._socket+'>'
            
        
    def __getattr__(self,name):
            return self.RPC(name,self._dorequest,timeout=self._timeout,target=self._socket)
 

由於是使用zmq所以很容易把上面的模式改進為，多進程+多線程的模型
因為python中是鼓勵使用多進程的(多進程可使用多核)
模型二: 多進程+多線程
Client             Server   
------       ----------------------------------------------          
client                       Server1________worker(thread)
             \                /(process)    \___worker(thread)
cleint   --   queue    
             /                \                                      
client                       Server2________worker(thread)
                                (process)    \___worker(thread)

要使用模型二需要把传输协议由INPROC改為IPC