https://github.com/geoffwatts/zmqrpc
傳輸資料格式:bson
模型: 多線程
Client Server
------ ------------------------------
client worker(thread)
\ /
cleint -- queue
/ \
client worker(thread)
server.py
client.py
由於是使用zmq所以很容易把上面的模式改進為,多進程+多線程的模型
因為python中是鼓勵使用多進程的(多進程可使用多核)
模型二: 多進程+多線程
Client Server
------ ----------------------------------------------
client Server1________worker(thread)
\ /(process) \___worker(thread)
cleint -- queue
/ \
client Server2________worker(thread)
(process) \___worker(thread)
要使用模型二需要把传输协议由INPROC改為IPC
傳輸資料格式:bson
模型: 多線程
Client Server
------ ------------------------------
client worker(thread)
\ /
cleint -- queue
/ \
client worker(thread)
server.py
"""
server: Implementing ZMQRPCServer class to export a user class via zmqrpc to ZMQRPC clients, and to arrange queued calls to server threads.
"""
import sys
if sys.version < '2.6':
sys.exit('ERROR: Sorry, python 2.6 is required for the way this module uses threading.')
import zmq
from bson import BSON
import threading
import os, sys, traceback
from zmqrpc import ZMQRPCError, ZMQRPCRemoteError
LISTEN=0
CONNECT=1
class ZMQRPCServer(object):
def _thread(self,context,worker_id,import_class,pid,serverid,counters,methods,target,stype,worker_args):
"""
Worker thread for zmqrpc server - binds to zmq socket (target) and works ZMQRPCServer import_class.
Instantiated by work() threading
Handles BSON in/out, zmq REP to zmq QUEUE or REQ
"""
socket = self.context.socket(zmq.REP)
job_count = 0
if stype == LISTEN:
socket.bind(target)
else:
socket.connect(target)
if worker_args:
nuclass = import_class(**worker_args)
else:
nuclass = import_class()
while True:
sockin = socket.recv()
message = BSON(sockin).decode()
result = None
fail = None
tb = None
method = str(message['method'])
args = message.get('args',[])
if self.export and (not method in self.export):
tb = "NameError: name '"+method+"' is not exported in ZMQRPC class '"+import_class.__name__+"'"
socket.send(BSON.encode({'fail':True,'result':None,'runner':None,'traceback':tb}))
return
# Convert kwargs from unicode strings to 8bit strings
if method == '__threadstatus__':
x = threading.current_thread()
socket.send(BSON.encode({'runner':None,'traceback':None,'fail':False,'result':{'id':serverid+':'+str(pid)+':'+str(x.name),'alive':x.is_alive(),'job_count':counters.get(x.name,0),'last_method':methods.get(x.name,''),}}))
else:
try:
kwargs = {}
for (k,v) in message.get('kwargs',{}).iteritems():
kwargs[str(k)] = v
job_count+=1
counters[threading.currentThread().name] = job_count
methods[threading.currentThread().name] = method
runner = {'job_count':job_count,'thread':threading.currentThread().name,'method':import_class.__name__+'.'+method,}
# Find the method in the module, run it.
try:
if hasattr(nuclass,method):
result = getattr(nuclass,method)(*args,**kwargs)
fail = False
else:
fail = True
tb = "NameError: name '"+method+"' is not defined in ZMQRPC class '"+import_class.__name__+"'"
except:
etype, evalue, etb = sys.exc_info()
fail = True
tb = "\n".join(traceback.format_exception(etype, evalue, etb))
socket.send(BSON.encode({'fail':fail,'result':result,'runner':runner,'traceback':tb}))
except:
etype, evalue, etb = sys.exc_info()
fail = True
tb = "\n".join(traceback.format_exception(etype, evalue, etb))
socket.send(BSON.encode({'fail':fail,'result':None,'runner':None,'traceback':tb}))
def __init__(self,import_class,export=None):
"""
Instantiate this class with your class to export via zmqrpc
"""
self.iclass = import_class
self.pid = os.getpid()
self.serverid = os.uname()[1]
self.context = zmq.Context(1)
self.export = export
def work(self,workers=1,target="inproc://workers",stype=CONNECT,worker_args={}):
"""
Call to spawn serverthreads that will then work forever.
stype: socket type, either zmqrpc.server.CONNECT or zmqrpc.server.LISTEN
target: zmq socket (eg: 'tcp://127.0.0.1:5000')
workers: number of worker threads to spwan
"""
counters = {}
methods = {}
for i in range(0,workers):
thread = threading.Thread(target=self._thread, name='zmqrpc-'+str(i), args=(self.context,i,self.iclass,self.pid,self.serverid,counters,methods,target,stype,worker_args))
thread.start()
def queue(self,listen,bind='inproc://workers',thread=False):
"""
Call to start a zmq queue device to disatch zmqrpc work.
listen: zmq socket to listen on for CLIENTS (eg: 'tcp://127.0.0.1:5 000')
target: zmq socket to listen on for worker threads (eg: 'tcp://127.0.0.1:6000')
workers: number of worker threads to spwan
"""
def q(listen,worker_target):
self.workers = self.context.socket(zmq.XREQ)
self.workers.bind(worker_target);
self.clients = self.context.socket(zmq.XREP)
self.clients.bind(listen)
zmq.device(zmq.QUEUE, self.clients, self.workers)
if thread:
thread = threading.Thread(target=q, name='zmqrpc-queue', args=(listen,bind ))
thread.start()
else:
q(listen,bind)
client.py
"""client: client class to export a class to an zmqrpc queue or client."""
import zmq
from bson import BSON
import os, sys, traceback
import time
from zmqrpc import ZMQRPCError, ZMQRPCRemoteError
class ZMQRPC(object):
"""
ZMQRPC: client class to export a class to an zmqrpc queue or client.
"""
def __init__(self,target,timeout=30):
"""
Instantiate this class with a zmq target (eg 'tcp://127.0.0.1:5000') and a timeout (in seconds) for method calls.
Then call zmqrpc server exported methods from the class.
"""
self._context = zmq.Context()
self._zmqsocket = self._context.socket(zmq.REQ)
# Connect to everything, or just one
if isinstance(target,list):
for t in target:
self._zmqsocket.connect(target)
else:
self._zmqsocket.connect(target)
self._socket = target
self._pollin = zmq.Poller()
self._pollin.register(self._zmqsocket,zmq.POLLIN)
self._pollout = zmq.Poller()
self._pollout.register(self._zmqsocket,zmq.POLLOUT)
self._timeout = timeout
self._lastrun = None
def _dorequest(self,msg,timeout=5):
"""
_dorequest: Set up a BSON string and send zmq REQ to ZMQRPC target
"""
# Set up bson message
bson = BSON.encode(msg)
# Send...
try:
self._pollout.poll(timeout=timeout*1000) # Poll for outbound send, then send
self._zmqsocket.send(bson,flags=zmq.NOBLOCK)
except:
raise ZMQRPCError('Request failure')
# Poll for inbound then rx
try:
for i in range(0,timeout*100):
if len(self._pollin.poll(timeout=1)) > 0:
break
time.sleep(0.01)
msg_in = self._zmqsocket.recv(flags=zmq.NOBLOCK)
except:
raise ZMQRPCError('Response timeout')
if msg_in == None:
raise ZMQRPCError('No response')
result = BSON(msg_in).decode()
self._lastrun = result.get('runner')
return result
def _debug_call(self,name,*args,**kwargs):
"""
_debug_call: Convenience method to call _dorequest with pre-filled dict with method name, args, kwargs and timeout
"""
return self._dorequest({'method':name,'args':args,'kwargs':kwargs},timeout=self._timeout)
def __serverstatus__(self,max_nodes=1000):
"""
__serverstatus__: Slightly hackish method to retreive threadstatus from all listening threads on a zmqrpc queue
"""
results = {}
try:
for r in range(0,max_nodes):
res = self._dorequest({'method':'__threadstatus__'},timeout=self._timeout)[u'result']
id = res[u'id']
if results.has_key(id): break
del res[u'id']
results[id] = res
except:
raise ZMQRPCError('Error finding server threads')
return results
class RPC(object):
"""
RPC: zmqrpc Remote procedure call class - encapsulates method calls to imported class
"""
def __init__(self,name,fn,timeout,target):
self._name = name
self._fn = fn
self._timeout = timeout
self._socket = target
def __call__(self,*args,**kwargs):
result = self._fn({'method':self._name,'args':args,'kwargs':kwargs},timeout=self._timeout)
if result['fail']:
raise ZMQRPCRemoteError(result['traceback']) #+" RUNNER:"+str(result['runner']))
else:
return result['result']
def __repr__(self):
return '<zmqrpc method '+self._name+' to zmq socket '+self._socket+'>'
def __getattr__(self,name):
return self.RPC(name,self._dorequest,timeout=self._timeout,target=self._socket)
由於是使用zmq所以很容易把上面的模式改進為,多進程+多線程的模型
因為python中是鼓勵使用多進程的(多進程可使用多核)
模型二: 多進程+多線程
Client Server
------ ----------------------------------------------
client Server1________worker(thread)
\ /(process) \___worker(thread)
cleint -- queue
/ \
client Server2________worker(thread)
(process) \___worker(thread)
要使用模型二需要把传输协议由INPROC改為IPC