本文主要分析使用oslo_messaging时,RPCClient的获取及发送消息流程
还需要补充的地方:
1.setevedore加载自定义插件方法
2.具体的driver发送消息send流程
这里只是简单给出了一个大致流程,还有很多地方需要补充
源码解析:
1.def _get_transport()
这里的DriverManager的参数:
namespace为oslo.messaging.drivers
name为conf中配置的transport_ul经过解析后的name,例如:transport_url="zmq+redis://127.0.0.1:6379",则经过解析后,name为zmq
invoke_args参数为conf和url
def _get_transport(conf, url=None, allowed_remote_exmods=None,
transport_cls=RPCTransport):
allowed_remote_exmods = allowed_remote_exmods or []
conf.register_opts(_transport_opts)
if not isinstance(url, TransportURL):
url = TransportURL.parse(conf, url)
kwargs = dict(default_exchange=conf.control_exchange,
allowed_remote_exmods=allowed_remote_exmods)
try:
mgr = driver.DriverManager('oslo.messaging.drivers',
url.transport.split('+')[0],
invoke_on_load=True,
invoke_args=[conf, url],
invoke_kwds=kwargs)
except RuntimeError as ex:
raise DriverLoadFailure(url.transport, ex)
return transport_cls(mgr.driver)
查看oslo_messaging的setup.cfg文件发现entry_points配置如下:
[entry_points]
console_scripts =
oslo-messaging-zmq-proxy = oslo_messaging._cmd.zmq_proxy:main
oslo-messaging-zmq-broker = oslo_messaging._cmd.zmq_proxy:main
oslo-messaging-send-notification = oslo_messaging.notify.notifier:_send_notification
oslo.messaging.drivers =
rabbit = oslo_messaging._drivers.impl_rabbit:RabbitDriver
zmq = oslo_messaging._drivers.impl_zmq:ZmqDriver
amqp = oslo_messaging._drivers.impl_amqp1:ProtonDriver
# This driver is supporting for only notification usage
kafka = oslo_messaging._drivers.impl_kafka:KafkaDriver
# To avoid confusion
kombu = oslo_messaging._drivers.impl_rabbit:RabbitDriver
# This is just for internal testing
fake = oslo_messaging._drivers.impl_fake:FakeDriver
我们找到namespace为oslo.messaging.drivers的部分,再根据name为zmq可以得到Driver了,即ZmqDriver
备注:可以了解setup.py、setup.cfg、pbr、setuptools这几个库的作用,这样就可以了解这个entry_points是什么。
2.ZmqDriver发送消息流程
以下是ZmqDriver代码,流程图中的Transport.send实际调用driver.send(),也就是下面ZmqDriver的send方法
class ZmqDriver(base.BaseDriver):
"""ZeroMQ Driver implementation.
Provides implementation of RPC and Notifier APIs by means
of ZeroMQ library.
See :doc:`zmq_driver` for details.
"""
def __init__(self, conf, url, default_exchange=None,
allowed_remote_exmods=None):
"""Construct ZeroMQ driver.
Initialize driver options.
Construct matchmaker - pluggable interface to targets management
Name Service
Construct client and server controllers
:param conf: oslo messaging configuration object
:type conf: oslo_config.CONF
:param url: transport URL
:type url: TransportUrl
:param default_exchange: Not used in zmq implementation
:type default_exchange: None
:param allowed_remote_exmods: remote exception passing options
:type allowed_remote_exmods: list
"""
//导入zmq
zmq = zmq_async.import_zmq()
if zmq is None:
raise ImportError(_LE("ZeroMQ is not available!"))
//注册配置项,可以了解oslo_config这个库
conf = zmq_options.register_opts(conf, url)
self.conf = conf
self.allowed_remote_exmods = allowed_remote_exmods
//根据配置选择matchmaker,详情介绍可以查看openstack的ZeroMq驱动官方介绍
//https://docs.openstack.org/oslo.messaging/ocata/zmq_driver.html#match-making-
//mandatory
self.matchmaker = driver.DriverManager(
'oslo.messaging.zmq.matchmaker',
self.get_matchmaker_backend(self.conf, url),
).driver(self.conf, url=url)
client_cls = zmq_client.ZmqClientProxy
if conf.oslo_messaging_zmq.use_pub_sub and not \
conf.oslo_messaging_zmq.use_router_proxy:
client_cls = zmq_client.ZmqClientMixDirectPubSub
elif not conf.oslo_messaging_zmq.use_pub_sub and not \
conf.oslo_messaging_zmq.use_router_proxy:
client_cls = zmq_client.ZmqClientDirect
self.client = LazyDriverItem(
client_cls, self.conf, self.matchmaker,
self.allowed_remote_exmods)
self.notifier = LazyDriverItem(
client_cls, self.conf, self.matchmaker,
self.allowed_remote_exmods)
super(ZmqDriver, self).__init__(conf, url, default_exchange,
allowed_remote_exmods)
@staticmethod
def get_matchmaker_backend(conf, url):
zmq_transport, _, matchmaker_backend = url.transport.partition('+')
assert zmq_transport == 'zmq', "Needs to be zmq for this transport!"
if not matchmaker_backend:
return conf.oslo_messaging_zmq.rpc_zmq_matchmaker
if matchmaker_backend not in zmq_options.MATCHMAKER_BACKENDS:
raise rpc_common.RPCException(
_LE("Incorrect matchmaker backend name %(backend_name)s! "
"Available names are: %(available_names)s") %
{"backend_name": matchmaker_backend,
"available_names": zmq_options.MATCHMAKER_BACKENDS})
return matchmaker_backend
def send(self, target, ctxt, message, wait_for_reply=None, timeout=None,
retry=None):
"""Send RPC message to server
:param target: Message destination target
:type target: oslo_messaging.Target
:param ctxt: Message context
:type ctxt: dict
:param message: Message payload to pass
:type message: dict
:param wait_for_reply: Waiting for reply flag
:type wait_for_reply: bool
:param timeout: Reply waiting timeout in seconds
:type timeout: int
:param retry: an optional default connection retries configuration
None or -1 means to retry forever
0 means no retry
N means N retries
:type retry: int
"""
client = self.client.get()
if wait_for_reply:
return client.send_call(target, ctxt, message, timeout, retry)
elif target.fanout:
client.send_fanout(target, ctxt, message, retry)
else:
client.send_cast(target, ctxt, message, retry)
查看setup.cfg, transport_url="zmq+redis://127.0.0.1:6379",则该方法self.get_matchmaker_backend(self.conf, url)最终为MatchmakerRedis
[entry_points]
oslo.messaging.zmq.matchmaker =
# Matchmakers for ZeroMQ
dummy = oslo_messaging._drivers.zmq_driver.matchmaker.zmq_matchmaker_base:MatchmakerDummy
redis = oslo_messaging._drivers.zmq_driver.matchmaker.zmq_matchmaker_redis:MatchmakerRedis
sentinel = oslo_messaging._drivers.zmq_driver.matchmaker.zmq_matchmaker_redis:MatchmakerSentinel
最终rpc client会通过ZmqClientDirect进行消息发送,相关类图如下:
3.最终经过调用,代码会在_drivers.zmq_driver.client.zmq_publisher_manager.PublisherManagerStatic的send_call/_send处开始
代码如下:
class PublisherManagerStatic(PublisherManagerBase):
@target_not_found_timeout
def send_call(self, request):
# 获取一个server的socket
socket = self.publisher.acquire_connection(request)
self.publisher.send_request(socket, request)
reply = self.publisher.receive_reply(socket, request)
return reply
@target_not_found_warn
def _send(self, request):
socket = self.publisher.acquire_connection(request)
self.publisher.send_request(socket, request)
send_cast = _send
send_fanout = _send
send_notify = _send
先看下socket的获取流程,可以看到他会调用publisher的acquire_connection方法,这self.publisher实际上就是class DealerPublisherDirectStatic这个类
代码如下:
class DealerPublisherDirectStatic(DealerPublisherDirect):
"""DEALER-publisher using direct static connections.
For some reason direct static connections may be also useful.
Assume a case when some agents are not connected with control services
over RPC (Ironic or Cinder+Ceph), and RPC is used only between controllers.
In this case number of RPC connections doesn't matter (very small) so we
can use static connections without fear and have all performance benefits
from it.
"""
def __init__(self, conf, matchmaker):
super(DealerPublisherDirectStatic, self).__init__(conf, matchmaker)
self.fanout_sockets = zmq_sockets_manager.SocketsManager(
conf, matchmaker, zmq.DEALER)
def acquire_connection(self, request):
# 获取target_key
# 如果未指定server,则获取target_key形如"'ROUTER/nova-compute"
# 指定server,则target_key形如"'ROUTER/nova-compute/{server}"
target_key = zmq_address.target_to_key(
request.target, zmq_names.socket_type_str(zmq.ROUTER))
if request.msg_type in zmq_names.MULTISEND_TYPES:
# 只有广播方式或通知方式才会进来
hosts = self.routing_table.get_fanout_hosts(request.target)
return self.fanout_sockets.get_cached_socket(target_key, hosts,
immediate=False)
else:
hosts = self.routing_table.get_all_round_robin_hosts(
request.target)
return self.sockets_manager.get_cached_socket(target_key, hosts)
继续看hosts的获取,self.routing_table为RoutingTableAdaptor
RoutingTableAdaptor:
class DealerPublisherDirect(zmq_dealer_publisher_base.DealerPublisherBase):
def __init__(self, conf, matchmaker):
sender = zmq_senders.RequestSenderDirect(conf, use_async=True)
receiver = zmq_receivers.ReceiverDirect(conf)
super(DealerPublisherDirect, self).__init__(conf, matchmaker,
sender, receiver)
self.routing_table = zmq_routing_table.RoutingTableAdaptor(
conf, matchmaker, zmq.ROUTER)
接着调用get_all_round_robin_hosts,在类RoutingTableAdaptor中
class RoutingTableAdaptor(object):
def __init__(self, conf, matchmaker, listener_type):
self.conf = conf
self.matchmaker = matchmaker
self.listener_type = listener_type
# 路由表,缓存了target和服务端的地址关系映射
self.routing_table = RoutingTable(conf)
# 周期性的更新路由表中的缓存
self.routing_table_updater = RoutingTableUpdater(
conf, matchmaker, self.routing_table)
self.round_robin_targets = {}
self._lock = threading.Lock()
def get_round_robin_host(self, target):
target_key = self._fetch_round_robin_hosts_from_matchmaker(target)
rr_gen = self.round_robin_targets[target_key]
host = next(rr_gen)
LOG.debug("Host resolved for the current connection is %s" % host)
return host
def get_all_round_robin_hosts(self, target):
# 若之前已经发送过请求,则从路由表的缓存返回服务端地址;
# 否则,从matchmaker(例如redis)获取最新的服务端地址
target_key = self._fetch_round_robin_hosts_from_matchmaker(target)
return self.routing_table.get_hosts_fanout(target_key)
def _fetch_round_robin_hosts_from_matchmaker(self, target):
target_key = zmq_address.target_to_key(
target, zmq_names.socket_type_str(self.listener_type))
LOG.debug("Processing target %s for round-robin." % target_key)
if target_key not in self.round_robin_targets:
# 若target_key不在round_robin_targets字典中
with self._lock:
if target_key not in self.round_robin_targets:
LOG.debug("Target %s is not in cache. Check matchmaker "
"server." % target_key)
# 从matchmaker中(redis)读取target_key对应的值
hosts = self.matchmaker.get_hosts_retry(
target, zmq_names.socket_type_str(self.listener_type))
LOG.debug("Received hosts %s" % hosts)
# 更新路由表
self.routing_table.update_hosts(target_key, hosts)
# 将target_key记录在round_robin_targets字典中
self.round_robin_targets[target_key] = \
self.routing_table.get_hosts_round_robin(target_key)
return target_key
若此前target_key在缓存中(在round_robin_targets字典中有记录),则直接返回,若不在,则在matchmaker(例如为redis)中获取,然后更新路由表(RoutingTable)并将该target_key记录在round_robin_targets中
RoutingTable的作用将请求的target转换为服务端地址,缓存了target与服务端地址的映射关系
我们可以看到get_all_round_robin_hosts中调用self.routing_table.get_hosts_fanout(target_key),最终就会调用RoutingTable的_get_hosts方法,获取服务端地址信息
class RoutingTable(object):
def __init__(self, conf):
self.conf = conf
self.targets = {}
self._lock = threading.Lock()
def get_targets(self):
with self._lock:
return list(self.targets.keys())
def update_hosts(self, target_key, hosts_updated):
with self._lock:
if target_key in self.targets and not hosts_updated:
self.targets.pop(target_key)
return
hosts_current, _ = self.targets.get(target_key, (set(), None))
hosts_updated = set(hosts_updated)
has_differences = hosts_updated ^ hosts_current
if has_differences:
self.targets[target_key] = (hosts_updated, self._create_tm())
def get_hosts_round_robin(self, target_key):
while self.contains(target_key):
for host in self._get_hosts_rr(target_key):
yield host
def get_hosts_fanout(self, target_key):
hosts, _ = self._get_hosts(target_key)
return hosts
def contains(self, target_key):
with self._lock:
return target_key in self.targets
def _get_hosts(self, target_key):
with self._lock:
hosts, tm = self.targets.get(target_key, ([], None))
hosts = list(hosts)
return hosts, tm
路由表的缓存也是会更新的,会通过RoutingTableUpdater进行周期性的更新,周期时期可以在配置中看到如下:
class RoutingTableUpdater(zmq_updater.UpdaterBase):
def __init__(self, conf, matchmaker, routing_table):
self.routing_table = routing_table
super(RoutingTableUpdater, self).__init__(
conf, matchmaker, self._update_routing_table,
conf.oslo_messaging_zmq.zmq_target_update)
def _update_routing_table(self):
target_keys = self.routing_table.get_targets()
try:
for target_key in target_keys:
hosts = self.matchmaker.get_hosts_by_key(target_key)
self.routing_table.update_hosts(target_key, hosts)
LOG.debug("Updating routing table from the matchmaker. "
"%d target(s) updated %s." % (len(target_keys),
target_keys))
except zmq_matchmaker_base.MatchmakerUnavailable:
LOG.warning(_LW("Not updated. Matchmaker was not available."))
周期时间配置,即zmq_target_update,默认180s
cfg.IntOpt('zmq_target_update', default=180,
deprecated_group='DEFAULT',
help='Update period in seconds of a name service record '
'about existing target.'),
未完待续。。。。。。。