oslo_messaging源码阅读(1)—RPCClient

本文主要分析使用oslo_messaging时，RPCClient的获取及发送消息流程

还需要补充的地方：

1.setevedore加载自定义插件方法

2.具体的driver发送消息send流程

这里只是简单给出了一个大致流程，还有很多地方需要补充

源码解析：

1.def _get_transport()

这里的DriverManager的参数：

namespace为oslo.messaging.drivers

name为conf中配置的transport_ul经过解析后的name，例如：transport_url="zmq+redis://127.0.0.1:6379",则经过解析后，name为zmq

invoke_args参数为conf和url

def _get_transport(conf, url=None, allowed_remote_exmods=None,
                   transport_cls=RPCTransport):
    allowed_remote_exmods = allowed_remote_exmods or []
    conf.register_opts(_transport_opts)

    if not isinstance(url, TransportURL):
        url = TransportURL.parse(conf, url)

    kwargs = dict(default_exchange=conf.control_exchange,
                  allowed_remote_exmods=allowed_remote_exmods)

    try:
        mgr = driver.DriverManager('oslo.messaging.drivers',
                                   url.transport.split('+')[0],
                                   invoke_on_load=True,
                                   invoke_args=[conf, url],
                                   invoke_kwds=kwargs)
    except RuntimeError as ex:
        raise DriverLoadFailure(url.transport, ex)

    return transport_cls(mgr.driver)

查看oslo_messaging的setup.cfg文件发现entry_points配置如下：

[entry_points]
console_scripts =
    oslo-messaging-zmq-proxy = oslo_messaging._cmd.zmq_proxy:main
    oslo-messaging-zmq-broker = oslo_messaging._cmd.zmq_proxy:main
    oslo-messaging-send-notification = oslo_messaging.notify.notifier:_send_notification

oslo.messaging.drivers =
    rabbit = oslo_messaging._drivers.impl_rabbit:RabbitDriver
    zmq = oslo_messaging._drivers.impl_zmq:ZmqDriver
    amqp = oslo_messaging._drivers.impl_amqp1:ProtonDriver

    # This driver is supporting for only notification usage
    kafka = oslo_messaging._drivers.impl_kafka:KafkaDriver

    # To avoid confusion
    kombu = oslo_messaging._drivers.impl_rabbit:RabbitDriver

    # This is just for internal testing
    fake = oslo_messaging._drivers.impl_fake:FakeDriver

我们找到namespace为oslo.messaging.drivers的部分，再根据name为zmq可以得到Driver了，即ZmqDriver

备注：可以了解setup.py、setup.cfg、pbr、setuptools这几个库的作用，这样就可以了解这个entry_points是什么。

2.ZmqDriver发送消息流程

以下是ZmqDriver代码，流程图中的Transport.send实际调用driver.send(),也就是下面ZmqDriver的send方法

class ZmqDriver(base.BaseDriver):

    """ZeroMQ Driver implementation.

    Provides implementation of RPC and Notifier APIs by means
    of ZeroMQ library.

    See :doc:`zmq_driver` for details.
    """

    def __init__(self, conf, url, default_exchange=None,
                 allowed_remote_exmods=None):
        """Construct ZeroMQ driver.

        Initialize driver options.

        Construct matchmaker - pluggable interface to targets management
        Name Service

        Construct client and server controllers

        :param conf: oslo messaging configuration object
        :type conf: oslo_config.CONF
        :param url: transport URL
        :type url: TransportUrl
        :param default_exchange: Not used in zmq implementation
        :type default_exchange: None
        :param allowed_remote_exmods: remote exception passing options
        :type allowed_remote_exmods: list
        """

        //导入zmq
        zmq = zmq_async.import_zmq()
        if zmq is None:
            raise ImportError(_LE("ZeroMQ is not available!"))

        //注册配置项，可以了解oslo_config这个库
        conf = zmq_options.register_opts(conf, url)
        self.conf = conf
        self.allowed_remote_exmods = allowed_remote_exmods

        //根据配置选择matchmaker,详情介绍可以查看openstack的ZeroMq驱动官方介绍        
        //https://docs.openstack.org/oslo.messaging/ocata/zmq_driver.html#match-making- 
        //mandatory
        self.matchmaker = driver.DriverManager(
            'oslo.messaging.zmq.matchmaker',
            self.get_matchmaker_backend(self.conf, url),
        ).driver(self.conf, url=url)

        client_cls = zmq_client.ZmqClientProxy
        if conf.oslo_messaging_zmq.use_pub_sub and not \
                conf.oslo_messaging_zmq.use_router_proxy:
            client_cls = zmq_client.ZmqClientMixDirectPubSub
        elif not conf.oslo_messaging_zmq.use_pub_sub and not \
                conf.oslo_messaging_zmq.use_router_proxy:
            client_cls = zmq_client.ZmqClientDirect

        self.client = LazyDriverItem(
            client_cls, self.conf, self.matchmaker,
            self.allowed_remote_exmods)

        self.notifier = LazyDriverItem(
            client_cls, self.conf, self.matchmaker,
            self.allowed_remote_exmods)

        super(ZmqDriver, self).__init__(conf, url, default_exchange,
                                        allowed_remote_exmods)

    @staticmethod
    def get_matchmaker_backend(conf, url):
        zmq_transport, _, matchmaker_backend = url.transport.partition('+')
        assert zmq_transport == 'zmq', "Needs to be zmq for this transport!"
        if not matchmaker_backend:
            return conf.oslo_messaging_zmq.rpc_zmq_matchmaker
        if matchmaker_backend not in zmq_options.MATCHMAKER_BACKENDS:
            raise rpc_common.RPCException(
                _LE("Incorrect matchmaker backend name %(backend_name)s! "
                    "Available names are: %(available_names)s") %
                {"backend_name": matchmaker_backend,
                 "available_names": zmq_options.MATCHMAKER_BACKENDS})
        return matchmaker_backend

    def send(self, target, ctxt, message, wait_for_reply=None, timeout=None,
             retry=None):
        """Send RPC message to server

        :param target: Message destination target
        :type target: oslo_messaging.Target
        :param ctxt: Message context
        :type ctxt: dict
        :param message: Message payload to pass
        :type message: dict
        :param wait_for_reply: Waiting for reply flag
        :type wait_for_reply: bool
        :param timeout: Reply waiting timeout in seconds
        :type timeout: int
        :param retry: an optional default connection retries configuration
                      None or -1 means to retry forever
                      0 means no retry
                      N means N retries
        :type retry: int
        """
        client = self.client.get()
        if wait_for_reply:
            return client.send_call(target, ctxt, message, timeout, retry)
        elif target.fanout:
            client.send_fanout(target, ctxt, message, retry)
        else:
            client.send_cast(target, ctxt, message, retry)

查看setup.cfg, transport_url="zmq+redis://127.0.0.1:6379"，则该方法self.get_matchmaker_backend(self.conf, url)最终为MatchmakerRedis

[entry_points]

oslo.messaging.zmq.matchmaker =
    # Matchmakers for ZeroMQ
    dummy = oslo_messaging._drivers.zmq_driver.matchmaker.zmq_matchmaker_base:MatchmakerDummy
    redis = oslo_messaging._drivers.zmq_driver.matchmaker.zmq_matchmaker_redis:MatchmakerRedis
    sentinel = oslo_messaging._drivers.zmq_driver.matchmaker.zmq_matchmaker_redis:MatchmakerSentinel

最终rpc client会通过ZmqClientDirect进行消息发送，相关类图如下：

 

3.最终经过调用，代码会在_drivers.zmq_driver.client.zmq_publisher_manager.PublisherManagerStatic的send_call/_send处开始

代码如下：

class PublisherManagerStatic(PublisherManagerBase):

    @target_not_found_timeout
    def send_call(self, request):
        # 获取一个server的socket
        socket = self.publisher.acquire_connection(request)
        self.publisher.send_request(socket, request)
        reply = self.publisher.receive_reply(socket, request)
        return reply

    @target_not_found_warn
    def _send(self, request):
        socket = self.publisher.acquire_connection(request)
        self.publisher.send_request(socket, request)

    send_cast = _send
    send_fanout = _send
    send_notify = _send

先看下socket的获取流程，可以看到他会调用publisher的acquire_connection方法，这self.publisher实际上就是class DealerPublisherDirectStatic这个类

代码如下：

class DealerPublisherDirectStatic(DealerPublisherDirect):
    """DEALER-publisher using direct static connections.

    For some reason direct static connections may be also useful.
    Assume a case when some agents are not connected with control services
    over RPC (Ironic or Cinder+Ceph), and RPC is used only between controllers.
    In this case number of RPC connections doesn't matter (very small) so we
    can use static connections without fear and have all performance benefits
    from it.
    """

    def __init__(self, conf, matchmaker):
        super(DealerPublisherDirectStatic, self).__init__(conf, matchmaker)
        self.fanout_sockets = zmq_sockets_manager.SocketsManager(
            conf, matchmaker, zmq.DEALER)

    def acquire_connection(self, request):
        # 获取target_key
        # 如果未指定server，则获取target_key形如"'ROUTER/nova-compute"
        # 指定server，则target_key形如"'ROUTER/nova-compute/{server}"
        target_key = zmq_address.target_to_key(
            request.target, zmq_names.socket_type_str(zmq.ROUTER))
        if request.msg_type in zmq_names.MULTISEND_TYPES:
            # 只有广播方式或通知方式才会进来
            hosts = self.routing_table.get_fanout_hosts(request.target)
            return self.fanout_sockets.get_cached_socket(target_key, hosts,
                                                         immediate=False)
        else:
            hosts = self.routing_table.get_all_round_robin_hosts(
                request.target)
            return self.sockets_manager.get_cached_socket(target_key, hosts)

继续看hosts的获取，self.routing_table为RoutingTableAdaptor

RoutingTableAdaptor：

class DealerPublisherDirect(zmq_dealer_publisher_base.DealerPublisherBase):

    def __init__(self, conf, matchmaker):
        sender = zmq_senders.RequestSenderDirect(conf, use_async=True)
        receiver = zmq_receivers.ReceiverDirect(conf)
        super(DealerPublisherDirect, self).__init__(conf, matchmaker,
                                                    sender, receiver)

        self.routing_table = zmq_routing_table.RoutingTableAdaptor(
            conf, matchmaker, zmq.ROUTER)

接着调用get_all_round_robin_hosts，在类RoutingTableAdaptor中

class RoutingTableAdaptor(object):

    def __init__(self, conf, matchmaker, listener_type):
        self.conf = conf
        self.matchmaker = matchmaker
        self.listener_type = listener_type
        
        # 路由表，缓存了target和服务端的地址关系映射
        self.routing_table = RoutingTable(conf)
        # 周期性的更新路由表中的缓存
        self.routing_table_updater = RoutingTableUpdater(
            conf, matchmaker, self.routing_table)
        
        self.round_robin_targets = {}
        self._lock = threading.Lock()

    def get_round_robin_host(self, target):
        target_key = self._fetch_round_robin_hosts_from_matchmaker(target)
        rr_gen = self.round_robin_targets[target_key]
        host = next(rr_gen)
        LOG.debug("Host resolved for the current connection is %s" % host)
        return host

    def get_all_round_robin_hosts(self, target):
        # 若之前已经发送过请求，则从路由表的缓存返回服务端地址；
        # 否则，从matchmaker（例如redis）获取最新的服务端地址
        target_key = self._fetch_round_robin_hosts_from_matchmaker(target)
        return self.routing_table.get_hosts_fanout(target_key)

    def _fetch_round_robin_hosts_from_matchmaker(self, target):
        target_key = zmq_address.target_to_key(
            target, zmq_names.socket_type_str(self.listener_type))

        LOG.debug("Processing target %s for round-robin." % target_key)

        if target_key not in self.round_robin_targets:
            # 若target_key不在round_robin_targets字典中
            with self._lock:
                if target_key not in self.round_robin_targets:
                    LOG.debug("Target %s is not in cache. Check matchmaker "
                              "server." % target_key)
                    # 从matchmaker中（redis）读取target_key对应的值
                    hosts = self.matchmaker.get_hosts_retry(
                        target, zmq_names.socket_type_str(self.listener_type))
                    LOG.debug("Received hosts %s" % hosts)
                    # 更新路由表
                    self.routing_table.update_hosts(target_key, hosts)
                    # 将target_key记录在round_robin_targets字典中
                    self.round_robin_targets[target_key] = \
                        self.routing_table.get_hosts_round_robin(target_key)
        return target_key

若此前target_key在缓存中(在round_robin_targets字典中有记录)，则直接返回，若不在，则在matchmaker(例如为redis)中获取，然后更新路由表（RoutingTable）并将该target_key记录在round_robin_targets中

RoutingTable的作用将请求的target转换为服务端地址，缓存了target与服务端地址的映射关系

我们可以看到get_all_round_robin_hosts中调用self.routing_table.get_hosts_fanout(target_key)，最终就会调用RoutingTable的_get_hosts方法，获取服务端地址信息

class RoutingTable(object):

    def __init__(self, conf):
        self.conf = conf
        self.targets = {}
        self._lock = threading.Lock()

    def get_targets(self):
        with self._lock:
            return list(self.targets.keys())

    def update_hosts(self, target_key, hosts_updated):
        with self._lock:
            if target_key in self.targets and not hosts_updated:
                self.targets.pop(target_key)
                return
            hosts_current, _ = self.targets.get(target_key, (set(), None))
            hosts_updated = set(hosts_updated)
            has_differences = hosts_updated ^ hosts_current
            if has_differences:
                self.targets[target_key] = (hosts_updated, self._create_tm())

    def get_hosts_round_robin(self, target_key):
        while self.contains(target_key):
            for host in self._get_hosts_rr(target_key):
                yield host

    def get_hosts_fanout(self, target_key):
        hosts, _ = self._get_hosts(target_key)
        return hosts

    def contains(self, target_key):
        with self._lock:
            return target_key in self.targets

    def _get_hosts(self, target_key):
        with self._lock:
            hosts, tm = self.targets.get(target_key, ([], None))
            hosts = list(hosts)
            return hosts, tm

路由表的缓存也是会更新的，会通过RoutingTableUpdater进行周期性的更新，周期时期可以在配置中看到如下：

class RoutingTableUpdater(zmq_updater.UpdaterBase):

    def __init__(self, conf, matchmaker, routing_table):
        self.routing_table = routing_table
        super(RoutingTableUpdater, self).__init__(
            conf, matchmaker, self._update_routing_table,
            conf.oslo_messaging_zmq.zmq_target_update)

    def _update_routing_table(self):
        target_keys = self.routing_table.get_targets()

        try:
            for target_key in target_keys:
                hosts = self.matchmaker.get_hosts_by_key(target_key)
                self.routing_table.update_hosts(target_key, hosts)
            LOG.debug("Updating routing table from the matchmaker. "
                      "%d target(s) updated %s." % (len(target_keys),
                                                    target_keys))
        except zmq_matchmaker_base.MatchmakerUnavailable:
            LOG.warning(_LW("Not updated. Matchmaker was not available."))

周期时间配置，即zmq_target_update，默认180s

cfg.IntOpt('zmq_target_update', default=180,
           deprecated_group='DEFAULT',
           help='Update period in seconds of a name service record '
           'about existing target.'),

 

 

 

未完待续。。。。。。。