Mitaka版本Cinder RPC机制
1. c-volume启动RPCServer机制
- cinder-volume进程在启动时，会创建RPCServer,并启动该RPCServer(代码文件：cinder/service.py)
def start(self):
version_string = version.version_string()
LOG.info(_LI('Starting %(topic)s node (version %(version_string)s)'),
{'topic': self.topic, 'version_string': version_string})
self.model_disconnected = False
self.manager.init_host()
LOG.debug("Creating RPC server for service %s", self.topic)
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
endpoints.extend(self.manager.additional_endpoints)
serializer = objects_base.CinderObjectSerializer()
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
- 在构造RPCServer时，使用一个transport来抽象底层使用的具体消息队列(代码文件：cinder/service.py)
def init(conf):
global TRANSPORT, NOTIFIER
exmods = get_allowed_exmods()
TRANSPORT = messaging.get_transport(conf,
allowed_remote_exmods=exmods,
aliases=TRANSPORT_ALIASES)
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor='eventlet',
serializer=serializer)
-
transport里面有一个使用底层消息队列的driver变量，对应底层不同的消息队列实现。
底层消息队列driver默认配置为rabbit（代码文件：oslo_messaging/transport.py）
_transport_opts = [ cfg.StrOpt('transport_url', secret=True, help='A URL representing the messaging driver to use and its ' 'full configuration. If not set, we fall back to the ' 'rpc_backend option and driver specific configuration.'), cfg.StrOpt('rpc_backend', default='rabbit', help='The messaging driver to use, defaults to rabbit. Other ' 'drivers include amqp and zmq.'), cfg.StrOpt('control_exchange', default='openstack', help='The default exchange under which topics are scoped. May ' 'be overridden by an exchange name specified in the ' 'transport_url option.'), ] def get_transport(conf, url=None, allowed_remote_exmods=None, aliases=None): """A factory method for Transport objects. This method will construct a Transport object from transport configuration gleaned from the user's configuration and, optionally, a transport URL. If a transport URL is supplied as a parameter, any transport configuration contained in it takes precedence. If no transport URL is supplied, but there is a transport URL supplied in the user's configuration then that URL will take the place of the URL parameter. In both cases, any configuration not supplied in the transport URL may be taken from individual configuration parameters in the user's configuration. An example transport URL might be:: rabbit://me:passwd@host:5672/virtual_host and can either be passed as a string or a TransportURL object. :param conf: the user configuration :type conf: cfg.ConfigOpts :param url: a transport URL :type url: str or TransportURL :param allowed_remote_exmods: a list of modules which a client using this transport will deserialize remote exceptions from :type allowed_remote_exmods: list :param aliases: A map of transport alias to transport name :type aliases: dict """ allowed_remote_exmods = allowed_remote_exmods or [] conf.register_opts(_transport_opts) if not isinstance(url, TransportURL): url = url or conf.transport_url parsed = TransportURL.parse(conf, url, aliases) if not parsed.transport: raise InvalidTransportURL(url, 'No scheme specified in "%s"' % url) url = parsed kwargs = dict(default_exchange=conf.control_exchange, allowed_remote_exmods=allowed_remote_exmods) try: mgr = driver.DriverManager('oslo.messaging.drivers', url.transport.split('+')[0], invoke_on_load=True, invoke_args=[conf, url], invoke_kwds=kwargs) except RuntimeError as ex: raise DriverLoadFailure(url.transport, ex) return Transport(mgr.driver)通过读取在安装时配置的消息驱动别名，对应具体的消息驱动代码路径（代码文件：oslo_messaging目录同级的setup.cfg）
[entry_points] oslo.messaging.drivers = rabbit = oslo_messaging._drivers.impl_rabbit:RabbitDriver zmq = oslo_messaging._drivers.impl_zmq:ZmqDriver amqp = oslo_messaging._drivers.protocols.amqp.driver:ProtonDriver # This driver is supporting for only notification usage kafka = oslo_messaging._drivers.impl_kafka:KafkaDriver # To avoid confusion kombu = oslo_messaging._drivers.impl_rabbit:RabbitDriver # This is just for internal testing fake = oslo_messaging._drivers.impl_fake:FakeDriver pika = oslo_messaging._drivers.impl_pika:PikaDriver -
RPCServer在执行start函数时，会启动transport的监听(代码文件:oslo_messaging/server.py)
@ordered(reset_after='stop') def start(self, override_pool_size=None): """Start handling incoming messages. This method causes the server to begin polling the transport for incoming messages and passing them to the dispatcher. Message processing will continue until the stop() method is called. The executor controls how the server integrates with the applications I/O handling strategy - it may choose to poll for messages in a new process, thread or co-operatively scheduled coroutine or simply by registering a callback with an event loop. Similarly, the executor may choose to dispatch messages in a new thread, coroutine or simply the current thread. """ # Warn that restarting will be deprecated if self._started: LOG.warning(_LW('Restarting a MessageHandlingServer is inherently ' 'racy. It is deprecated, and will become a noop ' 'in a future release of oslo.messaging. If you ' 'need to restart MessageHandlingServer you should ' 'instantiate a new object.')) self._started = True try: self.listener = self.dispatcher._listen(self.transport) except driver_base.TransportDriverError as ex: raise ServerListenError(self.target, ex) -
transport监听，最终通过消息队列的driver来实现监听(代码文件:oslo_messaging/transport.py)
def _listen(self, target): if not (target.topic and target.server): raise exceptions.InvalidTarget('A server\'s target must have ' 'topic and server names specified', target) return self._driver.listen(target) -
消息队列driver在监听时，会创3个队列:2个topic队列和1个fanout队列(代码文件:oslo_messaging/_drivers/amqpdriver.py)
def listen(self, target): conn = self._get_connection(rpc_common.PURPOSE_LISTEN) listener = AMQPListener(self, conn) conn.declare_topic_consumer(exchange_name=self._get_exchange(target), topic=target.topic, callback=listener) conn.declare_topic_consumer(exchange_name=self._get_exchange(target), topic='%s.%s' % (target.topic, target.server), callback=listener) conn.declare_fanout_consumer(target.topic, listener) return listener -
2个topic队列："cinder-volume"队列和"cinder-volume.host"队列，host为cinder service-list命令回显的host，比如:cinder@SATA(代码文件:oslo_messaging/_drivers/impl_rabbit.py)
def declare_topic_consumer(self, exchange_name, topic, callback=None, queue_name=None): """Create a 'topic' consumer.""" consumer = Consumer(exchange_name=exchange_name, queue_name=queue_name or topic, routing_key=topic, type='topic', durable=self.amqp_durable_queues, exchange_auto_delete=self.amqp_auto_delete, queue_auto_delete=self.amqp_auto_delete, callback=callback, rabbit_ha_queues=self.rabbit_ha_queues) self.declare_consumer(consumer) -
fanout队列会专门用一个ID来标识，即不同cinder-volume进程，会有不同的fanout队列，不过队列的前缀名都是"cinder-volume_fanout",并都绑定到"cinder-volume_fanout"这个exchange交换机上。(代码文件:oslo_messaging/_drivers/impl_rabbit.py)
def declare_fanout_consumer(self, topic, callback): """Create a 'fanout' consumer.""" unique = uuid.uuid4().hex exchange_name = '%s_fanout' % topic queue_name = '%s_fanout_%s' % (topic, unique) consumer = Consumer(exchange_name=exchange_name, queue_name=queue_name, routing_key=topic, type='fanout', durable=False, exchange_auto_delete=True, queue_auto_delete=False, callback=callback, rabbit_ha_queues=self.rabbit_ha_queues, rabbit_queue_ttl=self.rabbit_transient_queues_ttl) self.declare_consumer(consumer)
2. c-volume的RPCClient发送消息机制
- 指定发送client的server参数（代码文件：cinder/volume/rpcapi.py）
def create_volume(self, ctxt, volume, host, request_spec,
filter_properties, allow_reschedule=True):
request_spec_p = jsonutils.to_primitive(request_spec)
msg_args = {'volume_id': volume.id, 'request_spec': request_spec_p,
'filter_properties': filter_properties,
'allow_reschedule': allow_reschedule}
if self.client.can_send_version('2.0'):
version = '2.0'
msg_args['volume'] = volume
elif self.client.can_send_version('1.32'):
version = '1.32'
msg_args['volume'] = volume
else:
version = '1.24'
cctxt = self._get_cctxt(host, version)
request_spec_p = jsonutils.to_primitive(request_spec)
cctxt.cast(ctxt, 'create_volume', **msg_args)
def _get_cctxt(self, host, version):
new_host = utils.get_volume_rpc_host(host)
return self.client.prepare(server=new_host, version=version)
- self.client（RPCClient实例）在初始化时，对应的target，只设置了topic，没有设置server（代码文件:cinder/rpc.py）
class RPCAPI(object):
def __init__(self):
target = messaging.Target(topic=self.TOPIC,
version=self.RPC_API_VERSION)
obj_version_cap = self._determine_obj_version_cap()
serializer = base.CinderObjectSerializer(obj_version_cap)
rpc_version_cap = self._determine_rpc_version_cap()
self.client = get_client(target, version_cap=rpc_version_cap,
serializer=serializer)
def get_client(target, version_cap=None, serializer=None):
assert TRANSPORT is not None
serializer = RequestContextSerializer(serializer)
return messaging.RPCClient(TRANSPORT,
target,
version_cap=version_cap,
serializer=serializer)
- RPCClient的prepare函数，会返回_CallContext对象(代码文件：oslo_messaging/rpc/client.py)
class RPCClient(object):
def prepare(self, exchange=_marker, topic=_marker, namespace=_marker,
version=_marker, server=_marker, fanout=_marker,
timeout=_marker, version_cap=_marker, retry=_marker):
"""Prepare a method invocation context.
Use this method to override client properties for an individual method
invocation. For example::
def test(self, ctxt, arg):
cctxt = self.prepare(version='2.5')
return cctxt.call(ctxt, 'test', arg=arg)
:param exchange: see Target.exchange
:type exchange: str
:param topic: see Target.topic
:type topic: str
:param namespace: see Target.namespace
:type namespace: str
:param version: requirement the server must support, see Target.version
:type version: str
:param server: send to a specific server, see Target.server
:type server: str
:param fanout: send to all servers on topic, see Target.fanout
:type fanout: bool
:param timeout: an optional default timeout (in seconds) for call()s
:type timeout: int or float
:param version_cap: raise a RPCVersionCapError version exceeds this cap
:type version_cap: str
:param retry: an optional connection retries configuration
None or -1 means to retry forever
0 means no retry
N means N retries
:type retry: int
"""
return _CallContext._prepare(self,
exchange, topic, namespace,
version, server, fanout,
timeout, version_cap, retry)
class _CallContext(object):
@classmethod
def _prepare(cls, base,
exchange=_marker, topic=_marker, namespace=_marker,
version=_marker, server=_marker, fanout=_marker,
timeout=_marker, version_cap=_marker, retry=_marker):
"""Prepare a method invocation context. See RPCClient.prepare()."""
if version is not None and version is not cls._marker:
# quick sanity check to make sure parsable version numbers are used
try:
utils.version_is_compatible(version, version)
except (IndexError, ValueError):
raise exceptions.MessagingException(
"Version must contain a major and minor integer. Got %s"
% version)
kwargs = dict(
exchange=exchange,
topic=topic,
namespace=namespace,
version=version,
server=server,
fanout=fanout)
kwargs = dict([(k, v) for k, v in kwargs.items()
if v is not cls._marker])
target = base.target(**kwargs)
if timeout is cls._marker:
timeout = base.timeout
if retry is cls._marker:
retry = base.retry
if version_cap is cls._marker:
version_cap = base.version_cap
return _CallContext(base.transport, target,
base.serializer,
timeout, version_cap, retry)
- RPCClient的prepare函数返回的 _CallContext对象，会根据prepare的fanout、server等参数来重构Target对象(代码文件：oslo_messaging/target.py)
class Target(object):
def __call__(self, **kwargs):
for a in ('exchange', 'topic', 'namespace',
'version', 'server', 'fanout'):
kwargs.setdefault(a, getattr(self, a))
return Target(**kwargs)
- 调用_CallContext的cast/call发送方法，最终调用了transport的_send方法，call方法会设置wait_for_reply为True，用于获取响应结果(代码文件：oslo_messaging/rpc/client.py)
class _CallContext(object):
def cast(self, ctxt, method, **kwargs):
"""Invoke a method and return immediately. See RPCClient.cast()."""
msg = self._make_message(ctxt, method, kwargs)
ctxt = self.serializer.serialize_context(ctxt)
if self.version_cap:
self._check_version_cap(msg.get('version'))
try:
self.transport._send(self.target, ctxt, msg, retry=self.retry)
except driver_base.TransportDriverError as ex:
raise ClientSendError(self.target, ex)
def call(self, ctxt, method, **kwargs):
"""Invoke a method and wait for a reply. See RPCClient.call()."""
if self.target.fanout:
raise exceptions.InvalidTarget('A call cannot be used with fanout',
self.target)
msg = self._make_message(ctxt, method, kwargs)
msg_ctxt = self.serializer.serialize_context(ctxt)
timeout = self.timeout
if self.timeout is None:
timeout = self.conf.rpc_response_timeout
if self.version_cap:
self._check_version_cap(msg.get('version'))
try:
result = self.transport._send(self.target, msg_ctxt, msg,
wait_for_reply=True, timeout=timeout,
retry=self.retry)
except driver_base.TransportDriverError as ex:
raise ClientSendError(self.target, ex)
return self.serializer.deserialize_entity(ctxt, result)
- 最终调用transport里面的消息驱动的_send方法，并根据target时fanout还是topic，来发送不同类型的消息。（代码文件:oslo_messging/_drivers/amqpdriver.py）
def _send(self, target, ctxt, message,
wait_for_reply=None, timeout=None,
envelope=True, notify=False, retry=None):
# FIXME(markmc): remove this temporary hack
class Context(object):
def __init__(self, d):
self.d = d
def to_dict(self):
return self.d
context = Context(ctxt)
msg = message
if wait_for_reply:
msg_id = uuid.uuid4().hex
msg.update({'_msg_id': msg_id})
msg.update({'_reply_q': self._get_reply_q()})
rpc_amqp._add_unique_id(msg)
unique_id = msg[rpc_amqp.UNIQUE_ID]
rpc_amqp.pack_context(msg, context)
if envelope:
msg = rpc_common.serialize_msg(msg)
if wait_for_reply:
self._waiter.listen(msg_id)
log_msg = "CALL msg_id: %s " % msg_id
else:
log_msg = "CAST unique_id: %s " % unique_id
try:
with self._get_connection(rpc_common.PURPOSE_SEND) as conn:
if notify:
exchange = self._get_exchange(target)
log_msg += "NOTIFY exchange '%(exchange)s'" \
" topic '%(topic)s'" % {
'exchange': exchange,
'topic': target.topic}
LOG.debug(log_msg)
conn.notify_send(exchange, target.topic, msg, retry=retry)
elif target.fanout:
log_msg += "FANOUT topic '%(topic)s'" % {
'topic': target.topic}
LOG.debug(log_msg)
conn.fanout_send(target.topic, msg, retry=retry)
else:
topic = target.topic
exchange = self._get_exchange(target)
if target.server:
topic = '%s.%s' % (target.topic, target.server)
log_msg += "exchange '%(exchange)s'" \
" topic '%(topic)s'" % {
'exchange': exchange,
'topic': target.topic}
LOG.debug(log_msg)
conn.topic_send(exchange_name=exchange, topic=topic,
msg=msg, timeout=timeout, retry=retry)
if wait_for_reply:
result = self._waiter.wait(msg_id, timeout)
if isinstance(result, Exception):
raise result
return result
finally:
if wait_for_reply:
self._waiter.unlisten(msg_id)
3. oslo_messaging参考
https://blog.csdn.net/jxxiaohou/article/details/78386879
145
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
