Cinder volume服务启动以及创建挂载过程代码走读
1. 服务启动 (以cinder-volume为例)
服务进程启动如图
调用cinder/cmd/volume.py 的main方法启动服务
def main():
objects.register_all() //导入cinder 的相关orm object
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup(CONF, "cinder")
python_logging.captureWarnings(True)
priv_context.init(root_helper=shlex.split(utils.get_root_helper()))
utils.monkey_patch() //打猴子补丁
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF) //捕获异常日志相关
global LOG
LOG = logging.getLogger(__name__) //定义日志
if not CONF.enabled_backends:
LOG.error('Configuration for cinder-volume does not specify '
'"enabled_backends". Using DEFAULT section to configure '
'drivers is not supported since Ocata.')
sys.exit(1)
if os.name == 'nt':
# We cannot use oslo.service to spawn multiple services on Windows.
# It relies on forking, which is not available on Windows.
# Furthermore, service objects are unmarshallable objects that are
# passed to subprocesses.
_launch_services_win32()
else:
_launch_services_posix() //启动Linux服务
实例化launcher，Linux下实例化 ProcessLauncher
def _launch_services_posix():
launcher = service.get_launcher() //调用oslo_service
for backend in filter(None, CONF.enabled_backends):
_launch_service(launcher, backend)
_ensure_service_started()
launcher.wait()
class ProcessLauncher(object):
"""Launch a service with a given number of workers."""
................
调用_launch_service, 实例化server，创建cinder-volume的backend的service记录，并初始化sqlalchemy 数据库连接池，启动服务
def _launch_service(launcher, backend):
CONF.register_opt(host_opt, group=backend)
backend_host = getattr(CONF, backend).backend_host
host = "%s@%s" % (backend_host or CONF.host, backend)
# We also want to set cluster to None on empty strings, and we
# ignore leading and trailing spaces.
cluster = CONF.cluster and CONF.cluster.strip()
cluster = (cluster or None) and '%s@%s' % (cluster, backend)
try:
server = service.Service.create(host=host,
service_name=backend,
binary=constants.VOLUME_BINARY,
coordination=True,
cluster=cluster)
except Exception:
LOG.exception('Volume service %s failed to start.', host)
else:
# Dispose of the whole DB connection pool here before
# starting another process. Otherwise we run into cases where
# child processes share DB connections which results in errors.
session.dispose_engine()
launcher.launch_service(server)
_notify_service_started()
接着调用 ProcessLauncher实例的 launch_service 函数，传入cinder-volume的service，fork出子进程，默认workers=1,即每个backend service默认只启动一个worker干活
def launch_service(self, service, workers=1):
"""Launch a service with a given number of workers.
:param service: a service to launch, must be an instance of
:class:`oslo_service.service.ServiceBase`
:param workers: a number of processes in which a service
will be running
"""
_check_service_base(service)
wrap = ServiceWrapper(service, workers)
# Hide existing objects from the garbage collector, so that most
# existing pages will remain in shared memory rather than being
# duplicated between subprocesses in the GC mark-and-sweep. (Requires
# Python 3.7 or later.)
if hasattr(gc, 'freeze'):
gc.freeze()
LOG.info('Starting %d workers', wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
通过os.fork 出子进程，让子进程初始化eventlet协程池，子进程调用add方法，执行run_service, 最终执行cinder-volume的service.start()
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info('Forking too fast, sleeping')
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
self.launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(
self.launcher)
if not _is_sighup_and_daemon(signo):
self.launcher.wait()
break
self.launcher.restart()
os._exit(status)
LOG.debug('Started child %d', pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
支持cinder-volume 服务启动完成！
2. 系统盘Volume创建
-
nova client 发起http 请求创建云盘， 把镜像信息，az，volume type等发送给了cinder-api，
body: {“volume”: {“backup_id”: null, “description”: “”, “multiattach”: false, “source_volid”: null, "consistencygroup_id
": null, “snapshot_id”: null, “size”: 50, “name”: “”, “imageRef”: “14c674f6-3b05-4087-bbf6-2ba68112a0fd”, “availability_zone”: “nova”, “volume_type”: null, “metadata”: {}}}
volumeController 接受请求，校验镜像uuid，并调用volume_api创建volume
cinder.api.v3.volumes.VolumeController
def create(self, req, body):
..................
new_volume = self.volume_api.create(context,
size,
volume.get('display_name'),
volume.get('display_description'),
**kwargs)
..................................
volume_api中的create函数中校验size大小以及volume type等是否符合规范，拼装请求信息create_what 信息，然后执行api 的task_flow
cinder.volume.api.API
def create(self, context, size, name, description, snapshot=None,
image_id=None, volume_type=None, metadata=None,
availability_zone=None, source_volume=None,
scheduler_hints=None,
source_replica=None, consistencygroup=None,
cgsnapshot=None, multiattach=False, source_cg=None,
group=None, group_snapshot=None, source_group=None,
backup=None):
...........................................
try:
sched_rpcapi = (self.scheduler_rpcapi if (
not cgsnapshot and not source_cg and
not group_snapshot and not source_group)
else None)
volume_rpcapi = (self.volume_rpcapi if (
not cgsnapshot and not source_cg and
not group_snapshot and not source_group)
else None)
flow_engine = create_volume.get_flow(self.db,
self.image_service,
availability_zones,
create_what,
sched_rpcapi,
volume_rpcapi)
except Exception:
msg = _('Failed to create api volume flow.')
LOG.exception(msg)
raise exception.CinderException(msg)
with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
try:
flow_engine.run()
vref = flow_engine.storage.fetch('volume')
# NOTE(tommylikehu): If the target az is not hit,
# refresh the az cache immediately.
if flow_engine.storage.fetch('refresh_az'):
self.list_availability_zones(enable_cache=True,
refresh_cache=True)
# Refresh the object here, otherwise things ain't right
vref = objects.Volume.get_by_id(
context, vref['id'])
vref.save()
LOG.info("Create volume request issued successfully.",
resource=vref)
return vref
except exception.InvalidAvailabilityZone:
with excutils.save_and_reraise_exception():
self.list_availability_zones(enable_cache=True,
refresh_cache=True)
api的task_flow中，主要做提取校验input参数，创建数据库volume记录，提交quota，cast 请求给cinder-scheduler或者cinder-volume
cinder.volume.flows.api.create_volume
def get_flow(db_api, image_service_api, availability_zones, create_what,
scheduler_rpcapi=None, volume_rpcapi=None):
"""Constructs and returns the api entrypoint flow.
This flow will do the following:
1. Inject keys & values for dependent tasks.
2. Extracts and validates the input keys & values.
3. Reserves the quota (reverts quota on any failures).
4. Creates the database entry.
5. Commits the quota.
6. Casts to volume manager or scheduler for further processing.
"""
flow_name = ACTION.replace(":", "_") + "_api"
api_flow = linear_flow.Flow(flow_name)
api_flow.add(ExtractVolumeRequestTask(
image_service_api,
availability_zones,
rebind={'size': 'raw_size',
'availability_zone': 'raw_availability_zone',
'volume_type': 'raw_volume_type',
'multiattach': 'raw_multiattach'}))
api_flow.add(QuotaReserveTask(),
EntryCreateTask(),
QuotaCommitTask())
if scheduler_rpcapi and volume_rpcapi:
# This will cast it out to either the scheduler or volume manager via
# the rpc apis provided.
api_flow.add(VolumeCastTask(scheduler_rpcapi, volume_rpcapi, db_api))
# Now load (but do not run) the flow using the provided initial data.
return taskflow.engines.load(api_flow, store=create_what)
VolumeCastTask 最终调用scheduler的rpcapi 执行create_volume，发送异步请求cast , http请求完成
cinder.volume.flows.api.create_volume.VolumeCastTask
class VolumeCastTask(flow_utils.CinderTask):
......................
def _cast_create_volume(self, context, request_spec, filter_properties):
......................................
self.scheduler_rpcapi.create_volume(
context,
volume,
snapshot_id=snapshot_id,
image_id=image_id,
request_spec=request_spec,
filter_properties=filter_properties,
backup_id=backup_id)
def execute(self, context, **kwargs):
.................................
self._cast_create_volume(context, request_spec, filter_properties)
将volume相关信息作为参数 ，将msg 序列化发送cast 请求，msg指定router key 是 cinder-scheduler
cinder.scheduler.rpcapi.SchedulerAPI
def create_volume(self, ctxt, volume, snapshot_id=None, image_id=None,
request_spec=None, filter_properties=None,
backup_id=None):
volume.create_worker()
cctxt = self._get_cctxt()
msg_args = {'snapshot_id': snapshot_id, 'image_id': image_id,
'request_spec': request_spec,
'filter_properties': filter_properties,
'volume': volume, 'backup_id': backup_id}
if not self.client.can_send_version('3.10'):
msg_args.pop('backup_id')
return cctxt.cast(ctxt, 'create_volume', **msg_args)
某个cinder-scheduler oslo_message收到消息后，将消息传给了SchedulerManager.create_volume
cinder.scheduler.manager.SchedulerManager
def create_volume(self, context, volume, snapshot_id=None, image_id=None,
request_spec=None, filter_properties=None,
backup_id=None):
self._wait_for_scheduler()
try:
flow_engine = create_volume.get_flow(context,
self.driver,
request_spec,
filter_properties,
volume,
snapshot_id,
image_id,
backup_id)
except Exception:
msg = _("Failed to create scheduler manager volume flow")
LOG.exception(msg)
raise exception.CinderException(msg)
with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
flow_engine.run()
SchedulerManager.create_volume 执行自己的task_flow, 提取传入的参数并根据相关filter调度volume创建
def get_flow(context, driver_api, request_spec=None,
filter_properties=None,
volume=None, snapshot_id=None, image_id=None, backup_id=None):
"""Constructs and returns the scheduler entrypoint flow.
This flow will do the following:
1. Inject keys & values for dependent tasks.
2. Extract a scheduler specification from the provided inputs.
3. Use provided scheduler driver to select host and pass volume creation
request further.
"""
create_what = {
'context': context,
'raw_request_spec': request_spec,
'filter_properties': filter_properties,
'volume': volume,
'snapshot_id': snapshot_id,
'image_id': image_id,
'backup_id': backup_id,
}
flow_name = ACTION.replace(":", "_") + "_scheduler"
scheduler_flow = linear_flow.Flow(flow_name)
# This will extract and clean the spec from the starting values.
scheduler_flow.add(ExtractSchedulerSpecTask(
rebind={'request_spec': 'raw_request_spec'}))
# This will activate the desired scheduler driver (and handle any
# driver related failures appropriately).
scheduler_flow.add(ScheduleCreateVolumeTask(driver_api))
# Now load (but do not run) the flow using the provided initial data.
return taskflow.engines.load(scheduler_flow, store=create_what)
scheduler 调度创建volume时，先获取glance镜像id以及location等相关信息，再调用driver filter调度创建
cinder.scheduler.flows.create_volume.ScheduleCreateVolumeTask
def execute(self, context, request_spec, filter_properties, volume):
try:
if CONF.enable_multi_ceph:
get_remote_image_service = glance.get_remote_image_service
image_href = request_spec.get('image_id')
if image_href:
image_service, image_id = get_remote_image_service(context,
image_href)
image_location = image_service.get_location(context,
image_id)
filter_properties['image_location'] = image_location
self.driver_api.schedule_create_volume(context, request_spec,
filter_properties)
.......................................
筛选根据az，容量，特性等filter筛选合适的backend，并更新volume backend信息，最后调用cinde-volume的rpc_api create_volume
def schedule_create_volume(self, context, request_spec, filter_properties):
backend = self._schedule(context, request_spec, filter_properties)
if not backend:
raise exception.NoValidBackend(reason=_("No weighed backends "
"available"))
backend = backend.obj
volume_id = request_spec['volume_id']
updated_volume = driver.volume_update_db(
context, volume_id,
backend.host,
backend.cluster_name,
availability_zone=backend.service['availability_zone'])
self._post_select_populate_filter_properties(filter_properties,
backend)
# context is not serializable
filter_properties.pop('context', None)
self.volume_rpcapi.create_volume(context, updated_volume, request_spec,
filter_properties,
allow_reschedule=True)
msg指定router key 是 cinder-volume将请求cast 发出去
def create_volume(self, ctxt, volume, request_spec, filter_properties,
allow_reschedule=True):
cctxt = self._get_cctxt(volume.service_topic_queue)
cctxt.cast(ctxt, 'create_volume',
request_spec=request_spec,
filter_properties=filter_properties,
allow_reschedule=allow_reschedule,
volume=volume)
某个cinder-volume收到请求后，执行VolumeManager.create_volume，更新已分配容量信息，并执行task_flows
cinder.volume.manager.VolumeManager
@objects.Volume.set_workers
def create_volume(self, context, volume, request_spec=None,
filter_properties=None, allow_reschedule=True):
.............................
try:
# NOTE(flaper87): Driver initialization is
# verified by the task itself.
flow_engine = create_volume.get_flow(
context_elevated,
self,
self.db,
self.driver,
self.scheduler_rpcapi,
self.host,
volume,
allow_reschedule,
context,
request_spec,
filter_properties,
image_volume_cache=self.image_volume_cache,
)
except Exception:
msg = _("Create manager volume flow failed.")
LOG.exception(msg, resource={'type': 'volume', 'id': volume.id})
raise exception.CinderException(msg)
cinder-volume的task_flows 中提取volume的spec相关信息，通知volume创建，开始创建volume云盘，因为基于镜像创建系统盘，所以执行_create_from_image函数，调用rbd driver clone_image，同时更新volume_glance_metadata表
def execute(self, context, volume, volume_spec):
.......................................
elif create_type == 'image':
model_update = self._create_from_image(context,
volume,
**volume_spec)
............................................
def _create_from_image(self, context, volume,
image_location, image_id, image_meta,
image_service, **kwargs):
................................
if not volume_is_encrypted:
model_update, cloned = self.driver.clone_image(context,
volume,
image_location,
image_meta,
image_service)
self._handle_bootable_volume_glance_meta(context, volume,
image_id=image_id,
image_meta=image_meta)
return model_update
backend后端是ceph，所以调用了rbd clone_image, 根据image_location, 执行clone操作， 并resize 大小
cinder.volume.drivers.rbd.RBDVolumeProxy
def clone_image(self, context, volume,
image_location, image_meta,
image_service):
if image_location:
# Note: image_location[0] is glance image direct_url.
# image_location[1] contains the list of all locations (including
# direct_url) or None if show_multiple_locations is False in
# glance configuration.
if image_location[1]:
url_locations = [location['url'] for
location in image_location[1]]
else:
url_locations = [image_location[0]]
# iterate all locations to look for a cloneable one.
for url_location in url_locations:
if url_location and self._is_cloneable(
url_location, image_meta):
for location in image_meta['locations']:
if location['url'] == url_location and \
location.get('metadata', None):
image_meta['properties']['stores'] = \
location['metadata']['backend']
_prefix, pool, image, snapshot = \
self._parse_location(url_location)
volume_update = self._clone(volume, pool, image, snapshot)
volume_update['provider_location'] = None
self._resize(volume)
return volume_update, True
return ({}, False)
最终执行CreateVolumeOnFinishTask ，将volume状态变更为available，创建volume至此完成！
3. Volume挂载
-
nova 发起attach请求，调用cinderclient发起http请求，需要挂载系统盘 POST 请求 http://10.x.x.x:8776/v3/a6fea56ad90948af9339eecf83d6c9b0/volumes/c9dc6bdb-4f66-4aa6-8aef-bfb44eb13051/action
Action body: {"os-attach": {"instance_uuid": "32d86262-c223-482c-8f9a-94660fda437e", "mountpoint": "/dev/sda", "mode": "rw"}}- cinder api收到请求, 调用volume_api 执行attach操作，将attach模式写入volume_admin_metadata，再调用volume_rpcapi发起attach_volume请求
@wsgi.response(http\_client.ACCEPTED)
@wsgi.action('os-attach')
@validation.schema(volume\_action.attach)
def \_attach(self, req, id, body):
.................................
try:
self.volume\_api.attach(context, volume,
instance\_uuid, host\_name, mountpoint, mode)
except messaging.RemoteError as error:
if error.exc\_type in \['InvalidVolume', 'InvalidUUID',
'InvalidVolumeAttachMode']:
msg = \_("Error attaching volume - %(err\_type)s: "
"%(err\_msg)s") % {
'err\_type': error.exc\_type, 'err\_msg': error.value}
raise webob.exc.HTTPBadRequest(explanation=msg)
else:
\# There are also few cases where attach call could fail due to
\# db or volume driver errors. These errors shouldn't be exposed
\# to the user and in such cases it should raise 500 error.
raise
cinder-api 发起call请求，请求会同步阻塞，直至等待cinder-volume完成
def attach_volume(self, ctxt, volume, instance_uuid, host_name,
mountpoint, mode):
msg_args = {'volume_id': volume.id,
'instance_uuid': instance_uuid,
'host_name': host_name,
'mountpoint': mountpoint,
'mode': mode,
'volume': volume}
cctxt = self._get_cctxt(volume.service_topic_queue, ('3.3', '3.0'))
if not cctxt.can_send_version('3.3'):
msg_args.pop('volume')
return cctxt.call(ctxt, 'attach_volume', **msg_args)
cinder-volume attach volume时, 这里因为rbd 不需要做任何操作，所以self.driver.attach_volume不做任何操作，只操作了数据库，创建attachment信息，更新volume为in-use，同时更新volume-attachment表信息
cinder.volume.manager.VolumeManager
@coordination.synchronized('{volume_id}')
def attach_volume(self, context, volume_id, instance_uuid, host_name,
mountpoint, mode, volume=None):
.....................................................
attachment = volume.begin_attach(mode)
try:
if volume_metadata.get('readonly') == 'True' and mode != 'ro':
raise exception.InvalidVolumeAttachMode(mode=mode,
volume_id=volume.id)
# NOTE(flaper87): Verify the driver is enabled
# before going forward. The exception will be caught
# and the volume status updated.
utils.require_driver_initialized(self.driver)
LOG.info('Attaching volume %(volume_id)s to instance '
'%(instance)s at mountpoint %(mount)s on host '
'%(host)s.',
{'volume_id': volume_id, 'instance': instance_uuid,
'mount': mountpoint, 'host': host_name_sanitized},
resource=volume)
self.driver.attach_volume(context,
volume,
instance_uuid,
host_name_sanitized,
mountpoint)
except Exception as excep:
with excutils.save_and_reraise_exception():
self.message_api.create(
context,
message_field.Action.ATTACH_VOLUME,
resource_uuid=volume_id,
exception=excep)
attachment.attach_status = (
fields.VolumeAttachStatus.ERROR_ATTACHING)
attachment.save()
volume = attachment.finish_attach(
instance_uuid,
host_name_sanitized,
mountpoint,
mode)
self._notify_about_volume_usage(context, volume, "attach.end")
LOG.info("Attach volume completed successfully.",
至此，volume attach 操作完成！
扫一扫
985
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
