Cinder volume服务启动以及创建挂载过程代码走读

Cinder volume服务启动以及创建挂载过程代码走读

1. 服务启动 (以cinder-volume为例)

服务进程启动如图

调用cinder/cmd/volume.py 的main方法启动服务

def main():
    objects.register_all()  //导入cinder 的相关orm object
    gmr_opts.set_defaults(CONF) 
    CONF(sys.argv[1:], project='cinder',
         version=version.version_string())
    logging.setup(CONF, "cinder")
    python_logging.captureWarnings(True)
    priv_context.init(root_helper=shlex.split(utils.get_root_helper()))
    utils.monkey_patch() //打猴子补丁
    gmr.TextGuruMeditation.setup_autorun(version, conf=CONF) //捕获异常日志相关
    global LOG
    LOG = logging.getLogger(__name__) //定义日志

    if not CONF.enabled_backends:
        LOG.error('Configuration for cinder-volume does not specify '
                  '"enabled_backends". Using DEFAULT section to configure '
                  'drivers is not supported since Ocata.')
        sys.exit(1)

    if os.name == 'nt':
        # We cannot use oslo.service to spawn multiple services on Windows.
        # It relies on forking, which is not available on Windows.
        # Furthermore, service objects are unmarshallable objects that are
        # passed to subprocesses.
        _launch_services_win32()
    else:
        _launch_services_posix() //启动Linux服务

实例化launcher，Linux下实例化 ProcessLauncher

def _launch_services_posix():
    launcher = service.get_launcher() //调用oslo_service

    for backend in filter(None, CONF.enabled_backends):
        _launch_service(launcher, backend)

    _ensure_service_started()

    launcher.wait()



class ProcessLauncher(object):
    """Launch a service with a given number of workers."""
    ................

调用_launch_service, 实例化server，创建cinder-volume的backend的service记录，并初始化sqlalchemy 数据库连接池，启动服务

def _launch_service(launcher, backend):
    CONF.register_opt(host_opt, group=backend)
    backend_host = getattr(CONF, backend).backend_host
    host = "%s@%s" % (backend_host or CONF.host, backend)
    # We also want to set cluster to None on empty strings, and we
    # ignore leading and trailing spaces.
    cluster = CONF.cluster and CONF.cluster.strip()
    cluster = (cluster or None) and '%s@%s' % (cluster, backend)
    try:
        server = service.Service.create(host=host,
                                        service_name=backend,
                                        binary=constants.VOLUME_BINARY,
                                        coordination=True,
                                        cluster=cluster)
    except Exception:
        LOG.exception('Volume service %s failed to start.', host)
    else:
        # Dispose of the whole DB connection pool here before
        # starting another process.  Otherwise we run into cases where
        # child processes share DB connections which results in errors.
        session.dispose_engine()
        launcher.launch_service(server)
        _notify_service_started()

接着调用 ProcessLauncher实例的 launch_service 函数，传入cinder-volume的service，fork出子进程，默认workers=1,即每个backend service默认只启动一个worker干活

 def launch_service(self, service, workers=1):
        """Launch a service with a given number of workers.

       :param service: a service to launch, must be an instance of
              :class:`oslo_service.service.ServiceBase`
       :param workers: a number of processes in which a service
              will be running
        """
        _check_service_base(service)
        wrap = ServiceWrapper(service, workers)

        # Hide existing objects from the garbage collector, so that most
        # existing pages will remain in shared memory rather than being
        # duplicated between subprocesses in the GC mark-and-sweep. (Requires
        # Python 3.7 or later.)
        if hasattr(gc, 'freeze'):
            gc.freeze()

        LOG.info('Starting %d workers', wrap.workers)
        while self.running and len(wrap.children) < wrap.workers:
            self._start_child(wrap)

通过os.fork 出子进程，让子进程初始化eventlet协程池，子进程调用add方法，执行run_service, 最终执行cinder-volume的service.start()

 def _start_child(self, wrap):
        if len(wrap.forktimes) > wrap.workers:
            # Limit ourselves to one process a second (over the period of
            # number of workers * 1 second). This will allow workers to
            # start up quickly but ensure we don't fork off children that
            # die instantly too quickly.
            if time.time() - wrap.forktimes[0] < wrap.workers:
                LOG.info('Forking too fast, sleeping')
                time.sleep(1)

            wrap.forktimes.pop(0)

        wrap.forktimes.append(time.time())

        pid = os.fork()
        if pid == 0:
            self.launcher = self._child_process(wrap.service)
            while True:
                self._child_process_handle_signal()
                status, signo = self._child_wait_for_exit_or_signal(
                    self.launcher)
                if not _is_sighup_and_daemon(signo):
                    self.launcher.wait()
                    break
                self.launcher.restart()

            os._exit(status)

        LOG.debug('Started child %d', pid)

        wrap.children.add(pid)
        self.children[pid] = wrap

        return pid

支持cinder-volume 服务启动完成！

2. 系统盘Volume创建

1. nova client 发起http 请求创建云盘， 把镜像信息，az，volume type等发送给了cinder-api，

   body: {“volume”: {“backup_id”: null, “description”: “”, “multiattach”: false, “source_volid”: null, "consistencygroup_id
   ": null, “snapshot_id”: null, “size”: 50, “name”: “”, “imageRef”: “14c674f6-3b05-4087-bbf6-2ba68112a0fd”, “availability_zone”: “nova”, “volume_type”: null, “metadata”: {}}}

volumeController 接受请求，校验镜像uuid，并调用volume_api创建volume

cinder.api.v3.volumes.VolumeController

def create(self, req, body):
    ..................
    new_volume = self.volume_api.create(context,
                                            size,
                                            volume.get('display_name'),
                                            volume.get('display_description'),
                                            **kwargs)
    ..................................

volume_api中的create函数中校验size大小以及volume type等是否符合规范，拼装请求信息create_what 信息，然后执行api 的task_flow

cinder.volume.api.API 

def create(self, context, size, name, description, snapshot=None,
               image_id=None, volume_type=None, metadata=None,
               availability_zone=None, source_volume=None,
               scheduler_hints=None,
               source_replica=None, consistencygroup=None,
               cgsnapshot=None, multiattach=False, source_cg=None,
               group=None, group_snapshot=None, source_group=None,
               backup=None):
            
        ...........................................
        try:
            sched_rpcapi = (self.scheduler_rpcapi if (
                            not cgsnapshot and not source_cg and
                            not group_snapshot and not source_group)
                            else None)
            volume_rpcapi = (self.volume_rpcapi if (
                             not cgsnapshot and not source_cg and
                             not group_snapshot and not source_group)
                             else None)
            flow_engine = create_volume.get_flow(self.db,
                                                 self.image_service,
                                                 availability_zones,
                                                 create_what,
                                                 sched_rpcapi,
                                                 volume_rpcapi)
        except Exception:
            msg = _('Failed to create api volume flow.')
            LOG.exception(msg)
            raise exception.CinderException(msg)
         with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
            try:
                flow_engine.run()
                vref = flow_engine.storage.fetch('volume')
                # NOTE(tommylikehu): If the target az is not hit,
                # refresh the az cache immediately.
                if flow_engine.storage.fetch('refresh_az'):
                    self.list_availability_zones(enable_cache=True,
                                                 refresh_cache=True)
                # Refresh the object here, otherwise things ain't right
                vref = objects.Volume.get_by_id(
                    context, vref['id'])
                vref.save()
                LOG.info("Create volume request issued successfully.",
                         resource=vref)
                return vref
            except exception.InvalidAvailabilityZone:
                with excutils.save_and_reraise_exception():
                    self.list_availability_zones(enable_cache=True,
                                                 refresh_cache=True)

api的task_flow中，主要做提取校验input参数，创建数据库volume记录，提交quota，cast 请求给cinder-scheduler或者cinder-volume

cinder.volume.flows.api.create_volume
def get_flow(db_api, image_service_api, availability_zones, create_what,
             scheduler_rpcapi=None, volume_rpcapi=None):
    """Constructs and returns the api entrypoint flow.

    This flow will do the following:

    1. Inject keys & values for dependent tasks.
    2. Extracts and validates the input keys & values.
    3. Reserves the quota (reverts quota on any failures).
    4. Creates the database entry.
    5. Commits the quota.
    6. Casts to volume manager or scheduler for further processing.
    """

    flow_name = ACTION.replace(":", "_") + "_api"
    api_flow = linear_flow.Flow(flow_name)

    api_flow.add(ExtractVolumeRequestTask(
        image_service_api,
        availability_zones,
        rebind={'size': 'raw_size',
                'availability_zone': 'raw_availability_zone',
                'volume_type': 'raw_volume_type',
                'multiattach': 'raw_multiattach'}))
    api_flow.add(QuotaReserveTask(),
                 EntryCreateTask(),
                 QuotaCommitTask())

    if scheduler_rpcapi and volume_rpcapi:
        # This will cast it out to either the scheduler or volume manager via
        # the rpc apis provided.
        api_flow.add(VolumeCastTask(scheduler_rpcapi, volume_rpcapi, db_api))

    # Now load (but do not run) the flow using the provided initial data.
    return taskflow.engines.load(api_flow, store=create_what)

VolumeCastTask 最终调用scheduler的rpcapi 执行create_volume，发送异步请求cast , http请求完成

cinder.volume.flows.api.create_volume.VolumeCastTask
class VolumeCastTask(flow_utils.CinderTask):
    ......................
    def _cast_create_volume(self, context, request_spec, filter_properties):
        ......................................
        self.scheduler_rpcapi.create_volume(
            context,
            volume,
            snapshot_id=snapshot_id,
            image_id=image_id,
            request_spec=request_spec,
            filter_properties=filter_properties,
            backup_id=backup_id)
    def execute(self, context, **kwargs):
        .................................
        self._cast_create_volume(context, request_spec, filter_properties)

将volume相关信息作为参数 ，将msg 序列化发送cast 请求，msg指定router key 是 cinder-scheduler

cinder.scheduler.rpcapi.SchedulerAPI
    def create_volume(self, ctxt, volume, snapshot_id=None, image_id=None,
                      request_spec=None, filter_properties=None,
                      backup_id=None):
        volume.create_worker()
        cctxt = self._get_cctxt()
        msg_args = {'snapshot_id': snapshot_id, 'image_id': image_id,
                    'request_spec': request_spec,
                    'filter_properties': filter_properties,
                    'volume': volume, 'backup_id': backup_id}
        if not self.client.can_send_version('3.10'):
            msg_args.pop('backup_id')
        return cctxt.cast(ctxt, 'create_volume', **msg_args)

某个cinder-scheduler oslo_message收到消息后，将消息传给了SchedulerManager.create_volume

cinder.scheduler.manager.SchedulerManager
    def create_volume(self, context, volume, snapshot_id=None, image_id=None,
                      request_spec=None, filter_properties=None,
                      backup_id=None):
        self._wait_for_scheduler()

        try:
            flow_engine = create_volume.get_flow(context,
                                                 self.driver,
                                                 request_spec,
                                                 filter_properties,
                                                 volume,
                                                 snapshot_id,
                                                 image_id,
                                                 backup_id)
        except Exception:
            msg = _("Failed to create scheduler manager volume flow")
            LOG.exception(msg)
            raise exception.CinderException(msg)

        with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
            flow_engine.run()

SchedulerManager.create_volume 执行自己的task_flow, 提取传入的参数并根据相关filter调度volume创建

def get_flow(context, driver_api, request_spec=None,
             filter_properties=None,
             volume=None, snapshot_id=None, image_id=None, backup_id=None):

    """Constructs and returns the scheduler entrypoint flow.

    This flow will do the following:

    1. Inject keys & values for dependent tasks.
    2. Extract a scheduler specification from the provided inputs.
    3. Use provided scheduler driver to select host and pass volume creation
       request further.
    """
    create_what = {
        'context': context,
        'raw_request_spec': request_spec,
        'filter_properties': filter_properties,
        'volume': volume,
        'snapshot_id': snapshot_id,
        'image_id': image_id,
        'backup_id': backup_id,
    }

    flow_name = ACTION.replace(":", "_") + "_scheduler"
    scheduler_flow = linear_flow.Flow(flow_name)

    # This will extract and clean the spec from the starting values.
    scheduler_flow.add(ExtractSchedulerSpecTask(
        rebind={'request_spec': 'raw_request_spec'}))

    # This will activate the desired scheduler driver (and handle any
    # driver related failures appropriately).
    scheduler_flow.add(ScheduleCreateVolumeTask(driver_api))

    # Now load (but do not run) the flow using the provided initial data.
    return taskflow.engines.load(scheduler_flow, store=create_what)

scheduler 调度创建volume时，先获取glance镜像id以及location等相关信息，再调用driver filter调度创建

cinder.scheduler.flows.create_volume.ScheduleCreateVolumeTask
def execute(self, context, request_spec, filter_properties, volume):
        try:
            if CONF.enable_multi_ceph:
                get_remote_image_service = glance.get_remote_image_service
                image_href = request_spec.get('image_id')
                if image_href:
                    image_service, image_id = get_remote_image_service(context,
                                                                   image_href)
                    image_location = image_service.get_location(context,
                                                                image_id)
                    filter_properties['image_location'] = image_location
            self.driver_api.schedule_create_volume(context, request_spec,
                                                   filter_properties)
            .......................................

筛选根据az，容量，特性等filter筛选合适的backend，并更新volume backend信息，最后调用cinde-volume的rpc_api create_volume

 def schedule_create_volume(self, context, request_spec, filter_properties):
        backend = self._schedule(context, request_spec, filter_properties)

        if not backend:
            raise exception.NoValidBackend(reason=_("No weighed backends "
                                                    "available"))

        backend = backend.obj
        volume_id = request_spec['volume_id']

        updated_volume = driver.volume_update_db(
            context, volume_id,
            backend.host,
            backend.cluster_name,
            availability_zone=backend.service['availability_zone'])
        self._post_select_populate_filter_properties(filter_properties,
                                                     backend)

        # context is not serializable
        filter_properties.pop('context', None)

        self.volume_rpcapi.create_volume(context, updated_volume, request_spec,
                                         filter_properties,
                                         allow_reschedule=True)

msg指定router key 是 cinder-volume将请求cast 发出去

  def create_volume(self, ctxt, volume, request_spec, filter_properties,
                      allow_reschedule=True):
        cctxt = self._get_cctxt(volume.service_topic_queue)
        cctxt.cast(ctxt, 'create_volume',
                   request_spec=request_spec,
                   filter_properties=filter_properties,
                   allow_reschedule=allow_reschedule,
                   volume=volume)

某个cinder-volume收到请求后，执行VolumeManager.create_volume，更新已分配容量信息，并执行task_flows

cinder.volume.manager.VolumeManager
@objects.Volume.set_workers
    def create_volume(self, context, volume, request_spec=None,
                      filter_properties=None, allow_reschedule=True):
    .............................
        try:
            # NOTE(flaper87): Driver initialization is
            # verified by the task itself.
            flow_engine = create_volume.get_flow(
                context_elevated,
                self,
                self.db,
                self.driver,
                self.scheduler_rpcapi,
                self.host,
                volume,
                allow_reschedule,
                context,
                request_spec,
                filter_properties,
                image_volume_cache=self.image_volume_cache,
            )
        except Exception:
            msg = _("Create manager volume flow failed.")
            LOG.exception(msg, resource={'type': 'volume', 'id': volume.id})
            raise exception.CinderException(msg)

cinder-volume的task_flows 中提取volume的spec相关信息，通知volume创建，开始创建volume云盘，因为基于镜像创建系统盘，所以执行_create_from_image函数，调用rbd driver clone_image，同时更新volume_glance_metadata表

def execute(self, context, volume, volume_spec):
    .......................................
            elif create_type == 'image':
            model_update = self._create_from_image(context,
                                                   volume,
                                                   **volume_spec)
    ............................................
def _create_from_image(self, context, volume,
                           image_location, image_id, image_meta,
                           image_service, **kwargs):
    ................................            
    if not volume_is_encrypted:
        model_update, cloned = self.driver.clone_image(context,
                                                           volume,
                                                           image_location,
                                                           image_meta,
                                                           image_service)
                                                           
    self._handle_bootable_volume_glance_meta(context, volume,
                                                 image_id=image_id,
                                                 image_meta=image_meta)
    return model_update

backend后端是ceph，所以调用了rbd clone_image, 根据image_location, 执行clone操作， 并resize 大小

cinder.volume.drivers.rbd.RBDVolumeProxy
def clone_image(self, context, volume,
                    image_location, image_meta,
                    image_service):
        if image_location:
            # Note: image_location[0] is glance image direct_url.
            # image_location[1] contains the list of all locations (including
            # direct_url) or None if show_multiple_locations is False in
            # glance configuration.
            if image_location[1]:
                url_locations = [location['url'] for
                                 location in image_location[1]]
            else:
                url_locations = [image_location[0]]

            # iterate all locations to look for a cloneable one.
            for url_location in url_locations:
                if url_location and self._is_cloneable(
                        url_location, image_meta):
                    for location in image_meta['locations']:
                        if location['url'] == url_location and \
                                location.get('metadata', None):
                            image_meta['properties']['stores'] = \
                                location['metadata']['backend']
                    _prefix, pool, image, snapshot = \
                        self._parse_location(url_location)
                    volume_update = self._clone(volume, pool, image, snapshot)
                    volume_update['provider_location'] = None
                    self._resize(volume)
                    return volume_update, True
        return ({}, False)

最终执行CreateVolumeOnFinishTask ，将volume状态变更为available，创建volume至此完成！

3. Volume挂载

1. nova 发起attach请求，调用cinderclient发起http请求，需要挂载系统盘 POST 请求 http://10.x.x.x:8776/v3/a6fea56ad90948af9339eecf83d6c9b0/volumes/c9dc6bdb-4f66-4aa6-8aef-bfb44eb13051/action

   
   
   
   Action body: {"os-attach": {"instance_uuid": "32d86262-c223-482c-8f9a-94660fda437e", "mountpoint": "/dev/sda", "mode": "rw"}}
   

   1. cinder api收到请求, 调用volume_api 执行attach操作，将attach模式写入volume_admin_metadata，再调用volume_rpcapi发起attach_volume请求

    @wsgi.response(http\_client.ACCEPTED)
    @wsgi.action('os-attach')
    @validation.schema(volume\_action.attach)
    def \_attach(self, req, id, body):
    .................................
    try:
    self.volume\_api.attach(context, volume,
    instance\_uuid, host\_name, mountpoint, mode)
    except messaging.RemoteError as error:
    if error.exc\_type in \['InvalidVolume', 'InvalidUUID',
    'InvalidVolumeAttachMode']:
    msg = \_("Error attaching volume - %(err\_type)s: "
    "%(err\_msg)s") % {
    'err\_type': error.exc\_type, 'err\_msg': error.value}
    raise webob.exc.HTTPBadRequest(explanation=msg)
    else:
    \# There are also few cases where attach call could fail due to
    \# db or volume driver errors. These errors shouldn't be exposed
    \# to the user and in such cases it should raise 500 error.
    raise

cinder-api 发起call请求，请求会同步阻塞，直至等待cinder-volume完成

    def attach_volume(self, ctxt, volume, instance_uuid, host_name,
                          mountpoint, mode):
            msg_args = {'volume_id': volume.id,
                        'instance_uuid': instance_uuid,
                        'host_name': host_name,
                        'mountpoint': mountpoint,
                        'mode': mode,
                        'volume': volume}
            cctxt = self._get_cctxt(volume.service_topic_queue, ('3.3', '3.0'))
            if not cctxt.can_send_version('3.3'):
                msg_args.pop('volume')
            return cctxt.call(ctxt, 'attach_volume', **msg_args)

cinder-volume attach volume时, 这里因为rbd 不需要做任何操作，所以self.driver.attach_volume不做任何操作，只操作了数据库，创建attachment信息，更新volume为in-use，同时更新volume-attachment表信息

cinder.volume.manager.VolumeManager
@coordination.synchronized('{volume_id}')
    def attach_volume(self, context, volume_id, instance_uuid, host_name,
                      mountpoint, mode, volume=None):
    .....................................................
        attachment = volume.begin_attach(mode)
        try:
            if volume_metadata.get('readonly') == 'True' and mode != 'ro':
                raise exception.InvalidVolumeAttachMode(mode=mode,
                                                        volume_id=volume.id)
            # NOTE(flaper87): Verify the driver is enabled
            # before going forward. The exception will be caught
            # and the volume status updated.
            utils.require_driver_initialized(self.driver)

            LOG.info('Attaching volume %(volume_id)s to instance '
                     '%(instance)s at mountpoint %(mount)s on host '
                     '%(host)s.',
                     {'volume_id': volume_id, 'instance': instance_uuid,
                      'mount': mountpoint, 'host': host_name_sanitized},
                     resource=volume)
            self.driver.attach_volume(context,
                                      volume,
                                      instance_uuid,
                                      host_name_sanitized,
                                      mountpoint)
        except Exception as excep:
            with excutils.save_and_reraise_exception():
                self.message_api.create(
                    context,
                    message_field.Action.ATTACH_VOLUME,
                    resource_uuid=volume_id,
                    exception=excep)
                attachment.attach_status = (
                    fields.VolumeAttachStatus.ERROR_ATTACHING)
                attachment.save()

        volume = attachment.finish_attach(
            instance_uuid,
            host_name_sanitized,
            mountpoint,
            mode)

        self._notify_about_volume_usage(context, volume, "attach.end")
        LOG.info("Attach volume completed successfully.",

至此，volume attach 操作完成！