Openstack liberty 云主机迁移源码分析之在线迁移2

这是在线迁移源码分析的第二篇，在上一篇中提到：nova-compute从消息队列拿到live_migration消息后，会启动一个线程来执行后续的迁移工作，线程函数为：_do_live_migration；下文将以该方法为入口，详细分析nova-compute的处理过程；限于篇幅，整个执行过程分为下面3个部分：

• prepare准备阶段
• execute执行阶段
• complete完成阶段

将用三篇博文分别介绍上述三个主题，本文详细介绍prepare准备阶段的处理过，下面一起来看具体内容：

prepare准备阶段

#`nova/compute/manager.py/ComputeManager._do_live_migration`
def _do_live_migration(self, context, dest, instance, 
                            block_migration,
                            migration, migrate_data):
    # NOTE(danms): Remove these guards in v5.0 of the RPC API
    #更新`nova.migrations`记录，状态为:preparing
    if migration:
        """NOTE(danms): We should enhance the RT to account for 
        migrations and use the status field to denote when the 
        accounting has been done on source/destination. For 
        now, this is just here for status reporting
        """
        migration.status = 'preparing'
        migration.save()

    """ Create a local copy since we'll be modifying the 
    dictionary
    """
    migrate_data = dict(migrate_data or {})
    try:
        #如果是块迁移，需要先获取实例的non-volume类型磁盘列表
        if block_migration:
            """
            1.先从`nova.block_device_mapping`数据表中获取实例的块设
            备映射信息
            2.将实例的块设备映射转化为驱动格式（字典格式），如：
            {'root_device_name':u'/dev/vda','ephemerals'=[],
            'block_device_mapping':[......],'swap':=None}
            3.排除上述'block_device_mapping'中不包
            含'connection_info'信息的卷设备
            """
            block_device_info = self._get_instance_block_device_info(
                    context, instance)
            """从实例xml配置中获取non-volume类型的本地磁盘
            （<disk type='file/block'）,返回一个字典，如：
            {
            {'type':'raw','path':'$locak_disk_path',
            'virt_disk_size':'20','backing_file':'',
            'disk_size':20,'over_committed_disk_size':0},
            }
            """
            disk = self.driver.get_instance_disk_info(
                    instance, 
                    block_device_info=block_device_info)
        else:
            disk = None

        """迁移前的准备
        发起同步的rpc调用，将`pre_live_migration`消息发送到消息队列，
        消费者`nova-compute`会处理该消息，并等待应答，
        `pre_live_migration`消息请求的处理过程情况下文的具体分析
        """
        pre_migration_data = self.compute_rpcapi.pre_live_migration(
                context, instance,
                block_migration, disk, dest, migrate_data)
        migrate_data['pre_live_migration_result'] = 
                                        pre_migration_data  
    except Exception:
        with excutils.save_and_reraise_exception():
            LOG.exception(_LE('Pre live migration failed at 
                                %s'), dest, instance=instance)
            #更新`nova.migrations`数据表，设置迁移状态为failed
            if migration:
                migration.status = 'failed'
                migration.save()

            self._rollback_live_migration(context, instance, 
                                              dest,
                                              block_migration, 
                                              migrate_data)

    #准备工作完成后，执行迁移，下一篇博文继续分析
    .......

---------------------------------------------------------------
#接上文：迁移前的准备工作，处理`pre_live_migration`消息
#省略装饰器定义
#`nova/compute/manager.py/ComputeManager.pre_live_migration`
def pre_live_migration(self, context, instance, 
                            block_migration, disk,
                            migrate_data):

    """Preparations for live migration at dest host.
    :param context: security context
    :param instance: dict of instance data
    :param block_migration: if true, prepare for block 
    migration
    :param migrate_data: if not None, it is a dict which holds 
    data required for live migration without shared storage.
    """
    """获得实例的块设备信息
    1.先从`nova.block_device_mapping`数据表中获取实例的块设备映射信息
    2.将实例的块设备映射转化为驱动格式（字典格式），如：
    {'root_device_name'=u'/dev/vda','ephemerals'=[],
    'block_device_mapping'=[],'swap'= None}
    3.更新`block_device_mapping`中卷设备的`connection_info`信息（如
    果有的话），具体操作如下：
    """
    """
        1.获得主机上的设备连接器('connector')信息,返回如下格式的字典：
        {'ip'='', 'host'='', 'initiator'='','wwpn'='',
        'wwnn'='','multipath'=False/True,'platform'='x86_64',
        'os_type'='linux2'}, 具体过程如下
          1. 发起`sudo nova-rootwrap /etc/nova/rootwrap.conf cat 
          /etc/iscsi/initiatorname.iscsi`命令得到主机上的iscsi 
          initiator名称，如：iqn.1994-05.com.redhat:92c97b181e6
          2.发起`sudo nova-rootwrap /etc/nova/rootwrap.conf 
          systool -c fc_host -v`命令得到主机FC HBA的WWPN及WWNN信息
          3.通过`uname -a` 及`sys.platform`获得`platform`及
          'os_type'
    """
    """
        2.通过`cinder/volume/api.py/API`发起同步
        `initialize_connection`请求给 `cinder-volume`更新卷设备的
        `connection_info`信息, 具体处理过程如下：
          1. 通过卷id，从`cinder.volume`数据表获得卷信息
          2. 通过具体的卷驱动（如：`LVMVolumeDriver`,`RBDDriver`）
          验证输入的`connector`信息格式是否合法（如：
          `LVMVolumeDriver`调用`iscsi_helper`完成格证;`RBDDriver`
          是空操作）
          3.通过具体的卷驱动（如：`LVMVolumeDriver`,`RBDDriver`）导
          出卷设备（如：`LVMVolumeDriver`调用`iscsi_helper`导出卷路
          径，得到形如下的字典 {'provider_location': ''
                             'provider_auth'=''};
          `RBDDriver`是空操作）
          4. 将3.中的导出信息更新到`nova.volumes`数据库（如果有的话）
          5. 从`nova.driver_initiator_data`数据表获得卷的
          `initiator`信息
          6.调用具体的卷驱动（如：`LVMVolumeDriver`,`RBDDriver`）更
          新`connection`连接信息（如：`LVMVolumeDriver`调
          用`iscsi_helper`更新连接信息并发回字典格式信息；`RBDDriver`
          通过发起`sudo cinder-rootwrap 
          /etc/cinder/rootwrap.conf ceph mon dump --format=json 
          --id $rbd_user --conf $rbd_ceph_conf --cluster 
          $rbd_cluster_name`命令获得`ceph`集群的监视器信息并组
          合`cinder.conf`中配置的`rbd`信息得到`rbd`的连接信息）
          7.更新`nova.driver_initiator_data`数据表记录（如果有的话）
          8.从数据库获取卷的`qos_specs`信息（如果有的话）
          9.从`nova.volume_admin_metadata`数据表获得卷的元信息
    """

    block_device_info = self._get_instance_block_device_info(
                            context, instance, 
                            refresh_conn_info=True)
    """从neutron数据库（`neutron.ports`,`neutron.networks`,
    `neutron.subnets`,`neutron.floatingips`等）获取实
    例的网络信息（VIF）并更新`nova.instance_info_caches`数据表
    """
    network_info = self.network_api.get_instance_nw_info(context, instance)
    #发送`live_migration.pre.start`通知给ceilometer
    self._notify_about_instance_usage(
                     context, instance, 
                     "live_migration.pre.start",
                     network_info=network_info)
    """调用虚拟化驱动执行迁移前的准备工作（如：LibvirtDriver）
    主要实现VIF的安装及non-volume设备的复制，最后返回如下格式字典：  
    {'graphics_listen_addrs': {}, 'volume': {},
                    'serial_listen_addr': {}}
    请看下文的具体分析
    """
    pre_live_migration_data = self.driver.pre_live_migration(context,
                                       instance,
                                       block_device_info,
                                       network_info,
                                       disk,
                                       migrate_data)
     #空操作，真正迁移的时候再准备网络                               
     # NOTE(tr3buchet): setup networks on destination host
     self.network_api.setup_networks_on_host(context, instance,
                                                    self.host)
     """ Creating filters to hypervisors and firewalls.
     An example is that nova-instance-instance-xxx,
     which is written to libvirt.xml(Check "virsh nwfilter-
     list") This nwfilter is necessary on the destination host.
     In addition, this method is creating filtering rule onto 
     destination host.
     """
     #设置实例的网络及防火墙过滤规则（IptablesFirewallDriver）
     self.driver.ensure_filtering_rules_for_instance(instance,
                                            network_info)

    #发送`live_migration.pre.end`通知给ceilometer
    self._notify_about_instance_usage(
                     context, instance, 
                     "live_migration.pre.end",
                     network_info=network_info)

    return pre_live_migration_data

---------------------------------------------------------------
#接上文：`nova/virt/libvirt/driver.py/LibvirtDriver.pre_live_migration`
def pre_live_migration(self, context, instance, 
                            block_device_info,
                            network_info, disk_info, 
                            migrate_data=None):
    """Preparation live migration."""
    #non-volume设备
    if disk_info is not None:
        disk_info = jsonutils.loads(disk_info)

    # Steps for volume backed instance live migration w/o 
    #shared storage.
    is_shared_block_storage = True
    is_shared_instance_path = True
    is_block_migration = True
    #我的输入中migrate_data = None
    if migrate_data:
        LOG.debug('migrate_data in pre_live_migration: %s', 
                        migrate_data,
                       instance=instance)
        is_shared_block_storage = migrate_data.get(
                    'is_shared_block_storage', True)
        is_shared_instance_path = migrate_data.get(
                    'is_shared_instance_path', True)
        is_block_migration = migrate_data.get('block_migration'
                                             , True)
    #从实例对象的system_metadata中提取镜像元信息
    image_meta = objects.ImageMeta.from_instance(instance)

    """判断实例是否需要配置卷,满足下列条件之一：
    1.镜像属性中包含'img_config_drive'= 'mandatory'
    2.实例配置中包含 'config_drive'
    3.CONF.force_config_drive = True/'always'
    """
    if configdrive.required_by(instance):
        """ NOTE(sileht): configdrive is stored into the block 
        storage kvm is a block device, live migration will work
        NOTE(sileht): the configdrive is stored into a shared 
        path kvm don't need to migrate it, live migration will 
        work NOTE(dims): Using config drive with iso format 
        does not work because of a bug in libvirt with read 
        only devices. However one can use vfat as 
        config_drive_format which works fine. Please see 
        bug/1246201 for details on the libvirt bug.
        """
        if (is_shared_block_storage or
            is_shared_instance_path or
            CONF.config_drive_format == 'vfat'):
             pass
        else:
            raise exception.NoLiveMigrationForConfigDriveInLibVirt()

    """如果实例配置路径，不是共享的；获得目的主机上实例的配置路径
    1.通过CONF.instances_path+实例名得到配置路径
    2.创建配置路径
    3.创建'disk.info'文件并添加non-volume设备信息（如果有的话）
    4.在配置路径下创建本地设备
    5.在配置路径下创建实例kernel及ramdisk
    """
    if not is_shared_instance_path:
        #1.通过CONF.instances_path+实例名得到配置路径
        instance_dir = libvirt_utils.
                get_instance_path_at_destination(
                                    instance, migrate_data)

        #如果路径已经存在，抛异常
        if os.path.exists(instance_dir):
            raise 
            exception.DestinationDiskExists(path=instance_dir)
        #2.创建配置路径
        os.mkdir(instance_dir)

        """ Recreate the disk.info file and in doing so stop 
        the imagebackend from recreating it incorrectly by 
        inspecting the contents of each file when using the Raw 
        backend.
        """
        #3.创建'disk.info'文件并添加non-volume设备信息（如果有的话）
        if disk_info:
            image_disk_info = {}
            """info是如下格式的字典：
            {'type':'raw','path':'$locak_disk_path',
            'virt_disk_size':'20','backing_file':'',
            'disk_size':20,'over_committed_disk_size':0}
            """
            for info in disk_info:
                image_file = os.path.basename(info['path'])
                image_path = os.path.join(instance_dir, 
                                                image_file)
                image_disk_info[image_path] = info['type']

            image_disk_info_path = 
                                os.path.join(instance_dir,
                                               'disk.info')
            libvirt_utils.write_to_file(
                                    image_disk_info_path,
                         jsonutils.dumps(image_disk_info))
        """4.在配置路径下创建本地设备
        调用qemu-img create创建设备
        """
        if not is_shared_block_storage:
            # Ensure images and backing files are present.
            self._create_images_and_backing(
                    context, instance, instance_dir, disk_info,
                    fallback_from_host=instance.host)
        #5.在配置路径下创建实例kernel及ramdisk           
        if not (is_block_migration or is_shared_instance_path):
            """NOTE(angdraug): when block storage is shared 
            between source and destination and instance path 
            isn't (e.g. volume backed or rbd backed instance), 
            instance path on destination has to be prepared 
            Touch the console.log file, required by libvirt.
            """
            #通过CONF.instances_path+console.log得到控制台日志路径
            console_file = self._get_console_log_path(instance)

            libvirt_utils.file_open(console_file, 'a').close()

            # if image has kernel and ramdisk, just download
            # following normal way.
            #如果镜像有kernel和ramdisk就下载到配置路径下面
            self._fetch_instance_kernel_ramdisk(context, 
                                                    instance)
    # Establishing connection to volume server.
    block_device_mapping = 
                    driver.block_device_info_get_mapping(
                                        block_device_info)
    #调用具体的卷驱动连接卷（如：
    #LibvirtNetVolumeDriver,LibvirtISCSIVolumeDriver）                                   
    for vol in block_device_mapping:
        connection_info = vol['connection_info']
        disk_info = blockinfo.get_info_from_bdm(
                instance, CONF.libvirt.virt_type, image_meta, 
                vol)
        self._connect_volume(connection_info, disk_info)

    """ We call plug_vifs before the compute manager calls
    ensure_filtering_rules_for_instance, to ensure bridge is 
    set up.Retry operation is necessary because continuously 
    request comes,concurrent request occurs to iptables, then 
    it complains.
    """
    LOG.debug('Plugging VIFs before live migration.', 
                                            instance=instance)
    max_retry = CONF.live_migration_retry_count
    for cnt in range(max_retry):
        try:
            """安装VIF
            调用self.plug_vifs后，内部会通过判断vif的类型（我的例子中
            用的是bridge）来调用具体的接口，然后具体的调用是这样的：
            self.plug_vifs -> 
          nova/virt/libvirt/vif.py/LibvirtGenericVIFDriver.plug 
            -> LibvirtGenericVIFDriver.plug_bridge ->
            nova/network/linux_net.py/
            LinuxBridgeInterfaceDriver.ensure_bridge, 最后是通过
            brctl工具创建的bridge，具体的实现读者可以自行看看
            """
            self.plug_vifs(instance, network_info)
            break
        except processutils.ProcessExecutionError:
            if cnt == max_retry - 1:
                raise
            else:
                LOG.warn(_LW('plug_vifs() failed %(cnt)d.Retry'
                                'up to '
                                '%(max_retry)d.'),
                                {'cnt': cnt,
                                 'max_retry': max_retry},
                                instance=instance)
                greenthread.sleep(1)

      # Store vncserver_listen and latest disk device info
      res_data = {'graphics_listen_addrs': {}, 'volume': {},
                    'serial_listen_addr': {}}
      res_data['graphics_listen_addrs']['vnc'] = 
                                  CONF.vnc.vncserver_listen
      res_data['graphics_listen_addrs']['spice'] = 
                                  CONF.spice.server_listen
      res_data['serial_listen_addr'] = 
                    CONF.serial_console.proxyclient_address

      for vol in block_device_mapping:
          connection_info = vol['connection_info']
          if connection_info.get('serial'):
              serial = connection_info['serial']
              res_data['volume'][serial] = {'connection_info': 
                                              {},
                                              'disk_info': {}}
              res_data['volume'][serial]['connection_info'] = \
                    connection_info
              disk_info = blockinfo.get_info_from_bdm(
                    instance, CONF.libvirt.virt_type, 
                    image_meta, vol)
              res_data['volume'][serial]['disk_info'] = \
                                                      disk_info
    return res_data

总结：迁移的prepare准备阶段就分析完了，主要完成如下的工作：
1. 在目标主机安装实例的VIF网络
2. 复制non-volume设备到目标主机的实例配置目录
3. 配置实例的网络及防火墙过滤规则

下一篇文章继续分析execute执行阶段，敬请期待！！！