这是在线迁移
源码分析的第二篇,在上一篇中提到:nova-compute
从消息队列拿到live_migration
消息后,会启动一个线程来执行后续的迁移工作,线程函数为:_do_live_migration
;下文将以该方法为入口,详细分析nova-compute
的处理过程;限于篇幅,整个执行过程分为下面3个部分:
prepare
准备阶段execute
执行阶段complete
完成阶段
将用三篇博文分别介绍上述三个主题,本文详细介绍prepare
准备阶段的处理过,下面一起来看具体内容:
prepare
准备阶段
#`nova/compute/manager.py/ComputeManager._do_live_migration`
def _do_live_migration(self, context, dest, instance,
block_migration,
migration, migrate_data):
# NOTE(danms): Remove these guards in v5.0 of the RPC API
#更新`nova.migrations`记录,状态为:preparing
if migration:
"""NOTE(danms): We should enhance the RT to account for
migrations and use the status field to denote when the
accounting has been done on source/destination. For
now, this is just here for status reporting
"""
migration.status = 'preparing'
migration.save()
""" Create a local copy since we'll be modifying the
dictionary
"""
migrate_data = dict(migrate_data or {})
try:
#如果是块迁移,需要先获取实例的non-volume类型磁盘列表
if block_migration:
"""
1.先从`nova.block_device_mapping`数据表中获取实例的块设
备映射信息
2.将实例的块设备映射转化为驱动格式(字典格式),如:
{'root_device_name':u'/dev/vda','ephemerals'=[],
'block_device_mapping':[......],'swap':=None}
3.排除上述'block_device_mapping'中不包
含'connection_info'信息的卷设备
"""
block_device_info = self._get_instance_block_device_info(
context, instance)
"""从实例xml配置中获取non-volume类型的本地磁盘
(<disk type='file/block'),返回一个字典,如:
{
{'type':'raw','path':'$locak_disk_path',
'virt_disk_size':'20','backing_file':'',
'disk_size':20,'over_committed_disk_size':0},
}
"""
disk = self.driver.get_instance_disk_info(
instance,
block_device_info=block_device_info)
else:
disk = None
"""迁移前的准备
发起同步的rpc调用,将`pre_live_migration`消息发送到消息队列,
消费者`nova-compute`会处理该消息,并等待应答,
`pre_live_migration`消息请求的处理过程情况下文的具体分析
"""
pre_migration_data = self.compute_rpcapi.pre_live_migration(
context, instance,
block_migration, disk, dest, migrate_data)
migrate_data['pre_live_migration_result'] =
pre_migration_data
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Pre live migration failed at
%s'), dest, instance=instance)
#更新`nova.migrations`数据表,设置迁移状态为failed
if migration:
migration.status = 'failed'
migration.save()
self._rollback_live_migration(context, instance,
dest,
block_migration,
migrate_data)
#准备工作完成后,执行迁移,下一篇博文继续分析
.......
---------------------------------------------------------------
#接上文:迁移前的准备工作,处理`pre_live_migration`消息
#省略装饰器定义
#`nova/compute/manager.py/ComputeManager.pre_live_migration`
def pre_live_migration(self, context, instance,
block_migration, disk,
migrate_data):
"""Preparations for live migration at dest host.
:param context: security context
:param instance: dict of instance data
:param block_migration: if true, prepare for block
migration
:param migrate_data: if not None, it is a dict which holds
data required for live migration without shared storage.
"""
"""获得实例的块设备信息
1.先从`nova.block_device_mapping`数据表中获取实例的块设备映射信息
2.将实例的块设备映射转化为驱动格式(字典格式),如:
{'root_device_name'=u'/dev/vda','ephemerals'=[],
'block_device_mapping'=[],'swap'= None}
3.更新`block_device_mapping`中卷设备的`connection_info`信息(如
果有的话),具体操作如下:
"""
"""
1.获得主机上的设备连接器('connector')信息,返回如下格式的字典:
{'ip'='', 'host'='', 'initiator'='','wwpn'='',
'wwnn'='','multipath'=False/True,'platform'='x86_64',
'os_type'='linux2'}, 具体过程如下
1. 发起`sudo nova-rootwrap /etc/nova/rootwrap.conf cat
/etc/iscsi/initiatorname.iscsi`命令得到主机上的iscsi
initiator名称,如:iqn.1994-05.com.redhat:92c97b181e6
2.发起`sudo nova-rootwrap /etc/nova/rootwrap.conf
systool -c fc_host -v`命令得到主机FC HBA的WWPN及WWNN信息
3.通过`uname -a` 及`sys.platform`获得`platform`及
'os_type'
"""
"""
2.通过`cinder/volume/api.py/API`发起同步
`initialize_connection`请求给 `cinder-volume`更新卷设备的
`connection_info`信息, 具体处理过程如下:
1. 通过卷id,从`cinder.volume`数据表获得卷信息
2. 通过具体的卷驱动(如:`LVMVolumeDriver`,`RBDDriver`)
验证输入的`connector`信息格式是否合法(如:
`LVMVolumeDriver`调用`iscsi_helper`完成格证;`RBDDriver`
是空操作)
3.通过具体的卷驱动(如:`LVMVolumeDriver`,`RBDDriver`)导
出卷设备(如:`LVMVolumeDriver`调用`iscsi_helper`导出卷路
径,得到形如下的字典 {'provider_location': ''
'provider_auth'=''};
`RBDDriver`是空操作)
4. 将3.中的导出信息更新到`nova.volumes`数据库(如果有的话)
5. 从`nova.driver_initiator_data`数据表获得卷的
`initiator`信息
6.调用具体的卷驱动(如:`LVMVolumeDriver`,`RBDDriver`)更
新`connection`连接信息(如:`LVMVolumeDriver`调
用`iscsi_helper`更新连接信息并发回字典格式信息;`RBDDriver`
通过发起`sudo cinder-rootwrap
/etc/cinder/rootwrap.conf ceph mon dump --format=json
--id $rbd_user --conf $rbd_ceph_conf --cluster
$rbd_cluster_name`命令获得`ceph`集群的监视器信息并组
合`cinder.conf`中配置的`rbd`信息得到`rbd`的连接信息)
7.更新`nova.driver_initiator_data`数据表记录(如果有的话)
8.从数据库获取卷的`qos_specs`信息(如果有的话)
9.从`nova.volume_admin_metadata`数据表获得卷的元信息
"""
block_device_info = self._get_instance_block_device_info(
context, instance,
refresh_conn_info=True)
"""从neutron数据库(`neutron.ports`,`neutron.networks`,
`neutron.subnets`,`neutron.floatingips`等)获取实
例的网络信息(VIF)并更新`nova.instance_info_caches`数据表
"""
network_info = self.network_api.get_instance_nw_info(context, instance)
#发送`live_migration.pre.start`通知给ceilometer
self._notify_about_instance_usage(
context, instance,
"live_migration.pre.start",
network_info=network_info)
"""调用虚拟化驱动执行迁移前的准备工作(如:LibvirtDriver)
主要实现VIF的安装及non-volume设备的复制,最后返回如下格式字典:
{'graphics_listen_addrs': {}, 'volume': {},
'serial_listen_addr': {}}
请看下文的具体分析
"""
pre_live_migration_data = self.driver.pre_live_migration(context,
instance,
block_device_info,
network_info,
disk,
migrate_data)
#空操作,真正迁移的时候再准备网络
# NOTE(tr3buchet): setup networks on destination host
self.network_api.setup_networks_on_host(context, instance,
self.host)
""" Creating filters to hypervisors and firewalls.
An example is that nova-instance-instance-xxx,
which is written to libvirt.xml(Check "virsh nwfilter-
list") This nwfilter is necessary on the destination host.
In addition, this method is creating filtering rule onto
destination host.
"""
#设置实例的网络及防火墙过滤规则(IptablesFirewallDriver)
self.driver.ensure_filtering_rules_for_instance(instance,
network_info)
#发送`live_migration.pre.end`通知给ceilometer
self._notify_about_instance_usage(
context, instance,
"live_migration.pre.end",
network_info=network_info)
return pre_live_migration_data
---------------------------------------------------------------
#接上文:`nova/virt/libvirt/driver.py/LibvirtDriver.pre_live_migration`
def pre_live_migration(self, context, instance,
block_device_info,
network_info, disk_info,
migrate_data=None):
"""Preparation live migration."""
#non-volume设备
if disk_info is not None:
disk_info = jsonutils.loads(disk_info)
# Steps for volume backed instance live migration w/o
#shared storage.
is_shared_block_storage = True
is_shared_instance_path = True
is_block_migration = True
#我的输入中migrate_data = None
if migrate_data:
LOG.debug('migrate_data in pre_live_migration: %s',
migrate_data,
instance=instance)
is_shared_block_storage = migrate_data.get(
'is_shared_block_storage', True)
is_shared_instance_path = migrate_data.get(
'is_shared_instance_path', True)
is_block_migration = migrate_data.get('block_migration'
, True)
#从实例对象的system_metadata中提取镜像元信息
image_meta = objects.ImageMeta.from_instance(instance)
"""判断实例是否需要配置卷,满足下列条件之一:
1.镜像属性中包含'img_config_drive'= 'mandatory'
2.实例配置中包含 'config_drive'
3.CONF.force_config_drive = True/'always'
"""
if configdrive.required_by(instance):
""" NOTE(sileht): configdrive is stored into the block
storage kvm is a block device, live migration will work
NOTE(sileht): the configdrive is stored into a shared
path kvm don't need to migrate it, live migration will
work NOTE(dims): Using config drive with iso format
does not work because of a bug in libvirt with read
only devices. However one can use vfat as
config_drive_format which works fine. Please see
bug/1246201 for details on the libvirt bug.
"""
if (is_shared_block_storage or
is_shared_instance_path or
CONF.config_drive_format == 'vfat'):
pass
else:
raise exception.NoLiveMigrationForConfigDriveInLibVirt()
"""如果实例配置路径,不是共享的;获得目的主机上实例的配置路径
1.通过CONF.instances_path+实例名得到配置路径
2.创建配置路径
3.创建'disk.info'文件并添加non-volume设备信息(如果有的话)
4.在配置路径下创建本地设备
5.在配置路径下创建实例kernel及ramdisk
"""
if not is_shared_instance_path:
#1.通过CONF.instances_path+实例名得到配置路径
instance_dir = libvirt_utils.
get_instance_path_at_destination(
instance, migrate_data)
#如果路径已经存在,抛异常
if os.path.exists(instance_dir):
raise
exception.DestinationDiskExists(path=instance_dir)
#2.创建配置路径
os.mkdir(instance_dir)
""" Recreate the disk.info file and in doing so stop
the imagebackend from recreating it incorrectly by
inspecting the contents of each file when using the Raw
backend.
"""
#3.创建'disk.info'文件并添加non-volume设备信息(如果有的话)
if disk_info:
image_disk_info = {}
"""info是如下格式的字典:
{'type':'raw','path':'$locak_disk_path',
'virt_disk_size':'20','backing_file':'',
'disk_size':20,'over_committed_disk_size':0}
"""
for info in disk_info:
image_file = os.path.basename(info['path'])
image_path = os.path.join(instance_dir,
image_file)
image_disk_info[image_path] = info['type']
image_disk_info_path =
os.path.join(instance_dir,
'disk.info')
libvirt_utils.write_to_file(
image_disk_info_path,
jsonutils.dumps(image_disk_info))
"""4.在配置路径下创建本地设备
调用qemu-img create创建设备
"""
if not is_shared_block_storage:
# Ensure images and backing files are present.
self._create_images_and_backing(
context, instance, instance_dir, disk_info,
fallback_from_host=instance.host)
#5.在配置路径下创建实例kernel及ramdisk
if not (is_block_migration or is_shared_instance_path):
"""NOTE(angdraug): when block storage is shared
between source and destination and instance path
isn't (e.g. volume backed or rbd backed instance),
instance path on destination has to be prepared
Touch the console.log file, required by libvirt.
"""
#通过CONF.instances_path+console.log得到控制台日志路径
console_file = self._get_console_log_path(instance)
libvirt_utils.file_open(console_file, 'a').close()
# if image has kernel and ramdisk, just download
# following normal way.
#如果镜像有kernel和ramdisk就下载到配置路径下面
self._fetch_instance_kernel_ramdisk(context,
instance)
# Establishing connection to volume server.
block_device_mapping =
driver.block_device_info_get_mapping(
block_device_info)
#调用具体的卷驱动连接卷(如:
#LibvirtNetVolumeDriver,LibvirtISCSIVolumeDriver)
for vol in block_device_mapping:
connection_info = vol['connection_info']
disk_info = blockinfo.get_info_from_bdm(
instance, CONF.libvirt.virt_type, image_meta,
vol)
self._connect_volume(connection_info, disk_info)
""" We call plug_vifs before the compute manager calls
ensure_filtering_rules_for_instance, to ensure bridge is
set up.Retry operation is necessary because continuously
request comes,concurrent request occurs to iptables, then
it complains.
"""
LOG.debug('Plugging VIFs before live migration.',
instance=instance)
max_retry = CONF.live_migration_retry_count
for cnt in range(max_retry):
try:
"""安装VIF
调用self.plug_vifs后,内部会通过判断vif的类型(我的例子中
用的是bridge)来调用具体的接口,然后具体的调用是这样的:
self.plug_vifs ->
nova/virt/libvirt/vif.py/LibvirtGenericVIFDriver.plug
-> LibvirtGenericVIFDriver.plug_bridge ->
nova/network/linux_net.py/
LinuxBridgeInterfaceDriver.ensure_bridge, 最后是通过
brctl工具创建的bridge,具体的实现读者可以自行看看
"""
self.plug_vifs(instance, network_info)
break
except processutils.ProcessExecutionError:
if cnt == max_retry - 1:
raise
else:
LOG.warn(_LW('plug_vifs() failed %(cnt)d.Retry'
'up to '
'%(max_retry)d.'),
{'cnt': cnt,
'max_retry': max_retry},
instance=instance)
greenthread.sleep(1)
# Store vncserver_listen and latest disk device info
res_data = {'graphics_listen_addrs': {}, 'volume': {},
'serial_listen_addr': {}}
res_data['graphics_listen_addrs']['vnc'] =
CONF.vnc.vncserver_listen
res_data['graphics_listen_addrs']['spice'] =
CONF.spice.server_listen
res_data['serial_listen_addr'] =
CONF.serial_console.proxyclient_address
for vol in block_device_mapping:
connection_info = vol['connection_info']
if connection_info.get('serial'):
serial = connection_info['serial']
res_data['volume'][serial] = {'connection_info':
{},
'disk_info': {}}
res_data['volume'][serial]['connection_info'] = \
connection_info
disk_info = blockinfo.get_info_from_bdm(
instance, CONF.libvirt.virt_type,
image_meta, vol)
res_data['volume'][serial]['disk_info'] = \
disk_info
return res_data
总结:迁移的prepare
准备阶段就分析完了,主要完成如下的工作:
1. 在目标主机安装实例的VIF网络
2. 复制non-volume设备到目标主机的实例配置目录
3. 配置实例的网络及防火墙过滤规则
下一篇文章继续分析execute
执行阶段,敬请期待!!!