ARMv7 KVM 在 linux中的实现 1 代码静态分析

linux-5.6

从四个角度分析
	函数角度
	初始化角度
	运行时角度
	异常角度

函数角度

架构无关部分 在 virt
架构相关部分 在 arch/arm/kvm
hyp 模式下的函数 : lowvisor 
svc 模式下的函数 : highvisor

• arch/arm/kvm 的出口函数

__init_stage2_translation // 没被 virt 调用
__kvm_arm_vcpu_get_events
__kvm_arm_vcpu_set_events
__kvm_flush_vm_context
__kvm_hyp_init
__kvm_tlb_flush_local_vmid
__kvm_tlb_flush_vmid
__kvm_tlb_flush_vmid_ipa
__kvm_vcpu_run_nvhe
__vcpu_spsr // 没被 virt 调用
handle_exit
kvm_arch_vcpu_ioctl_get_fpu
kvm_arch_vcpu_ioctl_get_regs
kvm_arch_vcpu_ioctl_get_sregs
kvm_arch_vcpu_ioctl_set_fpu
kvm_arch_vcpu_ioctl_set_guest_debug
kvm_arch_vcpu_ioctl_set_regs
kvm_arch_vcpu_ioctl_set_sregs
kvm_arch_vcpu_ioctl_translate
kvm_arm_copy_reg_indices
kvm_arm_get_reg
kvm_arm_num_regs
kvm_arm_set_reg
kvm_arm_vcpu_arch_get_attr
kvm_arm_vcpu_arch_has_attr
kvm_arm_vcpu_arch_set_attr
kvm_coproc_table_init
kvm_inject_vabt
kvm_reset_vcpu
kvm_target_cpu
kvm_vcpu_preferred_target
vgic_v3_cpu_sysregs_uaccess
vgic_v3_has_cpu_sysregs_attr

初始化角度

virt/kvm/arm/arm.c
	module_init(arm_init);
		kvm_init
			kvm_arch_init
				init_hyp_mode
					kvm_mmu_init
				init_subsystems
					kvm_vgic_hyp_init
						kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq); // kvm [1]: vgic interrupt IRQ16
				kvm_info("Hyp mode initialized successfully\n");
			kvm_irqfd_init
			kvm_vcpu_cache = kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,SLAB_ACCOUNT,offsetof(struct kvm_vcpu, arch),sizeof_field(struct kvm_vcpu, arch), NULL);
			misc_register(&kvm_dev);
			kvm_init_debug
				kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
				

运行时角度

VM entry

// arm 的 kvm 代码(包括架构和非架构相关) 运行在 两个mode
// 有 __hyp_text 标识的 运行在 hyp mode
// 无 __hyp_text 标识的 运行在 svc mode

vcpu 对应一个 fd ,对应一个 file_operations , 为 kvm_vcpu_fops
static struct file_operations kvm_vcpu_fops = {
  .release        = kvm_vcpu_release,
  .unlocked_ioctl = kvm_vcpu_ioctl,
  .mmap           = kvm_vcpu_mmap,
  .llseek   = noop_llseek,
  KVM_COMPAT(kvm_vcpu_compat_ioctl),
};


kvm_vcpu_ioctl // virt/kvm/kvm_main.c
	switch (ioctl) {
		case KVM_RUN: {
			oldpid = rcu_access_pointer(vcpu->pid);
			kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); // virt/kvm/arm/arm.c
				kvm_vcpu_first_run_init
					kvm_arm_vcpu_is_finalized
					kvm_timer_enable
					kvm_arm_pmu_v3_enable
				vcpu_load
				while (ret > 0) {
					...
					/**************************************************************
					* Enter the guest
					* /
					trace_kvm_entry(*vcpu_pc(vcpu));
					guest_enter_irqoff();
					if (has_vhe()) {ret = kvm_vcpu_run_vhe(vcpu);} // 不会进入这个路径
					else {ret = kvm_call_hyp_ret(__kvm_vcpu_run_nvhe, vcpu);}
					vcpu->mode = OUTSIDE_GUEST_MODE;
					vcpu->stat.exits++;
					/*
					 * Back from guest
					 *************************************************************/
					...
					guest_exit
					handle_exit
				}
		}
	}

arch/arm/include/asm/virt.h
static inline bool has_vhe(void)
{
        return false;
}

kvm_call_hyp_ret // arch/arm/kvm/interrupts.S
ENTRY(__kvm_call_hyp)
	hvc	#0 // 下一句 就进入 arch/arm/kvm/hyp/hyp-entry.S 中的 __kvm_hyp_vector_ic_inv 中的 W(add)  sp, sp, #1      /* HVC            2 */
	bx	lr
ENDPROC(__kvm_call_hyp)


// 从 hvc	#0 开始的流程
-1.kvm_call_hyp_ret(__kvm_vcpu_run_nvhe, vcpu); //kvm_call_hyp_ret即__kvm_call_hyp
0. __kvm_call_hyp 中的 hvc	#0
1. arch/arm/kvm/hyp/hyp-entry.S 中的 __kvm_hyp_vector_ic_inv 中的 W(add)  sp, sp, #1      /* HVC            2 */
2. b       decode_vectors
3. vect_br 0, hyp_fiq 和 vect_br 1, hyp_irq 和 vect_br 2, hyp_hvc
4. hyp_hvc
5. blx     lr                      @ Call the HYP function
6. __kvm_vcpu_run_nvhe
7. __guest_enter
8. eret // 进入 guest

hyp_hvc // arch/arm/kvm/hyp/hyp-entry.S
	...
	blx lr          @ Call the HYP function
		// 即 __kvm_vcpu_run_nvhe // __hyp_text
			 /* Jump in the fire! */
			 again:
			 exit_code = __guest_enter(vcpu, host_ctxt); // 该行执行后,进入虚拟机, 下一行就是返回.
			 /* And we're baaack! */
	pop {r2, lr}
	eret



// arch/arm/kvm/hyp/entry.S
/* int __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host) */
ENTRY(__guest_enter)
	@ Save host registers
	add	r1, r1, #(USR_REGS_OFFSET + S_R4)
	stm	r1!, {r4-r12}
	str	lr, [r1, #4]	@ Skip SP_usr (already saved)

	@ Restore guest registers
	add	r0, r0,  #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R0)
	ldr	lr, [r0, #S_LR]
	ldm	r0, {r0-r12}

	clrex
	eret // 该指令后进入 虚拟机
ENDPROC(__guest_enter)

VM exit

我们知道返回时的第一条指令

__guest_enter 会进入,那什么时候返回呢?不知道
应该是 __guest_exit //TODO

__guest_exit  被 hyp_hvc 和 hyp_irq 调用 , hyp_hvc 和 hyp_irq 是 PL2 异常向量表的入口

但是什么时候会发生异常呢?

ENTRY(__guest_exit)
	/*
	 * return convention:
	 * guest r0, r1, r2 saved on the stack
	 * r0: vcpu pointer
	 * r1: exception code
	 */

	add	r2, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R3)
	stm	r2!, {r3-r12}
	str	lr, [r2, #4]
	add	r2, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R0)
	pop	{r3, r4, r5}		@ r0, r1, r2
	stm	r2, {r3-r5}

	ldr	r0, [r0, #VCPU_HOST_CTXT]
	add	r0, r0, #(USR_REGS_OFFSET + S_R4)
	ldm	r0!, {r4-r12}
	ldr	lr, [r0, #4]

	mov	r0, r1
	mrs	r1, SPSR
	mrs	r2, ELR_hyp
	mrc	p15, 4, r3, c5, c2, 0	@ HSR

	/*
	 * Force loads and stores to complete before unmasking aborts
	 * and forcing the delivery of the exception. This gives us a
	 * single instruction window, which the handler will try to
	 * match.
	 */
	dsb	sy
	cpsie	a

	.global	abort_guest_exit_start
abort_guest_exit_start:

	isb

	.global	abort_guest_exit_end
abort_guest_exit_end:

	/*
	 * If we took an abort, r0[31] will be set, and cmp will set
	 * the N bit in PSTATE.
	 */
	cmp	r0, #0
	msrmi	SPSR_cxsf, r1
	msrmi	ELR_hyp, r2
	mcrmi	p15, 4, r3, c5, c2, 0	@ HSR

	bx	lr
ENDPROC(__guest_exit)

• __guest_exit 的上层函数 : hyp_hvc 和 hyp_irq

arch/arm/kvm/hyp/hyp-entry.S 
guest_trap: // hyp_hvc
	load_vcpu r0			@ Load VCPU pointer to r0

#ifdef CONFIG_VFPv3
	@ Check for a VFP access
	lsr	r1, r1, #HSR_EC_SHIFT
	cmp	r1, #HSR_EC_CP_0_13
	beq	__vfp_guest_restore
#endif

	mov	r1, #ARM_EXCEPTION_HVC
	b	__guest_exit

hyp_irq:
	push	{r0, r1, r2}
	mov	r1, #ARM_EXCEPTION_IRQ
	load_vcpu r0			@ Load VCPU pointer to r0
	b	__guest_exit

• PL2 下的 异常向量表

arch/arm/kvm/hyp/hyp-entry.S
	.align 5
__kvm_hyp_vector:
	.global __kvm_hyp_vector

	@ Hyp-mode exception vector
	W(b)	hyp_reset
	W(b)	hyp_undef
	W(b)	hyp_svc
	W(b)	hyp_pabt
	W(b)	hyp_dabt
	W(b)	hyp_hvc
	W(b)	hyp_irq
	W(b)	hyp_fiq
.macro vect_br val, targ
ARM(	eor	sp, sp, #\val	)
ARM(	tst	sp, #7		)
ARM(	eorne	sp, sp, #\val	)

THUMB(	cmp	r1, #\val	)
THUMB(	popeq	{r1, r2}	)

	beq	\targ
.endm

	vect_br	0, hyp_fiq
	vect_br	1, hyp_irq
	vect_br	2, hyp_hvc
	vect_br	3, hyp_dabt
	vect_br	4, hyp_pabt
	vect_br	5, hyp_svc
	vect_br	6, hyp_undef
	vect_br	7, hyp_reset
#endif

.macro invalid_vector label, cause
	.align
\label:	mov	r0, #\cause
	b	__hyp_panic
.endm

	invalid_vector	hyp_reset	ARM_EXCEPTION_RESET
	invalid_vector	hyp_undef	ARM_EXCEPTION_UNDEFINED
	invalid_vector	hyp_svc		ARM_EXCEPTION_SOFTWARE
	invalid_vector	hyp_pabt	ARM_EXCEPTION_PREF_ABORT
	invalid_vector	hyp_fiq		ARM_EXCEPTION_FIQ

实现了以下三个异常向量,其他五个没实现
hyp_dabt
hyp_hvc
hyp_irq

虚拟机 退出 VM的流程
1. 执行敏感指令
2. __kvm_hyp_vector_ic_inv 中的 W(add)  sp, sp, #1      /* HVC            2 */
3. b       decode_vectors
4. vect_br 0, hyp_fiq 和 vect_br 1, hyp_irq 和 vect_br 2, hyp_hvc
5. hyp_hvc
6. bne     guest_trap              @ Not HVC instr.
7. mov     r1, #ARM_EXCEPTION_HVC 和 b       __guest_exit
8. abort_guest_exit_end: bx      lr
9. arch/arm/kvm/hyp/switch.c 中的 __populate_fault_info 中的 u32 hsr = read_sysreg(HSR); // ???
10. if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu)) goto again;

异常角度

实体机 有两组 异常向量表
	host_PL1(arch/arm/kernel/entry-armv.S L__vectors_start) 和 host_PL2(arch/arm/kvm/hyp/hyp-entry.S __kvm_hyp_vector)
虚拟机 有一组 异常向量表
	guest_PL1(arch/arm/kernel/entry-armv.S L__vectors_start)


虚拟机开始运行时
	host_PL1 -> host_PL2 // hyp_hvc 调用了 __kvm_vcpu_run_nvhe , __kvm_vcpu_run_nvhe  调用了  __guest_enter
虚拟机退出时
	host_PL2 //  hyp_hvc 和 hyp_irq 调用了 __guest_exit 
虚拟机正常运行时
	guest_PL1 // 虚拟机正常运行就是 PL0 -> PL1 PL1->PL0

问题:
	什么虚拟化相关的情况下会进入 host_PL1
		TODO
	什么虚拟化相关的情况下会进入 host_PL2
		1. 虚拟机开启时,ioctl KVM_RUN 进入 hyp_hvc
		2. 中断应该会进入 hyp_irq // TODO
		3. 内存异常应该会进入 hyp_dabt // TODO
	什么虚拟化相关的情况下会进入 guest_PL1
		1. 优化后的中断应该会进入 vector_irq // TODO
		2. 优化后的内存访问异常应该会进入 vector_dabt // TODO

其他 为什么放弃了 kvm 对 arm 的支持

• 截图链接
  

在 541ad0150ca4aa663a2dcb9c834ab493168fe494 将 arm 的kvm 代码 删除了
commit 541ad0150ca4aa663a2dcb9c834ab493168fe494
Author: Marc Zyngier <maz@kernel.org>
Date:   Fri Jan 24 22:42:15 2020 +0000

    arm: Remove 32bit KVM host support
    
    That's it. Remove all references to KVM itself, and document
    that although it is no more, the ABI between SVC and HYP still
    exists.


具体资料可查看 Documentation/virt/kvm/arm/hyp-abi.rst

作者认为arm-kvm有几个问题
- It is incomplete (no debug support, no PMU)
- It hasn't followed any of the architectural evolutions
- It has zero^W very few users (I don't count myself here)
- It is more and more getting in the way of new arm64 developments

作者认为
 I still don't think we should keep the 32bit port artificially alive for the sake of it, specially considering that the whole of the 32bit architecture is in best effort, maintenance mode.

 I'd rather drop KVM support now, while it is still in a decent shape, rather than seeing it bit-rot like it happened for ia64. 

作者决定
I also plan to keep maintaining it in the various stable kernels, up to (and including) 5.6.