OPTEE学习笔记 - 启动流程（二）

接上一篇OPTEE学习笔记 - 启动流程（一），本篇主要描述OPTEE被启动以后自己初始化的过程。

主核启动

由上篇文章可知，OPTEE第一个被执行的函数是__start

FUNC _start , :
	mov	x19, x0		/* Save pagable part address */
#if defined(CFG_DT_ADDR)
	ldr     x20, =CFG_DT_ADDR
#else
	mov	x20, x2		/* Save DT address */  /* 如果CFG_DT没配置，x2==0 */
#endif

	adr	x0, reset_vect_table
	msr	vbar_el1, x0
	isb

	set_sctlr_el1
	isb

#ifdef CFG_WITH_PAGER
	/* 忽略 */
#else
	/*
	 * The binary is built as:
	 * [Core, rodata and data] : In correct location
	 * [struct boot_embdata + data] : Should be moved to __end, first
	 * uint32_t tells the length of the struct + data
	 */
	adr_l	x0, __end		/* dst */
	adr_l	x1, __data_end		/* src */
	ldr	w2, [x1]   /* struct boot_embdata::total_len */ /* x2寄存器低32位保存struct长度 */
	/* Copy backwards (as memmove) in case we're overlapping */
	add	x0, x0, x2 /* x0指向dst的末尾 */
	add	x1, x1, x2 /* x1指向src的末尾 */
	adr	x3, cached_mem_end
	str	x0, [x3]
	adr_l	x2, __end

copy_init:
	ldp	x3, x4, [x1, #-16]!
	stp	x3, x4, [x0, #-16]!
	cmp	x0, x2
	b.gt	copy_init /* copy __data_end的数据到__end */
#endif

	/*
	 * Clear .bss, this code obviously depends on the linker keeping
	 * start/end of .bss at least 8 byte aligned.
	 */
	adr_l	x0, __bss_start
	adr_l	x1, __bss_end
clear_bss:
	str	xzr, [x0], #8
	cmp	x0, x1
	b.lt	clear_bss /* 清除bss段 */

#ifdef CFG_VIRTUALIZATION
	/* 忽略 */
#endif

	/* Setup SP_EL0 and SP_EL1, SP will be set to SP_EL0 */
	set_sp

	bl	thread_init_thread_core_local

	/* Enable aborts now that we can receive exceptions */
	msr	daifclr, #DAIFBIT_ABT

	/*
	 * Invalidate dcache for all memory used during initialization to
	 * avoid nasty surprices when the cache is turned on. We must not
	 * invalidate memory not used by OP-TEE since we may invalidate
	 * entries used by for instance ARM Trusted Firmware.
	 */
	adr_l	x0, __text_start
	ldr	x1, cached_mem_end
	sub	x1, x1, x0
	bl	dcache_cleaninv_range

	/* Enable Console */
	bl	console_init

#ifdef CFG_CORE_ASLR
	/* 忽略 */
#else
	mov	x0, #0
#endif

	adr	x1, boot_mmu_config
	bl	core_init_mmu_map

#ifdef CFG_CORE_ASLR
	/* 忽略 */
#endif

	bl	__get_core_pos
	bl	enable_mmu
#ifdef CFG_CORE_ASLR
	/* 忽略 */
#endif

	mov	x0, x19		/* pagable part address */
	mov	x1, #-1
	mov	x2, x20		/* DT address */
	bl	boot_init_primary

	/*
	 * In case we've touched memory that secondary CPUs will use before
	 * they have turned on their D-cache, clean and invalidate the
	 * D-cache before exiting to normal world.
	 */
	adr_l	x0, __text_start
	ldr	x1, cached_mem_end
	sub	x1, x1, x0
	bl	dcache_cleaninv_range


	/*
	 * Clear current thread id now to allow the thread to be reused on
	 * next entry. Matches the thread_init_boot_thread in
	 * boot.c.
	 */
#ifndef CFG_VIRTUALIZATION
	bl 	thread_clr_boot_thread
#endif

#ifdef CFG_CORE_FFA
	/* 忽略 */
#else
	/*
	 * Pass the vector address returned from main_init
	 * Compensate for the load offset since cpu_on_handler() is
	 * called with MMU off.
	 */
	ldr	x0, boot_mmu_config + CORE_MMU_CONFIG_LOAD_OFFSET
	adr	x1, thread_vector_table
	sub	x1, x1, x0
	mov	x0, #TEESMC_OPTEED_RETURN_ENTRY_DONE
	smc	#0
	b	.	/* SMC should not return */
#endif
END_FUNC _start
DECLARE_KEEP_INIT _start

_start主要做了以下几件事：

1. 复制struct boot_embdata到__end以后，这里我没太明白struct boot_embdata的作用。从这段copy的逻辑看，这段数据在编译阶段是保存在bss段，copy到__end以后就清除掉了bss段。
2. 跳转thread_init_thread_core_local初始化thread。OPTEE的thread设计会在其他文章中描述
3. 初始化console
4. 初始化mmu
5. 跳转boot_init_primary。前篇提到，_start函数只会在primary core上被调用，secondary core的初始化是通过vector_cpu_on_entry函数实现的，因此我们目前说到的都是主核

/*
 * Note: this function is weak just to make it possible to exclude it from
 * the unpaged area so that it lies in the init area.
 */
void __weak boot_init_primary(unsigned long pageable_part,
			      unsigned long nsec_entry __maybe_unused,
			      unsigned long fdt)
{
	unsigned long e = PADDR_INVALID;

#if !defined(CFG_WITH_ARM_TRUSTED_FW)
	e = nsec_entry;
#endif

	init_primary(pageable_part, e);
	paged_init_primary(fdt);
}

static void init_primary(unsigned long pageable_part, unsigned long nsec_entry)
{
	/*
	 * Mask asynchronous exceptions before switch to the thread vector
	 * as the thread handler requires those to be masked while
	 * executing with the temporary stack. The thread subsystem also
	 * asserts that the foreign interrupts are blocked when using most of
	 * its functions.
	 */
	thread_set_exceptions(THREAD_EXCP_ALL);
	primary_save_cntfrq(); /* 空函数 */
	init_vfp_sec(); /* 空函数 */
	/*
	 * Pager: init_runtime() calls thread_kernel_enable_vfp() so we must
	 * set a current thread right now to avoid a chicken-and-egg problem
	 * (thread_init_boot_thread() sets the current thread but needs
	 * things set by init_runtime()).
	 */
	thread_get_core_local()->curr_thread = 0;
	init_runtime(pageable_part); /* 初始化堆池 */

	if (IS_ENABLED(CFG_VIRTUALIZATION)) {
		/*
		 * Virtualization: We can't initialize threads right now because
		 * threads belong to "tee" part and will be initialized
		 * separately per each new virtual guest. So, we'll clear
		 * "curr_thread" and call it done.
		 */
		thread_get_core_local()->curr_thread = -1;
	} else {
		thread_init_boot_thread();
	}
	thread_init_primary();
	thread_init_per_cpu();
	init_sec_mon(nsec_entry); /* 配置CFG_WITH_ARM_TRUSTED_FW，则是空函数 */
}

init_primary()函数主要做了以下几件事：

1. 初始化和设置本地线程
2. 初始化堆池
3. 设置本地线程栈空间

关于thread的设置将会在其他章节中描述，以上函数只简单说明一下thread_init_per_cpu

void thread_init_per_cpu(void)
{
	size_t pos = get_core_pos();
	struct thread_core_local *l = thread_get_core_local();

	init_sec_mon_stack(pos);

	set_tmp_stack(l, GET_STACK_BOTTOM(stack_tmp, pos) - STACK_TMP_OFFS);
	set_abt_stack(l, GET_STACK_BOTTOM(stack_abt, pos));

	thread_init_vbar(get_excp_vect());
}

这个函数主要做的事情是对当前线程设置栈地址，然后设置了EL1的中断向量表。可以看到的是，在_start函数开始的地方也设置了中断向量表，设置的是reset_vect_table，在这里重新设置了中断向量表，是thread_excp_vect

从核启动

从OPTEE学习笔记 - 启动流程（一）中的最后部分可知，从核启动是转入init_secondary_helper函数

static void init_secondary_helper(unsigned long nsec_entry)
{
	IMSG("Secondary CPU %zu initializing", get_core_pos());
 
	/*
	 * Mask asynchronous exceptions before switch to the thread vector
	 * as the thread handler requires those to be masked while
	 * executing with the temporary stack. The thread subsystem also
	 * asserts that the foreign interrupts are blocked when using most of
	 * its functions.
	 */
	thread_set_exceptions(THREAD_EXCP_ALL);
 
	secondary_init_cntfrq();
	thread_init_per_cpu();
	init_sec_mon(nsec_entry);
	main_secondary_init_gic();
	init_vfp_sec();
	init_vfp_nsec();
 
	IMSG("Secondary CPU %zu switching to normal world boot", get_core_pos());
}

对比主核启动的初始化函数，可以看到做的内容差不多。

到目前为止，OPTEE的主从核就启动完成了。