DPDK学习记录11 - 内存初始化4之rte_eal_hugepage_init

1 rte_eal_hugepage_init

rte_eal_hugepage_init有两种模式：legacy和dynamic，默认是后者。我运行dpdk app时没有指定模式，所以gdb跟代码到了后者函数中。

int
rte_eal_hugepage_init(void)
{
	return internal_config.legacy_mem ?
			eal_legacy_hugepage_init() :
			eal_hugepage_init();
}

2 eal_hugepage_init

internal_config.socket_mem[hp_sz_idx] = 0；
internal_config.force_socket_limits = 0；

好像什么都没做，分支都不会进去。

static int
eal_hugepage_init(void)
{
	struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
	uint64_t memory[RTE_MAX_NUMA_NODES];
	int hp_sz_idx, socket_id;
	test_phys_addrs_available();
	memset(used_hp, 0, sizeof(used_hp));
	for (hp_sz_idx = 0;hp_sz_idx < (int) internal_config.num_hugepage_sizes;hp_sz_idx++) {
		/* also initialize used_hp hugepage sizes in used_hp */
		struct hugepage_info *hpi;
		hpi = &internal_config.hugepage_info[hp_sz_idx];
		used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz;
	}

	/* make a copy of socket_mem, needed for balanced allocation. */
	for (hp_sz_idx = 0; hp_sz_idx < RTE_MAX_NUMA_NODES; hp_sz_idx++)
		memory[hp_sz_idx] = internal_config.socket_mem[hp_sz_idx];

	/* calculate final number of pages */ //calc_num_pages_per_socket返回值为0，且used_hp没有被修改。
	if (calc_num_pages_per_socket(memory,internal_config.hugepage_info,used_hp,internal_config.num_hugepage_sizes) < 0)
		return -1;

	for (hp_sz_idx = 0;	hp_sz_idx < (int)internal_config.num_hugepage_sizes;hp_sz_idx++) {
		for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES;socket_id++) {
			struct rte_memseg **pages;
			struct hugepage_info *hpi = &used_hp[hp_sz_idx];
			unsigned int num_pages = hpi->num_pages[socket_id]; 
			int num_pages_alloc, i;

			if (num_pages == 0) //这里都是0，每次都是被continue;
				continue;

			pages = malloc(sizeof(*pages) * num_pages);

			num_pages_alloc = eal_memalloc_alloc_seg_bulk(pages,num_pages, hpi->hugepage_sz,socket_id, true);
			if (num_pages_alloc < 0) {
				free(pages);
				return -1;
			}
			/* mark preallocated pages as unfreeable */
			for (i = 0; i < num_pages_alloc; i++) {
				struct rte_memseg *ms = pages[i];
				ms->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE;
			}
			free(pages);
		}
	}
	
	/* if socket limits were specified, set them */
	if (internal_config.force_socket_limits) {  //internal_config.force_socket_limits = 0
		unsigned int i; 
		for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
			uint64_t limit = internal_config.socket_limit[i];
			if (limit == 0)
				continue;
			if (rte_mem_alloc_validator_register("socket-limit",
					limits_callback, i, limit))
				RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n");
		}
	}
	return 0;
}

3 calc_num_pages_per_socket

internal_config.force_sockets = 0;
total_mem = internal_config.memory = 0;
total_size = internal_config.memory = 0;
internal_config.socket_mem[socket] = 0

上面几个都会0，会导致所有的分支都不能进去，所以hp_used不会被改变，返回值total_num_pages为初始值0。

static int calc_num_pages_per_socket(uint64_t * memory,	struct hugepage_info *hp_info,struct hugepage_info *hp_used,unsigned num_hp_info)
{
	int total_num_pages = 0;
	uint64_t total_mem = internal_config.memory;
	if (num_hp_info == 0)
		return -1;

	/* if specific memory amounts per socket weren't requested */
	if (internal_config.force_sockets == 0) {
		total_size = internal_config.memory;
		for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
			/* Set memory amount per socket */
			default_size = (internal_config.memory * cpu_per_socket[socket])
					/ rte_lcore_count();

			/* Limit to maximum available memory on socket */
			default_size = RTE_MIN(default_size, get_socket_mem_size(socket));

			/* Update sizes */
			memory[socket] = default_size;
			total_size -= default_size;
		}

		for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
			/* take whatever is available */
			default_size = RTE_MIN(get_socket_mem_size(socket) - memory[socket],
					       total_size);

			/* Update sizes */
			memory[socket] += default_size;
			total_size -= default_size;
		}
	}

	for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_mem != 0; socket++) {
		/* skips if the memory on specific socket wasn't requested */
		for (i = 0; i < num_hp_info && memory[socket] != 0; i++){
			strlcpy(hp_used[i].hugedir, hp_info[i].hugedir,
				sizeof(hp_used[i].hugedir));
			hp_used[i].num_pages[socket] = RTE_MIN(
					memory[socket] / hp_info[i].hugepage_sz,
					hp_info[i].num_pages[socket]);

			cur_mem = hp_used[i].num_pages[socket] *
					hp_used[i].hugepage_sz;

			memory[socket] -= cur_mem;
			total_mem -= cur_mem;

			total_num_pages += hp_used[i].num_pages[socket];

			/* check if we have met all memory requests */
			if (memory[socket] == 0)
				break;

			/* check if we have any more pages left at this size, if so
			 * move on to next size */
			if (hp_used[i].num_pages[socket] == hp_info[i].num_pages[socket])
				continue;
			/* At this point we know that there are more pages available that are
			 * bigger than the memory we want, so lets see if we can get enough
			 * from other page sizes.
			 */
			remaining_mem = 0;
			for (j = i+1; j < num_hp_info; j++)
				remaining_mem += hp_info[j].hugepage_sz *
				hp_info[j].num_pages[socket];

			/* is there enough other memory, if not allocate another page and quit */
			if (remaining_mem < memory[socket]){
				cur_mem = RTE_MIN(memory[socket],
						hp_info[i].hugepage_sz);
				memory[socket] -= cur_mem;
				total_mem -= cur_mem;
				hp_used[i].num_pages[socket]++;
				total_num_pages++;
				break; /* we are done with this socket*/
			}
		}
		/* if we didn't satisfy all memory requirements per socket */
		if (memory[socket] > 0 &&
				internal_config.socket_mem[socket] != 0) {
			/* to prevent icc errors */
			requested = (unsigned) (internal_config.socket_mem[socket] /
					0x100000);
			available = requested -
					((unsigned) (memory[socket] / 0x100000));
			RTE_LOG(ERR, EAL, "Not enough memory available on socket %u! "
					"Requested: %uMB, available: %uMB\n", socket,
					requested, available);
			return -1;
		}
	}

	/* if we didn't satisfy total memory requirements */
	if (total_mem > 0) {
		requested = (unsigned) (internal_config.memory / 0x100000);
		available = requested - (unsigned) (total_mem / 0x100000);
		RTE_LOG(ERR, EAL, "Not enough memory available! Requested: %uMB,"
				" available: %uMB\n", requested, available);
		return -1;
	}
	return total_num_pages;
}

4 总结

在我当前什么都没有配置的情况下，貌似rte_eal_hugepage_init什么都没有操作。