postgresql之内存池-AllocsetContext

一、简介

postgresql大部分的内存分配管理都是通过MemoryContext进行操作的，
多个相关的MemoryContext构成了一个树型结构，
多个树构成了一个森林。

实现了三种MemoryContext:

• SlabContext
• GenerationContext
• AllocSetContext

使用全局变量CurrentMemoryContext进行MemoryContext切换

二、编程范式

2.1 创建MemoryContext

newcontext = AllocSetContextCreate((MemoryContext) NULL,
										"TestMemoryContext",
										ALLOCSET_DEFAULT_SIZES);

2.2 切换需要使用的MemoryContext

MemoryContextSwitchTo(newcontext);

2.3 使用相应API进行分配/释放空间

//分配空间
item = (text *) palloc(VARHDRSZ + len + 1);
//分配空间，并且清空
plabel = palloc0(sizeof(pending_label));
//扩容/缩容
es->steps = repalloc(es->steps, sizeof(ExprEvalStep) * es->steps_alloc);
//释放空间
pfree(r);

2.4 重置MemoryContext

MemoryContextReset(argContext);

2.5 销毁MemoryContext

MemoryContextDelete(tmpCxt);

三、实现原理

3.1 数据结构简介

AllocSetContext由三部分构成， MemoryContextData + AllocBlock + AllocChunk
MemoryContextData：

• 由如下字段将各个MemoryContext构成树型结构

MemoryContext parent;		/* NULL if no parent (toplevel context) */
MemoryContext firstchild;	/* head of linked list of children */
MemoryContext prevchild;	/* previous child of same parent */
MemoryContext nextchild;	/* next child of same parent */

• 虚函数表，实现统一接口，不同实现

const MemoryContextMethods *methods;	/* virtual function table */

AllocBlock：

• 由如下字段将block构成一个双向链表

AllocBlock	prev;			/* prev block in aset's blocks list, if any */
AllocBlock	next;			/* next block in aset's blocks list, if any */

• 标识可用空间范围

char	   *freeptr;		/* start of free space in this block */
char	   *endptr;			/* end of space in this block */

AllocChunk：

• chunk中可用大小

Size		size;

3.2 初始化

#define AllocSetContextCreate \
	AllocSetContextCreateInternal

#define ALLOCSET_DEFAULT_MINSIZE   0
#define ALLOCSET_DEFAULT_INITSIZE  (8 * 1024)
#define ALLOCSET_DEFAULT_MAXSIZE   (8 * 1024 * 1024)
#define ALLOCSET_DEFAULT_SIZES \
	ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE

TopMemoryContext = AllocSetContextCreate((MemoryContext) NULL,
										 "TopMemoryContext",
										 ALLOCSET_DEFAULT_SIZES);

MemoryContext
AllocSetContextCreateInternal(MemoryContext parent,
							  const char *name,
							  Size minContextSize,
							  Size initBlockSize,
							  Size maxBlockSize)
{
	int			freeListIndex;
	Size		firstBlockSize;
	AllocSet	set;
	AllocBlock	block;
...
	/* Determine size of initial block */
	firstBlockSize = MAXALIGN(sizeof(AllocSetContext)) +
		ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
	if (minContextSize != 0)
		firstBlockSize = Max(firstBlockSize, minContextSize);
	else
		firstBlockSize = Max(firstBlockSize, initBlockSize);

	/*
	 * Allocate the initial block.  Unlike other aset.c blocks, it starts with
	 * the context header and its block header follows that.
	 */
	set = (AllocSet) malloc(firstBlockSize);
	...
	/* Fill in the initial block's block header */
	block = (AllocBlock) (((char *) set) + MAXALIGN(sizeof(AllocSetContext)));
	block->aset = set;
	block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
	block->endptr = ((char *) set) + firstBlockSize;
	block->prev = NULL;
	block->next = NULL;
...
	/* Remember block as part of block list */
	set->blocks = block;
	/* Mark block as not to be released at reset time */
	set->keeper = block;

...
	set->initBlockSize = initBlockSize;
	set->maxBlockSize = maxBlockSize;
	set->nextBlockSize = initBlockSize;
	set->freeListIndex = freeListIndex;
...
	set->allocChunkLimit = ALLOC_CHUNK_LIMIT;
	while ((Size) (set->allocChunkLimit + ALLOC_CHUNKHDRSZ) >
		   (Size) ((maxBlockSize - ALLOC_BLOCKHDRSZ) / ALLOC_CHUNK_FRACTION))
		set->allocChunkLimit >>= 1;

	/* Finally, do the type-independent part of context creation */
	MemoryContextCreate((MemoryContext) set,
						T_AllocSetContext,
						&AllocSetMethods,
						parent,
						name);

	((MemoryContext) set)->mem_allocated = firstBlockSize;

	return (MemoryContext) set;
}

3.3 分配空间

void *
palloc(Size size)
{
	/* duplicates MemoryContextAlloc to avoid increased overhead */
	void	   *ret;
	MemoryContext context = CurrentMemoryContext;
...
	context->isReset = false;

	ret = context->methods->alloc(context, size);
...
	return ret;
}

3.3.1 分配空间大于allocChunkLimit

static void *
AllocSetAlloc(MemoryContext context, Size size)
{
	AllocSet	set = (AllocSet) context;
	AllocBlock	block;
	AllocChunk	chunk;
	int			fidx;
	Size		chunk_size;
	Size		blksize;

...
	/*
	 * If requested size exceeds maximum for chunks, allocate an entire block
	 * for this request.
	 */
	if (size > set->allocChunkLimit)
	{
		chunk_size = MAXALIGN(size);
		blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
		block = (AllocBlock) malloc(blksize);
		if (block == NULL)
			return NULL;

		context->mem_allocated += blksize;

		block->aset = set;
		block->freeptr = block->endptr = ((char *) block) + blksize;

		chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ);
		chunk->aset = set;
		chunk->size = chunk_size;

		/*
		 * Stick the new block underneath the active allocation block, if any,
		 * so that we don't lose the use of the space remaining therein.
		 */
		if (set->blocks != NULL)
		{
			block->prev = set->blocks;
			block->next = set->blocks->next;
			if (block->next)
				block->next->prev = block;
			set->blocks->next = block;
		}
		else
		{
			block->prev = NULL;
			block->next = NULL;
			set->blocks = block;
		}

		return AllocChunkGetPointer(chunk);
	}
...
}

例如分配16K

char *ptr = palloc(16 * 1024);

例如分配32K

char *ptr = palloc(32 * 1024);

3.3.2 分配空间小于等于allocChunkLimit

#define ALLOC_MINBITS		3	/* smallest chunk size is 8 bytes */

static inline int
AllocSetFreeIndex(Size size)
{
	int			idx;

	if (size > (1 << ALLOC_MINBITS))
	{
...
		idx = 31 - __builtin_clz((uint32) size - 1) - ALLOC_MINBITS + 1;
...
	}
	else
		idx = 0;

	return idx;
}

fidx = AllocSetFreeIndex(size);

/*
 * Choose the actual chunk size to allocate.
 */
chunk_size = (1 << ALLOC_MINBITS) << fidx;

• 分配小于等于8字节
  小于等于8的，最终将分配8字节

• 分配大于8字节
  通过__builtin_clz计算size-1的前导0的个数，然后通过减法计算size的对其位数（非常高效的方式）

3.3.2.1 当前block有足够空间

if ((block = set->blocks) != NULL)
{
...
}
/*
	 * OK, do the allocation
	 */
	chunk = (AllocChunk) (block->freeptr);
...
	block->freeptr += (chunk_size + ALLOC_CHUNKHDRSZ);
...
	chunk->aset = (void *) set;
	chunk->size = chunk_size;
...
	return AllocChunkGetPointer(chunk);

例如分配20byte

char *ptr = palloc(20);

通过对其，实际分配32byte

3.3.2.2 当前block没有足够的空间

随着分配，当前block中的可用空间越来越小，再次分配时，将不够当前分配

Size availspace = block->endptr - block->freeptr;
if (availspace < (chunk_size + ALLOC_CHUNKHDRSZ))
{
...
}

3.3.2.2.1 当前block还有一些空间

if (availspace < (chunk_size + ALLOC_CHUNKHDRSZ))
	{
		while (availspace >= ((1 << ALLOC_MINBITS) + ALLOC_CHUNKHDRSZ))
		{
			Size		availchunk = availspace - ALLOC_CHUNKHDRSZ;
			int			a_fidx = AllocSetFreeIndex(availchunk);

...
			if (availchunk != ((Size) 1 << (a_fidx + ALLOC_MINBITS)))
			{
				a_fidx--;
				Assert(a_fidx >= 0);
				availchunk = ((Size) 1 << (a_fidx + ALLOC_MINBITS));
			}

			chunk = (AllocChunk) (block->freeptr);
...
			block->freeptr += (availchunk + ALLOC_CHUNKHDRSZ);
			availspace -= (availchunk + ALLOC_CHUNKHDRSZ);

			chunk->size = availchunk;

			chunk->aset = (void *) set->freelist[a_fidx];
			set->freelist[a_fidx] = chunk;
		}

		/* Mark that we need to create a new block */
		block = NULL;
	}

将剩余空间大小，按照对其的大小放到freelist中，构成一个单链表，用于后续分配时，直接从这里获取。
可能会有一小块用不了的空间

3.3.2.2.2 创建新block

• 第一次创建时，使用initBlockSize
• 后续将会是上次block的两倍，当大于maxBlockSize，使用maxBlockSize
• 如果新block大小依然小于需要分配的空间，则继续扩大

/*
	 * Time to create a new regular (multi-chunk) block?
	 */
	if (block == NULL)
	{
		Size		required_size;

		/*
		 * The first such block has size initBlockSize, and we double the
		 * space in each succeeding block, but not more than maxBlockSize.
		 */
		blksize = set->nextBlockSize;
		set->nextBlockSize <<= 1;
		if (set->nextBlockSize > set->maxBlockSize)
			set->nextBlockSize = set->maxBlockSize;

		/*
		 * If initBlockSize is less than ALLOC_CHUNK_LIMIT, we could need more
		 * space... but try to keep it a power of 2.
		 */
		required_size = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
		while (blksize < required_size)
			blksize <<= 1;

		/* Try to allocate it */
		block = (AllocBlock) malloc(blksize);
...
		if (block == NULL)
			return NULL;

		context->mem_allocated += blksize;

		block->aset = set;
		block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
		block->endptr = ((char *) block) + blksize;

		/* Mark unallocated space NOACCESS. */
		VALGRIND_MAKE_MEM_NOACCESS(block->freeptr,
								   blksize - ALLOC_BLOCKHDRSZ);

		block->prev = NULL;
		block->next = set->blocks;
		if (block->next)
			block->next->prev = block;
		set->blocks = block;
	}

3.3.3 从freelist中分配

fidx = AllocSetFreeIndex(size);
chunk = set->freelist[fidx];
if (chunk != NULL)
{
	set->freelist[fidx] = (AllocChunk) chunk->aset;

	chunk->aset = (void *) set;
...
	return AllocChunkGetPointer(chunk);
}

3.4 释放空间

void
pfree(void *pointer)
{
	MemoryContext context = GetMemoryChunkContext(pointer);

	context->methods->free_p(context, pointer);
...
}

3.4.1 直接释放

static void
AllocSetFree(MemoryContext context, void *pointer)
{
	AllocSet	set = (AllocSet) context;
	AllocChunk	chunk = AllocPointerGetChunk(pointer);
...
	if (chunk->size > set->allocChunkLimit)
	{
		AllocBlock	block = (AllocBlock) (((char *) chunk) - ALLOC_BLOCKHDRSZ);

...
		/* OK, remove block from aset's list and free it */
		if (block->prev)
			block->prev->next = block->next;
		else
			set->blocks = block->next;
		if (block->next)
			block->next->prev = block->prev;

		context->mem_allocated -= block->endptr - ((char *) block);
...
		free(block);
	}
	else
	{
		...
	}
}

见 3.3.1, 大于chunklimit的直接malloc的空间，而不是从block中分配的，因此直接释放，并且将此block从双向链表中删除。

3.4.2 加入freelist中

static void
AllocSetFree(MemoryContext context, void *pointer)
{
	AllocSet	set = (AllocSet) context;
	AllocChunk	chunk = AllocPointerGetChunk(pointer);

	/* Allow access to private part of chunk header. */
	VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOCCHUNK_PRIVATE_LEN);

#ifdef MEMORY_CONTEXT_CHECKING
	/* Test for someone scribbling on unused space in chunk */
	if (chunk->requested_size < chunk->size)
		if (!sentinel_ok(pointer, chunk->requested_size))
			elog(WARNING, "detected write past chunk end in %s %p",
				 set->header.name, chunk);
#endif

	if (chunk->size > set->allocChunkLimit)
	{
		...
	}
	else
	{
		/* Normal case, put the chunk into appropriate freelist */
		int			fidx = AllocSetFreeIndex(chunk->size);

		chunk->aset = (void *) set->freelist[fidx];
...
		set->freelist[fidx] = chunk;
	}
}