前一篇文章追踪heap使用,检测内存泄漏?既然没有解决问题(上一篇主要是帮助发现问题),那么能不能避开缺省堆呢,使用自己管理的堆,调用接口又能完全兼容呢?于是,有了如下想法:
1. 使用一大段全局区作为自定义的堆(代码中的g_heap[MAX_HEAP_SIZE]),所有的堆分配请求,和用于堆管理的内存(HEAP_BLOCK)都使用这段内存。
2. 将这一大段区域,处理成HEAP_BLOCK为结点的双向链表,每个结点HEAP_BLOCK要么处于已分配状态(ALLOCATED),要么可分配状态(FREE),每个结点还包含内存大小等信息,具体见HEAP_BLOCK定义。注意:为了避免使用系统缺省堆,结点在自定义heap区域是依次连续的,只是结点状态和内存大小会随着分配和释放不停变化。初始状态下,整个自定义heap区域被视为一个FREE态的HEAP_BLOCK。
3. 分配请求的处理:在链表中寻找第一个未分配且buff足够大的HEAP_BLOCK,如找到就进行分配(即1个FREE的HEAP_BLOCK可能分裂成1个ALLOCATED的HEAP_BLOCK和1个FREE的HEAP_BLOCK);如果没找到,先尝试合并连续的FREE态的HEAP_BLOCK(即将多个连续的FREE态HEAP_BLOCK合并成一个大的FREE态HEAP_BLOCK)。
4. 释放请求的处理:根据传入的地址,将对应的HEAP_BLOCK标记为FREE。
5. 已考虑:统计自定义堆的内存使用峰值,避免多个很小的FREE态HEAP_BLOCK(避免太多内存碎片),等等,具体见代码。
6. 未考虑:线程安全;已尽可能考虑数据大小溢出的问题。
以下是代码(macro USE_MY_HEAP包含的是本文描述的内容,macro USE_SYS_HEAP_TRACK包含的是上一篇文章描述的内容),和遇到问题的简单总结。
完整的头文件的内容如下(heap_wrapper.h):
#pragma once
#define USE_MY_HEAP //use the heap implemented based on a big range of global data
//#define USE_SYS_HEAP_TRACK //use the system heap and we track the heap usage
//#define USE_SYS_HEAP_NO_TRACK //just the system heap
#if defined(USE_MY_HEAP)
void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num);
void custom_free(void* ptr);
void print_heap_info();
#define NEW_CALLOC(num,size) custom_calloc(num,size,__FUNCTION__,__LINE__)
#define NEW_FREE(ptr) custom_free(ptr)
#define PRINT_HEAP_INFO() print_heap_info()
#endif
#if defined(USE_SYS_HEAP_TRACK)
void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num);
void custom_free(void* ptr);
void print_heap_info();
#define NEW_CALLOC(num,size) custom_calloc(num,size,__FUNCTION__,__LINE__)
#define NEW_FREE(ptr) custom_free(ptr)
#define PRINT_HEAP_INFO() print_heap_info()
#endif
#if defined(USE_SYS_HEAP_NO_TRACK)
#define NEW_CALLOC calloc
#define NEW_FREE free
#define PRINT_HEAP_INFO() print_heap_info()
#endif
完整的源代码文件如下(heap_wrapper.c):
#include "heap_wrapper.h"
#if defined(USE_MY_HEAP)
//use a big range of global area as the heap
//all heap request from callers and heap management is based on it
//the global heap area is a list of blocks, either in allocated state (used for a heap memory request) or free state(not used and available for request)
//each block begins with a fixed-size block information (state, start/size, function_name/line_num, next/previous) and a variable buffer (specified by start/size in block information)
//initially, the whole heap area is one block of free area
//Process a calloc request:
//step 1: search the first free block that is big enough,
//step 2: if found,
//step 2.1: do the allocation: that is 1 free block changed into 1 allocated block and 1 free block(if more bytes left)
//Step 2.2: inserted the 2 new blocks into the list to replace the original one
//Step 3: if not found, merge the continous free blocks: multiple continous free block are replaced with 1 big free block in the list, goto step 1 and try again
//Process a free request:
//per the input address, find the block in the list and mark it as free
//limitation not thread safe
//if after a allocate request, the left memory is not big enough, not to mark it as a new free block
//to avoid too many small free blocks
#define MIN_BLOCK_AVAILABLE_BUFF_SIZE (32)
#define MAX_HEAP_SIZE (2*1024*1024) //2MB
#dfine MAX_FUNCTION_NAME_LEN (32)
static char g_function_name[MAX_FUNCTION_NAME_LEN] = "unkown";
static uint8_t g_heap[MAX_HEAP_SIZE];
typedef enum _HEAP_BLOCK_STATE_
{
HEAP_BLOCK_FREE = 0,
HEAP_BLOCK_ALLOCATED,
}HEAP_BLOCK_STATE;
typedef struct _HEAP_BLOCK_ HEAP_BLOCK, *PHEAP_BLOCK;
typedef struct _HEAP_BLOCK_
{
HEAP_BLOCK_STATE state;
void* start; //start of buffer for caller's request (not including this HEAP_BLOCK)
uint32_t size; //buffer size available for caller's request (not including this HEAP_BLOCK)
uint32_t line_num;
char function_name[MAX_FUNCTION_NAME_LEN]; //NOTE: trunc it if too long
PHEAP_BLOCK next;
PHEAP_BLOCK previous;
}HEAP_BLOCK, *PHEAP_BLOCK;
typedef struct _HEAP_CONTEXT_
{
uint64_t heap_size; //size of all the reserved heap memory
uint64_t peak_size; //peak size of all the requests from callers
uint64_t total_alloc_size; //current allocated size from callers
uint32_t total_alloc_count; //increment for every request, decrement for every free
uint32_t total_unexpected; //tracking unexpected cases
uint32_t total_merge_count; //count of merging continous free areas
PHEAP_BLOCK head;
PHEAP_BLOCK tail;
}HEAP_CONTEXT;
static HEAP_CONTEXT g_heap_context;
//initially, the whole heap area is one block of free area
static void init_heap_context()
{
if(NULL == g_heap_context.head)
{
PHEAP_BLOCK first_block = (PHEAP_BLOCK)g_heap;
memset(first_block, 0, sizeof(HEAP_BLOCK));
first_block->state = HEAP_BLOCK_FREE;
first_block->start = (uint8_t*)first_block + sizeof(HEAP_BLOCK);
first_block->size = (uint32_t)(sizeof(g_heap) - sizeof(HEAP_BLOCK));
//populate the g_heap_context
memset(&g_heap_context, 0, sizeof(g_heap_context));
g_heap_context.heap_size = (uint64_t)sizeof(g_heap);
g_heap_context.head = first_block;
g_heap_context.tail = first_block;
}
}
//merge the continous free blocks: multiple continous free block are replaced with 1 big free block in the list
static void merge_continuous_free_block()
{
PHEAP_BLOCK free_block = NULL; //first free block found
PHEAP_BLOCK next_allocated_block = NULL; //next allocated block after <free_block> in the list,
PHEAP_BLOCK temp = NULL;
uint32_t total_size = 0;
uint8_t meet_merge_condition = 0;
if(g_heap_context.head)
{
free_block = g_heap_context.head;
while(free_block)
{
while(free_block && (HEAP_BLOCK_FREE != free_block.state)) //to find the free block
{
free_block = free_block.next;
}
if(free_block) //found a free block, then to find next allocated block
{
next_allocated_block = free_block.next;
while(next_allocated_block && (HEAP_BLOCK_ALLOCATED != next_allocated_block.state)) //to find the allocated block
{
next_allocated_block = next_allocated_block->next;
}
//merge condition: more than 1 free blocks (including <free_block>), multiple free blocks in the end of list
//merge condition: more than 1 free blocks (including <free_block>) | allocated block
meet_merge_condition = 0;
if(NULL == next_allocated_block && NULL != free_block->next)
{
meet_merge_condition = 1;
}
if((NULL != next_allocated_block) && (next_allocated_block != free_block->next)) //not the merge condition
{
meet_merge_condition = 1;
}
if(!meet_merge_condition) //not the merge condition
{
free_block = next_allocated_block;
continue;
}
else //meet the merge condition: from <free_block> to revious of <next_allocated_block> (could be NULL) to merge into 1 big free block
{
temp = free_block;
total_size = 0;
//calc the total size
while(temp->next != next_allocated_block)
{
total_size += (sizeof(HEAP_BLOCK) + temp->size);
temp = temp->next;
}
free_block->size = total_size - sizeof(HEAP_BLOCK);
free_block->next = next_allocated_block;
//no changes to other fields
memset(free_block->start, 0, free_block->size);
if(NULL != next_allocated_block)
{
next_allocated_block->previous = free_block;
}
else
{
g_heap_context.tail = free_block;
}
free_block = next_allocated_block;
g_heap_context.total_merge_count++;
}
}
}
}
}
void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num)
{
void* ret_addr = NULL;
uint32_t request_size = num * size;
uint32_t temp_available_size = 0;
uint32_t merge_count = 0;
uint8_t retried_flag = 0;
PHEAP_BLOCK temp_block = NULL;
PHEAP_BLOCK new_block = NULL;
char* func_name = NULL;
if(request_size <= 0 || request_size > sizeof(g_heap))
{
LOGD("heap input wrong size %d", request_size);
return NULL;
}
if(NULL == g_heap_context.head)
{
init_heap_context();
}
func_name = (char*)function_name;
if(NULL == func_name)
{
func_name = g_function_name;
}
LOGD("heap input size %d, %s:%d", request_size, func_name, line_num);
try_again:
//search the first free block that is big enough
temp_block = g_heap_context.head;
while(temp_block)
{
if((HEAP_BLOCK_FREE == temp_block->state) && (temp_block->size >= request_size))
{
break;
}
else
{
temp_block = temp_block->next;
}
}
//if not found, merge the continous free blocks: multiple continous free block are replaced with 1 big free block in the list, goto step 1 and try again
if(NULL == temp_block)
{
if(0 == retried_flag)
{
retried_flag = 1;
g_heap_context.total_unexpected++;
merge_count = g_heap_context.total_merge_count;
merge_continuous_free_block();
if(merge_count != g_heap_context.total_merge_count)
{
goto try_again;
}
}
}
else
{
//step 2.1: do the allocation: that is 1 free block changed into 1 allocated block and 1 free block(if more bytes left)
//Step 2.2: inserted the 2 new blocks into the list to replace the original one
//big enough but not too much, not split it
if(temp_block->size < (request_size + sizeof(HEAP_BLOCK) + MIN_BLOCK_AVAILABLE_BUFF_SIZE)) //NOTE: this is to avoid too many small free blocks
{
ret_addr = temp_block->start;
temp_block->state = HEAP_BLOCK_ALLOCATED;
//temp_block->size not changed //the whold block is allocated and no new free block generated
temp_block->line_num = line_num;
strncpy(temp_block->function_name, func_name, sizeof(temp_block->function_name));
}
else //big enough and too much, split into 2
{
ret_addr = temp_block->start;
temp_available_size = temp_block->size; //split into request_size | new_block | available size of new_block
//first block of the 2
temp_block->state = HEAP_BLOCK_ALLOCATED;
//temp_block->start //not changed
temp_block->size = request_size;
temp_block->line_num = line_num;
strncpy(temp_block->function_name, func_name, sizeof(temp_block->function_name));
//second block of the 2
new_block = (PHEAP_BLOCK)((uint8_t*)temp_block->start + temp_block->size);
new_block->state = HEAP_BLOCK_FREE;
new_block->start = (uint8_t*)new_block + sizeof(HEAP_BLOCK);
new_block->size = temp_available_size - request_size - sizeof(HEAP_BLOCK);
new_block->next = temp_block->next;
new_block->previous = temp_block;
temp_block->next = new_block;
if(g_heap_context.tail == temp_block)
{
g_heap_context.tail = new_block;
}
}
g_heap_context.total_alloc_count++;
g_heap_context.total_alloc_size += temp_block->size;
if(g_heap_context.total_alloc_size > g_heap_context.peak_size)
{
g_heap_context.peak_size = g_heap_context.total_alloc_size;
}
}
if(ret_addr)
{
print_heap_info();
}
else
{
LOGD("heap alloc failed");
}
return ret_addr;
}
void custom_free(void* ptr)
{
PHEAP_BLOCK temp_block = NULL;
if(ptr)
{
temp_block = g_heap_context.head;
while(temp_block && (ptr != temp_block->start))
{
temp_block = temp_block->next;
}
if(NULL == temp_block)
{
g_heap_context.total_unexpected++;
LOGD("heap address not found");
}
else
{
if(HEAP_BLOCK_ALLOCATED != temp_block->state)
{
g_heap_context.total_unexpected++;
}
LOGD("heap free node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
temp_block->state = HEAP_BLOCK_FREE;
memset(temp_block->function_name, 0, sizeof(temp_block->function_name));
temp_block->line_num = 0;
memset(temp_block->start, 0, temp_block->size);
g_heap_context.total_alloc_count--;
g_heap_context.total_alloc_size -= temp_block->size;
}
}
}
void print_heap_info()
{
PHEAP_BLOCK temp_block = g_heap_context.head;
while(NULL != temp_block)
{
if(HEAP_BLOCK_ALLOCATED == temp_block->state)
{
LOGD("heap alloc node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
}
else
{
LOGD("heap free node size %d", temp_block->size);
}
temp_block = temp_block->next;
}
LOGD("heap count %d, size %lld, peak %lld, fail %d, merge %d", g_heap_context.total_alloc_count,
g_heap_context.total_alloc_size, g_heap_context.peak_size, g_heap_context.total_unexpected, g_heap_context.total_merge_count);
}
#endif
#if defined(USE_SYS_HEAP_TRACK) //track the system heap usage information
//just track the function_name:line_number which requests buffer from heap
//track when it's freed
//then when know if there is any heap memory leak
#define MAX_FUNCTION_NAME_LEN (32)
typedef struct _HEAP_BLOCK_ HEAP_BLOCK, *PHEAP_BLOCK;
typedef struct _HEAP_BLOCK_
{
void* start; //system allocated buffer address
uint32_t size; //request size from callers
uint32_t line_num;
char function_name[MAX_FUNCTION_NAME_LEN]; //NOTE: trunc it if too long
PHEAP_BLOCK next;
}HEAP_BLOCK, *PHEAP_BLOCK;
typedef struct _HEAP_CONTEXT_
{
uint64_t total_alloc_size; //current allocated size from callers
uint32_t total_alloc_count; //increment for every request, decrement for every free
uint32_t total_unexpected; //tracking unexpected cases
PHEAP_BLOCK head;
PHEAP_BLOCK tail;
}HEAP_CONTEXT;
static HEAP_CONTEXT g_heap_context;
void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num)
{
void* ret_addr = calloc(num * size);
LOGD("heap input size %d, %s:%d", num * size, function_name, line_num);
if(ret_addr)
{
PHEAP_BLOCK temp_block = (PHEAP_BLOCK)calloc(1, sizeof(HEAP_BLOCK));
if(temp_block)
{
temp_block->start = ret_addr;
temp_block->size = num * size;
temp_block->line_num = line_num;
strncpy(temp_block->function_name, func_name, sizeof(temp_block->function_name));
g_heap_context.total_alloc_count++;
g_heap_context.total_alloc_size += temp_block->size;
if(NULL == g_heap_context.head)
{
g_heap_context.head = temp_block;
g_heap_context.tail = temp_block;
}
else
{
g_heap_context.tail->next = temp_block;
g_heap_context.tail = temp_block;
}
}
else
{
g_heap_context.total_unexpected++;
}
}
else
{
g_heap_context.total_unexpected++;
}
if(ret_addr)
{
print_heap_info();
}
else
{
LOGD("heap alloc failed");
}
return ret_addr;
}
void custom_free(void* ptr)
{
PHEAP_BLOCK temp_block = NULL;
PHEAP_BLOCK previous_block = NULL;
if(ptr)
{
temp_block = g_heap_context.head;
previous_block = NULL;
while(temp_block && (ptr != temp_block->start))
{
previous_block = temp_block;
temp_block = temp_block->next;
}
if(NULL == temp_block)
{
g_heap_context.total_unexpected++;
LOGD("heap address not found");
}
else
{
LOGD("heap free node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
if(g_heap_context.head != temp_block && g_heap_context.tail != temp_block )
{
previous_block->next = temp_block->next;
}
else //found block could be head or tail or head and tail(only 1 block)
{
if(g_heap_context.head == temp_block)
{
g_heap_context.head = temp_block->next;
}
if(g_heap_context.tail == temp_block)
{
g_heap_context.tail = previous_block;
if(NULL != previous_block)
{
previous_block->next = NULL;
}
}
}
free(temp_block);
g_heap_context.total_alloc_count--;
g_heap_context.total_alloc_size -= temp_block->size;
}
free(ptr);
}
}
void print_heap_info()
{
PHEAP_BLOCK temp_block = g_heap_context.head;
while(NULL != temp_block)
{
LOGD("heap alloc node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
temp_block = temp_block->next;
}
LOGD("heap count %d, size %lld, fail %d", g_heap_context.total_alloc_count, g_heap_context.total_alloc_size, g_heap_context.total_unexpected);
}
#endif
调试过程中的问题总结:
1. 使用自定义堆,会直接增大编译生成的二进制文件大小;可以减少编译脚本(链接文件)中的HEAP_SIZE,有时可能要权衡一下。问题来了,这个HEAP_SZIE可以减少到0吗? 肯定是不行的,虽然我们代码不用缺省堆,但调用我们代码的,和我们代码调用的,对我们来说都是黑盒子,能保证他们都不用缺省堆吗?保险起见,可以先使用缺省值吧。
2. 那么这个全局区域最大设为多少呢? 可以根据peak_size统计进行设置,甚至也可以实现方案略加修改,变为自定义堆+缺省堆的方案。
3. 低级错误1:日志打印的%s和%d与实际的参数不对应导致的问题。由此可见,本地搭建完整的测试环境的重要性。
4. 低级错误2:最初内存分配custom_calloc的入参num/size为uint16_t,程序实际使用过程中会溢出。由此可见,测试驱动开发,测试用例设计的重要性。
5. 低级错误3:custom_calloc中最初实现的查找“第一个未分配且buff足够大的HEAP_BLOCK”的while循环的条件没有指定正确。这类条件设定,逻辑一定要越简单越清晰越好。
那最终问题解决了吗? 解决了,不是内存泄漏,不是一个问题。
【2022年5.1节 重庆】