Nginx 内存池剖析

为什么要使用 Nginx 内存池

传统直接调用内存分配函数的弊端

弊端一:
高并发时较小的内存块使用导致系统调用频繁,降低了系统的执行效率

弊端2:
频繁使用时增加了系统内存的碎片，降低内存使用效率
内部碎片 － 已经被分配出去（能明确指出属于哪个进程）却不能被利用的内存空间；
产生根源：1.内存分配必须起始于可被 4、8 或 16 整除（视处理器体系结构而定）的地址
2.MMU的分页机制的限制

弊端三:
没有垃圾回收机制，容易造成内存泄漏，导致内存枯竭

弊端4:
内存分配与释放的逻辑在程序中相隔较远时，降低程序的稳定性,就是当指针做为函数的返回值时，无法确定指针是否指向的是栈中的内存还是堆上的内存

弊端的解决之道

系统层:使用高性能内存管理组件 Tcmalloc Jemalloc（优化效率和碎片问题）使用动态链接库接管 Glibc Ptmalloc 实现.这里不做多介绍
应用层:使用内存池技术

什么是Nginx 内存池

什么是内存池技术

就是在使用内存之前先分配一定数量、大小相等(一般情况下)的内存，当有内存需求时再从内存块中取出一部分，若内存块不够再继续申请新内存，统一对内存进行分配和释放，一个显著的优点是对内存的的分配效率大大提升.

内存池如何解决弊端

高并发时较小的内存块使用导致系统调用频繁,降低了系统的执行效率
解决方案：内存池提前预先分配大块内存，统一释放，极大的减少了malloc 和 free 等函数的调用。
频繁使用时增加了系统内存的碎片，降低内存使用效率
解决方案:内存池每次请求分配大小适度的内存块，避免了碎片的产生
没有垃圾回收机制，容易造成内存泄漏，导致内存枯竭
解决方案:在生命周期结束后统一释放内存，完全避免了内存泄露的产生
内存分配与释放的逻辑在程序中相隔较远时，降低程序的稳定性,就是当指针做为函数的返回值时，无法确定指针是否指向的是栈中的内存还是堆上的内存
解决方案:在生命周期结束后统一释放内存，避免重复释放指针或释放空指针等情况

内存池的设计思想

分而治之

对于每个请求或者连接都会建立相应的内存池，建立好内存池之后，我们可以直接从内存池中申请所需要的内存，不用去管内存的释放，当内存池使用完成之后一次性销毁内存池。

区分大小内存块的申请和释放，大于池尺寸的定义为大内存块，使用单独的大内存块链表保存，即时分配和释放；小于等于池尺寸的定义为小内存块，直接从预先分配的内存块中提取，不够就扩充池中的内存，在生命周期内对小块内存不做释放，直到最后统一销毁。

Nginx 内存池结构体图

结构体:
typedef struct ngx_pool_data_t{ //小结构体模块
uchar last; //保存当前内存块起始位置
uchar end; //保存当前内存块结束位置
ngx_pool_t *next; //内存块由多快数据快构成，指向下一个分配块
int failed; //内存分配失败的次数
};
//分配快
typedef struct ngx_pool_t{ //内存分配模块
ngx_pool_data_t d; //指向小结构模块
size_t max; //小结构体能够分配的最大内存
ngx_pool_t *current; //当前正在工作的小结构体模块
ngx_large_pool_t *alloc; //大内存块
};

typedef struct large_pool_t{
large_pool_t *next; //指向下一块大内存块 类似单链表中的结构体定义
void *alloc; //分配的数据块
}large_pool_t;

内存池模块部分源代码

mem_core.h

/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) Nginx, Inc.
 */


#ifndef _NGX_CORE_H_INCLUDED_
#define _NGX_CORE_H_INCLUDED_


#define NGX_HAVE_POSIX_MEMALIGN  1


typedef struct ngx_pool_s            ngx_pool_t;


#define  NGX_OK          0
#define  NGX_ERROR      -1
#define  NGX_AGAIN      -2
#define  NGX_BUSY       -3
#define  NGX_DONE       -4
#define  NGX_DECLINED   -5
#define  NGX_ABORT      -6



#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <error.h>
#include <sys/stat.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>

typedef intptr_t        ngx_int_t;
typedef uintptr_t       ngx_uint_t;

#define NGX_ALIGNMENT   sizeof(unsigned long)    /* platform word */
#define ngx_align(d, a)     (((d) + (a - 1)) & ~(a - 1))
#define ngx_align_ptr(p, a)                                                   \
    (u_char *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1))

#define ngx_memzero(buf, n)       (void) memset(buf, 0, n)

#include "mem_alloc.h"
#include "mem_pool_palloc.h"


#endif /* _NGX_CORE_H_INCLUDED_ */

mem_alloc.h

/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) Nginx, Inc.
 */

#ifndef _NGX_ALLOC_H_INCLUDED_
#define _NGX_ALLOC_H_INCLUDED_



#include "mem_core.h"


void *ngx_alloc(size_t size);
void *ngx_calloc(size_t size);

#define ngx_free         free


/*
 * Linux has memalign() or posix_memalign()
 * Solaris has memalign()
 * FreeBSD 7.0 has posix_memalign(), besides, early version's malloc()
 * aligns allocations bigger than page size at the page boundary
 */

/*#if (NGX_HAVE_POSIX_MEMALIGN || NGX_HAVE_MEMALIGN)

void *ngx_memalign(size_t alignment, size_t size);

#else
*/
#define ngx_memalign(alignment, size)  ngx_alloc(size)
/*
#endif
*/

extern ngx_uint_t  ngx_pagesize;
extern ngx_uint_t  ngx_pagesize_shift;
extern ngx_uint_t  ngx_cacheline_size;


#endif /* _NGX_ALLOC_H_INCLUDED_ */

mem_pool_palloc

/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) Nginx, Inc.
 */


#include "mem_core.h"


static inline void *ngx_palloc_small(ngx_pool_t *pool, size_t size,
    ngx_uint_t align);
static void *ngx_palloc_block(ngx_pool_t *pool, size_t size);
static void *ngx_palloc_large(ngx_pool_t *pool, size_t size);


ngx_pool_t *
ngx_create_pool(size_t size)
{
    ngx_pool_t  *p;

    p = ngx_memalign(NGX_POOL_ALIGNMENT, size);	
    if (p == NULL) {
        return NULL;
    }

    p->d.last = (u_char *) p + sizeof(ngx_pool_t);
    p->d.end =  (u_char *) p + size;
    p->d.next = NULL;
    p->d.failed = 0;

    size = size - sizeof(ngx_pool_t);

    p->max = (size < NGX_MAX_ALLOC_FROM_POOL) ? size : NGX_MAX_ALLOC_FROM_POOL;

    p->current = p;
    //p->chain = NULL;
    p->large = NULL;
    //p->cleanup = NULL;
    //p->log = log;

    return p;
}


void
ngx_destroy_pool(ngx_pool_t *pool)
{
    ngx_pool_t          *p, *n;
    ngx_pool_large_t    *l;
    //ngx_pool_cleanup_t  *c;

    /*for (c = pool->cleanup; c; c = c->next) {
        if (c->handler) {
            ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, pool->log, 0,
                           "run cleanup: %p", c);
            c->handler(c->data);
        }
    }*/

#if (NGX_DEBUG)

    /*
     * we could allocate the pool->log from this pool
     * so we cannot use this log while free()ing the pool
     */

    for (l = pool->large; l; l = l->next) {
        fprintf(stderr,"free: %p", l->alloc);
    }

    for (p = pool, n = pool->d.next; /* void */; p = n, n = n->d.next) {
        fprintf(stderr,"free: %p, unused: %zu", p, p->d.end - p->d.last);

        if (n == NULL) {
            break;
        }
    }

#endif

    for (l = pool->large; l; l = l->next) {
        if (l->alloc) {
            ngx_free(l->alloc);
        }
    }

    for (p = pool, n = pool->d.next; /* void */; p = n, n = n->d.next) {
        ngx_free(p);

        if (n == NULL) {
            break;
        }
    }
}


void
ngx_reset_pool(ngx_pool_t *pool)
{
    ngx_pool_t        *p;
    ngx_pool_large_t  *l;

    for (l = pool->large; l; l = l->next) {
        if (l->alloc) {
            ngx_free(l->alloc);
        }
    }

    for (p = pool; p; p = p->d.next) {
        p->d.last = (u_char *) p + sizeof(ngx_pool_t);
        p->d.failed = 0;
    }

    pool->current = pool;
    //pool->chain = NULL;
    pool->large = NULL;
}


void *
ngx_palloc(ngx_pool_t *pool, size_t size)
{
#if !(NGX_DEBUG_PALLOC)
    if (size <= pool->max) {
        return ngx_palloc_small(pool, size, 1);
    }
#endif

    return ngx_palloc_large(pool, size);
}


void *
ngx_pnalloc(ngx_pool_t *pool, size_t size)
{
#if !(NGX_DEBUG_PALLOC)
    if (size <= pool->max) {
        return ngx_palloc_small(pool, size, 0);
    }
#endif

    return ngx_palloc_large(pool, size);
}


static inline void *
ngx_palloc_small(ngx_pool_t *pool, size_t size, ngx_uint_t align)
{
    u_char      *m;
    ngx_pool_t  *p;

    p = pool->current;

    do {
        m = p->d.last;

        if (align) {
            m = ngx_align_ptr(m, NGX_ALIGNMENT);
        }

        if ((size_t) (p->d.end - m) >= size) {
            p->d.last = m + size;

            return m;
        }

        p = p->d.next;

    } while (p);

    return ngx_palloc_block(pool, size);
}


static void *
ngx_palloc_block(ngx_pool_t *pool, size_t size)
{
    u_char      *m;

    size_t       psize;
    
	ngx_pool_t  *p, *new;

    psize = (size_t) (pool->d.end - (u_char *) pool);

    m = ngx_memalign(NGX_POOL_ALIGNMENT, psize);

    if (m == NULL) {
        return NULL;
    }

    new = (ngx_pool_t *) m;

    new->d.end = m + psize;
    new->d.next = NULL;
    new->d.failed = 0;

    m += sizeof(ngx_pool_data_t);
    m = ngx_align_ptr(m, NGX_ALIGNMENT);
    new->d.last = m + size;

    for (p = pool->current; p->d.next; p = p->d.next) {
        if (p->d.failed++ > 4) {
            pool->current = p->d.next;
        }
    }

    p->d.next = new;

    return m;
}


static void *
ngx_palloc_large(ngx_pool_t *pool, size_t size)
{
    void              *p;

    ngx_uint_t         n;
    ngx_pool_large_t  *large;

    p = ngx_alloc(size);
    if (p == NULL) {
        return NULL;
    }

    n = 0;

    for (large = pool->large; large; large = large->next) {
        if (large->alloc == NULL) {
            large->alloc = p;
            return p;
        }

        if (n++ > 3) {
            break;
        }
    }

    large = ngx_palloc_small(pool, sizeof(ngx_pool_large_t), 1);
    if (large == NULL) {
        ngx_free(p);
        return NULL;
    }

    large->alloc = p;
    large->next = pool->large;
    pool->large = large;

    return p;
}


void *
ngx_pmemalign(ngx_pool_t *pool, size_t size, size_t alignment)
{
    void              *p;
    ngx_pool_large_t  *large;

    p = ngx_memalign(alignment, size);
    if (p == NULL) {
        return NULL;
    }

    large = ngx_palloc_small(pool, sizeof(ngx_pool_large_t), 1);
    if (large == NULL) {
        ngx_free(p);
        return NULL;
    }

    large->alloc = p;
    large->next = pool->large;
    pool->large = large;

    return p;
}


ngx_int_t
ngx_pfree(ngx_pool_t *pool, void *p)
{
    ngx_pool_large_t  *l;

    for (l = pool->large; l; l = l->next) {
        if (p == l->alloc) {
            fprintf(stderr,"free: %p", l->alloc);
            ngx_free(l->alloc);
            l->alloc = NULL;

            return NGX_OK;
        }
    }

    return NGX_DECLINED;
}


void *
ngx_pcalloc(ngx_pool_t *pool, size_t size)
{
    void *p;

    p = ngx_palloc(pool, size);
    if (p) {
        ngx_memzero(p, size);
    }

    return p;
}

mem_alloc.c

/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) Nginx, Inc.
 */


#include "mem_core.h"

static int debug = 0;

ngx_uint_t  ngx_pagesize;
ngx_uint_t  ngx_pagesize_shift;
ngx_uint_t  ngx_cacheline_size;



void *
ngx_alloc(size_t size)
{
    void  *p;

    p = malloc(size);	//直接malloc
    if (p == NULL) {
        fprintf(stderr,"malloc(%zu) failed", size);
    }

    if(debug) fprintf(stderr, "malloc: %p:%zu", p, size);

    return p;
}


void *
ngx_calloc(size_t size)
{
    void  *p;

    p = ngx_alloc(size);

    if (p) {
        ngx_memzero(p, size);
    }

    return p;
}

/*
#if (NGX_HAVE_POSIX_MEMALIGN)

void *
ngx_memalign(size_t alignment, size_t size)
{
    void  *p;
    int    err;

    err = posix_memalign(&p, alignment, size);

    if (err) {
        fprintf(stderr,"posix_memalign(%zu, %zu) failed", alignment, size);
        p = NULL;
    }

    if(debug) fprintf(stderr,"posix_memalign: %p:%zu @%zu", p, size, alignment);

    return p;
}

#elif (NGX_HAVE_MEMALIGN)

void *
ngx_memalign(size_t alignment, size_t size)
{
    void  *p;

    p = memalign(alignment, size);
    if (p == NULL) {
        fprintf(stderr,"memalign(%zu, %zu) failed", alignment, size);
    }

    if(debug) fprintf(stderr,"memalign: %p:%zu @%zu", p, size, alignment);

    return p;
}

#endif
*/

//测试 main.cpp

#include "mem_core.h"


#define BLOCK_SIZE  16   //每次分配内存块大小

#define MEM_POOL_SIZE (1024 * 4) //内存池每块大小



int main(int argc, char **argv)
{
	int i = 0, k = 0;
	int use_free = 0;   

	ngx_pagesize = getpagesize();
	//printf("pagesize: %zu\n",ngx_pagesize);

	
	if(argc >= 2){
		use_free = 1;
		printf("use malloc/free\n");		
	} else {
             printf("use mempool.\n");
        }

	if(!use_free){
	    char * ptr = NULL;
	
	    for(k = 0; k< 1024 * 500; k++)
	    {
	        ngx_pool_t * mem_pool = ngx_create_pool(MEM_POOL_SIZE);
		    
		for(i = 0; i < 1024 ; i++)
		{
		    ptr = ngx_palloc(mem_pool,BLOCK_SIZE);

		    if(!ptr) fprintf(stderr,"ngx_palloc failed. \n");
		    else {
		         *ptr = '\0';
			 *(ptr + BLOCK_SIZE -1) = '\0';
                    }
		}
		    
                ngx_destroy_pool(mem_pool);
	    }
	} else {
	    char * ptr[1024];
	    for(k = 0; k< 1024 * 500; k++){
		for(i = 0; i < 1024 ; i++)
                {
                    ptr[i] = malloc(BLOCK_SIZE);
		    if(!ptr[i]) fprintf(stderr,"malloc failed. reason:%s\n",strerror(errno));
		    else{
		         *ptr[i] = '\0';
			 *(ptr[i] +  BLOCK_SIZE - 1) = '\0';
		    }
                }

		for(i = 0; i < 1024 ; i++){
		    if(ptr[i]) free(ptr[i]);
		}
	    }

	}
	return 0;
}

要点剖析：
分配内存块的初始化，ngx_pool_t 结构体初始化。
ngx_pool_t *
ngx_create_pool(size_t size)
{
ngx_pool_t *p;

p = ngx_memalign(NGX_POOL_ALIGNMENT, size);	//最大的大小
if (p == NULL) {
    return NULL;
}

p->d.last = (u_char *) p + sizeof(ngx_pool_t);//下次分配内存的起始位置
p->d.end =  (u_char *) p + size;                     //内存块的结束位置
p->d.next = NULL;										   
p->d.failed = 0;

size = size - sizeof(ngx_pool_t);

p->max = (size < NGX_MAX_ALLOC_FROM_POOL) ? size : NGX_MAX_ALLOC_FROM_POOL;//一个内存块能够分配的最大内存，

p->current = p;
//p->chain = NULL;							
p->large = NULL;
//p->cleanup = NULL;
//p->log = log;

return p;

}
//申请内存
void *
ngx_pnalloc(ngx_pool_t *pool, size_t size)
{ //判断 size 的大小和小内存块最大能够分配的内存相比，如果 小于等于使用小内存块，否则使用大内存块分配.
#if !(NGX_DEBUG_PALLOC)
if (size <= pool->max) {
return ngx_palloc_small(pool, size, 0);
}
#endif

return ngx_palloc_large(pool, size);

}
//尝试从大内存块获取内存
static void *
ngx_palloc_large(ngx_pool_t *pool, size_t size)
{
void *p;

ngx_uint_t         n;
ngx_pool_large_t  *large;

p = ngx_alloc(size);		//大内存块直接 malloc
if (p == NULL) {
    return NULL;
}

n = 0;

//需要的内存已经 malloc 了，但是 需要 large_pool_t 结构体的内存还未分配
for (large = pool->large; large; large = large->next) {
if (large->alloc == NULL) { //检查是否有闲置的 large_pool_t 类型的结构体
large->alloc = p;
return p;
}

    if (n++ > 3) {	// 查找 next 三次，直接退出手动分配这样更省时
        break;
    }
}

large = ngx_palloc_small(pool, sizeof(ngx_pool_large_t), 1);// 尝试从小内存块中 获取  sizeof(ngx_pool_large_t),大小的结构体
if (large == NULL) {
    ngx_free(p);
    return NULL;
}

large->alloc = p;
large->next = pool->large;
pool->large = large;

return p;

}

从小内存块 获取内存
static inline void *
ngx_palloc_small(ngx_pool_t *pool, size_t size, ngx_uint_t align)
{
u_char *m;
ngx_pool_t *p;

p = pool->current;
//从小内存块获取内存，逐个遍历
do {
    m = p->d.last;

    if (align) {
        m = ngx_align_ptr(m, NGX_ALIGNMENT);
    }

    if ((size_t) (p->d.end - m) >= size) {
        p->d.last = m + size;

        return m;
    }

    p = p->d.next;

} while (p);

return ngx_palloc_block(pool, size);  //小内存块剩余内存无法满足

}

//从新分配小内存块这样的结构体
static void *
ngx_palloc_block(ngx_pool_t *pool, size_t size)
{
u_char *m;

size_t       psize;

ngx_pool_t  *p, *new;

psize = (size_t) (pool->d.end - (u_char *) pool);

m = ngx_memalign(NGX_POOL_ALIGNMENT, psize);

if (m == NULL) {
    return NULL;
}

new = (ngx_pool_t *) m;

new->d.end = m + psize;
new->d.next = NULL;
new->d.failed = 0;

m += sizeof(ngx_pool_data_t);
m = ngx_align_ptr(m, NGX_ALIGNMENT);
new->d.last = m + size;


for (p = pool->current; p->d.next; p = p->d.next) {
    if (p->d.failed++ > 4) {   //对于已经，查找过失败四次的结构体 果断 将工作内存块往后偏移
        pool->current = p->d.next;
    }
}

p->d.next = new;	    //头插法

return m;

}
总结: Nginx的内存池设计的相当精巧，自 2004 年第一个版本上线，中间经过多次迭代，到今日仍被大量使用，可见研究 Nginx 线程池具有价值.