nginx源码学习1——扒内存池代码

学nginx源码有一段时间了，如果学习曲线是看——调——写，那么这一次走到第二步了。

nginx源码有不少可取之处，事件驱动模型、读conf方法、父子进程通信、顺序处理module、conf层级配置、内存池等。其中，内存池算是比较清晰和其它模块依赖较少的。

就扒下来了，把依赖相关处给改了，加了注释和一个打印函数，写了一个不完善的测试程序。

希望本文作为一个开始，后面坚定地走下去。

mytool_palloc.h

#ifndef MYTOOL_PALLOC_H

#define MYTOOL_PALLOC_H
#define MYTOOL_MAX_ALLOC_FROM_POOL (1024 * 4 - 1)
#define MYTOOL_DEFAULT_POOL_SIZE (16 * 1024)
#define MYTOOL_ALIGNMENT sizeof(unsigned long)
#define MYTOOL_POOL_ALIGNMENT 16
#define mytool_align(d, a) (((d) + ((a) - 1)) & ~((a) - 1))
#define MYTOOL_MIN_POOL_SIZE \
	mytool_align((sizeof(mytool_pool_t) + 2 * sizeof(mytool_pool_large_t)),MYTOOL_POOL_ALIGNMENT)
#define mytool_align_ptr(p, a) \
	(u_char *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1))
#define mytool_free free
#define mytool_alloc malloc 
#define mytool_memzero(buf, n) (void)memset(buf,0,n)

#include <cstdlib> //for malloc\free\posix_memalign
#include <cstdint> //for uintptr_t 
#include <cstddef> //for size_t 

//其实已经有了mytool命名空间函数名不用加前缀，我下次注意。
namespace mytool{
	static void *mytool_memalign(size_t alignment,size_t size){
	    void  *p;
	    int    err;
	
	    err = posix_memalign(&p, alignment, size);
	    if(err)
	        p = NULL;
	    return p;
	}

	struct mytool_pool_t;
	struct mytool_pool_cleanup_t;
	
	//size并不是真实能够存储的空间，而是包含了管理信息mytool_pool_t的大小，并且实际能够分配的内存还得对齐，那么实际能一次分配的最大内存为(size - sizeof(mytool_pool_t) - 对齐padding)
	//size - sizeof(mytool_pool_t) == max，但max貌似没考虑padding，怀疑某种边界分配内存会失败(只是有这种可能，因为后面分配的块管理头部要小点todo)
	mytool_pool_t *mytool_create_pool(size_t size);
	void mytool_destroy_pool(mytool_pool_t *pool);
	释放大块（大块被认为不可重用），重置小块，回收pool
	void mytool_reset_pool(mytool_pool_t *pool);
	
	//如果size大于max，直接large，否则挨个检测current及以后的块看有没有能用的，没有就重新分配小块
	void *mytool_palloc(mytool_pool_t *pool, size_t size);
	//类似mytool_pnalloc,只是不用align，如果知道自己的结构对齐为1或者比如直接请求char数组，可以调用这个
	void *mytool_pnalloc(mytool_pool_t *pool, size_t size);
	//带清零的mytool_palloc
	void *mytool_pcalloc(mytool_pool_t *pool, size_t size);
	//同mytool_palloc_large，不过是给外部使用必须带alignment
	void *mytool_pmemalign(mytool_pool_t *pool, size_t size, size_t alignment);
	//释放某一个的large，这命名基本也看不出来了
	int mytool_pfree(mytool_pool_t *pool, void *p);
	
	mytool_pool_cleanup_t *mytool_pool_cleanup_add(mytool_pool_t *p, size_t size);

	//额外加的打印函数
	void mytool_print_pool(mytool_pool_t *);
}

#endif

mytool_palloc.cpp

#include "mytool_palloc.h"
#include <cstring>
#include <cstdio>
namespace mytool{
	//————之前是在头文件中定义，现在移到实现中
	typedef void (*mytool_pool_cleanup_pt)(void *data);
    struct mytool_pool_cleanup_t{
        mytool_pool_cleanup_pt handler;
        void *data;
        mytool_pool_cleanup_t *next;
    };

	//large分配的内存是放入新指针alloc的
    struct mytool_pool_large_t; 
    struct mytool_pool_large_t{
        mytool_pool_large_t *next;
        void *alloc;
    };
    
	//小块分配的内存是和管理头部mytool_pool_data_t一起分配的
    typedef struct{
        u_char *last;
        u_char *end;
        mytool_pool_t *next;
        unsigned failed;
    } mytool_pool_data_t;
    
    struct mytool_pool_t{
        mytool_pool_data_t d;
        size_t max;
        mytool_pool_t *current;
        //mytool_chain_t *chain;
        mytool_pool_large_t *large;
        mytool_pool_cleanup_t *cleanup;
        //mytool_log_t *log;
    };
	//————头文件定义到此

	static void *mytool_palloc_block(mytool_pool_t *pool, size_t size);
	static void *mytool_palloc_large(mytool_pool_t *pool, size_t size);
	
	void mytool_print_pool(mytool_pool_t *pmpt){
		if(pmpt == NULL){
			printf("empty pool\n");
			return;
		}
		printf("\npool info : max=%u,current=%x,first large=%x\n",pmpt->max,(unsigned)pmpt->current,(unsigned)pmpt->large);

		mytool_pool_data_t *pn;
		pn = (mytool_pool_data_t *)pmpt;
		int i = 1;
		while(pn != NULL){
			if(i == 1)
				printf("%dth pool data : begin=%x,max=%u,used=%u,left=%u,failed=%u\n",i++,(unsigned)pn,pn->end - (u_char *)pn - sizeof(mytool_pool_t),pn->last - (u_char *)pn - sizeof(mytool_pool_t),pn->end - pn->last,pn->failed);
			else
				printf("%dth pool data : begin=%x,max=%u,used=%u,left=%u,failed=%u\n",i++,(unsigned)pn,pn->end - (u_char *)pn - sizeof(mytool_pool_data_t),pn->last - (u_char *)pn - sizeof(mytool_pool_data_t),pn->end - pn->last,pn->failed);
			pn = (mytool_pool_data_t *)pn->next;
		}

		printf("\n");
		mytool_pool_large_t *pln = pmpt->large;
		i = 1;
		while(pln != NULL){
			if(pln->alloc != NULL)
				printf("%dth large pool is used\n",i++);
			else
				printf("%dth large pool is empty\n",i++);
			pln = pln->next;
		}
		printf("\n");
	}

	mytool_pool_t *mytool_create_pool(size_t size){
	    mytool_pool_t *p;
	
		//malloc也会一样align进行最大对齐保证分配的内存一定能用
	    p = (mytool_pool_t *)mytool_memalign(MYTOOL_POOL_ALIGNMENT,size);
	    if (p == NULL) {
	        return NULL;
	    }
	
	    p->d.last = (u_char *)p + sizeof(mytool_pool_t);
	    p->d.end = (u_char *)p + size;
	    p->d.next = NULL;
	    p->d.failed = 0;
	
	    size = size - sizeof(mytool_pool_t);
	    p->max = (size < MYTOOL_MAX_ALLOC_FROM_POOL) ? size : MYTOOL_MAX_ALLOC_FROM_POOL;
	
	    p->current = p;
	    //p->chain = NULL;
	    p->large = NULL;
	    p->cleanup = NULL;
	
	    return p;
	}
	
	//pool只有分配，没有回收，只有destroy，是针对短周期线程（一段逻辑上独立代码从开始到结束）如带超时的http交互
	void mytool_destroy_pool(mytool_pool_t *pool){
	    mytool_pool_t *p, *n;
	    mytool_pool_large_t *l;
	    mytool_pool_cleanup_t *c; 
	
	    for(c = pool->cleanup; c; c = c->next)
	        if (c->handler) 
	            c->handler(c->data);
	
	    for(l = pool->large; l; l = l->next)
	        if (l->alloc)
	            mytool_free(l->alloc);
	
	    for (p = pool, n = pool->d.next;; p = n, n = n->d.next){
	        mytool_free(p);
	        if (n == NULL)
	            break;
	    }
	}

	//释放大块（大块被认为不可重用），重置小块，回收pool
	void mytool_reset_pool(mytool_pool_t *pool){
	    mytool_pool_t        *p;
	    mytool_pool_large_t  *l;
	
	    for(l = pool->large; l; l = l->next)
	        if(l->alloc)
	            mytool_free(l->alloc);
	
	    for(p = pool;p;p = p->d.next){
	        p->d.last = (u_char *)p + sizeof(mytool_pool_t);
	        p->d.failed = 0;
	    }
	
	    pool->current = pool;
	    //pool->chain = NULL;
	    pool->large = NULL;
	}

	//如果size大于max，直接large，否则挨个检测current及以后的块看有没有能用的，没有就重新分配小块
	void *mytool_palloc(mytool_pool_t *pool,size_t size){
	    u_char *m;
	    mytool_pool_t *p;
	
	    if(size <= pool->max){
	        p = pool->current;
	        do{
	            m = mytool_align_ptr(p->d.last,MYTOOL_ALIGNMENT);
	            if((size_t) (p->d.end - m) >= size){
	                p->d.last = m + size;
	                return m;
	            }
	            p = p->d.next;
	        } while (p);
			//重新分配的小块是按照pool的大小来的，并不是size
	        return mytool_palloc_block(pool, size);
	    }
	    return mytool_palloc_large(pool, size);
	}

	//类似mytool_pnalloc,只是不用align，如果知道自己的结构对齐为1或者比如直接请求char数组，可以调用这个。
	void *mytool_pnalloc(mytool_pool_t *pool,size_t size){
	    u_char *m;
	    mytool_pool_t *p;
	
	    if(size <= pool->max){
	        p = pool->current;
	        do {
	            m = p->d.last;
	            if((size_t) (p->d.end - m) >= size){
	                p->d.last = m + size;
	                return m;
	            }
	            p = p->d.next;
	        }while(p);
	        return mytool_palloc_block(pool, size);
	    }
	    return mytool_palloc_large(pool, size);
	}

	//重新分配的小块是按照pool的大小来的，并不是size
	static void *mytool_palloc_block(mytool_pool_t *pool,size_t size){
	    u_char *m;
	    size_t psize;
	    mytool_pool_t *p, *newp, *current;
	
	    psize = (size_t)(pool->d.end - (u_char *)pool);
	
	    m = (u_char *)mytool_memalign(MYTOOL_POOL_ALIGNMENT,psize);
	    if (m == NULL)
	        return NULL;
	
	    newp = (mytool_pool_t *)m;
	    newp->d.end = m + psize;
	    newp->d.next = NULL;
	    newp->d.failed = 0;
	
	    m += sizeof(mytool_pool_data_t);
	    m = mytool_align_ptr(m,MYTOOL_ALIGNMENT);
	    newp->d.last = m + size;
	
	    current = pool->current;
	
		//并不是一次分配失败该块failed增加，而是所有的已有块均失败才一起增加
	    for (p = current; p->d.next; p = p->d.next)
	        if (p->d.failed++ > 4)
	            current = p->d.next;
	    p->d.next = newp;
	    pool->current = current ? current : newp;
	
	    return m;
	}

	static void *mytool_palloc_large(mytool_pool_t *pool, size_t size){
	    void *p;
	    unsigned n;
	    mytool_pool_large_t *large;
	
	    p = mytool_alloc(size);
	    if(p == NULL)
	        return NULL;
	
	    n = 0;
		//reset之后会有空large
	    for(large = pool->large; large; large = large->next){
	        if (large->alloc == NULL){
	            large->alloc = p;
	            return p;
	        }
	
			//小块防止遍历过度用current，大块用强制次数
	        if (n++ > 3)
	            break;
	    }
	
	    large = (mytool_pool_large_t *)mytool_palloc(pool,sizeof(mytool_pool_large_t));
	    if (large == NULL){
	        mytool_free(p);
	        return NULL;
	    }
	
		//large是插入到头部的
	    large->alloc = p;
	    large->next = pool->large;
	    pool->large = large;
	
	    return p;
	}

	//同mytool_palloc_large，不过是给外部使用必须带alignment
	void *mytool_pmemalign(mytool_pool_t *pool, size_t size, size_t alignment){
	    void *p;
	    mytool_pool_large_t *large;
	
	    p = mytool_memalign(alignment,size);
	    if (p == NULL)
	        return NULL;
	
	    large = (mytool_pool_large_t *)mytool_palloc(pool,sizeof(mytool_pool_large_t));
	    if (large == NULL){
	        mytool_free(p);
	        return NULL;
	    }
	
	    large->alloc = p;
	    large->next = pool->large;
	    pool->large = large;
	
	    return p;
	}

	//释放某一个的large，这命名基本也看不出来了
	int mytool_pfree(mytool_pool_t *pool, void *p){
	    mytool_pool_large_t *l;
	
	    for (l = pool->large;l;l = l->next){
	        if(p == l->alloc){
	            mytool_free(l->alloc);
	            l->alloc = NULL;
	            return 0;
	        }
	    }
	
	    return 1;
	}

	//带清零的mytool_palloc
	void *mytool_pcalloc(mytool_pool_t *pool, size_t size){
	    void *p;
	
	    p = mytool_palloc(pool, size);
	    if(p)
	        mytool_memzero(p, size);

	    return p;
	}

	//先不考虑
	mytool_pool_cleanup_t *mytool_pool_cleanup_add(mytool_pool_t *p,size_t size){
	    mytool_pool_cleanup_t *c;
	
	    c = (mytool_pool_cleanup_t *)mytool_palloc(p,sizeof(mytool_pool_cleanup_t));
	    if (c == NULL)
	        return NULL;
	
	    if (size){
	        c->data = mytool_palloc(p, size);
	        if (c->data == NULL)
	            return NULL;
	    }else{
	        c->data = NULL;
	    }
	
	    c->handler = NULL;
	    c->next = p->cleanup;
	
	    p->cleanup = c;
	    return c;
	}
}

测试程序：

#include "mytool_palloc.h"
#include <iostream>
#include <cstdio>

using namespace mytool;
int main(){
    //初始化
    mytool_pool_t *pmpt = mytool_create_pool(MYTOOL_DEFAULT_POOL_SIZE);
    mytool_print_pool(pmpt);

    //分配200
    mytool_pnalloc(pmpt,200);
    printf("\n200 allocated\n");
    mytool_print_pool(pmpt);

    //分配10000，第一个大块
    mytool_pnalloc(pmpt,10000);
    printf("\n10000 allocated\n");
    mytool_print_pool(pmpt);

    //连续分配8个2000，第一个小块到临界状态
    int i = 0;
    while(i < 8){
        i++;
        mytool_pnalloc(pmpt,2000);
    }
    printf("\n8*2000 allocated\n");
    mytool_print_pool(pmpt);

    //分配2000，第一个小块分配失败，第二小块
    mytool_pnalloc(pmpt,2000);
    printf("\n2000 allocated\n");
    mytool_print_pool(pmpt);

    //分配5000，第二个大块
    void *p = mytool_pnalloc(pmpt,5000);
    printf("\n5000 allocated\n");
    mytool_print_pool(pmpt);

    //释放分配的5000，实际是释放链表中第一个大块
    mytool_pfree(pmpt,p);
    printf("\n5000 freed\n");
    mytool_print_pool(pmpt);

    //再分配5000，重新利用链表中第一个空大块
    p = mytool_pnalloc(pmpt,5000);
    printf("\n5000 allocated\n");
    mytool_print_pool(pmpt);
}

测试输出：

pool info : max=4095,current=8655020,first large=0
1th pool data : begin=8655020,max=16352,used=0,left=16352,failed=0



200 allocated

pool info : max=4095,current=8655020,first large=0
1th pool data : begin=8655020,max=16352,used=200,left=16152,failed=0



10000 allocated

pool info : max=4095,current=8655020,first large=8655108
1th pool data : begin=8655020,max=16352,used=208,left=16144,failed=0

1th large pool is used


8*2000 allocated

pool info : max=4095,current=8655020,first large=8655108
1th pool data : begin=8655020,max=16352,used=16208,left=144,failed=0

1th large pool is used


2000 allocated

pool info : max=4095,current=8655020,first large=8655108
1th pool data : begin=8655020,max=16352,used=16208,left=144,failed=0
2th pool data : begin=865b750,max=16368,used=2000,left=14368,failed=0

1th large pool is used


5000 allocated

pool info : max=4095,current=8655020,first large=8658f90
1th pool data : begin=8655020,max=16352,used=16216,left=136,failed=0
2th pool data : begin=865b750,max=16368,used=2000,left=14368,failed=0

1th large pool is used
2th large pool is used


5000 freed

pool info : max=4095,current=8655020,first large=8658f90
1th pool data : begin=8655020,max=16352,used=16216,left=136,failed=0
2th pool data : begin=865b750,max=16368,used=2000,left=14368,failed=0

1th large pool is empty
2th large pool is used


5000 allocated

pool info : max=4095,current=8655020,first large=8658f90
1th pool data : begin=8655020,max=16352,used=16216,left=136,failed=0
2th pool data : begin=865b750,max=16368,used=2000,left=14368,failed=0

1th large pool is used
2th large pool is used

符合预期。