bacula之内存池实现

关于内存池方面的文章很多，bacula中的实现相比与STL中的内存池与MemCached的内存池实现要简单的多。主要思想是，把申请的内存块大小分为固定大小的四块，另外用户也可自由申请指定大小的内存。

mem_pool.h

/*
memory pool design
*/

#ifndef __MEM_POOL_H_
#define __MEM_POOL_H_ 1

#include<stdio.h>
#include<stdlib.h>
#include<pthread.h>
#include<assert.h>
#include<time.h>
#include<string.h>


#define int32_t  int

#define PM_NOPOOL  0                  /* nonpooled memory */
#define PM_NAME    1                  /* system name */
#define PM_FNAME   2                  /* file name buffer */
#define PM_MESSAGE 3                  /* daemon message */
#define PM_EMSG    4                  /* error message */
#define PM_MAX     PM_EMSG            /* Number of types */

#define MAX_NAME_LENGTH 128            /*the length of name */
#define NLEN (MAX_NAME_LENGTH+2)

/*  Memory allocation control structures and storage.  */
struct mbufhead {
   int32_t ablen;                     /* Buffer length in bytes */
   int32_t pool;                      /* pool type*/
   struct mbufhead *next;             /* pointer to next free buffer */
};

struct s_pool_ctl {
   char type[7];
   int32_t size;                      /* default size */
   int32_t max_allocated;             /* the max space which has been allocated by*/
   int32_t max_used;                  /* the max buffer which is used */
   int32_t in_use;                    /* number in use */
   struct mbufhead *free_buf;         /* pointer to free buffers */
};

#define ALIGN_SIZE (sizeof(double))
#define BALIGN(x)  (((x)+ALIGN_SIZE-1)&~(ALIGN_SIZE-1)) // 与8字节对齐
#define M_HEAD_SIZE  BALIGN(sizeof(struct mbufhead))

char *get_pool_memory( int pool);  //得到指定类型的内存
char *get_memory( int32_t size); /*nonpool type*/
int32_t sizeof_pool_memory(char *obuf);  // 检查内存的大小
char *realloc_pool_memory( char *obuf, int32_t size);
void free_pool_memory( char *obuf);
void print_pool_memory_stats();
void garbage_collect_memory_pool(); //定时回收垃圾内存
void close_pool_memory();  // 关闭内存池

#endif

.mem_pool.c

#include "mem_pool.h"

static struct s_pool_ctl pool_ctl[] = {
   {"NoPool", 256,  256, 0, 0, NULL },        /* PM_NOPOOL no pooling */
   {"NAME  ", NLEN, NLEN,0, 0, NULL },        /* PM_NAME System name */
   {"FNAME ",256,  256, 0, 0, NULL },         /* PM_FNAME filename buffers */
   {"MSG   ", 512,  512, 0, 0, NULL },        /* PM_MESSAGE message buffer */
   {"EMSG  ",1024, 1024, 0, 0, NULL }         /* PM_EMSG error message buffer */
};

static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;

char *get_pool_memory( int pool)
{
   struct mbufhead *buf;

   if (pool > PM_MAX)
   {
      printf("MemPool index %d larger than max %d\n", pool, PM_MAX);
   }
   P(&mutex);
   /* if it has free buffer */
   if (pool_ctl[pool].free_buf) {
      buf = pool_ctl[pool].free_buf;
      pool_ctl[pool].free_buf = buf->next;
      pool_ctl[pool].in_use++;
      if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
         pool_ctl[pool].max_used = pool_ctl[pool].in_use;
      }
      V(&mutex);
      printf("sm_get_pool_memory reuse %p to %s:%d\n", buf, __FILE__, __LINE__);
      //snew_owner(__FILE__, __LINE__, (char *)buf);
      return (char *)((char *)buf+M_HEAD_SIZE);
   }
    /* or,it will generate a new buffer */
   if ((buf = (struct mbufhead *)malloc(pool_ctl[pool].size+M_HEAD_SIZE)) == NULL)
   {
      V(&mutex);
      printf("Out of memory requesting %d bytes\n", pool_ctl[pool].size);
   }
   buf->ablen = pool_ctl[pool].size;
   buf->pool = pool;
   pool_ctl[pool].in_use++;
   if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
      pool_ctl[pool].max_used = pool_ctl[pool].in_use;
   }
   V(&mutex);
   printf("sm_get_pool_memory give %p to %s:%d\n", buf, __FILE__, __LINE__);
   return (char *)((char *)buf+M_HEAD_SIZE);
}

/* Get nonpool memory of size requested */
char *get_memory( int32_t size)
{
   struct mbufhead *buf;
   int pool = 0;

   if ((buf = (struct mbufhead *)malloc(size+M_HEAD_SIZE)) == NULL)
   {
      printf("Out of memory requesting %d bytes\n", size);
   }
   buf->ablen = size;
   buf->pool = pool;
   buf->next = NULL;
   pool_ctl[pool].in_use++;
   if (pool_ctl[pool].in_use > pool_ctl[pool].max_used)
      pool_ctl[pool].max_used = pool_ctl[pool].in_use;
   return (char *)(((char *)buf)+M_HEAD_SIZE);
}


/* Return the size of a memory buffer */
int32_t sizeof_pool_memory(char *obuf)
{
   char *cp = (char *)obuf;

   assert(obuf); /*check*/

   cp -= M_HEAD_SIZE;
   return ((struct mbufhead *)cp)->ablen;
}

/* Realloc pool memory buffer ,size must be greater than before*/
char *realloc_pool_memory( char *obuf, int32_t size)
{
   assert(obuf); 
   if (size <= sizeof_pool_memory(obuf)) {
      return obuf;
   }
   char *cp = (char *)obuf;
   void *buf;
   int pool;

   assert(obuf);

   P(&mutex);
   cp -= M_HEAD_SIZE;
   buf = realloc(cp, size+M_HEAD_SIZE);
   if (buf == NULL) {
      V(&mutex);
      printf("Out of memory requesting %d bytes\n", size);
   }
   ((struct mbufhead *)buf)->ablen = size;
   pool = ((struct mbufhead *)buf)->pool;
   if (size > pool_ctl[pool].max_allocated) {
      pool_ctl[pool].max_allocated = size;
   }
   V(&mutex);
   return (char *)(((char *)buf)+M_HEAD_SIZE);
}

/* Free a memory buffer */
void free_pool_memory(char *obuf)
{
   struct mbufhead *buf;
   int pool;

   assert(obuf);

   P(&mutex);
   buf = (struct mbufhead *)((char *)obuf - M_HEAD_SIZE);
   pool = buf->pool;
   pool_ctl[pool].in_use--;
   if(pool == 0) {
      free((char *)buf);              /* free nonpooled memory */
   } else {                           /* otherwise link it to the free pool chain */
      struct mbufhead *next;
      /* Don't let him free the same buffer twice */
      for (next=pool_ctl[pool].free_buf; next; next=next->next) {
         if (next == buf) {
            printf("free_pool_memory %p pool=%d from %s:%d\n", buf, pool,__FILE__,__LINE__);
            printf("bad free_pool_memory %p pool=%d from %s:%d\n", buf, pool,__FILE__, __LINE__);
            V(&mutex);                 /* unblock the pool */
            assert(next != buf);      /* attempt to free twice */
         }
      }
      buf->next = pool_ctl[pool].free_buf;
      pool_ctl[pool].free_buf = buf;
   }
   printf("free_pool_memory %p pool=%d from %s:%d\n", buf, pool, __FILE__, __LINE__);
   V(&mutex);
}


/* Release all pooled memory */
void close_pool_memory()
{
   struct mbufhead *buf, *next;
   int count = 0;
   unsigned long long int  bytes = 0;

   //bcheck(__FILE__, __LINE__, false);
   P(&mutex);
   int i=0;
   for (i=1; i<=PM_MAX; i++) {
      buf = pool_ctl[i].free_buf;
      while (buf) {
         next = buf->next;
         count++;
         //bytes += sizeof_pool_memory((char *)buf);
  bytes+=buf->ablen;
         free((char *)buf);
         buf = next;
      }
      pool_ctl[i].free_buf = NULL;
   }
   printf("Freed mem_pool count=%d size=%llu\n", count, bytes);
   V(&mutex);

}


/*
 * Clean up memory pool periodically
 *
 */
static time_t last_garbage_collection = 0;
const int garbage_interval = 24 * 60 * 60;  /* garbage collect every 24 hours */

/* create a thread to excute it */
void garbage_collect_memory_pool()
{
   time_t now;

   printf( "garbage collect memory pool\n");
   P(&mutex);
   if (last_garbage_collection == 0) {
      last_garbage_collection = time(NULL);
      V(&mutex);
      return;
   }
   now = time(NULL);
   if (now >= last_garbage_collection + garbage_interval) {
      last_garbage_collection = now;
      V(&mutex);
      close_pool_memory();
   } else {
      V(&mutex);
   }
}


/* Print staticstics on memory pool usage
 */
void print_pool_memory_stats()
{
   printf("Pool   Maxsize  Maxused  Inuse\n");
    int i=0;
   for (; i<=PM_MAX; i++)
      printf("%5s  %7d  %7d  %5d\n", pool_ctl[i].type, pool_ctl[i].max_allocated,
         pool_ctl[i].max_used, pool_ctl[i].in_use);

  
}

测试程序

 

int main()
{
 char *s;
 s=get_pool_memory(3);
 printf("sizeof_pool_memory:%d\n",sizeof_pool_memory(s));
 print_pool_memory_stats();
 
 char *s1=get_pool_memory(1);
 char *s2=get_memory(100);
 print_pool_memory_stats();

 free_pool_memory(s1);
        free_pool_memory(s2);

 s=realloc_pool_memory(s,536);
  free_pool_memory(s);
 print_pool_memory_stats();
 
 close_pool_memory();
 print_pool_memory_stats();
 
 char *s3=get_pool_memory(3);
 print_pool_memory_stats();
 
 sscanf("zhoupj","%s",s3);
 printf("%s\n",s3);
 pm_strcat(&s3, "hello");
   printf("%s\n",s3);
 pm_strcpy(&s3, "XYZSDF") ;
   printf("%s\n",s3);

}

相关程序

 

.mutex.c

#include "mem_pool.h"

/*
 * These are mutex routines that do error checking
 *  for deadlock and such.  Normally not turned on.
 */

/* enclose P  V operation */
void P(pthread_mutex_t *m)
{
   int errstat;
   if ((errstat = pthread_mutex_trylock(m))) {
      printf("Possible mutex deadlock.\n");
      /* We didn't get the lock, so do it definitely now */
      if ((errstat=pthread_mutex_lock(m))) {
         printf("Mutex lock failure.\n");
      } else {
         printf("Possible mutex deadlock resolved.\n");
      }

   }
}

void V(pthread_mutex_t *m)
{
   int errstat;

   /* Note, this trylock *should* fail if the mutex is locked */
   if ((errstat=pthread_mutex_trylock(m)) == 0) {
      printf("Mutex unlock not locked.\n");
    }
    if ((errstat=pthread_mutex_unlock(m))) {
       printf("Mutex unlock failure. \n");
    }
}