Linux下对libevent的evbuffer模块裁剪移植

1. 背景

libevent作为一款高效的网络开发库，内部模块也有许多优秀的实现。
evbuffer为libevent的核心缓冲器功能，提供了与I/O的操作的高效结合：数据拷贝、移动、读写。
上篇文章《Linux下使用gtest对接口进行单元测试》对evbuffer准备了单元测试，本节尝试将evbuffer模块单独裁剪出来，学习一下evbuffer的实现。

2. 源码分析

2.1 结构体分析

主要结构体为 struct evbuffer 和 struct evbuffer_chain 两个数据结构，
通过链表的形式对碎片数据进行组织。

struct evbuffer {
    /** The first chain in this buffer's linked list of chains. */
    struct evbuffer_chain *first;
    /** The last chain in this buffer's linked list of chains. */
    struct evbuffer_chain *last;

    /** Pointer to the next pointer pointing at the 'last_with_data' chain.
     *
     * To unpack:
     *
     * The last_with_data chain is the last chain that has any data in it.
     * If all chains in the buffer are empty, it is the first chain.
     * If the buffer has no chains, it is NULL.
     *
     * The last_with_datap pointer points at _whatever 'next' pointer_
     * points at the last_with_datap chain.  If the last_with_data chain
     * is the first chain, or it is NULL, then the last_with_datap pointer
     * is &buf->first.
     */
    struct evbuffer_chain **last_with_datap;

    /** Total amount of bytes stored in all chains.*/
    size_t total_len;
    ...
};

主要成员为first、last、last_with_datap 这些指针，用于构建链表的处理，
另外，evbuffer还支持操作过程中添加、删除数据调用回调函数处理，本节对该功能进行裁剪。

/** A single item in an evbuffer. */
struct evbuffer_chain {
    /** points to next buffer in the chain */
    struct evbuffer_chain *next;

    /** total allocation available in the buffer field. */
    size_t buffer_len;

    /** unused space at the beginning of buffer or an offset into a
     * file for sendfile buffers. */
    ssize_t misalign;

    /** Offset into buffer + misalign at which to start writing.
     * In other words, the total number of bytes actually stored
     * in buffer. */
    size_t off;

    /** Set if special handling is required for this chain */
    unsigned flags;

    /** number of references to this chain */
    int refcnt;

    /** Usually points to the read-write memory belonging to this
     * buffer allocated as part of the evbuffer_chain allocation.
     * For mmap, this can be a read-only buffer and
     * EVBUFFER_IMMUTABLE will be set in flags.  For sendfile, it
     * may point to NULL.
     */
    unsigned char *buffer;
};

chain结构为单个节点的信息，主要维护了指针域数据，并且buffer挂载用户数据。

关于evbuffer链表的设计的思路，evbuffer为何使用链表的形式实现，而不是使用单个地址的大堆来完成？

翻看了接口使用后，我理解是这几个方面考虑：

1. 链表方便对数据节点进行添加、删除操作（类似链表与数组的区别）；
2. 两个evbuffer的数据移动，可以理解为两个链表的操作，避免内存拷贝；
3. 链表为分散的内存地址，evbuffer配合iovec的思路，实现了writev、readv的功能；
4. 在系统调用函数read、write中，操作大块数据时（如10M），是通过多次调用完成的（如每次4KB，多次循环）；

2.2 主要接口

接口名称	说明
struct evbuffer *evbuffer_new(void);	为新的evbuffer分配空间
void evbuffer_free(struct evbuffer *buf);	对evbuffer释放空间
size_t evbuffer_get_length(const struct evbuffer *buf);	获取evbuffer总长度
int evbuffer_add(struct evbuffer *buf, const void *data, size_t datlen);	追加数据到evbuffer尾部
int evbuffer_prepend(struct evbuffer *buf, const void *data, size_t size);	在evbuffer头部插入数据
unsigned char *evbuffer_pullup(struct evbuffer *buf, ssize_t size);	使evbuffer开头的数据连续
int evbuffer_remove(struct evbuffer *buf, void *data, size_t datlen);	读取并删除数据
ssize_t evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen);	读取但保留数据
int evbuffer_drain(struct evbuffer *buf, size_t len);	删除数据

3.裁剪

我们主要工作是裁剪出核心功能进行学习，其他的高级功能裁剪功能如下：

1. 回调函数，通过HAVE_CALLBACK注释掉
2. 文件操作，EVBUFFER_FILESEGMENT、EVBUFFER_SENDFILE
3. 只读属性，EVBUFFER_IMMUTABLE
4. 多处引用，EVBUFFER_REFERENCE、EVBUFFER_MULTICAST
5. 内存驻留，EVBUFFER_MEM_PINNED_R、EVBUFFER_MEM_PINNED_W
6. 线程安全，evbuffer可以支持线程安全的，我们暂时通过宏将 evthread-internal.h 替换掉

3.1 兼容性定义

对应evbuffer的属性，我们进行调整定义

/*
 * Minimum allocation for a chain.
 * We define this so that we're burning no
 * more than 5% of each allocation on overhead.
 * It would be nice to lose even less space, though.
 */
#define MIN_BUFFER_SIZE 1024
#define EVBUFFER_CHAIN_MAX  (1024 * 1024)

替换掉libevent的内部数据结构（目前仅考虑Linux-x86_64平台）

#ifdef __cplusplus
# define __STDC_LIMIT_MACROS
# define __STDC_CONSTANT_MACROS
# include <stdint.h>
#endif

#define EV_UINT64_MAX UINT64_MAX
#define EV_INT64_MAX  INT64_MAX
#define EV_INT64_MIN  INT64_MIN

#define EV_UINT32_MAX UINT32_MAX
#define EV_INT32_MAX  INT32_MAX
#define EV_INT32_MIN  INT32_MIN

#define EV_UINT16_MAX UINT16_MAX
#define EV_INT16_MAX  INT16_MAX
#define EV_INT16_MIN  INT16_MIN

#define EV_UINT8_MAX  UINT8_MAX
#define EV_INT8_MAX   INT8_MAX
#define EV_INT8_MIN   INT8_MIN

#define EV_SIZE_MAX EV_UINT64_MAX
#define EV_SSIZE_MAX EV_INT64_MAX

/*
 * include/event2/util.h
 */
typedef unsigned char        ev_uint8_t;
typedef unsigned short       ev_uint16_t;
typedef unsigned int         ev_uint32_t;
typedef unsigned long long   ev_uint64_t;

#define ev_uintptr_t uintptr_t
#define ev_intptr_t intptr_t

typedef ssize_t ev_ssize_t;
typedef ev_int64_t ev_off_t;

替换掉libevent内存管理，更换为标准的系统调用

/*
 * mm-internal.h
 */
#define mm_malloc(sz) malloc(sz)
#define mm_calloc(n, sz) calloc((n), (sz))
#define mm_strdup(s) strdup(s)
#define mm_realloc(p, sz) realloc((p), (sz))
#define mm_free(p) free(p)

/*
 * evutil.c
 */
#define evutil_vsnprintf vsnprintf
#define evutil_snprintf   snprintf

不考虑多线程操作，把线程安全函数宏定义空

/*
 * util-internal.h
 */
/* A good no-op to use in macro definitions. */
#define EVUTIL_NIL_STMT_ ((void)0)

/*
 * include/event2/util.h
 */
/** Assert that we are holding the lock on an evbuffer */
#define ASSERT_EVBUFFER_LOCKED(buffer)          \
    EVLOCK_ASSERT_LOCKED((buffer)->lock)

#define EVBUFFER_LOCK(buffer)                       \
    do {                                \
        EVLOCK_LOCK((buffer)->lock, 0);             \
    } while (0)
#define EVBUFFER_UNLOCK(buffer)                     \
    do {                                \
        EVLOCK_UNLOCK((buffer)->lock, 0);           \
    } while (0)
#define EVBUFFER_LOCK2(buffer1, buffer2)                \
    do {                                \
        EVLOCK_LOCK2((buffer1)->lock, (buffer2)->lock, 0, 0);   \
    } while (0)
#define EVBUFFER_UNLOCK2(buffer1, buffer2)              \
    do {                                \
        EVLOCK_UNLOCK2((buffer1)->lock, (buffer2)->lock, 0, 0); \
    } while (0)
    
/*
 * evthread-internal.h
 */
#define EVENT__DISABLE_THREAD_SUPPORT

#ifdef  EVENT__DISABLE_THREAD_SUPPORT
# define EVTHREAD_GET_ID()  1
# define EVTHREAD_ALLOC_LOCK(lockvar, locktype) EVUTIL_NIL_STMT_
# define EVTHREAD_FREE_LOCK(lockvar, locktype) EVUTIL_NIL_STMT_

# define EVLOCK_LOCK(lockvar, mode) EVUTIL_NIL_STMT_
# define EVLOCK_UNLOCK(lockvar, mode) EVUTIL_NIL_STMT_
# define EVLOCK_LOCK2(lock1,lock2,mode1,mode2) EVUTIL_NIL_STMT_
# define EVLOCK_UNLOCK2(lock1,lock2,mode1,mode2) EVUTIL_NIL_STMT_

# define EVBASE_IN_THREAD(base) 1
# define EVBASE_NEED_NOTIFY(base) 0
# define EVBASE_ACQUIRE_LOCK(base, lock) EVUTIL_NIL_STMT_
# define EVBASE_RELEASE_LOCK(base, lock) EVUTIL_NIL_STMT_
# define EVLOCK_ASSERT_LOCKED(lock) EVUTIL_NIL_STMT_

# define EVLOCK_TRY_LOCK_(lock) 1

# define EVTHREAD_ALLOC_COND(condvar) EVUTIL_NIL_STMT_
# define EVTHREAD_FREE_COND(cond) EVUTIL_NIL_STMT_
# define EVTHREAD_COND_SIGNAL(cond) EVUTIL_NIL_STMT_
# define EVTHREAD_COND_BROADCAST(cond) EVUTIL_NIL_STMT_
# define EVTHREAD_COND_WAIT(cond, lock) EVUTIL_NIL_STMT_

3.2 裁剪实现

3.1 chain相关操作

static struct evbuffer_chain *
evbuffer_chain_new(size_t size)
{
    struct evbuffer_chain *chain;
    size_t to_alloc;

    if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE) {
        return (NULL);
    }    
    size += EVBUFFER_CHAIN_SIZE; // 注意这个地方 ！！！！！！

    /* get the next largest memory that can hold the buffer */
    if (size < EVBUFFER_CHAIN_MAX / 2) { 
        to_alloc = MIN_BUFFER_SIZE;
        while (to_alloc < size) {
            to_alloc <<= 1;
        }    
    }    
    else {
        to_alloc = size;
    }    

    /* we get everything in one chunk */
    if ((chain = (struct evbuffer_chain *)mm_malloc(to_alloc)) == NULL) {
        return (NULL);
    }    
    memset(chain, 0, EVBUFFER_CHAIN_SIZE);
    chain->buffer_len = to_alloc - EVBUFFER_CHAIN_SIZE;

    /* this way we can manipulate the buffer to different addresses,
     * which is required for mmap for example.
     */
    chain->buffer = EVBUFFER_CHAIN_EXTRA(unsigned char, chain);
    chain->refcnt = 1; 

    return (chain);
}

注意buffer地址这块，不是二段式指针的实现，而是连续的内存空间，类似柔性数组的实现方式：
EVBUFFER_CHAIN_EXTRA 返回buffer的内存地址

/** Return a pointer to extra data allocated along with an evbuffer. */
#define EVBUFFER_CHAIN_EXTRA(t, c) (t *)((struct evbuffer_chain *)(c) + 1)

对应alloc有free的动作，我们在里面将flags的特殊处理裁剪掉。

static inline void
evbuffer_chain_free(struct evbuffer_chain *chain)
{
    EVUTIL_ASSERT(chain->refcnt > 0);
    if (--chain->refcnt > 0) {
        /* chain is still referenced by other chains */
        return;
    }
    // 中间裁剪了特殊属性的chain的处理，我们这仅处理通用chain
    mm_free(chain);
}

static void
evbuffer_free_all_chains(struct evbuffer_chain *chain)
{
    struct evbuffer_chain *next;
    for (; chain; chain = next) {
        next = chain->next;  
        evbuffer_chain_free(chain);
    }
}

插入函数，主要处理两个逻辑：

• 插入节点到空链表；
• 插入节点到现有链表，里面通过额外的 evbuffer_free_trailing_empty_chains 函数，保证链表尾是纯净的链表尾；

/* Add a single chain 'chain' to the end of 'buf', freeing trailing empty                                                                           
 * chains as necessary.  Requires lock.  Does not schedule callbacks.                                                                               
 */                                                                                                                                                 
static void                                                                                                                                         
evbuffer_chain_insert(struct evbuffer *buf,                                                                                                         
                      struct evbuffer_chain *chain)                                                                                                 
{                                                                                                                                                   
    ASSERT_EVBUFFER_LOCKED(buf);                                                                                                                    
    if (*buf->last_with_datap == NULL) {                                                                                                            
        /* There are no chains data on the buffer at all. */                                                                                        
        EVUTIL_ASSERT(buf->last_with_datap == &buf->first);                                                                                         
        EVUTIL_ASSERT(buf->first == NULL);                                                                                                          
        buf->first = buf->last = chain;                                                                                                             
    }                                                                                                                                               
    else {                                                                                                                                          
        struct evbuffer_chain **chp;                                                                                                                
        chp = evbuffer_free_trailing_empty_chains(buf);                                                                                             
        *chp = chain;                                                                                                                               
        if (chain->off) {                                                                                                                           
            buf->last_with_datap = chp;                                                                                                             
        }                                                                                                                                           
        buf->last = chain;                                                                                                                          
    }                                                                                                                                               
    buf->total_len += chain->off;                                                                                                                   
} 

3.2 evbuffer的操作

初始化函数

struct evbuffer *
evbuffer_new(void)
{
    struct evbuffer *buffer;

    buffer = (struct evbuffer *)mm_calloc(1, sizeof(struct evbuffer));
    if (buffer == NULL) {
        return (NULL);       
    }

#if HAVE_CALLBACK
    LIST_INIT(&buffer->callbacks);
#endif
    buffer->refcnt = 1;  
    buffer->last_with_datap = &buffer->first;

    return (buffer);
}

释放函数

void
evbuffer_decref_and_unlock_(struct evbuffer *buffer)
{
    struct evbuffer_chain *chain, *next;
    ASSERT_EVBUFFER_LOCKED(buffer);

    EVUTIL_ASSERT(buffer->refcnt > 0);

    if (--buffer->refcnt > 0) {
        EVBUFFER_UNLOCK(buffer);
        return;
    }

    for (chain = buffer->first; chain != NULL; chain = next) {
        next = chain->next;
        evbuffer_chain_free(chain);
    }
#if HAVE_CALLBACK
    evbuffer_remove_all_callbacks(buffer);
    if (buffer->deferred_cbs) {
        event_deferred_cb_cancel_(buffer->cb_queue, &buffer->deferred);
    }
#endif

    EVBUFFER_UNLOCK(buffer);
    if (buffer->own_lock) {
        EVTHREAD_FREE_LOCK(buffer->lock, EVTHREAD_LOCKTYPE_RECURSIVE);
    }
    mm_free(buffer);
}


void
evbuffer_free(struct evbuffer *buffer)
{
    EVBUFFER_LOCK(buffer);
    evbuffer_decref_and_unlock_(buffer);
}

添加数据到evbuffer中，这个处理比较简单好理解：

/* Adds data to an event buffer */
int
evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen)
{
    struct evbuffer_chain *chain, *tmp; 
    const unsigned char *data = (const unsigned char *)data_in;
    size_t remain, to_alloc; 
    int result = -1;
    EVBUFFER_LOCK(buf);

    if (buf->freeze_end) {   
        goto done;
    }
    /* Prevent buf->total_len overflow */
    if (datlen > EV_SIZE_MAX - buf->total_len) {
        goto done;
    }

    if (*buf->last_with_datap == NULL) {
        chain = buf->last;                                                                                                                          
    }
    else {
        chain = *buf->last_with_datap;
    }

    /* If there are no chains allocated for this buffer, allocate one
     * big enough to hold all the data. */
    if (chain == NULL) {     
        chain = evbuffer_chain_new(datlen);
        if (!chain) {            
            goto done;       
        }
        evbuffer_chain_insert(buf, chain);
    }
        /* we need to add another chain */
    to_alloc = chain->buffer_len;
    if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE / 2) {
        to_alloc <<= 1;
    }
    if (datlen > to_alloc) {
        to_alloc = datlen;
    }
    tmp = evbuffer_chain_new(to_alloc);
    if (tmp == NULL) {
        goto done;
    }

    if (remain) {
        memcpy(chain->buffer + chain->misalign + chain->off,
               data, remain);
        chain->off += remain;
        buf->total_len += remain;
        buf->n_add_for_cb += remain;
    }

    data += remain;
    datlen -= remain;

    memcpy(tmp->buffer, data, datlen);
    tmp->off = datlen;
    evbuffer_chain_insert(buf, tmp);
    buf->n_add_for_cb += datlen;
    
    result = 0;
done:
    EVBUFFER_UNLOCK(buf);
    return result;
}

跟evbuffer_add 异曲同工的是 evbuffer_prepend，将数据添加到链表头部

int
evbuffer_prepend(struct evbuffer *buf, const void *data, size_t datlen)
{
    struct evbuffer_chain *chain, *tmp;
    int result = -1;

    EVBUFFER_LOCK(buf);

    if (buf->freeze_start) {
        goto done;
    }
    if (datlen > EV_SIZE_MAX - buf->total_len) {
        goto done;
    }

    chain = buf->first;

    if (chain == NULL) {
        chain = evbuffer_chain_new(datlen);
        if (!chain) {
            goto done;
        }
        evbuffer_chain_insert(buf, chain);
    }
    /* we need to add another chain */
    if ((tmp = evbuffer_chain_new(datlen)) == NULL) {
        goto done;
    }
    buf->first = tmp;
    if (buf->last_with_datap == &buf->first) {
        buf->last_with_datap = &tmp->next;
    }

    tmp->next = chain;

    tmp->off = datlen;
    EVUTIL_ASSERT(datlen <= tmp->buffer_len);
    tmp->misalign = tmp->buffer_len - datlen;

    memcpy(tmp->buffer + tmp->misalign, data, datlen);
    buf->total_len += datlen;
    buf->n_add_for_cb += datlen;
    
    result = 0;
done:
    EVBUFFER_UNLOCK(buf);
    return result;
}

接下来看删除相关的函数，可以这么理解：
evbuffer_remove = evbuffer_copyout + evbuffer_drain

拷贝数据的流程，其实是遍历链表，挨个拷贝出所需的数据。

ev_ssize_t
evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen)
{
    return evbuffer_copyout_from(buf, NULL, data_out, datlen);
}

ev_ssize_t
evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos,
                      void *data_out, size_t datlen)
{
    /*XXX fails badly on sendfile case. */
    struct evbuffer_chain *chain;
    char *data = (char *)data_out;
    size_t nread;
    ev_ssize_t result = 0;
    size_t pos_in_chain;

    EVBUFFER_LOCK(buf);

    if (pos) {
        if (datlen > (size_t)(EV_SSIZE_MAX - pos->pos)) {
            result = -1;
            goto done;
        }
        chain = (struct evbuffer_chain *)pos->internal_.chain;
        pos_in_chain = pos->internal_.pos_in_chain;
        if (datlen + pos->pos > buf->total_len) {
            datlen = buf->total_len - pos->pos;
        }
    }
    else {
        chain = buf->first;
        pos_in_chain = 0;
        if (datlen > buf->total_len) {
            datlen = buf->total_len;
        }
    }
    
    if (datlen == 0) {
        goto done;
    }

    if (buf->freeze_start) {
        result = -1;
        goto done;
    }

    nread = datlen;

    while (datlen && datlen >= chain->off - pos_in_chain) {
        size_t copylen = chain->off - pos_in_chain;
        memcpy(data,
               chain->buffer + chain->misalign + pos_in_chain,
               copylen);
        data += copylen;
        datlen -= copylen;

        chain = chain->next;
        pos_in_chain = 0;
        EVUTIL_ASSERT(chain || datlen == 0);
    }

    if (datlen) {
        EVUTIL_ASSERT(chain);
        EVUTIL_ASSERT(datlen + pos_in_chain <= chain->off);

        memcpy(data, chain->buffer + chain->misalign + pos_in_chain,
               datlen);
    }

    result = nread;
done:
    EVBUFFER_UNLOCK(buf);
    return result;
}

最后，我们看一下链表的移动操作，代表函数evbuffer_add_buffer 和evbuffer_remove_buffer

• 目的chain链可能为不带任何数据的空链表，进行整理后进行 COPY_CHAIN
• 如果不是空链，那么追加到最后，APPEND_CHAIN

int
evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
{                   
    struct evbuffer_chain *pinned, *last;
    size_t in_total_len, out_total_len;
    int result = 0;

    EVBUFFER_LOCK2(inbuf, outbuf);
    in_total_len = inbuf->total_len;
    out_total_len = outbuf->total_len; 

    if (in_total_len == 0 || outbuf == inbuf) {
        goto done;
    }

    if (outbuf->freeze_end || inbuf->freeze_start) {
        result = -1;
        goto done;
    }

    if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
        result = -1;
        goto done;
    }

    if (out_total_len == 0) {
        /* There might be an empty chain at the start of outbuf; free
         * it. */
        evbuffer_free_all_chains(outbuf->first);
        COPY_CHAIN(outbuf, inbuf);
    }
    else {
        APPEND_CHAIN(outbuf, inbuf);
    }

    RESTORE_PINNED(inbuf, pinned, last);

    inbuf->n_del_for_cb += in_total_len;
    outbuf->n_add_for_cb += in_total_len;

#if HAVE_CALLBACK
    evbuffer_invoke_callbacks_(inbuf);
    evbuffer_invoke_callbacks_(outbuf);
#endif

done:
    EVBUFFER_UNLOCK2(inbuf, outbuf);
    return result;
}

static inline void
COPY_CHAIN(struct evbuffer *dst, struct evbuffer *src)
{
    ASSERT_EVBUFFER_LOCKED(dst);
    ASSERT_EVBUFFER_LOCKED(src);
    dst->first = src->first;
    if (src->last_with_datap == &src->first) {
        dst->last_with_datap = &dst->first;
    }
    else {
        dst->last_with_datap = src->last_with_datap;
    }
    dst->last = src->last;
    dst->total_len = src->total_len;
}

static void
APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
{
    struct evbuffer_chain **chp;

    ASSERT_EVBUFFER_LOCKED(dst);
    ASSERT_EVBUFFER_LOCKED(src);

    chp = evbuffer_free_trailing_empty_chains(dst);
    *chp = src->first;

    if (src->last_with_datap == &src->first) {
        dst->last_with_datap = chp;
    }
    else {
        dst->last_with_datap = src->last_with_datap;
    }
    dst->last = src->last;
    dst->total_len += src->total_len;
}

evbuffer_remove_buffer 稍微复杂一些，主要体现在：

• 移动一部分节点到目的链表
• 有可能出现最后的节点移动了一半的数据

/*  XXXX should return ev_ssize_t */
int
evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst,
    size_t datlen)
{
    /*XXX We should have an option to force this to be zero-copy.*/

    /*XXX can fail badly on sendfile case. */
    struct evbuffer_chain *chain, *previous;
    size_t nread = 0;
    int result;

    EVBUFFER_LOCK2(src, dst);

    chain = previous = src->first;

    if (datlen == 0 || dst == src) {
        result = 0;
        goto done;
    }

    if (dst->freeze_end || src->freeze_start) {
        result = -1;
        goto done;
    }

    /* short-cut if there is no more data buffered */
    if (datlen >= src->total_len) {
        datlen = src->total_len;
        evbuffer_add_buffer(dst, src);
        result = (int)datlen; /*XXXX should return ev_ssize_t*/
        goto done;
    }
    /* removes chains if possible */
    while (chain->off <= datlen) {
        /* We can't remove the last with data from src unless we
         * remove all chains, in which case we would have done the if
         * block above */
        EVUTIL_ASSERT(chain != *src->last_with_datap);
        nread += chain->off;
        datlen -= chain->off;
        previous = chain;
        if (src->last_with_datap == &chain->next)
            src->last_with_datap = &src->first;
        chain = chain->next; 
    }

    if (nread) {
        /* we can remove the chain */
        struct evbuffer_chain **chp;
        chp = evbuffer_free_trailing_empty_chains(dst);

        if (dst->first == NULL) {
            dst->first = src->first;
        } else {
            *chp = src->first;
        }
        dst->last = previous;
        previous->next = NULL;
        src->first = chain;
        advance_last_with_data(dst);

        dst->total_len += nread;
        dst->n_add_for_cb += nread;
    }

    /* we know that there is more data in the src buffer than
     * we want to read, so we manually drain the chain */
    evbuffer_add(dst, chain->buffer + chain->misalign, datlen);
    chain->misalign += datlen;
    chain->off -= datlen;
    nread += datlen;

    /* You might think we would want to increment dst->n_add_for_cb
     * here too.  But evbuffer_add above already took care of that.
     */
    src->total_len -= nread;
    src->n_del_for_cb += nread;    
    
#if HAVE_CALLBACK
    if (nread) {
        evbuffer_invoke_callbacks_(dst);
        evbuffer_invoke_callbacks_(src);
    }
#endif
    result = (int)nread;/*XXXX should change return type */

done:
    EVBUFFER_UNLOCK2(src, dst);
    return result;
}

3.3 运行测试

单元测试使用gtest来做，
测试有个核心函数，来验证chain链结构是否是正常的（防止中间操作破坏）

#define tt_assert assert
/*
 * Validates that an evbuffer is good.
 * Returns false if it isn't, true if it is
 */
static int
evbuffer_validate_(struct evbuffer *buf)
{
    struct evbuffer_chain *chain;
    size_t sum = 0;
    int found_last_with_datap = 0;

    if (buf->first == NULL) {
        tt_assert(buf->last == NULL);
        tt_assert(buf->total_len == 0);
    }

    chain = buf->first;

    tt_assert(buf->last_with_datap);
    if (buf->last_with_datap == &buf->first) {
        found_last_with_datap = 1;
    }
        while (chain != NULL) {
        if (&chain->next == buf->last_with_datap) {
            found_last_with_datap = 1;
        }
        sum += chain->off;
        if (chain->next == NULL) {
            tt_assert(buf->last == chain);
        }
        tt_assert(chain->buffer_len >= chain->misalign + chain->off);
        chain = chain->next;
    }

    if (buf->first) {
        tt_assert(*buf->last_with_datap);
    }

    if (*buf->last_with_datap) {
        chain = (*buf->last_with_datap);
        if (chain->off == 0 || buf->total_len == 0) {
            tt_assert(chain->off == 0);
            tt_assert(chain == buf->first);
            tt_assert(buf->total_len == 0);
        }

        chain = chain->next;
        while (chain != NULL) {
            tt_assert(chain->off == 0);
            chain = chain->next;
        }
    }
    else {
        tt_assert(buf->last_with_datap == &buf->first);
    }
    tt_assert(found_last_with_datap);
    tt_assert(sum == buf->total_len);
    return TRUE;
}

通过分别编译原版-levent的实现，以及裁剪版的my_buffer.c的实现，我们可以做出对比：

g++ -Wall -g2 -o ut_buffer \
                my_buffer.c ut_buffer.cc \
                -I../include -DMLEVEL=5 -lcrypto -lgtest -pthread
g++ -Wall -g2 -o ut_buffer2 \
                ut_buffer.cc \
                -I../include -DMLEVEL=5 -levent -lcrypto -lgtest -pthread

[==========] Running 4 tests from 1 test case.
[----------] Global test environment set-up.
[----------] 4 tests from sdk_buffer
[ RUN      ] sdk_buffer.evbuffer_add_printf
[       OK ] sdk_buffer.evbuffer_add_printf (1 ms)
[ RUN      ] sdk_buffer.evbuffer_add_buffer
[       OK ] sdk_buffer.evbuffer_add_buffer (0 ms)
[ RUN      ] sdk_buffer.evbuffer_add_remove
[       OK ] sdk_buffer.evbuffer_add_remove (0 ms)
[ RUN      ] sdk_buffer.evbuffer_add_buffer_big
[       OK ] sdk_buffer.evbuffer_add_buffer_big (0 ms)
[----------] 4 tests from sdk_buffer (1 ms total)
[----------] Global test environment tear-down
[==========] 4 tests from 1 test case ran. (1 ms total)
[  PASSED  ] 4 tests.

初步测试是ok的，证明裁剪成功了。

4. 结论

通过本次的裁剪移植的实践，对evbuffer内部操作又有了进一步的了解。
evbuffer作为libevent的核心函数，对上层的 buffer_evnent进行支撑，裁剪的evbuffer可以在我们其他I/O操作的程序中去使用。
如果还涉及多线程、文件I/O、网络I/O的处理，直接引入libevent库进行操作也是不错的选择。