网络编程之半同步/半异步模式

这种模式是一个主线程，主线程自己创建一个epoll然后负责接收连接，也只负责接收连接，当接收到连接之后，将连接的fd分发给子线程。
子线程有多个，类似于线程池的模式，不同的是这里的子线程每个子线程都有自己的一个epoll，从主线程分发过来的fd会被加入到子线程的epoll中然后由子线程接管后续的读写。

整个Server的工作流程是首先初始化num_threads个工作线程，工作线程的信息用LIBEVENT_THREAD结构体保存，其中包括eventbase、管道的fd、以及属于该线程的消息队列。然后创建主线程自己的main_base，只负责监听连接事件，然后启动主循环监听。
主线程接收到连接之后将连接的fd分发给子线程处理，使用dispatch_conn_new(int sfd, int event_flags)函数，该函数中使用的是轮询分配的方式，这里主线程通知子线程的方式是通过管道，子线程在刚启动时就一直监听着管道上的读事件，主线程首先将消息CQ_ITEM放入子线程的消息队列，然后往该子线程的管道里写入一个字符c，表示有新的fd分配给子线程。
子线程监听到管道的读事件之后，回调thread_libevent_process(int fd, short which, void *arg)这个函数，在这个函数当中子线程读取消息队列，然后将消息队列里面读出来的fd加入到自己的loop中，监听其读事件。
最后就是在子线程自己的回调读函数中处理包了，这里用的是包长度+包内容的协议，每当读取到一个完整的包，就可以调用自己的业务逻辑了。

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <ctype.h>
#include <time.h>
#include <signal.h>
#include <unistd.h>

#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>

#include <pthread.h>

#include <event.h>
#include <event2/listener.h>
#include <event2/util.h>

typedef struct LIBEVENT_THREAD LIBEVENT_THREAD;

/* An item in the connection queue. */
typedef struct conn_queue_item CQ_ITEM;
struct conn_queue_item {
    int               sfd;
    int               event_flags;
    CQ_ITEM          *next;
};

/* A connection queue. */
typedef struct conn_queue CQ;
struct conn_queue {
    CQ_ITEM *head;
    CQ_ITEM *tail;
    pthread_mutex_t lock;
    pthread_cond_t  cond;
};

typedef struct conn conn;
struct conn {
    int sfd;

    int nBytes;     // 还要读nBytes个字节才是一个完整的消息
    int nReads;     // 缓冲区里保存了多少字节的消息
    int len;        // 缓冲区大小的长度
    char *buf;      // 保存当前消息的缓冲区大小

    LIBEVENT_THREAD *thread;    /* Pointer to the thread object serving this connection */
};

static int sigignore(int sig);

static conn *conn_new(void);
static void conn_free(conn *con);

static CQ_ITEM *cqi_new(void);
static void cqi_free(CQ_ITEM *item);
static void cq_init(CQ *cq);
static CQ_ITEM *cq_pop(CQ *cq);
static void cq_push(CQ *cq, CQ_ITEM *item);

// 多个线程，每个线程一个 event_base
struct LIBEVENT_THREAD {
    pthread_t thread_id;        /* unique ID of this thread */
    struct event_base *base;    /* libevent handle this thread uses */

    // 用于管道读写事件的监听
    struct event notify_event;  /* listen event for notify pipe */

    // 读写管道文件描述符
    int notify_receive_fd;      /* receiving end of notify pipe */
    int notify_send_fd;         /* sending end of notify pipe */

    // 线程需要处理的连接队列
    // TODO: conn_queue
    struct conn_queue *new_conn_queue;  /* queue of new connections to handle */
} ;

typedef struct {
    pthread_t thread_id;        /* unique ID of this thread */
    struct event_base *base;    /* libevent handle this thread uses */
} LIBEVENT_DISPATCHER_THREAD;

int num_threads = 4;
static struct event_base *main_base;

static LIBEVENT_DISPATCHER_THREAD dispatcher_thread;
static LIBEVENT_THREAD *threads;

/* 新连接最近一次被分配到的线程 */
static int last_thread = -1;

/**
 * Dispatches a new connection to another thread. This is only ever called
 * from the main thread.
 */
void dispatch_conn_new(int sfd, int event_flags) {
    CQ_ITEM *item = cqi_new();
    char buf[1];

    int tid = (last_thread + 1) % num_threads;

    LIBEVENT_THREAD *thread = threads + tid;

    last_thread = tid;

    item->sfd = sfd;
    item->event_flags = event_flags;

    cq_push(thread->new_conn_queue, item);

    // 当管道中被写入数据后，worker thread的事件会被触发，然后thread_libevent_process()函数会被调用
    buf[0] = 'c';
    if (write(thread->notify_send_fd, buf, 1) != 1) {
        perror("Writing to thread notify pipe");
    }
}

static void
listener_cb(struct evconnlistener *listener, evutil_socket_t fd,
    struct sockaddr *sa, int socklen, void *user_data)
{
	struct event_base *base = user_data;

    // 接受新来的连接，然后将新来的连接分配到子线程的event_base中
    dispatch_conn_new(fd, EV_READ | EV_PERSIST);
}

static void 
conn_readcb(struct bufferevent *bev, void *user_data) 
{
    conn *con = user_data;
    struct evbuffer *input = bufferevent_get_input(bev);
    uint32_t len;

    // buffer里最多有4096个字节，需要用循环一次性将所有的包都读出来
    // 否则如果有两个包在缓冲区内，只读了第一个，后续没有新来的包，回调函数不触发
    while (1) {
        // 上一个包还未完全读完
        if (con->nBytes > 0) {
            // buf的偏移量为 nReads
            int nRead = bufferevent_read(bev, con->buf + con->nReads, con->nBytes);
            con->nBytes -= nRead;
            con->nReads += nRead;
            // 读取到了一个完整的包
            if (con->nBytes == 0) {
                // 调用业务逻辑代码
                bufferevent_write(bev, con->buf, con->nReads);
            }
        }

        /**
         * 1. 上一个包读完了，缓冲区里已经有了头长度，把头长度读出来
         * 2. 上一个包未读完，也就是这一次drain(用光了)缓冲区，即break
         */
        if (evbuffer_get_length(input) >= 4) {
            bufferevent_read(bev, (uint8_t *)&len, sizeof(uint32_t));
            len = ntohl(len);
            con->nBytes = len;
            con->nReads = 0;    // 要读取一个新的包，把已读取长度置为0
            if (len > con->len) {
                // 大小不够，可采取各种扩容方式
                con->len = len;
                con->buf = realloc(con->buf, len);
                // TODO: check alloc error
                if (con->buf == NULL) {
                    fprintf(stderr, "alloc con->buf error!");
                    exit(1);
                }
            }
        } else {
            // 缓冲区中不够四个字节的头部长度，下次再读取
            break;
        }
    }
}

static void 
conn_eventcb(struct bufferevent *bev, short events, void *user_data)
{
    conn *con = user_data;
    if (events & BEV_EVENT_EOF) {
		// 客户端关闭了连接，正常关闭
		// printf("Connection closed.\n");
	} 
    if (events & BEV_EVENT_ERROR) {
		printf("Got an error on the connection: %s\n",
		    strerror(errno));
	} 
    if (events & BEV_EVENT_TIMEOUT) {
		// 连接超时
		// printf("Connection timeout!\n");
	}
    // 只会出现上述三类情况，其他的事件都未注册
    // bufferevent_free自动释放与其关联的socket
    bufferevent_free(bev);
    conn_free(con);
}

/*
 * Processes an incoming "handle a new connection" item. This is called when
 * input arrives on the libevent wakeup pipe.
 *
 * 当管道有数据可读的时候会触发此函数的调用
 * fd: 是管道的描述符
 * item->sfd: 是新建立的连接的描述符
 */
static void thread_libevent_process(int fd, short which, void *arg) {
    LIBEVENT_THREAD *me = arg;
    CQ_ITEM *item;
    conn *con;
    struct bufferevent *bev;
    char buf[1];

    if (read(fd, buf, 1) != 1)
        fprintf(stderr, "Can't read from libevent pipe\n");
    
    switch (buf[0])
    {
    case 'c':
        item = cq_pop(me->new_conn_queue);

        if (NULL != item) {

            bev = bufferevent_socket_new(me->base, item->sfd, BEV_OPT_CLOSE_ON_FREE);
            if (!bev) {
                fprintf(stderr, "Error constructing bufferevent!");
                close(item->sfd);
                goto clean;
            }

            con = conn_new();
            con->sfd = item->sfd;
            con->thread = me;

            bufferevent_setcb(bev, conn_readcb, NULL, conn_eventcb, con);
            bufferevent_enable(bev, EV_READ | EV_PERSIST);

            // 这里应该改成从配置文件里读
            // 设置超时时间
            struct timeval tv = {30, 0};
            bufferevent_set_timeouts(bev, &tv, NULL);

            clean:
            cqi_free(item);
        }
        break;
    
    }
}

/**
 * 填充struct event
 * 初始化线程工作队列
 */
static void setup_thread(LIBEVENT_THREAD *me) {
    me->base = event_init();
    if (!me->base) {
        fprintf(stderr, "Can't allocate event base\n");
        exit(1);
    }

    event_set(&me->notify_event, me->notify_receive_fd,
                EV_READ | EV_PERSIST, thread_libevent_process, me);
    event_base_set(me->base, &me->notify_event);

    if (event_add(&me->notify_event, 0) == -1) {
        fprintf(stderr, "Can't monitor libevent notify pipe\n");
        exit(1);
    }

    // 初始化该线程的工作队列
    me->new_conn_queue = malloc(sizeof(struct conn_queue));
    if (me->new_conn_queue == NULL) {
        perror("Failed to allocate memory for connection queue");
        exit(EXIT_FAILURE);
    }
    cq_init(me->new_conn_queue);

}

/**
 * Worker thread: main event loop
 * 工作线程: 启动事件循环
 */
static void *worker_libevent(void *arg) {
    LIBEVENT_THREAD *me = arg;

    event_base_loop(me->base, 0);
    return NULL;
}

// 启动工作线程
static void create_worker(void *(*func)(void *), void *arg) {
    pthread_t       thread;
    pthread_attr_t  attr;
    int             ret;

    pthread_attr_init(&attr);

    if ((ret = pthread_create(&thread, &attr, func, arg)) != 0) {
        fprintf(stderr, "Can't create thread: %s\n",
                strerror(ret));
        exit(1);
    }
}

// 初始化线程子系统，创建工作线程
void thread_init(int nthreads, struct event_base *main_base) {
    threads = calloc(nthreads, sizeof(LIBEVENT_THREAD));
    if (!threads) {
        perror("Can't allocate thread descriptors");
        exit(1);
    }

    dispatcher_thread.base = main_base;
    dispatcher_thread.thread_id = pthread_self();

    // 创建用于通知消息的管道
    for (int i = 0; i < nthreads; i++) {
        int fds[2];
        if (pipe(fds)) {
            perror("Can't create notify pipe");
            exit(1);
        }

        threads[i].notify_receive_fd = fds[0];
        threads[i].notify_send_fd = fds[1];

        // 初始化线程信息数据结构，最重要的是将event的回调函数设置为thread_libevent_process()
        setup_thread(&threads[i]);
    }

    // 启动每一个工作线程，工作线程都是 work_libevent()
    for (int i = 0; i < nthreads; i++) {
        create_worker(worker_libevent, &threads[i]);
    }
}

int main() {
    const int port = 2323;
    struct evconnlistener *listener;
    struct sockaddr_in sin = {0};

    setvbuf(stdout, 0, _IONBF, 0);
    setbuf(stderr, NULL);

    main_base = event_init();

    if (sigignore(SIGPIPE) == -1) {
        perror("failed to ignore SIGPIPE; sigaction");
        exit(1);
    }

    // 启动工作线程
    thread_init(num_threads, main_base);

    // 启动主线程事件循环，监听连接事件
    sin.sin_family = AF_INET;
    sin.sin_port = htons(port);

    listener = evconnlistener_new_bind(main_base, listener_cb, (void *)main_base, 
		LEV_OPT_REUSEABLE|LEV_OPT_CLOSE_ON_FREE, -1,
	    (struct sockaddr*)&sin,
	    sizeof(sin));
	
	if (!listener) {
		fprintf(stderr, "Could not create a listener!\n");
		exit(1);
	}

    // 启动事件循环
    if (event_base_loop(main_base, 0) != 0) {
        perror("start event_base_loop error!");
    }

    // 清理资源
    evconnlistener_free(listener);
    event_base_free(main_base);

    // TODO: 线程资源也要清理
    
}

static conn *conn_new(void) {
    conn *con = NULL;

    con = malloc(sizeof(conn));
    if (NULL == con) {
        fprintf(stderr, "conn_new: malloc error!");
        exit(1);
    }

    // init it
    con->buf = NULL;
    con->len = 0;
    con->nBytes = 0;
    con->nReads = 0;

    return con;
}

static void conn_free(conn *con) {
    if (con)
        free(con->buf);
    free(con);
}

/**
 * Returns a fresh connection queue item.
 */
static CQ_ITEM *cqi_new(void) {
    // 使用空闲链表的方式更好，这里为了方便起见不用了
    // TODO: 加上空闲链表的方式
    CQ_ITEM *item = NULL;

    item = malloc(sizeof(CQ_ITEM));
    if (NULL == item) {
        fprintf(stderr, "cqi_new: malloc error!");
        exit(1);
    }

    return item;
}

static void cqi_free(CQ_ITEM *item) {
    // TODO: 空闲链表
    free(item);
}

/*
 * Initializes a connection queue.
 */
static void cq_init(CQ *cq) {
    pthread_mutex_init(&cq->lock, NULL);
    pthread_cond_init(&cq->cond, NULL);
    cq->head = NULL;
    cq->tail = NULL;
}

/*
 * Looks for an item on a connection queue, but doesn't block if there isn't
 * one.
 * Returns the item, or NULL if no item is available
 */
static CQ_ITEM *cq_pop(CQ *cq) {
    CQ_ITEM *item;

    pthread_mutex_lock(&cq->lock);
    item = cq->head;
    if (NULL != item) {
        cq->head = item->next;
        if (NULL == cq->head)
            cq->tail = NULL;
    }
    pthread_mutex_unlock(&cq->lock);

    return item;
}

/*
 * Adds an item to a connection queue.
 * 在连接队列中增加一个项
 */
static void cq_push(CQ *cq, CQ_ITEM *item) {
    item->next = NULL;

    pthread_mutex_lock(&cq->lock);
    // 在队尾添加
    if (NULL == cq->tail)
        cq->head = item;
    else
        cq->tail->next = item;
    cq->tail = item;
    pthread_cond_signal(&cq->cond);
    pthread_mutex_unlock(&cq->lock);
}

static int sigignore(int sig) {
    struct sigaction sa = { .sa_handler = SIG_IGN, .sa_flags = 0 };

    if (sigemptyset(&sa.sa_mask) == -1 || sigaction(sig, &sa, 0) == -1) {
        return -1;
    }
    return 0;
}