Linux - Epoll的基础使用

目录

一、Epoll 事件驱动流程

二、Reactor设计模式

三、Epoll相关函数

1. 创建：epoll_create

2. 向EPOLL对象中添加、修改或者删除感兴趣的事件：epoll_ctl

3. 收集指定epoll监视的事件中已发生的事件：epoll_wait

四、触发模式

1. 水平触发(Level_triggered)

2. 边缘触发(Edge_triggered)

五、使用Epoll实现简单的web服务器

1. C语言源码

2.   ./Resources/index.html源码

3.   ./Reources/reply.html源码

六、使用Epoll封装的库的使用

1. 库源码(globals.h)

2. epoll相关接口的实现文件(comm_epoll.c)

3. 对外相关接口的实现(comm.c)

4. 使用库实现回声服务器(main.c)

附：Linux下默认最大文件描述符的查询和修改

---

---

一、Epoll 事件驱动流程

事件驱动流程中的未就绪事件队列和就绪事件队列采用红黑树实现。

二、Reactor设计模式

1. Epoll使用此设计模式实现
2. 优点：

1. 响应快，不必为单个同步事件所阻塞，虽然Reactor本身依然是同步的

2. 编程相对简单，可以最大程度的避免复杂的多线程及同步问题，并且避免了多线程/进程的切换开销

3. 可扩展性，可以方便的通过增加Reactor实例个数来充分利用CPU资源

4. 可复用性，reactor框架本身与具体事件处理逻辑无关，具有很高的复用性

三、Epoll相关函数

1. 创建：epoll_create

/*************************************************************************************************
函数：int epoll_create(int size);
头部：#include
功能：创建一个新的epoll实例
参数：
    size        - 最大监听数。从Linux2.6.8开始，这个参数被忽略，但必须大于 0
返回：
    成功        - 返回创建的文件描述符
    失败        - 返回-1，errno表示出错信息
**************************************************************************************************/

2. 向EPOLL对象中添加、修改或者删除感兴趣的事件：epoll_ctl

/*************************************************************************************************
函数：int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
头部：#include
功能：向EPOLL对象中添加、修改或者删除感兴趣的事件
参数：
    epfd        - 通过epoll_create()创建的实例
    op            - 动作，可用如下宏：
                            EPOLL_CTL_ADD - 注册新的fd到epfd中
                            EPOLL_CTL_MOD - 修改已注册的fd的监听事件
                            EPOLL_CTL_DEL - 从epfd中删除一个fd
    fd            - 需要监听的fd
    event        - 需要监听的事件
返回：
    成功        - 返回0
    失败        - 返回-1，errno表示出错信息
**************************************************************************************************/

// 参数event对应的数据结构
typedef union epoll_data {
    void         *ptr;
    int          fd;
    uint32_t      u32;
    uint64_t      u64;
} epoll_data_t;

struct epoll_event {
    uint32_t events;    // Epoll events
    epoll_data_t data;    // User data variable
};

// events取值
EPOLLIN     // 表示关联fd有可读操作
EPOLLOUT    // 表示关联fd有可写操作
EPOLLERR    // 对应的fd发送错误
EPOLLHUP    // 对应的连接被挂起

3. 收集指定epoll监视的事件中已发生的事件：epoll_wait

/*************************************************************************************************
函数：int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout);
头部：#include
功能：收集指定epoll监视的事件中已发生的事件
参数：
    epfd        - 通过epoll_create()创建的实例
    events      - 则是分配好的 epoll_event结构体数组，epoll将会把发生的事件复制到 events数组中(events不可以是空指针，内核只负责把数据复制到这个 events数组中，不会去帮助我们在用户态中分配内存。内核这种做法效率很高)
    maxevents   - 本次可以返回的最大事件数目，通常 maxevents参数与预分配的events数组的大小是相等的
    timeout     - 表示在没有检测到事件发生时最多等待的时间（单位为毫秒），如果 timeout为0，立刻返回，不会等待。-1表示无限期阻塞
返回：
    > 0         - 有事件的文件描述符数量
    = 0         - 超时
    -1          - 出错，errno表示出错信息
**************************************************************************************************/

// events参数对应的数据结构
struct epoll_event {
    __uint32_t      events;
    epoll_data_t    data;
};

typedef enum epoll_data {
    void         *ptr;
    int           fd;
    uint32_t      u32;
    uint64_t      u64;
} epoll_data_t;

四、触发模式

1. 水平触发(Level_triggered)

当被监控的文件描述符上有可读写事件发生时，epoll_wait()会通知处理程序去读写。如果这次没有把数据一次性全部读写完(如读写缓冲区太小)，那么下次调用 epoll_wait()时，它还会通知你在上没读写完的文件描述符上继续读写，当然如果你一直不去读写，它会一直通知你！！！如果系统中有大量你不需要读写的就绪文件描述符，而它们每次都会返回，这样会大大降低处理程序检索自己关心的就绪文件描述符的效率。

设置方式: 默认即水平触发

2. 边缘触发(Edge_triggered)

当被监控的文件描述符上有可读写事件发生时，epoll_wait()会通知处理程序去读写。如果这次没有把数据全部读写完(如读写缓冲区太小)，那么下次调用epoll_wait()时，它不会通知你，也就是它只会通知你一次，直到该文件描述符上出现第二次可读写事件才会通知你！！！这种模式比水平触发效率高，系统不会充斥大量你不关心的就绪文件描述符。

设置方式: stat->_ev.events = EPOLLIN | EPOLLET

五、使用Epoll实现简单的web服务器

1. C语言源码

#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/epoll.h>
#include <sys/socket.h>

typedef struct connect_stat connect_stat_t;
typedef void (*page_process_func)(connect_stat_t*);

struct connect_stat {
    int fd;
    char name[64];
    char age[64];
    struct epoll_event _ev;
    int status;                 // 0 - 未登录  1 - 已登录
    page_process_func handler;  // 不同页面的处理函数
};

static int epfd = 0;

static const char* main_header = "HTTP/1.0 200 OK\r\nServer: Martin Server\r\nContent-Type: text/html\r\nConnection: Close\r\n";

// 获取默认connect_stat_t节点
connect_stat_t* get_stat(int fd);
void set_nonblock(int fd);

// 成功返回服务器socket，失败返回 -1
int init_server(const char *ip, unsigned short port);
// 初始化连接，然后等待浏览器发送请求
void add_event_to_epoll(int newfd);

void do_http_request(connect_stat_t* p);
void do_http_respone(connect_stat_t* p);

void welcome_response_handler(connect_stat_t* p);
void commit_respone_handler  (connect_stat_t* p);

int main(int argc, char* argv[]) {
    if (argc < 3) {
        fprintf(stderr, "%s:please input [ip][port]!\n", argv[0]);
        exit(1);
    }

    int sock = init_server(argv[1], atoi(argv[2]));
    if (sock < 0) { exit(2); }

    // 1. 创建epoll
    epfd = epoll_create(256);
    if (epfd < 0) {
        fprintf(stderr, "epoll_create(): failed! reason: %s!\n", strerror(errno));
        exit(3);
    }

    struct epoll_event _ev; // epoll 结构填充
    connect_stat_t* stat = get_stat(sock);
    _ev.events = EPOLLIN;   // 初始监听事件为读
    _ev.data.ptr = stat;

    // 托管
    epoll_ctl(epfd, EPOLL_CTL_ADD, sock, &_ev);  //将sock添加到epfd中，监听读事件

    struct epoll_event revents[64]; // 有事件时，事件存放的数组

    int timeout = -1;   // -1 - 无限期阻塞   0 - 立即返回
    int count = 0;      // 事件数量
    int done = 0;

    while (!done) {
        switch ((count = epoll_wait(epfd, revents, sizeof(revents)/sizeof(revents[0]), -1)))
        {
        case -1:
            fprintf(stderr, "epoll_wait(): failed! reason: %s!\n", strerror(errno));
            exit(4);
            break;
        case 0:
            fprintf(stderr, "timeout!\n");
            exit(5);
        default:
            for (int i = 0; i < count; ++i) {
                connect_stat_t* p = (connect_stat_t*)revents[i].data.ptr;
                int fd = p->fd;
                // 如果是服务器fd并且是读事件，则接收连接
                if (fd == sock && (revents[i].events & EPOLLIN)) {
                    struct sockaddr_in client;
                    int client_len = sizeof(struct sockaddr_in);
                    int newfd = accept(sock, (struct sockaddr*)(&client), &client_len);
                    if (newfd < 0) {
                        fprintf(stderr, "accept(): failed! reason: %s!\n", strerror(errno));
                        continue;
                    }
                    printf("get a new client: %d\n", newfd);
                    add_event_to_epoll(newfd);
                }
                else {  // 对非服务器socket进行处理
                    if (revents[i].events & EPOLLIN) {
                        do_http_request((connect_stat_t*)revents[i].data.ptr);
                    }
                    else if(revents[i].events & EPOLLOUT) {
                        do_http_respone((connect_stat_t*)revents[i].data.ptr);
                    }
                    else {

                    }
                }
            }
            break;
        }   // end switch
    }   // end while

    return 0;
}

connect_stat_t* get_stat(int fd) {
    connect_stat_t* p = (connect_stat_t*)malloc(sizeof(connect_stat_t));
    if (!p) { return NULL; }

    bzero(p, sizeof(connect_stat_t));
    p->fd = fd;

    return p;
}

void set_nonblock(int fd) {
    int fl = fcntl(fd, F_GETFL);
    fcntl(fd, F_SETFL, fl | O_NONBLOCK);
}

int init_server(const char* ip, unsigned short port) {
    if (!ip) {
        fprintf(stderr, "func:%s(): Parameter[ip] is empty!\n", __FUNCTION__);
        return -1;
    }

    int ret = 0;
    int sock = socket(AF_INET, SOCK_STREAM, 0);
    if (sock < 0) {
        fprintf(stderr, "func:%s() - socket(): failed! reason: %s!\n", __FUNCTION__, strerror(errno));
        return -1;
    }

    int opt = 1;
    setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));

    // 绑定IP地址端口号
    struct sockaddr_in local;
    local.sin_family = AF_INET;
    local.sin_port = htons(port);
    local.sin_addr.s_addr = inet_addr(ip);
    ret = bind(sock, (struct sockaddr*)(&local), sizeof(local));
    if (ret < 0) {
        fprintf(stderr, "func:%s() - bind(): failed! reason: %s!\n", __FUNCTION__, strerror(errno));
        return -1;
    }

    // 监听
    ret = listen(sock, 5);
    if (ret < 0) {
        fprintf(stderr, "func:%s() - listen(): failed! reason: %s!\n", __FUNCTION__, strerror(errno));
        return -1;
    }

    return sock;
}

void add_event_to_epoll(int newfd) {
    if (newfd < 0) { return; }

    connect_stat_t* p = get_stat(newfd);
    set_nonblock(newfd);

    p->_ev.events = EPOLLIN;
    p->_ev.data.ptr = p;

    epoll_ctl(epfd, EPOLL_CTL_ADD, newfd, &p->_ev);
}

void do_http_request(connect_stat_t* p) {
    if (!p) { return; }

    char buffer[4096] = { 0 };

    ssize_t len = read(p->fd, buffer, sizeof(buffer) - 1);
    if (len == 0) {
        printf("close client[%d] socket!\n", p->fd);
        epoll_ctl(epfd, EPOLL_CTL_DEL, p->fd, NULL);
        close(p->fd);
        free(p);
        return;
    }
    else if (len < 0) {
        fprintf(stderr, "%s() - read(): failed! reason: %s\n", __FUNCTION__, strerror(errno));
        return;
    }

    buffer[len] = '\0';
    printf("recv request data: %s\n", buffer);

    char* pos = buffer;

    if (!strncasecmp(pos, "GET", 3)) {
        p->handler = welcome_response_handler;
    }
    else if (!strncasecmp(pos, "POST", 4)) {
        printf("---data: %s\n", buffer);
        // 获取URL
        printf("---post---\n");
        pos += strlen("POST");
        while (*pos == ' ' || *pos == '/') ++pos;

        if (!strncasecmp(pos, "commit", 6)) {
            int len = 0;

            printf("post commit --------\n");
            pos = strstr(buffer, "\r\n\r\n");
            char* end = NULL;
            if (end = strstr(pos, "name=")) {
                pos = end + strlen("name=");
                end = pos;
                while (('a' <= *end && *end <= 'z') || ('A' <= *end && *end <= 'Z') || ('0' <= *end && *end <= '9'))	end++;
                len = end - pos;
                if (len > 0) {
                    memcpy(p->name, pos, end - pos);
                    p->name[len] = '\0';
                }
            }

            if (end = strstr(pos, "age=")) {
                pos = end + strlen("age=");
                end = pos;
                while ('0' <= *end && *end <= '9')	end++;
                len = end - pos;
                if (len > 0) {
                    memcpy(p->age, pos, end - pos);
                    p->age[len] = '\0';
                }
            }
            p->handler = commit_respone_handler;

        }
        else {
            p->handler = welcome_response_handler;
        }
    }
    else {
        p->handler = welcome_response_handler;
    }

    // 生成处理结果
    p->_ev.events = EPOLLOUT;
    epoll_ctl(epfd, EPOLL_CTL_MOD, p->fd, &p->_ev);

}

void do_http_respone(connect_stat_t* p) {
    if (!p) { return; }
    p->handler(p);
}

void welcome_response_handler(connect_stat_t* p) {
    int fd = open("./Resources/index.html", O_RDONLY);
    if (fd < 0) {
        fprintf(stderr, "commit_respone_handler() - open: failed! reason: %s\n", strerror(errno));
        return;
    }

    char buffer[4096] = { 0 };
    int rlen = read(fd, buffer, sizeof(buffer));
    close(fd);
    if (rlen < 1) {
        fprintf(stderr, "commit_respone_handler() - read: failed! reason: %s\n", strerror(errno));
        return;
    }

    char* content = (char*)malloc(strlen(main_header) + 128 + rlen);
    char len_buf[64] = { 0 };
    strcpy(content, main_header);
    snprintf(len_buf, sizeof(len_buf), "Content-Length: %d\r\n\r\n", (int)rlen);
    strcat(content, len_buf);
    strcat(content, buffer);
    printf("send reply to client!\n");

    int wlen = write(p->fd, content, strlen(content));
    if (wlen < 1) {
        fprintf(stderr, "commit_respone_handler() - write: failed! reason: %s\n", strerror(errno));
        return;
    }
    
    p->_ev.events = EPOLLIN;
    epoll_ctl(epfd, EPOLL_CTL_MOD, p->fd, &p->_ev);
    free(content);
}

void commit_respone_handler(connect_stat_t* p) {
    int fd = open("./Resources/reply.html", O_RDONLY);
    if (fd < 0) {
        fprintf(stderr, "commit_respone_handler() - open: failed! reason: %s\n", strerror(errno));
        return;
    }

    char buffer[1024] = { 0 };
    int rlen = read(fd, buffer, sizeof(buffer));
    close(fd);
    if (rlen < 1) {
        fprintf(stderr, "commit_respone_handler() - read: failed! reason: %s\n", strerror(errno));
        return;
    }

    char* content = (char*)malloc(strlen(main_header) + 128 + rlen);
    char* tmp = (char*)malloc(rlen + 128);
    char len_buf[64] = { 0 };
    strcpy(content, main_header);
    snprintf(len_buf, sizeof(len_buf), "Content-Length: %d\r\n\r\n", (int)rlen);
    strcat(content, len_buf);
    snprintf(tmp, rlen + 128, buffer, p->name, p->age);
    strcat(content, tmp);
    printf("send reply to client!\n");

    //printf("write data: %s\n", content);
    int wlen = write(p->fd, content, strlen(content));
    if (wlen < 1) {
        fprintf(stderr, "commit_respone_handler() - write: failed! reason: %s\n", strerror(errno));
        return;
    }

    p->_ev.events = EPOLLIN;
    epoll_ctl(epfd, EPOLL_CTL_MOD, p->fd, &p->_ev);
    free(tmp);
    free(content);

}

2.   ./Resources/index.html源码

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>登录</title>
    <style type="text/css">
        
html {
    width: 100%;
    height: 100%;
    overflow: hidden;
    font-style: sans-serif;
}

body{
    width: 100%;
    height: 100%;
    font-family: 'Open Sans',sans-serif;
    margin: 0;
    background-color: #4A374A;
}

#login {
    position: absolute;
    top: 50%;
    left: 50%;
    margin: -150px 0 0 -150px;
    width: 300px;
    height: 300px;
}

#login h1{
    color: #fff;
    text-shadow: 0 0 10px;
    letter-spacing: 1px;
    text-align: center;
}

h1 {
    font-size: 2em;
    margin: 0.67em 0;
}

input {
    width: 278px;
    height: 18px;
    margin-bottom: 10px;
    outline: none;
    padding: 10px;
    font-size: 13px;
    color: #fff;
    text-shadow: 1px 1px 1px;
    border-top: 1px solid #31213D;
    border-left: 1px solid #31213D;
    border-right: 1px solid #31213D;
    border-bottom: 1px solid #56536A;
    border-radius: 4px;
    background-color: #2D2D3F;
}

#login .but {
    width: 300px;
    min-height: 20px;
    display: block;
    background-color: #4A77D4;
    border: 1px solid #3762BC;
    color: #FFF;
    padding: 9px 14px;
    font-size: 15px;
    line-height: normal;
    border-radius: 5px;
    margin: 0;
}
</style>
</head>

<body>
    <div id="login">
        <h1>登录</h1>
        <form action="commit" method="POST">
            <input type="text" required="required" placeholder="用户名" name="name"></input>
            <input type="password" required="required" placeholder="密码" name="age"></input>
            <button class="but" type="submit">登录</button>
        </form>
    </div>
    
</body>
</html>

3.   ./Reources/reply.html源码

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>欢迎访问</title>
</head>
<body>
    <div align="center" height="500px" >
        <br/><br/><br/>
        <h2>欢迎学霸同学&nbsp;%s &nbsp;,你的芳龄是&nbsp;%s！</h2><br/><br/>
        </div>
</body>
</html>

六、使用Epoll封装的库的使用

1. 提供超时踢出的功能

1. 库源码(globals.h)

#ifndef GLOBALS_H
#define GLOBALS_H

#include <sys/time.h>
#include <sys/resource.h>	
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/stat.h>

#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <errno.h>
#include <fcntl.h>
#include <assert.h>

#define FD_DESC_SZ		64

#define COMM_OK		  (0)
#define COMM_ERROR	 (-1)
#define COMM_NOMESSAGE	 (-3)
#define COMM_TIMEOUT	 (-4)
#define COMM_SHUTDOWN	 (-5)
#define COMM_INPROGRESS  (-6)
#define COMM_ERR_CONNECT (-7)
#define COMM_ERR_DNS     (-8)
#define COMM_ERR_CLOSING (-9)


//调试相关
#define DEBUG_LEVEL  0
#define DEBUG_ONLY   8
#define debug(m, n)    if( m >= DEBUG_LEVEL && n <= DEBUG_ONLY  ) printf
	
#define safe_free(x)	if (x) { free(x); x = NULL; }

typedef void PF(int, void *);

struct _fde {
    unsigned int type;
    u_short local_port;
    u_short remote_port;
    struct in_addr local_addr;
 
    char ipaddr[16];		/* dotted decimal address of peer */
   
   
    PF *read_handler;
    void *read_data;
    PF *write_handler;
    void *write_data;
    PF *timeout_handler;
    time_t timeout;
    void *timeout_data;
};

typedef struct _fde fde;

extern fde *fd_table;		// 例如:fd=1, 则访问方式为 fd_table[1]
extern int Biggest_FD;		

/*系统时间相关,设置成全局变量，供所有模块使用*/
extern struct timeval current_time;
extern double current_dtime;
extern time_t sys_curtime;


/* epoll 相关接口实现 */
extern void do_epoll_init(int max_fd);
extern void do_epoll_shutdown();
extern void epollSetEvents(int fd, int need_read, int need_write);
extern int do_epoll_select(int msec);

/*框架外围接口*/
void comm_init(int max_fd);
extern int comm_select(int msec);
extern void comm_call_handlers(int fd, int read_event, int write_event);
void  commUpdateReadHandler(int fd, PF * handler, void *data);
void commUpdateWriteHandler(int fd, PF * handler, void *data);



extern const char *xstrerror(void);
int ignoreErrno(int ierrno);

#endif /* GLOBALS_H */

2. epoll相关接口的实现文件(comm_epoll.c)

#include "globals.h"
#include <sys/epoll.h>

#define MAX_EVENTS	256	/* 一次处理的最大的事件 */

/* epoll structs */
static int kdpfd;
static struct epoll_event events[MAX_EVENTS];
static int epoll_fds = 0;
static unsigned *epoll_state;	/* 保存每个epoll 的事件状态 */

static const char *
epolltype_atoi(int x)
{
    switch (x) {

    case EPOLL_CTL_ADD:
	return "EPOLL_CTL_ADD";

    case EPOLL_CTL_DEL:
	return "EPOLL_CTL_DEL";

    case EPOLL_CTL_MOD:
	return "EPOLL_CTL_MOD";

    default:
	return "UNKNOWN_EPOLLCTL_OP";
    }
}

void do_epoll_init(int max_fd)
{

    
    kdpfd = epoll_create(max_fd);
    if (kdpfd < 0)
	  fprintf(stderr,"do_epoll_init: epoll_create(): %s\n", xstrerror());
    //fd_open(kdpfd, FD_UNKNOWN, "epoll ctl");
    //commSetCloseOnExec(kdpfd);

    epoll_state = calloc(max_fd, sizeof(*epoll_state));
}


void do_epoll_shutdown()
{
    
    close(kdpfd);
    kdpfd = -1;
    safe_free(epoll_state);
}



void epollSetEvents(int fd, int need_read, int need_write)
{
    int epoll_ctl_type = 0;
    struct epoll_event ev;

    assert(fd >= 0);
    debug(5, 8) ("commSetEvents(fd=%d)\n", fd);

	memset(&ev, 0, sizeof(ev));
    
    ev.events = 0;
    ev.data.fd = fd;

    if (need_read)
	ev.events |= EPOLLIN;

    if (need_write)
	ev.events |= EPOLLOUT;

    if (ev.events)
	ev.events |= EPOLLHUP | EPOLLERR;

    if (ev.events != epoll_state[fd]) {
	/* If the struct is already in epoll MOD or DEL, else ADD */
	if (!ev.events) {
	    epoll_ctl_type = EPOLL_CTL_DEL;
	} else if (epoll_state[fd]) {
	    epoll_ctl_type = EPOLL_CTL_MOD;
	} else {
	    epoll_ctl_type = EPOLL_CTL_ADD;
	}

	/* Update the state */
	epoll_state[fd] = ev.events;

	if (epoll_ctl(kdpfd, epoll_ctl_type, fd, &ev) < 0) {
	    debug(5, 1) ("commSetEvents: epoll_ctl(%s): failed on fd=%d: %s\n",
		epolltype_atoi(epoll_ctl_type), fd, xstrerror());
	}
	switch (epoll_ctl_type) {
	case EPOLL_CTL_ADD:
	    epoll_fds++;
	    break;
	case EPOLL_CTL_DEL:
	    epoll_fds--;
	    break;
	default:
	    break;
	}
    }
}

int do_epoll_select(int msec)
{
    int i;
    int num;
    int fd;
    struct epoll_event *cevents;

    /*if (epoll_fds == 0) {
	assert(shutting_down);
	return COMM_SHUTDOWN;
    }
    statCounter.syscalls.polls++;
    */
    num = epoll_wait(kdpfd, events, MAX_EVENTS, msec);
    if (num < 0) {
	getCurrentTime();
	if (ignoreErrno(errno))
	    return COMM_OK;

	debug(5, 1) ("comm_select: epoll failure: %s\n", xstrerror());
	return COMM_ERROR;
    }
    //statHistCount(&statCounter.select_fds_hist, num);

    if (num == 0)
	return COMM_TIMEOUT;

    for (i = 0, cevents = events; i < num; i++, cevents++) {
		fd = cevents->data.fd;
		comm_call_handlers(fd, cevents->events & ~EPOLLOUT, cevents->events & ~EPOLLIN);
    }

    return COMM_OK;
}

3. 对外相关接口的实现(comm.c)

#include "globals.h"
#include <string.h>

double current_dtime;
time_t sys_curtime;
struct timeval current_time;


int Biggest_FD = 1024;  /*默认的最大文件描述符数量 1024*/
static int MAX_POLL_TIME = 1000;	/* see also comm_quick_poll_required() */
fde *fd_table = NULL;	

time_t getCurrentTime(void)
{
    gettimeofday(&current_time, NULL);
    current_dtime = (double) current_time.tv_sec +
	(double) current_time.tv_usec / 1000000.0;
    return sys_curtime = current_time.tv_sec;
}


void
comm_close(int fd)
{
	assert(fd>0);
	fde *F = &fd_table[fd];
	if(F) memset((void *)F,'\0',sizeof(fde));
	epollSetEvents(fd, 0, 0);
	close(fd);
}

void
comm_init(int max_fd)
{
	if(max_fd > 0 ) Biggest_FD = max_fd;
	fd_table = calloc(Biggest_FD, sizeof(fde));
    do_epoll_init(Biggest_FD);
}



void
comm_select_shutdown(void)
{
    do_epoll_shutdown();
	if(fd_table) free(fd_table);
}


//static int comm_select_handled;


inline void
comm_call_handlers(int fd, int read_event, int write_event)
{
    fde *F = &fd_table[fd];
    
    debug(5, 8) ("comm_call_handlers(): got fd=%d read_event=%x write_event=%x F->read_handler=%p F->write_handler=%p\n"
	,fd, read_event, write_event, F->read_handler, F->write_handler);
    if (F->read_handler && read_event) {
	    PF *hdl = F->read_handler;
	    void *hdl_data = F->read_data;
	    /* If the descriptor is meant to be deferred, don't handle */

		debug(5, 8) ("comm_call_handlers(): Calling read handler on fd=%d\n", fd);
		//commUpdateReadHandler(fd, NULL, NULL);
		hdl(fd, hdl_data);
    }
	
    if (F->write_handler && write_event) {
	
	    PF *hdl = F->write_handler;
	    void *hdl_data = F->write_data;
	
	    //commUpdateWriteHandler(fd, NULL, NULL);
	    hdl(fd, hdl_data);
    }
}


int
commSetTimeout(int fd, int timeout, PF * handler, void *data)
{
    fde *F;
    debug(5, 3) ("commSetTimeout: FD %d timeout %d\n", fd, timeout);
    assert(fd >= 0);
    assert(fd < Biggest_FD);
    F = &fd_table[fd];

	
    if (timeout < 0) {
	F->timeout_handler = NULL;
	F->timeout_data = NULL;
	return F->timeout = 0;
    }
    assert(handler || F->timeout_handler);
    if (handler || data) {
	F->timeout_handler = handler;
	F->timeout_data = data;
    }
    return F->timeout = sys_curtime + (time_t) timeout;
}

void
commUpdateReadHandler(int fd, PF * handler, void *data)
{
    fd_table[fd].read_handler = handler;
    fd_table[fd].read_data = data;
    
    epollSetEvents(fd,1,0); 
}

void
commUpdateWriteHandler(int fd, PF * handler, void *data)
{
    fd_table[fd].write_handler = handler;
    fd_table[fd].write_data = data;
	
    epollSetEvents(fd,0,1); 
}



static void
checkTimeouts(void)
{
    int fd;
    fde *F = NULL;
    PF *callback;

    for (fd = 0; fd <= Biggest_FD; fd++) {
	F = &fd_table[fd];
	/*if (!F->flags.open)
	    continue;
	*/
	
	if (F->timeout == 0)
	    continue;
	if (F->timeout > sys_curtime)
	    continue;
	debug(5, 5) ("checkTimeouts: FD %d Expired\n", fd);
	
	if (F->timeout_handler) {
	    debug(5, 5) ("checkTimeouts: FD %d: Call timeout handler\n", fd);
	    callback = F->timeout_handler;
	    F->timeout_handler = NULL;
	    callback(fd, F->timeout_data);
	} else {
	    debug(5, 5) ("checkTimeouts: FD %d: Forcing comm_close()\n", fd);
	    comm_close(fd);
	}
    }
}


int
comm_select(int msec)
{
    static double last_timeout = 0.0;
    int rc;
    double start = current_dtime;

    debug(5, 3) ("comm_select: timeout %d\n", msec);

    if (msec > MAX_POLL_TIME)
	msec = MAX_POLL_TIME;


    //statCounter.select_loops++;
    /* Check timeouts once per second */
    if (last_timeout + 0.999 < current_dtime) {
	last_timeout = current_dtime;
	checkTimeouts();
    } else {
	int max_timeout = (last_timeout + 1.0 - current_dtime) * 1000;
	if (max_timeout < msec)
	    msec = max_timeout;
    }
    //comm_select_handled = 0;

    rc = do_epoll_select(msec);


    getCurrentTime();
    //statCounter.select_time += (current_dtime - start);

    if (rc == COMM_TIMEOUT)
	debug(5, 8) ("comm_select: time out\n");

    return rc;
}


const char *
xstrerror(void)
{
    static char xstrerror_buf[BUFSIZ];
    const char *errmsg;

    errmsg = strerror(errno);

    if (!errmsg || !*errmsg)
	errmsg = "Unknown error";

    snprintf(xstrerror_buf, BUFSIZ, "(%d) %s", errno, errmsg);
    return xstrerror_buf;
}


int
ignoreErrno(int ierrno)
{
    switch (ierrno) {
    case EINPROGRESS:
    case EWOULDBLOCK:
#if EAGAIN != EWOULDBLOCK
    case EAGAIN:
#endif
    case EALREADY:
    case EINTR:
#ifdef ERESTART
    case ERESTART:
#endif
	return 1;
    default:
	return 0;
    }
    /* NOTREACHED */
}

4. 使用库实现回声服务器(main.c)

#include "globals.h"
typedef struct _ConnectStat  ConnectStat;

#define BUFLEN  1024

struct _ConnectStat {
	int fd;
	char send_buf[BUFLEN];
	PF *handler;//不同页面的处理函数
};

//echo 服务实现相关代码
ConnectStat * stat_init(int fd);
void accept_connection(int fd, void *data);
void do_echo_handler(int fd, void  *data);
void do_echo_response(int fd,void *data);
void do_echo_timeout(int fd, void *data);



void usage(const char* argv)
{
	printf("%s:[ip][port]\n", argv);
}

void set_nonblock(int fd)
{
	int fl = fcntl(fd, F_GETFL);
	fcntl(fd, F_SETFL, fl | O_NONBLOCK);
}

int startup(char* _ip, int _port)  //创建一个套接字，绑定，检测服务器
{
	//sock
	//1.创建套接字
	int sock = socket(AF_INET, SOCK_STREAM, 0);
	if (sock < 0)
	{
		perror("sock");
		exit(2);
	}

	int opt = 1;
	setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));

	//2.填充本地 sockaddr_in 结构体（设置本地的IP地址和端口）
	struct sockaddr_in local;
	local.sin_port = htons(_port);
	local.sin_family = AF_INET;
	local.sin_addr.s_addr = inet_addr(_ip);

	//3.bind（）绑定
	if (bind(sock, (struct sockaddr*)&local, sizeof(local)) < 0)
	{
		perror("bind");
		exit(3);
	}
	//4.listen（）监听 检测服务器
	if (listen(sock, 5) < 0)
	{
		perror("listen");
		exit(4);
	}
	
	return sock;    //这样的套接字返回
}


ConnectStat * stat_init(int fd) {
	ConnectStat * temp = NULL;
	temp = (ConnectStat *)malloc(sizeof(ConnectStat));

	if (!temp) {
		fprintf(stderr, "malloc failed. reason: %m\n");
		return NULL;
	}

	memset(temp, '\0', sizeof(ConnectStat));
	temp->fd = fd;
	//temp->status = 0;
}

void do_welcome_handler(int fd, void  *data) {
	const char * WELCOME= "Welcome.\n";
	int wlen = strlen(WELCOME);
	int n ;
	ConnectStat * stat = (ConnectStat *)(data);
	
	if( (n = write(fd, "Welcome.\n",wlen)) != wlen ){
		
		if(n<=0){
		    fprintf(stderr, "write failed[len:%d], reason: %s\n",n,strerror(errno));
		}else fprintf(stderr, "send %d bytes only ,need to send %d bytes.\n",n,wlen);
		
	}else {
		commUpdateReadHandler(fd, do_echo_handler,(void *)stat);
		commSetTimeout(fd, 10, do_echo_timeout, (void *)stat);
	}
}


void do_echo_handler(int fd, void  *data) {
	ConnectStat * stat = (ConnectStat *)(data);
	char * p = NULL;
	
	assert(stat!=NULL);
	
	
	p = stat->send_buf;
	*p++ = '-';
	*p++ = '>';
	ssize_t _s = read(fd, p, BUFLEN-(p-stat->send_buf)-1); //2字节"->" +字符结束符.
    if (_s > 0)
    {
		
		*(p+_s) = '\0';
		printf("receive from client: %s\n", p);
		//_s--;
		//while( _s>=0 && ( stat->send_buf[_s]=='\r' || stat->send_buf[_s]=='\n' ) ) stat->send_buf[_s]='\0';
		
		if(!strncasecmp(p, "quit", 4)){//退出.
			comm_close(fd);
                        free(stat);
			return ;
		}
		//write(fd,
		commUpdateWriteHandler(fd, do_echo_response, (void *)stat);
		commSetTimeout(fd, 10, do_echo_timeout, (void *)stat);
	}else if (_s == 0)  //client:close
    {
        fprintf(stderr,"Remote connection[fd: %d] has been closed\n", fd);
        comm_close(fd);
        free(stat);
    }
    else //err occurred.
    {
        fprintf(stderr,"read faield[fd: %d], reason:%s [%d]\n",fd , strerror(errno), _s);
    }
}

void do_echo_response(int fd, void  *data) {
	ConnectStat * stat = (ConnectStat *)(data);
	int len = strlen(stat->send_buf);
	int _s = write(fd, stat->send_buf, len);
	
	if(_s>0){
		commSetTimeout(fd, 10, do_echo_timeout, (void *)stat);
		commUpdateReadHandler(fd, do_echo_handler, (void *)stat);
		
	}else if(_s==0){
		fprintf(stderr,"Remote connection[fd: %d] has been closed\n", fd);
                comm_close(fd);
                free(stat);
	}else {
		fprintf(stderr,"read faield[fd: %d], reason:%s [%d]\n",fd ,_s ,strerror(errno));
	}
}

//read()
//注册写事件
  
//写事件就绪
//write()

void accept_connection(int fd, void *data){
	struct sockaddr_in peer;
	socklen_t len = sizeof(peer);

	ConnectStat * stat = (ConnectStat *)data;
	
	int new_fd = accept(fd, (struct sockaddr*)&peer, &len);

	if (new_fd > 0)
	{
		ConnectStat *stat = stat_init(new_fd);
		set_nonblock(new_fd);

		printf("new client: %s:%d\n", inet_ntoa(peer.sin_addr), ntohs(peer.sin_port));
		commUpdateWriteHandler(new_fd, do_welcome_handler, (void *)stat);
		commSetTimeout(new_fd, 30,do_echo_timeout, (void *)stat);
	}
}

void do_echo_timeout(int fd, void *data){
	fprintf(stdout,"---------timeout[fd:%d]----------\n",fd);
	comm_close(fd);
        free(data);
}



int main(int argc,char **argv){

	if (argc != 3)     //检测参数个数是否正确
	{
		usage(argv[0]);
		exit(1);
	}

	int listen_sock = startup(argv[1], atoi(argv[2]));      //创建一个绑定了本地 ip 和端口号的套接字描述符
	//初始化异步事件处理框架epoll
	
	comm_init(102400);
	
	ConnectStat * stat = stat_init(listen_sock);
	commUpdateReadHandler(listen_sock,accept_connection,(void *)stat);

	do{
		//不断循环处理事件
		comm_select(1000);
		
	}while(1==1);

	comm_select_shutdown();



}

附：Linux下默认最大文件描述符的查询和修改

1. 查询文件描述符的最大值：

    ulimit -a (查询所有)

    ulimit -n (查询最大可打开的文件描述符)

2. 修改文件描述符的最大值：

     (1) ulimit -n 102400                (这种方式适用于单个终端单次使用)

     (2) 编辑 /etc/rc.local文件，步骤如下：

          a. vi /etc/rc.local

          b. 输入 ulimit -n 102400

          c. 保存退出即可