基于linux下的epoll服务端和客户端编程实例

epoll概念及基本函数介绍:
对epoll的详细介绍说明可查看此链接：
http://blog.csdn.net/h514434485/article/details/78151090

---

epoll服务端代码实例:

#include <stdio.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/socket.h>   
#include <sys/epoll.h>   
#include <netinet/in.h>   
#include <arpa/inet.h>   
#include <fcntl.h>
#include <errno.h>

#ifdef __cplusplus
#if __cplusplus
extern "C"{
#endif
#endif /* End of #ifdef __cplusplus */

#define SERV_PORT           (5555)   
#define MAX_LISTENQ         (32)    
#define MAX_EVENT           (128) 
#define MAX_BUFSIZE         (512)   

int setNonBlocking(int p_nSock)
{   
    int nOpts;   
    nOpts=fcntl(p_nSock,F_GETFL);   
    if(nOpts<0)   
    {   
        printf("[%s %d] Fcntl Sock GETFL fail!\n",__FUNCTION__,__LINE__);
        return -1;
    }   

    nOpts = nOpts|O_NONBLOCK;   
    if(fcntl(p_nSock,F_SETFL,nOpts)<0)   
    {  
        printf("[%s %d] Fcntl Sock SETFL fail!\n",__FUNCTION__,__LINE__);
        return -1;   
    } 

    return 0;
}   

int main()
{
    int nRet,i;
    int nSockfd;
    int nAcceptfd;
    int nEventNum;   
    int nEpollfd;
    int nListenfd;

    struct sockaddr_in clientaddr;     
    struct sockaddr_in serveraddr;
    socklen_t clilen = sizeof(struct sockaddr_in);

    struct epoll_event ev;
    struct epoll_event events[MAX_EVENT];

    //生成用epoll专用文件描述符   
    nEpollfd=epoll_create(MAX_EVENT);
    if (nEpollfd <= 0)
    {
        printf("[%s %d] Epoll Create fail return:%d!\n",__FUNCTION__,__LINE__,nEpollfd);
        return 0;
    }

    nListenfd = socket(AF_INET, SOCK_STREAM, 0);
    if (nListenfd < 0)
    {
        printf("[%s %d] Socket Create fail return:%d!\n",__FUNCTION__,__LINE__,nListenfd);
        return 0;
    }

    if (setNonBlocking(nListenfd) < 0)
    {
        return 0;
    }

    memset(&serveraddr, 0x00, sizeof(serveraddr));
    serveraddr.sin_family = AF_INET;   
    serveraddr.sin_addr.s_addr=inet_addr("0.0.0.0");   
    serveraddr.sin_port=htons(SERV_PORT);   

    if (bind(nListenfd,(struct sockaddr *)&serveraddr, sizeof(serveraddr)) < 0)
    {
        printf("[%s %d] Bind fd fail!\n",__FUNCTION__,__LINE__);
        return 0;
    }

    if (listen(nListenfd,MAX_LISTENQ) < 0)
    {
        printf("[%s %d] Listen fd fail!\n",__FUNCTION__,__LINE__);
        return 0;
    } 

    ev.data.fd=nListenfd;        
    ev.events=EPOLLIN;//默认使用EPOLLLT模式    
    if (epoll_ctl(nEpollfd,EPOLL_CTL_ADD,nListenfd,&ev) < 0)
    {
        printf("[%s %d] Epoll ctl error!\n",__FUNCTION__,__LINE__);
        return 0;
    }

    //置位true
    char szRecvBuf[MAX_BUFSIZE];
    memset(szRecvBuf,0x0,MAX_BUFSIZE); 

    for (;;)    
    {   
        nEventNum = epoll_wait(nEpollfd, events, MAX_EVENT, -1);
        printf("events num:%d\n",nEventNum);
        for ( i = 0;  i<nEventNum; i++ )
        {   

            if ((events[i].events & EPOLLERR) ||  
                    (events[i].events & EPOLLHUP))  
            {  
                //客户端异常被关掉
                printf("[%s %d] Epoll Event Error!\n",__FUNCTION__,__LINE__);
                close (events[i].data.fd);  
                continue;  
            }
            else if (nListenfd == events[i].data.fd)  
            {
                printf("[%d] Listenfd:%d,events[i].data.fd:%d\n", __LINE__,nListenfd,events[i].data.fd);

                nAcceptfd = accept(nListenfd, (struct sockaddr*)&clientaddr, &clilen);
                if(nAcceptfd < 0)
                {
                    printf("[%s %d] Accept fd fail return:%d!\n",__FUNCTION__,__LINE__,nAcceptfd);
                    continue;
                }

                printf("[%d] Acceptfd:%d,IP:%s,Port:%d\n",__LINE__,nAcceptfd,inet_ntoa(clientaddr.sin_addr),ntohs(clientaddr.sin_port));
                if (setNonBlocking(nAcceptfd) < 0)
                {
                    continue;
                }

                ev.data.fd = nAcceptfd;// 将新连接也加入EPOLL的监听队列
                ev.events = EPOLLIN | EPOLLET ;
                if(epoll_ctl(nEpollfd, EPOLL_CTL_ADD, nAcceptfd, &ev)< 0)
                {
                    printf("[%s %d] Epoll ctl error!\n",__FUNCTION__,__LINE__);
                    continue;
                }

                continue;
            }

            if(events[i].events&EPOLLIN)                   
            {   
                if ((nSockfd = events[i].data.fd) < 0)
                {
                    continue;
                }

                printf("[%s %d] EPOLLIN Sockfd:%d\n",__FUNCTION__,__LINE__,nSockfd);

                memset(szRecvBuf, 0x00, sizeof(szRecvBuf));
                if ( (nRet = recv(nSockfd, szRecvBuf, MAX_BUFSIZE,0)) < 0)   
                {  
                    printf("[%s %d] Recv Data nRet:%d, error:%s!\n",__FUNCTION__,__LINE__,nRet,strerror(errno));
                    if (errno == ECONNRESET)
                    {
                        close(nSockfd);  
                        epoll_ctl(nEpollfd,EPOLL_CTL_DEL,nSockfd,&ev);  
                        events[i].data.fd = -1;
                    }

                    continue;
                } 
                else if (nRet == 0)   
                {   
                    //连接关闭
                    printf("[%s %d] Recv error, client had closed!\n",__FUNCTION__,__LINE__);
                    close(nSockfd);  
                    epoll_ctl(nEpollfd,EPOLL_CTL_DEL,nSockfd,&ev);  
                    events[i].data.fd = -1;
                    continue;
                } 

                printf("\n******************************\n");
                printf("**RecvData:%s\n",szRecvBuf);
                printf("******************************\n");

                //修改sockfd上要处理的事件为EPOLLOUT               
                ev.data.fd = nSockfd;            
                ev.events = EPOLLOUT | EPOLLET;
                epoll_ctl(nEpollfd, EPOLL_CTL_MOD, nSockfd, &ev);
            }
            else if(events[i].events&EPOLLOUT)                 
            {    
                if ((nSockfd = events[i].data.fd) < 0)
                {
                    continue;
                }

                printf("[%s %d] EPOLLOUT Sockfd:%d\n", __FUNCTION__, __LINE__, nSockfd);

                nRet = send(nSockfd, szRecvBuf, strlen(szRecvBuf), 0);
                printf("[%s %d] Send Len:%d\n", __FUNCTION__, __LINE__, nRet);

                //修改sockfd上要处理的事件为EPOLIN                   
                ev.data.fd = nSockfd;
                ev.events = EPOLLIN | EPOLLET;
                epoll_ctl(nEpollfd, EPOLL_CTL_MOD, nSockfd, &ev);
            }   
        }   
    }  

    close(nListenfd);
}

#ifdef __cplusplus
#if __cplusplus
}
#endif
#endif /* End of #ifdef __cplusplus */

---

epoll客户端代码实例:

 #include<stdio.h>
 #include<string.h>
 #include<stdlib.h>
 #include<errno.h>
 #include <fcntl.h>
 #include <sys/epoll.h> 
 #include<sys/types.h>
 #include<sys/socket.h>
 #include<unistd.h>//close()
 #include<netinet/in.h>//struct sockaddr_in
 #include<arpa/inet.h>//inet_ntoa

 #define DEST_PORT 5555
 #define DEST_IP_ADDRESS "0.0.0.0"

#define MAX_EVENT           (128) 
#define MAX_BUFSIZE         (512)   

int setNonBlocking(int p_nSock)
{
    int nOpts;
    nOpts = fcntl(p_nSock, F_GETFL);
    if (nOpts<0)
    {
        printf("[%s %d] Fcntl Sock GETFL fail!\n", __FUNCTION__, __LINE__);
        return -1;
    }

    nOpts = nOpts | O_NONBLOCK;
    if (fcntl(p_nSock, F_SETFL, nOpts)<0)
    {
        printf("[%s %d] Fcntl Sock SETFL fail!\n", __FUNCTION__, __LINE__);
        return -1;
    }

    return 0;
}

int main(int argc,char *argv[])
{
    int sock_fd;
    sock_fd = socket(AF_INET,SOCK_STREAM,0);
    if(sock_fd < 0)
    {
        printf("[%s %d] Socket Create fail return:%d!\n", __FUNCTION__, __LINE__, sock_fd);
        return 0;
    } 

    struct sockaddr_in addr_serv;//服务器端地址
    memset(&addr_serv,0,sizeof(addr_serv));

    addr_serv.sin_family = AF_INET;
    addr_serv.sin_port =  htons(DEST_PORT);
    addr_serv.sin_addr.s_addr = inet_addr(DEST_IP_ADDRESS);

    setNonBlocking(sock_fd);

    if( connect(sock_fd,(struct sockaddr *)&addr_serv,sizeof(struct sockaddr)) < 0)
    {
        printf("[%s %d] Client connect errno:%d!\n",__FUNCTION__,__LINE__,errno);
        if (errno != EINPROGRESS) 
        {
            printf("[%s %d] Connnect Remote Server fail.\n",__FUNCTION__, __LINE__);
            return(0);
        } 
    } 

    int nEpollfd;
    struct epoll_event ev;
    struct epoll_event events[MAX_EVENT];

    //生成用epoll专用文件描述符   
    nEpollfd=epoll_create(MAX_EVENT);
    if (nEpollfd <= 0)
    {
        printf("[%s %d] Epoll create fail return:%d!\n",__FUNCTION__,__LINE__,nEpollfd);
        return 0;
    }

    ev.data.fd=sock_fd;        
    ev.events=EPOLLIN|EPOLLET|EPOLLOUT;     
    if (epoll_ctl(nEpollfd,EPOLL_CTL_ADD,sock_fd,&ev) < 0)
    {
        printf("[%s %d] Epoll ctl error!\n",__FUNCTION__,__LINE__);
        return 0;
    }

    int i;
    int nSockfd;
    int nEventNum;
    int nSendNum;
    int nRecvNum;
    char szSendBuf[]="NonBlocking Epoll Test!";
    char szRecvBuf[MAX_BUFSIZE];

    for (;;)
    {
        nEventNum = epoll_wait(nEpollfd, events, MAX_EVENT, -1);
        printf(" epoll events num:%d\n", nEventNum);

        for (i = 0; i<nEventNum; i++)
        {
            if ((events[i].events & EPOLLERR) ||  
                    (events[i].events & EPOLLHUP))  
            {  
                printf("[%s %d] Epoll event error!\n",__FUNCTION__,__LINE__);
                close (events[i].data.fd);  
                continue;  
            }

            if (events[i].events & EPOLLOUT) 
            {
                printf("[%d] EPOLLOUT Event.\n",__LINE__);
                if ((nSockfd = events[i].data.fd) < 0)
                {
                    continue;
                }

                nSendNum = send(nSockfd, szSendBuf, sizeof(szSendBuf), 0);
                if (nSendNum < 0)
                {
                    printf("[%s %d] Send Error", __FUNCTION__,__LINE__);
                    return 0;
                } 

                //修改sockfd上要处理的事件为EPOLLIN              
                ev.data.fd = nSockfd;
                ev.events = EPOLLIN | EPOLLET;
                epoll_ctl(nEpollfd, EPOLL_CTL_MOD, nSockfd, &ev);
            }
            else if (events[i].events&EPOLLIN)
            {
                printf("[%d] EPOLLIN Event.\n", __LINE__);
                if ((nSockfd = events[i].data.fd) < 0)
                {
                    continue;
                }

                memset(szRecvBuf, 0x00, sizeof(szRecvBuf));
                nRecvNum = recv(nSockfd, szRecvBuf, sizeof(szRecvBuf), 0);
                if (nRecvNum < 0)
                {
                    printf("[%s %d] Client Recv Data Error!\n",__FUNCTION__, __LINE__);
                    return 0;
                }

                printf("\n******************************\n");
                printf("**RecvData:%s\n", szRecvBuf);
                printf("******************************\n");

                //修改sockfd上要处理的事件为EPOLOUT                   
                ev.data.fd = nSockfd;
                ev.events = EPOLLOUT | EPOLLET;
                epoll_ctl(nEpollfd, EPOLL_CTL_MOD, nSockfd, &ev);
            }
        }
    }

    close(sock_fd);

    return 0;
}

效果如下:

---

注意点:
在服务端创建监听listen sock并添加到epoll中时要注意，代码提示如下：

    ev.data.fd=nListenfd;        
    ev.events=EPOLLIN;//不要写成ev.events=EPOLLIN|EPOLLET;     
    if (epoll_ctl(nEpollfd,EPOLL_CTL_ADD,nListenfd,&ev) < 0)
    {
        printf("[%s %d] Epoll ctl error!\n",__FUNCTION__,__LINE__);
        return 0;
    }

具体可能出现的问题及原因引用网上作者文章一段：
1.listen_sock使用LT模式，其他使用ET模式，这样就不会遗漏后续的connect请求
(其实还有另外一种方式，listen_sock也使用ET模式，只是要用while包裹accept语句，直到返回-1，这里我们不讨论)

在遗落后续的connect请求这个问题上花费了很多时间。
具体测试表现是：1000个client发起tcp connect，并在连接成功之后发送数据。测试中，1000个client正常连接并发送数据，但epoll中未收到部分client的连接请求。
使用netstat命令查看有以下发现：

netstat -apn|grep 7777|wc -l
1001
netstat -apn|grep 7777|grep -|wc -l
139

系统已经建立了1000个tcp连接，但是其中有小部分连接不是由我们的程序负责建立的。

netstat -apn|grep 7777|grep -

使用上面的命令能够发现，有小部分连接处于ESTABLISHED状态，但进程信息那一栏却是”-“

这应该是ET模式下连接就绪触发了一次，但我们的程序没有来及处理，ET不会再触发，导致这小部分连接再没有机会得到accept处理。
解决这个问题的话，把listen_sock(负责listen和accept新连接的socket)设成LT模式，这样没来得及处理的connect请求会在下次执行到epoll_wait时继续得到处理。

---

参考资料：
http://www.cnblogs.com/bugchecker/p/tips_of_using_epoll.html