epoll的LT、ET模式和EPOLLONESHOT事件实例介绍

LT、ET模式

LT（Level Trigger）电平触发模式 -> 默认的， 相当于效率较高的poll

epoll_wait检测到事件发生后通知了应用程序 -> 应用程序可以不立即处理该事件。
下一次调用epoll_wait时，还会再次向应用程序通知此事件，直到该事件被处理

ET（Edge Trigger）边沿触发模式 epoll的高效工作模式（需向epoll内核事件表中注册一fd上的EPOLLET事件）

epoll_wait检测到事件发生后通知了应用程序 ->应用程序必须立即处理该事件，因为后续的epoll_wait调用将不再向应用程序通知这一事件。

注：
我的ubuntu是12.04，查看/usr/include/sys/epoll.h为：

struct epoll_event
{
    uint32_t events;           /*Epoll events*/
    epoll_data_t data;         /*User data variable*/
}__attribute__((__packed__));

所以不能像书里写的那样直接使用epoll_event，而要使用struct epoll_event

代码示例：

#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/epoll.h>
#include <pthread.h>

#define MAX_EVENT_NUMBER 1024
#define BUFFER_SIZE 10

int setnonblocking( int fd )
{
    int old_option = fcntl( fd, F_GETFL );
    int new_option = old_option | O_NONBLOCK;
    fcntl( fd, F_SETFL, new_option );
    return old_option;
}

void addfd( int epollfd, int fd, bool enable_et )
{
    struct epoll_event event;
    event.data.fd = fd;
    event.events = EPOLLIN;
    if( enable_et )
    {
        event.events |= EPOLLET;
    }
    epoll_ctl( epollfd, EPOLL_CTL_ADD, fd, &event );
    setnonblocking( fd );
}

void lt(struct epoll_event* events, int number, int epollfd, int listenfd )
{
    char buf[ BUFFER_SIZE ];
    for ( int i = 0; i < number; i++ )
    {
        int sockfd = events[i].data.fd;
        if ( sockfd == listenfd )
        {
            struct sockaddr_in client_address;
            socklen_t client_addrlength = sizeof( client_address );
            int connfd = accept( listenfd, ( struct sockaddr* )&client_address, &client_addrlength );
            addfd( epollfd, connfd, false );
        }
        else if ( events[i].events & EPOLLIN )
        {
            printf( "event trigger once\n" );
            memset( buf, '\0', BUFFER_SIZE );
            int ret = recv( sockfd, buf, BUFFER_SIZE-1, 0 );
            if( ret <= 0 )
            {
                close( sockfd );
                continue;
            }
            printf( "get %d bytes of content: %s\n", ret, buf );
        }
        else
        {
            printf( "something else happened \n" );
        }
    }
}

void et(struct epoll_event* events, int number, int epollfd, int listenfd )
{
    char buf[ BUFFER_SIZE ];
    for ( int i = 0; i < number; i++ )
    {
        int sockfd = events[i].data.fd;
        if ( sockfd == listenfd )
        {
            struct sockaddr_in client_address;
            socklen_t client_addrlength = sizeof( client_address );
            int connfd = accept( listenfd, ( struct sockaddr* )&client_address, &client_addrlength );
            addfd( epollfd, connfd, true );
        }
        else if ( events[i].events & EPOLLIN )
        {
            printf( "event trigger once\n" );
            while( 1 )
            {
                memset( buf, '\0', BUFFER_SIZE );
                int ret = recv( sockfd, buf, BUFFER_SIZE-1, 0 );
                if( ret < 0 )
                {
                    if( ( errno == EAGAIN ) || ( errno == EWOULDBLOCK ) )
                    {
                        printf( "read later\n" );
                        break;
                    }
                    close( sockfd );
                    break;
                }
                else if( ret == 0 )
                {
                    close( sockfd );
                }
                else
                {
                    printf( "get %d bytes of content: %s\n", ret, buf );
                }
            }
        }
        else
        {
            printf( "something else happened \n" );
        }
    }
}

int main( int argc, char* argv[] )
{
    if( argc <= 2 )
    {
        printf( "usage: %s ip_address port_number\n", basename( argv[0] ) );
        return 1;
    }
    const char* ip = argv[1];
    int port = atoi( argv[2] );

    int ret = 0;
    struct sockaddr_in address;
    bzero( &address, sizeof( address ) );
    address.sin_family = AF_INET;
    inet_pton( AF_INET, ip, &address.sin_addr );
    address.sin_port = htons( port );

    int listenfd = socket( PF_INET, SOCK_STREAM, 0 );
    assert( listenfd >= 0 );

    ret = bind( listenfd, ( struct sockaddr* )&address, sizeof( address ) );
    assert( ret != -1 );

    ret = listen( listenfd, 5 );
    assert( ret != -1 );

    struct epoll_event events[ MAX_EVENT_NUMBER ];
    int epollfd = epoll_create( 5 );
    assert( epollfd != -1 );
    addfd( epollfd, listenfd, true );

    while( 1 )
    {
        int ret = epoll_wait( epollfd, events, MAX_EVENT_NUMBER, -1 );
        if ( ret < 0 )
        {
            printf( "epoll failure\n" );
            break;
        }

        lt( events, ret, epollfd, listenfd );
        //et( events, ret, epollfd, listenfd );
    }

    close( listenfd );
    return 0;
}

客户端测试代码：

#include <sys/socket.h>
#include <netinet/in.h>
#include <assert.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <stdlib.h>
#include <errno.h>
#include <time.h>

#include <pthread.h>

int main(int argc, char **argv)
{
    int connectfd;
    struct sockaddr_in servaddr;
    char sendline[20] = "hello world!";
    int i;

    int port = atoi(argv[2]);

    if(argc != 3)
        printf("usage: tcpcli <IP addresss> <port number>\n");

    bzero(&servaddr, sizeof(servaddr));

    connectfd = socket(AF_INET, SOCK_STREAM, 0);

    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(port);
    inet_pton(AF_INET, argv[1], &servaddr.sin_addr);

    printf("start connect\n");

    connect(connectfd, (struct sockaddr *)&servaddr, sizeof(servaddr) );
    if(errno == EINTR)
        printf("connect is interupted\n");
    if(errno == ETIMEDOUT)
        printf("connect is interupted2\n");


    for(i = 0; i < 4; i++)
    {
        sleep(1);
        send(connectfd, sendline, strlen(sendline), 0);
    }
    exit(0);
}

执行结果：

kip@dou:~/Desktop/ServerCoding$ ./dou_cli 127.0.0.1 9877
start connect

lt模式下：

kip@dou:~/Desktop/ServerCoding$ ./epoll_et_lt 127.0.0.1 9877
event trigger once
get 9 bytes of content:hello wor
event trigger once
get 9 bytes of content:ld!hello 
event trigger once
get 9 bytes of content:world!hel
event trigger once
get 9 bytes of content:lo world!
event trigger once
get 9 bytes of content:hello wor
event trigger once
get 3 bytes of content:ld!
event trigger once

et模式下：

kip@dou:~/Desktop/ServerCoding$ ./epoll_et_lt 127.0.0.1 9877
event trigger once
get 9 bytes of content:hello wor
get 3 bytes of content:ld!
read later
event trigger once
get 9 bytes of content:hello wor
get 3 bytes of content:ld!
read later
event trigger once
get 9 bytes of content:hello wor
get 3 bytes of content:ld!
read later
event trigger once
get 9 bytes of content:hello wor
get 3 bytes of content:ld!
read later
event trigger once

EPOLLONESHOT事件

一线程读完某socket上数据后开始处理这些数据，此时该socket上又有新数据可读（即EPOLLIN再次被触发） -> 另一线程被唤醒读新的数据 =>造成2个线程同时操作一个socket的局面
EPOLLONESHOT事件：保证一个socket连接在任一时刻只被一个线程处理。
注册了EPOLLONESHOT事件的文件描述符，操作系统最多触发其上注册的一个可读、可写/异常事件。
且只触发一次，除非使用epoll_ctl重置该fd上注册的EPOLLONESHOT事件。=>所以，注册了EPOLLONESHOT事件的socket一旦被某个线程处理完，该线程应立即重置该fd上注册的EPOLLONESHOT事件，以确保这个socket下次可读时，其EPOLLIN事件能被触发。

示例代码如下：
客户端代码同上，服务器端代码：

#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <pthread.h>

#define MAX_EVENT_NUMBER 10
#define BUFFER_SIZE 10


typedef struct fds_t
{
    int epollfd;
    int sockfd;
}fds;

void reset_oneshot(int epollfd, int fd)
{
    struct epoll_event event;
    event.data.fd = fd;
    event.events = EPOLLIN | EPOLLET | EPOLLONESHOT;
    epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &event);
}

int setnobloking(int fd)
{
    int old_opt = fcntl(fd, F_GETFL);
    int new_op = old_opt | O_NONBLOCK;
    fcntl(fd, F_SETFL, new_op);
    return old_opt;
}

void addfd(int epollfd, int fd, int flag)
{
    struct epoll_event event;
    event.data.fd = fd;
    event.events = EPOLLIN | EPOLLET;
    if(flag)
    {
        event.events |= EPOLLONESHOT;
    }
    epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event);
    setnobloking(fd);
}

void *worker(void *arg)
{
    int sockfd = ((fds*)arg)->sockfd;
    int epollfd = ((fds*)arg)->epollfd;

    printf("start new thread to receive data on fd:%d\n", sockfd);

    char buf[BUFFER_SIZE];
    memset(buf, '\0', BUFFER_SIZE);
    while(1)
    {
        int ret = recv(sockfd, buf, BUFFER_SIZE-1, 0);
        if(ret < 0)
        {
            if(EAGAIN == errno)
            {
                reset_oneshot(epollfd, sockfd);
                printf("read later\n");
                break;
            }
        }
        else if(0 == ret)
        {
            close(sockfd);  
            printf("foreiner closed the connection\n");
            break;      
        }
        else
        {
            printf("get %d bytes of content on:%s\n", ret, buf);    
            sleep(5);     //模拟处理事件所消耗的时间
        }
    }
    printf("end thread receiving data on fd:%d\n", sockfd);
}

int main( int argc, char * argv[])
{
    int i;
    if(argc <= 2)
    {
        printf("usage: %s ip_address port_number\n", basename(argv[0]));
        return 1;
    }   
    const char *ip = argv[1];
    int port  = atoi(argv[2]);

    int ret = 0;
    struct sockaddr_in address;
    bzero(&address, sizeof(address));
    address.sin_family = AF_INET;
    inet_pton(AF_INET, ip, &address.sin_addr);
    address.sin_port = htons(port);

    int listenfd = socket(AF_INET, SOCK_STREAM, 0);
    assert(listenfd >= 0);

    ret = bind(listenfd, (struct sockaddr *)&address, sizeof(address));
    assert(ret != -1);

    ret = listen(listenfd, 5);
    assert(ret != -1);

    struct epoll_event events[MAX_EVENT_NUMBER];

    int epollfd = epoll_create(5);
    assert(epollfd != -1);

    addfd(epollfd, listenfd, 0);

    while(1)
    {
        int ret = epoll_wait(epollfd, events, MAX_EVENT_NUMBER, -1);
        if(ret < 0)
        {
            printf("epoll failure\n");
            break;
        }
    //  lt(events, ret, epollfd, listenfd);
        //et(events, ret, epollfd, listenfd);
        for(i = 0; i < ret; i++)
        {
            int sockfd = events[i].data.fd;
            if(sockfd == listenfd)
            {
                struct sockaddr_in cli_addr;
                socklen_t cli_addr_len = sizeof(cli_addr);

                int connfd = accept(listenfd, (struct sockaddr *)&cli_addr, &cli_addr_len);

                addfd(epollfd, connfd, 1);
            }
            else if(events[i].events & EPOLLIN)
            {
                pthread_t thread;
                fds fds_for_new_worker;
                fds_for_new_worker.epollfd = epollfd;
                fds_for_new_worker.sockfd = sockfd;
                pthread_create(&thread, NULL, worker, (void *)&fds_for_new_worker);
            }
            else
            {
                printf("sth else happened1\n");
            }
        }
    }
    close(listenfd);
    return 0;

}

addfd(epollfd, connfd, 0)，即当不注册EPOLLONESHOT事件时，服务器端反应为：

kip@dou:~/Desktop/ServerCoding$ ./epoll_one_shot 127.0.0.1 9877start new thread to receive data on fd:5
get 9 bytes of content on:hello wor
start new thread to receive data on fd:5
get 9 bytes of content on:ld!hello 
start new thread to receive data on fd:5
get 9 bytes of content on:world!hel
start new thread to receive data on fd:5
get 9 bytes of content on:lo world!
start new thread to receive data on fd:5
get 9 bytes of content on:hello wor
get 3 bytes of content on:ld!lo wor
foreiner closed the connection
end thread receiving data on fd:5

addfd(epollfd, connfd, 1)，对连接socket描述符注册EPOLLONESHOT事件时，服务器端反应为：

kip@dou:~/Desktop/ServerCoding$ ./epoll_one_shot 127.0.0.1 9877
start new thread to receive data on fd:5
get 9 bytes of content on:hello wor
get 9 bytes of content on:ld!hello 
get 9 bytes of content on:world!hel
get 9 bytes of content on:lo world!
get 9 bytes of content on:hello wor
get 3 bytes of content on:ld!lo wor
foreiner closed the connection
end thread receiving data on fd:5

可以看出明显区别，在一个线程处理事件未完成时，如果注册了EPOLLONESHOT，该连接socket有新数据到了，不会触发新的线程去读这些数据。而是处理完，本线程继续读，直至（errno==EAGAIN）无数据可读 -> 该线程才放弃为该socket服务，同时要重置该fd上的EPOLLONESHOT事件（reset_oneshot()）