MY EPOLL

/**
* MoreStor SuperVault
* Copyright (c), 2008, Sierra Atlantic, Dream Team.
*
* epoll server head file
*
* Author(s): wxlin  <weixuan.lin@sierraatlantic.com>
*
* $Id:epoll.h,v 1.4 2008-12-30 08:14:55 wxlin Exp $
*
*/
#ifndef _EPOLL_H_
#define _EPOLL_H_

#include "hashmap.h"
#include "buffer.h"
#include "queue.h"
#include "threadpool.h"
#include <sys/epoll.h>
#include <pthread.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/epoll.h>

#define EPOLL_MAX_POOL_THREADS 10

struct Network;
struct epoll_server;
struct epoll_client;

enum epoll_type
{
    EPOLL_ACCEPT,
    EPOLL_CONNECT
};

enum epoll_flags
{
    EPOLL_READ_TPOLL    = 0x00,  /* use read thread poll*/
    EPOLL_WRITE_TPOLL   = 0x02,  /* use write thread poll*/
    EPOLL_UPCALL_TPOLL  = 0x04   /* use upcall thread poll*/
};

enum epoll_status
{
    EPOLL_CONNECTED        = 0x00,  /* default is avaiable*/
    EPOLL_EAGAIN        = 0x02,  /* no more data read */
    EPOLL_NOMEM            = 0x04,  /* no memory for epoll*/
    EPOLL_READING        = 0x08,  /* epoll event reading*/
    EPOLL_WRITING        = 0x10,  /* epoll event writing*/
    EPOLL_READNEXT        = 0x20,  /* epoll next event*/
    EPOLL_WRITNEXT        = 0x40,  /* epoll next writing*/
    EPOLL_SCHEDULING    = 0x80,  /* epoll event schduling*/
    EPOLL_CLOSED        = 0x100  /* event scoket closed*/
};

struct epoll_task
{
    int id;                     /* epoll event array index*/
    int fd;                     /* event socket file handle*/
    struct sockaddr_in addr;    /* peer socket address*/
    struct epoll_task *next;    /* next epoll task ptr*/
};

struct epoll_ops
{
    int (*add)(struct epoll_server* srv,int fd,unsigned int ip);
    int (*del)(struct epoll_server* srv,int fd);
    int (*dispatch)(struct epoll_server* srv,int fd);
    void* (*loop)(void *args);
    int (*read)(struct epoll_server* srv,int fd,Buffer* buf);
    struct epoll_server* (*create)();
    int (*init)(struct epoll_server* srv);
    int (*start)(struct epoll_server* srv);
    int (*stop)(struct epoll_server* srv);
    void (*destroy)(struct epoll_server* srv);
    int (*recv)(struct epoll_server* srv,struct epoll_client *client);
    int (*send)(struct epoll_server* srv,struct epoll_client *client,Buffer* buf,int n);
    int (*input)(Buffer* buf);
    int (*output)(Buffer* buf);
};

struct epoll_server
{
    int flags;                  /* epoll server init flags*/
    int running;                /* server running flag*/
    int port;                   /* listening socket port*/
    int listen_fd;              /* listening socket handle*/
    int epoll_fd;               /* epoll socket handle*/
    int nevents;                /* epoll max listen events*/
    int timeout;                /* epoll listen wait timeout*/
    unsigned int seq;           /* epoll buffer sequence*/
    struct epoll_event* events; /* epoll event array*/
    struct queue_t* rtask;      /* read event task queue*/
    struct queue_t* wtask;      /* write event task queue*/
    struct queue_t* rqueue;     /* input buffer queue*/
    Hashmap* ipmap;             /* connection ip map*/
    Hashmap* fdmap;             /* epoll socket handle map*/
    pthread_t acceptor;         /* socket data reader thread*/
    pthread_t reader;           /* socket data reader thread*/
    pthread_t epoller;          /* epoll loop thread*/
    pthread_t writer;           /* socket data writer thread*/
    pthread_t sender;           /* socket data send thread*/
    pthread_t scheduler;        /* upcall buffer thread*/
    struct epoll_ops* ops;      /* server operation set*/
    unsigned int rbuf_cnt;      /* receive buffer counter*/
    unsigned int wbuf_cnt;      /* send buffer counter*/
    unsigned int wbuf_limit;    /* limitation of send buffer*/
    pthread_cond_t wcond;        /* write queue condition*/
    pthread_mutex_t rlock;      /* event read lock*/
    pthread_mutex_t wlock;      /* event write lock*/
    pthread_mutex_t lock;       /* server object lock*/
    unsigned int thrmax;        /* threads max in pool*/
    ThreadPool threadpool;      /* thread pool for rw*/
    struct Network *net;
};

struct epoll_client
{
    int fd;                     /* client socket handle*/
    short int flag;             /* client current flag*/
    short int type;             /* accept/connect type*/
    int event;                  /* client epoll event*/
    unsigned int ip;            /* client ip address*/
    unsigned short port;        /* client socket port*/
    short int state;            /* epoll client status*/
    short int rstate;           /* client read status*/
    short int wstate;           /* client write status*/
    Buffer* rbuf;               /* socket read buffer*/
    Buffer* wbuf;               /* socket write buffer*/
    volatile unsigned int write;/* write request count*/
    volatile int refcnt;        /* client reference count*/
};

struct epoll_server* eserver_create();
int eserver_init(struct epoll_server* srv);
int eserver_start(struct epoll_server* srv);
int eserver_stop(struct epoll_server* srv);
void eserver_destroy(struct epoll_server* srv);
int epoll_output(Buffer* buf);

#endif /* EPOLL_H_ */

 

/*
* MoreStor SuperVault
* Copyright (c), 2008, Sierra Atlantic, Dream Team.
*
* tcp epoll interface
*
* Author(s): wxlin    <weixuan.lin@sierraatlantic.com>
*
* $Id: epoll.h,v 1.2 2008-09-02 09:30:22 wlin Exp $
*
*/

#include <stdio.h>
#include <time.h>
#include <errno.h>
#include <fcntl.h>
#include <sys/resource.h>
#include "log.h"
#include "buffer.h"
#include "network.h"
#include "epoll.h"
#include "upcall.h"
#include "list.h"

#define TCP_CORK         3
#define SOCKET_ERROR     -1

extern struct epoll_ops _eserver_ops;
void* epoll_schedule(void* param);

Buffer* buffer_realloc(Buffer* buf,int size)
{
    int new_size = size;
    Buffer* rbuf = NULL;
    new_size += HEADER_SIZE(PacketDesc);
    rbuf = buffer.alloc(new_size);
    if(rbuf){
        /* buffer form cache maybe larger new_size,
        we use actual capacity of buffer*/
        new_size = buffer_capacity(rbuf);
        rbuf->desc = MAKE_DESC(rbuf,new_size);
        rbuf->pkt = MAKE_PACKET(rbuf);
        rbuf->hdr = MAKE_HEAD(rbuf);
        rbuf->data = (char*)rbuf->pkt;
        rbuf->flag = buf->flag;
        memcpy(rbuf->pkt,buf->pkt,buf->dlen);
        rbuf->dlen = buf->dlen;
        rbuf->data += rbuf->dlen;
    }
    buffer_free(buf);
    return rbuf;
}

inline
void buffer_add(Buffer* buf,Buffer* head)
{
    buf->next = head;
    buf->prev = head->prev;
    head->prev->next = buf;
    head->prev = buf;
}

inline
void buffer_add_tail(Buffer* buf,Buffer* head)
{
    buf->next = head->prev->next;
    buf->prev = head->prev;
    head->prev->next = buf;
    head->prev = buf;
}

inline
void buffer_remove(Buffer* buf)
{
    if (buf->next != buf) {
        buf->next->prev = buf->prev;
        buf->prev->next = buf->next;
        buf->prev = buf->next = NULL;
    }
}

inline
int buffer_release(Buffer* buf)
{
    int count = 0;
    Buffer *next,*head = buf;
    do {
        next = buf->next;
        buffer_free(buf);
        buf = next;
        count++;
    } while (next != head);
    return count;
}

Buffer* buffer_next(Buffer* buf)
{
    if (buf->prev == buf &&
        buf->next == buf) {
            return NULL; /* last one */
    } else {
        return buf->next;
    }
}

int setnonblocking(int sock)
{
    int opts;

    opts = fcntl(sock,F_GETFL);
    if (opts < 0) {
        DBG("EPoll: get nonblocking fcntl error %d./n",opts);
        return opts;
    }

    opts = opts|O_NONBLOCK;
    if (fcntl(sock,F_SETFL,opts)<0) {
        DBG("EPoll: set nonblocking fcntl error %d./n",opts);
    }

    return opts;
}

int socket_init()
{
    int ret;
    int sock;
    int optval;
    socklen_t optlen;

    /* create port socket*/
    sock = socket(AF_INET, SOCK_STREAM, 0);
    if(sock < 0) {
        DBG("EPoll: cannot open socket./n");
        return -1;
    }

    /* Set socket send buffer size    */
    optval = 1024*1024;
    optlen = sizeof(int);
    ret = setsockopt( sock, SOL_SOCKET, SO_SNDBUF,&optval, optlen);
    if(ret == SOCKET_ERROR){
        DBG("EPoll: Can't set the socket send buffer size,err %d/n",ret);
        goto erro;
    }

    /* Set socket receive buffer size */
    ret = setsockopt( sock, SOL_SOCKET, SO_RCVBUF, &optval, optlen);
    if(ret == SOCKET_ERROR){
        DBG("EPoll: Can't set the socket receive buffer size,err %d!/n",ret);
        goto erro;
    }else{
        LOG("EPoll: Set socket buffer size %dk./n",optval/1024);
    }

    optval = 1;
    ret = setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &optval, optlen);
    if(ret == SOCKET_ERROR){
        DBG("EPoll: Unable to set addr reuse,err %d!/n",ret);
        goto erro;
    }

    ret = setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen);
    if(ret == SOCKET_ERROR){
        DBG("EPoll: Unable to keepalive,err %d!/n",ret);
        goto erro;
    }
    /*
    ret = setsockopt(sock, SOL_SOCKET, SO_LINGER, &optval, optlen);
    if(ret == SOCKET_ERROR){
    DBG("EPoll: Unable to set linger,err %d!/n",ret);
    goto erro;
    }
    */
    /* init socket successfully */
    return sock;

erro:
    /* close socket,if failure*/
    if(sock){
        close(sock);
    }

    return -1;
}

int socket_bind(int sock,int port,int max)
{
    int ret;
    struct sockaddr_in addr;

    /* prepare socket address*/
    bzero(&addr, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = INADDR_ANY;

    /* bind socket address*/
    ret = bind(sock, (struct sockaddr*)&addr, sizeof(addr));
    if (ret < 0) {
        DBG("EPoll: Bind scoket address error %d!/n",ret);
        return ret;
    }

    /* start listen socket */
    if ( (ret = listen(sock, max)) < 0) {
        DBG("EPoll: Listen the socket error %d!/n",ret);
        return ret;
    }

    /* bind socket successfully */
    return ret;
}

int socket_connect(int fd,unsigned int ip,int port)
{
    int ret,retry=0;
    struct sockaddr_in addr;

    addr.sin_family = AF_INET;
    addr.sin_addr.s_addr = htonl(ip);
    addr.sin_port = htons(port);

    while (retry++ < 1){
        ret = connect(fd,(struct sockaddr*)&addr,sizeof(addr));
        if(ret == SOCKET_ERROR){
            DBG("EPoll: Can't connect peer fd.!/n");
            /* usleep(300); */
        }else{
            DBG("EPoll: Connect %s fd %d ok./n",inet_ntoa(addr.sin_addr),fd);
            break;
        }
    }
    return ret;
}

int socket_read(int fd,void *buffer,unsigned int length)
{
    unsigned int nleft;
    int nread;
    char *ptr;
    ptr = (char *)buffer;
    nleft = length;

    while(nleft > 0)
    {
        if((nread = read(fd, ptr, nleft)) < 0){
            if(errno == EINTR)
                nread = 0;
            else
                return -1;
        }
        else if(nread == 0){
            break;
        }
        nleft -= nread;
        ptr += nread;
    }
    return length - nleft;
}

int socket_write(int fd,const void *buffer,unsigned int length)
{
    unsigned int nleft;
    int nwritten;
    const char *ptr;
    ptr = (const char *)buffer;
    nleft = length;

    while(nleft > 0){
        if((nwritten = send(fd, ptr, nleft,0)) <= 0){
            if(errno == EINTR)
                nwritten=0;
            else
                return -1;
        }
        nleft -= nwritten;
        ptr += nwritten;
    }
    return length;
}

int epoll_config(struct epoll_server* srv)
{
    int size,nfiles;
    struct rlimit rl;

    /* disable environment variable is set */
    if (getenv("EVENT_NOEPOLL"))
        return -1;

    if (getrlimit(RLIMIT_NOFILE, &rl) == 0 &&
        rl.rlim_cur != RLIM_INFINITY) {
            nfiles = rl.rlim_cur - 1;
    }

    /* init the epoll max event*/
    srv->nevents = nfiles;

    /* init the epoll timeout*/
    srv->timeout = 500;

    /* init threadpool threads*/
    srv->thrmax = EPOLL_MAX_POOL_THREADS;

    /* alloc the event array*/
    size = sizeof(struct epoll_event)* srv->nevents;
    srv->events = (struct epoll_event*)malloc(size);

    return srv->events ? 0 : -1;
}

int epoll_init(struct epoll_server* srv)
{
    int ret,epfd;
    struct epoll_event event = {0, {0}};

    /* create epoll handle*/
    epfd = epoll_create(srv->nevents);
    if (epfd == -1) {
        if (errno != ENOSYS)
            DBG("EPoll: epoll_create error!/n");
        return (epfd);
    }

    /* register listen event */
    event.data.fd = srv->listen_fd;
    event.events = EPOLLIN | EPOLLET;

    ret = epoll_ctl(epfd,EPOLL_CTL_ADD,srv->listen_fd,&event);
    if ( ret == -1){ /* fail to add listen fd*/
        close(epfd);
        return ret;
    }

    /* return epoll handle*/
    srv->epoll_fd = epfd;
    return epfd;
}

inline
int epoll_lock(struct epoll_server* srv)
{
    return pthread_mutex_lock(&srv->lock);
}

inline
int epoll_unlock(struct epoll_server* srv)
{
    return pthread_mutex_unlock(&srv->lock);
}

inline
void epoll_ref(struct epoll_client* client)
{
    client->refcnt++;
}

void epoll_unref(struct epoll_server* srv,struct epoll_client* client)
{
    int n;
    client->refcnt--;

    /* no more one using,have setted close*/
    if (client->refcnt == 0 &&
        client->state & EPOLL_CLOSED)
    {
        /* remove client fd map*/
        hash_remove(srv->fdmap,client->fd);

        /* release buffers*/
        if(client->rbuf){
            buffer_free(client->rbuf);
            srv->rbuf_cnt--;
        }
        if(client->wbuf) {
            n = buffer_release(client->wbuf);
            srv->wbuf_cnt -= n;
        }
        /* close socket*/
        close(client->fd);
        free(client);
    }
}

inline
int epoll_unref_safe(struct epoll_server* srv,struct epoll_client* client)
{
    pthread_mutex_lock(&srv->lock);
    epoll_unref(srv,client);
    pthread_mutex_unlock(&srv->lock);
    return 0;
}

int epoll_del(struct epoll_server* srv,int fd)
{
    int ret;
    struct epoll_client *client;
    struct epoll_event event = {0, {0}};

    /* unregister socket events*/
    event.data.fd = fd;
    event.events = 0;
    ret = epoll_ctl(srv->epoll_fd, EPOLL_CTL_DEL,fd,&event);
    if (ret == -1) {
        DBG("EPoll: epoll_ctl del error!/n");
    }

    /* flag client is closed*/
    client = hash_get(srv->fdmap,fd);
    if (client != NULL) {
        hash_remove(srv->ipmap,client->ip);
        client->state |= EPOLL_CLOSED;
    }

    return ret;
}

int epoll_add(struct epoll_server* srv,int fd,unsigned int ip)
{
    int ret;
    struct epoll_client *client;
    struct epoll_event event = {0, {0}};

    /* exist active connection*/
    client = hash_get(srv->ipmap,ip);
    if (client != NULL) {
        if(client->type == EPOLL_CONNECT){
            client->state |= EPOLL_CLOSED;
            hash_remove(srv->ipmap,client->ip);
            if (client->refcnt == 0){
                client->refcnt++;
                epoll_unref(srv,client);
            }
        }
    }

    /* create accept connection*/
    client = malloc(sizeof(struct epoll_client));
    if (client != NULL) {
        memset(client,0,sizeof(struct epoll_client));
        client->fd = fd;
        client->ip = ip;
        client->type = EPOLL_ACCEPT;
        hash_set(srv->fdmap,fd,client);
        hash_set(srv->ipmap,ip,client);
    }

    /* register socket events*/
    event.data.fd = fd;  
    event.events = EPOLLIN | EPOLLET;
    client->event = event.events;
    ret = epoll_ctl(srv->epoll_fd,EPOLL_CTL_ADD,fd,&event);
    if (ret == -1) {
        DBG("EPoll: epoll_ctl add error!/n");
    }
    return ret;
}

int epoll_rmod(struct epoll_server* srv,struct epoll_client *client)
{
    int ret;
    struct epoll_event event;
    event.data.fd = client->fd;

    /* nobody read,set write event*/
    if (!(client->event & EPOLLOUT) && client->write) {
        event.events = EPOLLOUT | EPOLLET;
        client->event = event.events;
    }else{
        return -1; /* keep orignal*/
    }

    /* change write event*/
    ret = epoll_ctl(srv->epoll_fd,EPOLL_CTL_MOD,
        client->fd,&event);
    if(ret == -1) {
        DBG("EPoll: epoll_ctl mod write error!/n");
    }

    return ret;
}

int epoll_wmod(struct epoll_server* srv,struct epoll_client *client)
{
    int ret;
    struct epoll_event event;
    event.data.fd = client->fd;

    /* nobody read,set write event*/
    if (!(client->event & EPOLLOUT) && client->write) {
        event.events = EPOLLOUT | EPOLLET;
        client->event = event.events;
    }else{
        return -1; /* keep orignal*/
    }

    /* change write event*/
    ret = epoll_ctl(srv->epoll_fd,EPOLL_CTL_MOD,
        client->fd,&event);
    if(ret == -1) {
        DBG("EPoll: epoll_ctl mod write error!/n");
    }

    return ret;
}

int epoll_wmod_reset(struct epoll_server* srv,struct epoll_client *client)
{
    int ret;
    struct epoll_event event;

    /* set event*/
    event.data.fd = client->fd;
    event.events = EPOLLIN | EPOLLET;
    client->event = event.events;

    /* change to read event*/
    ret = epoll_ctl(srv->epoll_fd,EPOLL_CTL_MOD,
        client->fd,&event);
    if(ret == -1) {
        DBG("EPoll: epoll_ctl mod write error!/n");
    }

    return ret;
}

int epoll_mod_reset(struct epoll_server* srv,int fd)
{
    int ret;
    struct epoll_event event;
    event.data.fd = fd;
    event.events = EPOLLIN | EPOLLET;

    /* change to read event*/
    ret = epoll_ctl(srv->epoll_fd,EPOLL_CTL_MOD,fd,&event);
    if(ret == -1) {
        DBG("EPoll: epoll_ctl mod write error!/n");
    }

    return ret;
}

int epoll_read(struct epoll_server* srv,int fd,Buffer* buf)
{
    char *ptr;
    int n,len,nleft;

    ptr = buf->data;
    nleft = buf->dlen;
    len = buf->dlen;

    while(nleft > 0)
    {
        n = read(fd, ptr, nleft);
        if (n  < 0) {
            /* interrupt signal*/
            if(errno == EINTR){
                continue;
                //return -1;
            }
            /* no more data */
            if(errno == EAGAIN){
                buf->flag = EAGAIN;
                break;
            } /* socket error*/
            else{
                return -1;
            }
        } /* socket closed*/      
        else if(n == 0){
            return -1;
        } /* read success*/
        else{
            nleft -= n;
            ptr += n;
        }
    }

    return len - nleft;;
}

int epoll_write(struct epoll_server* srv,int fd,Buffer* buf)
{
    char *ptr;
    int n,len,nleft;

    ptr = buf->data;
    nleft = buf->dlen;
    len = buf->dlen;

    while(nleft > 0){
        n = send(fd, ptr, nleft,0);
        if ( n < 0) {
            /* interrupt signal*/
            if(errno == EINTR){
                continue;
                //return -1;
            }
            /* fd write full*/
            if(errno == EAGAIN){
                buf->flag = EAGAIN;
                break;
            }
            /* other errors*/
            return -1;
        }
        else if (n == 0){
            DBG("EPOLL: send return 0/n");
            return -1;
        }
        nleft -= n;
        ptr += n;
    }

    buf->dlen = nleft;
    buf->data = ptr;
    return len - nleft;
}

int packet_recv(struct epoll_server* srv,struct epoll_client *client)
{  
    Buffer* buf;
    int desc_size,head_size,total_size;
    int n,len,size,fd,capacity;

    buf = client->rbuf;
    fd = client->fd;
    desc_size = HEADER_SIZE(PacketDesc);
    head_size = size = HEADER_SIZE(Packet);

    /* receive packet head*/
    if(buf->dlen < size){
        len = buf->dlen;
        size -= len;
        buf->dlen = size;
        n = epoll_read(srv,fd,buf);
        if (n == -1) {  /*read error*/
            return n;
        }
        if(buf->flag == EAGAIN){ /*no more data!*/
            client->rstate |= EPOLL_EAGAIN;
            buf->dlen = len+n;
            buf->data += n;
            return n;
        }
        if(n != size){ /*not enough,this not exist*/
            buf->dlen = len+n;
            buf->data += n;
            return n;
        }
        buf->dlen += len;
        buf->data += n;
    }

    /* check buffer capacity */
    capacity = buffer_capacity(buf)-desc_size;
    total_size = head_size+buf->pkt->size;
    if(total_size > capacity){
        client->rstate &= ~EPOLL_NOMEM;
        buf = buffer_realloc(buf,total_size);
        if(buf == NULL){
            client->rstate |= EPOLL_NOMEM;
            DBG("EPoll: fail to realloc mem/n");
            return -1;
        }
        client->rbuf = buf;
    }

    /* receive packet body*/
    if(buf->dlen < total_size){
        len = buf->dlen;
        size = total_size-buf->dlen;
        buf->dlen = size;
        n = epoll_read(srv,fd,buf);
        if (n == -1)  /*read error*/
            return n;
        if(buf->flag == EAGAIN){ /*no more data!*/
            client->rstate |= EPOLL_EAGAIN;
            buf->dlen = len+n;
            buf->data += n;
            return n;
        }
        if(n != size){/*not enough,error*/
            buf->dlen = len+n;
            buf->data += n;
            return n;
        }
        buf->dlen += len;
        buf->data = (char*)buf->pkt;
        DBG("EPoll: recv dlen %d/n",buf->dlen);
    }

    return buf->dlen;
}

int epoll_recv(struct epoll_server* srv,struct epoll_client *client)
{
    int n,size;
    struct Buffer *buf;

    /* check previous buf*/
    if (client->rbuf == NULL) {
        /* alloc receive buffer*/
        client->rstate &= ~EPOLL_NOMEM;
        buf = buffer.alloc(BIG_ENOUGH);
        if(buf == NULL){
            client->rstate |= EPOLL_NOMEM;
            DBG("EPoll: receiver fail to alloc mem/n");
            return -1;
        }

        /* build buffer members,here we use actual capacity
        of buffer,let desc place at top of buffer*/
        size = buffer_capacity(buf);
        buf->desc = MAKE_DESC(buf,size);
        buf->pkt = MAKE_PACKET(buf);
        buf->hdr = MAKE_HEAD(buf);
        buf->data = (char*)buf->pkt;
        buf->flag = 0;
        buf->dlen = 0;
        client->rbuf = buf;
    }

    /* receive packet head data*/
    client->rbuf->flag = 0;
    client->rstate &= ~EPOLL_EAGAIN;
    n = packet_recv(srv,client);
    return n;
}

int epoll_send(struct epoll_server* srv,struct epoll_client *client,Buffer* buf,int write)
{
    int n,fd,size,count;
    int agian,error;
    Buffer* next;  

    count = 0; /* sent count*/
    agian = error = 0;
    fd = client->fd;
    next = buf;/* go until last*/

    while (next) {
        /*check buffer len*/
        next = buffer_next(buf);
        size = buf->dlen;
        if (size <= 0) {
            DBG("EPoll: buffer size zero/n");
            buffer_free(buf);
            buf = next;          
            count++;
            continue;
        }

        /* write(send) buffer*/
        buf->flag = 0;
        n = epoll_write(srv,fd,buf);
        //DBG("EPoll: fd %d buf len%d,writed %d,left %d./n",
        //      client->fd,size,n,buf->dlen);

        /* socket write full*/
        if(buf->flag == EAGAIN){
            client->wstate |= EPOLL_EAGAIN;
            epoll_lock(srv);
            if(buf->dlen == 0){
                DBG("EPoll: write EAGAIN dlen =0/n");
            }
            if (client->wbuf) {
                buffer_add(buf,client->wbuf);
                client->wbuf = buf;
            }else{
                buf->prev = buf->next = buf;
                client->wbuf = buf;
            }
            client->write = write - count;
            epoll_unlock(srv);
            agian = 1;
            //DBG("EPoll: write EAGAIN %d./n",n);
            break; /*wait next send*/
        }
        /* write ok,continue*/
        else if (n==size && n>0) {
            buffer_remove(buf);
            buffer_free(buf);
            /* fuck next buffer*/
            buf = next;
            count++;
        }
        /* socket write error */
        else{
            DBG("EPoll: write buffer error!/n");
            error = -1;
            break; /* error */
        }
    } /*while*/

    /* socket write failure or closed */
    if (error == -1 && buf != NULL) {
        /* release buffer link*/
        int free = buffer_release(buf);
        count += free;
        client->write -= free;
    }
    else{ /* write full,remove EPOLOUT */
        epoll_lock(srv);
        if(agian){
            epoll_wmod_reset(srv,client);
        }
        /* nobody write,remove EPOLOUT */
        else if(client->write==0) {
            epoll_wmod_reset(srv,client);
            //DBG("EPoll: fd %d nobody write.../n",client->fd);
        }
        epoll_unlock(srv);
    }
    /* return buffer count*/
    return count;
}

int epoll_dispatch(struct epoll_server* srv,int fd)
{
    int ret = 0;
    struct epoll_task* task;

    /* create new read task*/
    task = malloc(sizeof(struct epoll_task));
    if (task == NULL)
        return -1;

    /* add new read task*/
    task->fd = fd;
    ret = queue_push(srv->rtask,task);
    //if(srv->rtask->count >= 2) {
    //    request_threadpool(srv->threadpool,epoll_reader,srv);
    //}
    return ret;
}

int epoll_activate(struct epoll_server* srv,int fd)
{
    int ret = 0;
    struct epoll_task* task;

    /* create new read task*/
    task = malloc(sizeof(struct epoll_task));
    if (task == NULL) {
        DBG("EPoll: fail to alloc write task/n");
        return -1;
    }

    /* add new read task*/
    task->fd = fd;
    ret = queue_push(srv->wtask,task);

    return ret;
}

struct epoll_client*
    epoll_connect(struct epoll_server* srv,unsigned int ip,int port)
{
    int ret,fd;
    struct epoll_client* conn;
    struct epoll_event event = {0, {0}};

    /* create epoll_client of connect*/
    conn = malloc(sizeof(struct epoll_client));
    if(conn == NULL)
        return conn;

    /* set socket ip and port*/
    memset(conn,0,sizeof(struct epoll_client));
    conn->ip = ip;
    conn->port = port;
    conn->state = EPOLL_CLOSED;
    conn->type = EPOLL_CONNECT;

    /* create clinet connect socket*/
    fd = socket_init();
    if(fd < 0){
        free(conn);
        return NULL;
    }

    /* connect to the server*/
    conn->fd = fd;
    ret = socket_connect(fd,ip,port);
    if(ret < 0){
        free(conn);
        close(fd);
        return NULL;
    }

    /* hash ip/fd epoll_client*/
    conn->state = EPOLL_CONNECTED;
    conn->wstate = EPOLL_READING;
    hash_set(srv->ipmap,ip,conn);
    hash_set(srv->fdmap,fd,conn);

    /* init epoll event */
    setnonblocking(fd);
    event.data.fd = fd;
    event.events = EPOLLIN | EPOLLET;
    conn->event = event.events;
    ret = epoll_ctl(srv->epoll_fd,EPOLL_CTL_ADD,fd,&event);
    if ( ret == -1){ /* fail to add client fd*/
        DBG("EPoll: add client error!/n");
    }

    /* return client conn*/
    return conn;
}

void* epoll_accept(void *args)
{
    int fd;
    unsigned int ip;
    socklen_t addrlen;
    struct sockaddr_in addr;
    struct epoll_server* srv;

    srv = (struct epoll_server*)args;
    addrlen = sizeof(struct sockaddr);
    while (srv->running)
    { 
        fd = accept(srv->listen_fd,(struct sockaddr*)&addr, &addrlen);
        if (fd < 0) {
            DBG("EPoll: accept client error!/n");
        }
        else{
            ip = ntohl(addr.sin_addr.s_addr);
            DBG("EPoll: accept %s./n",inet_ntoa(addr.sin_addr));

            /* register socket events*/
            if(ip){
                /* set fd non blocking*/
                setnonblocking(fd);

                /* add epoll fd */
                epoll_lock(srv);
                epoll_add(srv,fd,ip);
                epoll_unlock(srv);
            }else{
                close(fd);
                DBG("EPoll: accept ip err./n");
            }
        }
    }
    return 0;
}

void* epoll_loop(void *args)
{
    int i,fd,nfds;
    unsigned int ip;
    socklen_t addrlen;
    struct sockaddr_in addr;
    struct epoll_server* srv;

    srv = (struct epoll_server*)args;
    addrlen = sizeof(struct sockaddr);

    while(srv->running)
    {
        nfds = epoll_wait(srv->epoll_fd,srv->events,srv->nevents,srv->timeout);
        if (nfds == -1) {
            if (errno != EINTR) {
                DBG("EPoll: epoll_wait EINTR!/n");
                return 0; /*interrupt*/
            } else {
                //DBG("EPoll: epoll_wait!/n");
                if(!srv->running)
                    break;
                continue;
            }
        }

        for (i=0; i<nfds; ++i)
        {
            if(srv->events[i].data.fd == srv->listen_fd)
            {
                fd = accept(srv->listen_fd,(struct sockaddr*)&addr, &addrlen);
                if (fd < 0) {
                    DBG("EPoll: accept client error!/n");
                }
                else{
                    ip = ntohl(addr.sin_addr.s_addr);
                    DBG("EPoll: accept %s fd %d/n",inet_ntoa(addr.sin_addr),fd);

                    /* register socket events*/
                    if(ip){
                        /* set fd non blocking*/
                        setnonblocking(fd);

                        /* add epoll fd */
                        epoll_lock(srv);
                        epoll_add(srv,fd,ip);
                        epoll_unlock(srv);
                    }else{
                        close(fd);
                    }
                }
            }
            else if(srv->events[i].events & EPOLLIN)
            {
                //DBG("EPoll: reading./n");
                if ((fd = srv->events[i].data.fd) < 0)
                    continue;

                /* create new read task*/
                epoll_dispatch(srv,fd);
            }
            else if(srv->events[i].events & EPOLLOUT)
            {
                if ((fd = srv->events[i].data.fd) < 0)
                    continue;

                /* activate write task*/
                epoll_activate(srv,fd);
            }
        }
    }

    return (0);
}

void* epoll_reader(void *args)
{
    int n,fd,next;
    struct Buffer* buf;
    struct sockaddr_in* addr;
    struct epoll_task *task;
    struct epoll_server* srv;
    struct epoll_client *client;
    srv = (struct epoll_server*)args;

    while(srv->running)
    {
        /* pop read event task */
        task = NULL; next = 0;
        queue_pop(srv->rtask,(void**)&task);
        if(srv->rtask->terminated)
            break;

        /* check task pop*/
        if(task == NULL)
            continue;

        /* get read event fd */
        fd = task->fd;
        free(task);

        /* get fd client ptr */
        epoll_lock(srv);
        client = hash_get(srv->fdmap,fd);
        if (client != NULL) {
            epoll_ref(client);
            /* only allow a thread read*/
            if(client->rstate & EPOLL_READING){
                DBG("EPoll: wait reading done./n");
                client->rstate |= EPOLL_READNEXT;
                next = 1; /* handle by reading*/
            }else{
                client->rstate |= EPOLL_READING;
            }
        }
        /* client unavailable?*/
        epoll_unlock(srv);
        if (!client || next)
            continue;

        /* receive socket data */
        read_next: do{
            /* receive client data*/
            n = epoll_recv(srv,client);

            /* error,closed,nomem*/
            if (n == -1){
                /* delete epoll event*/
                epoll_lock(srv);
                epoll_del(srv,fd);
                epoll_unlock(srv);
                DBG("EPoll: Client close connect fd %d!/n",fd);
                break; /*exit*/
            }

            /* handle receive buffer*/
            if (client->rbuf->flag != EAGAIN) {
                buf = client->rbuf;
                client->rbuf = NULL;
                addr = &buf->desc->from;
                addr->sin_addr.s_addr = htonl(client->ip);
                addr->sin_port = htons(client->port);
                buf->owner = srv;
                srv->ops->input(buf);
            }

            /* check receive status*/
        }while(!(client->rstate & EPOLL_EAGAIN));       

        /* read finished,check next*/       
        epoll_lock(srv);
        client->rstate &= ~EPOLL_READING;
        if(client->rstate & EPOLL_READNEXT){
            client->rstate &= ~EPOLL_READNEXT;
            epoll_unlock(srv);
            goto read_next;
        }

        /* switch to write event*/
        epoll_wmod(srv,client);
        epoll_unref(srv,client);
        epoll_unlock(srv);
    }
    return 0;
}

void* epoll_writer(void *args)
{
    int fd,n,write,next;
    struct Buffer* buf;  
    struct epoll_task *task;
    struct epoll_server* srv;  
    struct epoll_client *client;
    srv = (struct epoll_server*)args;

    while(srv->running)
    {
        /* pop write event fd*/
        task = NULL; next = 0;
        queue_pop(srv->wtask,(void**)&task);
        if(srv->wtask->terminated)
            break;

        /* check task pop*/
        if(task == NULL)
            continue;

        /* get write event fd*/
        fd = task->fd;
        free(task);
        /* next write loop*/
        write_next:

        /* get fd client ptr*/       
        buf = NULL;
        epoll_lock(srv);       
        client = hash_get(srv->fdmap,fd);
        if (client != NULL) {
            epoll_ref(client);
            /* only allow a thread write*/
            if(client->wstate & EPOLL_WRITING){
                DBG("EPoll: wait writing done./n");
                client->wstate |= EPOLL_WRITNEXT;
                next = 1; /* handle by writing*/
            }else{
                client->wstate |= EPOLL_WRITING;
            }
            /* take write buffer list*/
            if (client->wbuf) {
                buf = client->wbuf;
                client->wbuf = NULL;
                write = client->write;
                client->write = 0;
            }else{
                DBG("EPoll: write task client null./n");
                //...
            }
        }

        /* if client buf null*/
        if (buf == NULL && client){
            epoll_wmod_reset(srv,client);
            DBG("EPoll: client wbuf null./n");
        }

        /* if client unavailable*/
        epoll_unlock(srv);
        if (client == NULL) {
            DBG("EPoll: not found client,reset./n");
            epoll_mod_reset(srv,fd);
            continue;
        }

        /* handle client buffer*/
        if (buf != NULL) {
            client->wstate &= ~EPOLL_EAGAIN;
            n = epoll_send(srv,client,buf,write);
            epoll_lock(srv);
            srv->wbuf_cnt -= n;
            client->wstate &= ~EPOLL_WRITING;
            if(client->wstate & EPOLL_WRITNEXT){
                client->wstate &= ~EPOLL_WRITNEXT;
                next = 1; /*next event following*/
            }
            epoll_unref(srv,client);
            epoll_unlock(srv);
            if (next){ /* next write loop*/
                goto write_next;
            }
        }
    }

    return 0;
}

void epoll_close(struct epoll_server* srv)
{
    if (srv->listen_fd >= 0)
        close(srv->listen_fd);
    if (srv->epoll_fd >= 0)  
        close(srv->epoll_fd);
    if (srv->events != NULL)
        free(srv->events);
}

void epoll_free(struct epoll_server* srv)
{
    HashPair* hash;
    struct epoll_client* client;

    hash = hash_first(srv->fdmap);
    while (hash) {
        client = (struct epoll_client*)hash->value;
        if (client) {
            if (client->rbuf)
                buffer_free(client->rbuf);      
            if (client->wbuf)
                buffer_release(client->wbuf);
            hash_remove(srv->fdmap,client->fd);
            free(client);
        }
        hash = hash_next(srv->fdmap);
    }
}

struct epoll_server* eserver_create(int port)
{
    /* create epoll server */
    struct epoll_server* srv = NULL;
    srv = malloc(sizeof(struct epoll_server));
    if(srv != NULL){
        memset(srv,0,sizeof(struct epoll_server));
        srv->flags = EPOLL_READ_TPOLL;
        srv->port = port;
        srv->ipmap = hash_new(10);
        srv->fdmap = hash_new(10);
        srv->rtask = queue_create(1000);
        srv->wtask = queue_create(1000);
        srv->rqueue = queue_create(1000);
        /* alloc thread pool*/
        srv->threadpool = alloc_threadpool();
    }
    return srv;
}

int eserver_init(struct epoll_server* srv)
{
    int ret = 0;

    /* config epoll resource*/
    if(epoll_config(srv) != 0)
        return -1;

    /* init listen socket*/
    srv->listen_fd = socket_init();
    if(srv->listen_fd < 0)
        return -1;

    /* set fd non blocking*/
    setnonblocking(srv->listen_fd);

    /* bind and listen sock*/
    if(socket_bind(srv->listen_fd,
        srv->port,srv->nevents) <0)
        return -1;

    /* init epoll handle*/
    if(epoll_init(srv) < 0)
        return -1;

    /* init read lock*/
    if(pthread_mutex_init(&srv->rlock, NULL) != 0)
        return -1;

    /* init server ipc */
    if(pthread_mutex_init(&srv->lock, NULL) != 0)
        return -1;

    /* init server write condition */
    if(pthread_cond_init(&srv->wcond, NULL) != 0)
        return -1;

    /* init thread pool*/
    init_threadpool(srv->threadpool,srv->thrmax);

    /* server operations*/
    srv->ops = &_eserver_ops;

    /* success return */
    return ret;
}

int eserver_start(struct epoll_server* srv)
{
    int i;
    /* start run epoll server*/
    srv->running = 1;

    /* create epoll thread*/
    if(pthread_create(&srv->epoller,NULL,
        epoll_loop,(void*)srv) != 0)
        return -1;

    /* create epoll read thread*/
    for(i=0;i<2;i++)
        if(pthread_create(&srv->reader,NULL,
            epoll_reader,(void*)srv) != 0)
            return -1;

    /* create epoll writer thread*/
    for(i=0;i<2;i++)
        if(pthread_create(&srv->writer,NULL,
            epoll_writer,(void*)srv) != 0)
            return -1;

    /* create epoll read thread
    if(pthread_create(&srv->acceptor,NULL,
    epoll_accept,(void*)srv) != 0)
    return -1;*/

    /* create epoll upcall thread */
    if(srv->flags & EPOLL_UPCALL_TPOLL){
        if(pthread_create(&srv->scheduler,NULL,
            epoll_schedule,(void*)srv) != 0)
            return -1;
    }
    return 0;
}

int eserver_stop(struct epoll_server* srv)
{
    /* set server stop flag*/
    srv->running = 0;

    /* close listen/epoll */
    epoll_close(srv);

    /* stop read task*/
    queue_term(srv->rtask);

    /* stop write task*/
    queue_term(srv->wtask);

    /* stop input queue*/
    queue_term(srv->rqueue);

    /* stop write condition*/
    if(pthread_cond_broadcast(&srv->wcond) != 0)
        return -1;

    /* waiting accept exist
    if(pthread_join(srv->acceptor, NULL) !=0)
    return -1;*/

    /* waiting loop exist*/
    if(pthread_join(srv->epoller, NULL) !=0)
        return -1;

    /* waiting reader exist*/
    if(pthread_join(srv->reader, NULL) !=0)
        return -1;

    /* waiting writer exist*/
    if(pthread_join(srv->writer, NULL) !=0)
        return -1;

    if(srv->flags & EPOLL_UPCALL_TPOLL){
        if(pthread_join(srv->scheduler, NULL) !=0)
            return -1;
    }
    return 0;
}

void eserver_destroy(struct epoll_server* srv)
{
    /* release task queue*/
    queue_destroy(srv->rtask);
    queue_destroy(srv->wtask);

    /* release clients*/
    epoll_free(srv);

    /* release hash map*/
    hash_destroy(srv->ipmap);
    hash_destroy(srv->fdmap);

    /* destroy thread pool */
    destroy_threadpool(srv->threadpool);

    /* release write condition*/
    pthread_cond_destroy(&srv->wcond);

    /* release read lock*/
    pthread_mutex_destroy(&srv->rlock);

    /* release server mutex */
    pthread_mutex_destroy(&srv->lock);

    /* release server instance*/
    free(srv);
}

int epoll_input(Buffer* buf)
{
    int ret;
    struct epoll_server* srv;
    srv = (struct epoll_server*)buf->owner;

    /* if we have upcall queue*/
    if(srv->flags & EPOLL_UPCALL_TPOLL){
        ret = queue_push(srv->rqueue,buf);
    }
    else{
        buffer.pop(buf, HEADER_SIZE(PacketDesc));
        buffer.pop(buf, HEADER_SIZE(Packet));
        __network_onreceived(srv->net, &buf->desc->from, buf);
    }

    return ret;
}

static inline
int epoll_upcall(Buffer* buf)
{
    struct epoll_server* srv;
    srv = (struct epoll_server*)buf->owner;

    buffer.pop(buf, HEADER_SIZE(PacketDesc));
    buffer.pop(buf, HEADER_SIZE(Packet));
    __network_onreceived(srv->net, &buf->desc->from, buf);
    return 0;
}

void* epoll_schedule(void* param)
{
    struct Buffer* buf;
    struct epoll_server* srv;
    srv = (struct epoll_server*)param;

    /* dequeue and upcall buffer */
    while (srv->running){
        /* pop read receive buffer */
        queue_pop(srv->rqueue,(void**)&buf);
        if(buf != NULL){
            epoll_upcall(buf);
        }
    }
    return 0;
}

int epoll_output(Buffer* buf)
{
    int ret = -1;
    unsigned int ip;
    struct epoll_server* srv;
    struct epoll_client *client;
    srv = (struct epoll_server*)buf->owner;

    /* check buffer length*/
    if (!buf->dlen) {
        buffer.free(buf);
        return ret;
    }

    /* get buffer destination*/
    ip = buf->dst;  
    epoll_lock(srv);
    client = (struct epoll_client*)hash_get(srv->ipmap,ip);
    if (client == NULL) {
        client = epoll_connect(srv,ip,srv->port);
    }

    /* if client still unavaiable*/
    if (client == NULL){
        DBG("EPoll: client unavaiable/n");
        goto err;
    }

    /* send buffer to destination */
    epoll_ref(client);
    if (!(client->state & EPOLL_CLOSED)) {
        buf->priority = srv->seq++;
        srv->wbuf_cnt++;
        client->write++;
        //DBG("EPoll: fd %d output s%d,wc%d, cw%d, dlen %d../n",
        //    client->fd,srv->seq,srv->wbuf_cnt,client->write,buf->dlen);
        if (client->wbuf == NULL) {
            /* set head buffer*/
            buf->prev = buf->next = buf;
            client->wbuf = buf;
        }
        else{ /* add to tail*/
            buffer_add_tail(buf,client->wbuf);
        }
        ret = 0; /* add buffer ok*/

        /* request write,set event*/
        epoll_rmod(srv,client);
    }
    epoll_unref(srv,client);

err:
    epoll_unlock(srv);
    if (ret == -1) {
        buffer.free(buf);
    }
    return ret;
}

struct epoll_ops _eserver_ops =
{
    .add = epoll_add,
    .del = epoll_del,
    .dispatch = epoll_dispatch,
    .loop = epoll_loop,
    .read = epoll_read,
    .create = eserver_create,
    .init = eserver_init,
    .start = eserver_start,
    .stop = eserver_stop,
    .destroy = eserver_destroy,
    .recv = epoll_recv,
    .send = epoll_send,
    .input = epoll_input,
    .output = epoll_output,
};