一个基于线程池的网络处理服务器demo

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <unistd.h>
#include <arpa/inet.h>
//#include <openssl/ssl.h>
//#include <openssl/err.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <pthread.h>
#include <assert.h>

//#define DEBUG_TILERA


#ifdef DEBUG_TILERA
#include <tmc/alloc.h>

#include <arch/cycle.h>
#include <arch/spr.h>

#include <tmc/cpus.h>
#include <tmc/sync.h>
#include <tmc/task.h>
#endif

 /* These are non-NULL pointers that will result in page faults
 * under normal circumstances, used to verify that nobody uses
 * non-initialized list entries.
 */
#define LIST_POISON1  ((void *) 0x00100100)
#define LIST_POISON2  ((void *) 0x00200200)

#define MAXBUF 1024
#define MAXEPOLLSIZE 100000

#define MAX_THREAD_NUMBER 200
int  THREAD_NUMBER = 50;


int kdpfd;
struct epoll_event events[MAXEPOLLSIZE];


int msgcount = 0;
int timecount = 0;

int count_packet= 0;

int connect_count = 0;

pthread_mutex_t curfds_lock;
int curfds;

char buffer[MAX_THREAD_NUMBER][MAXBUF + 1];
pthread_t thread_count;


cpu_set_t cpus;

#define offset_of(type, memb) \
	((unsigned long)(&((type *)0)->memb))



#define container_of(obj, type, memb) \
	((type *)(((char *)obj) - offset_of(type, memb)))


typedef struct list_head {
    struct list_head *next, *prev;
}list_head;

#define LIST_HEAD_INIT(name) { &(name), &(name) }

#define LIST_HEAD(name) \
    struct list_head name = LIST_HEAD_INIT(name)

static inline void INIT_LIST_HEAD(struct list_head *list)
{
    list->next = list;
    list->prev = list;
}



/*
 * Insert a new entry between two known consecutive entries.
 *
 * This is only for internal list manipulation where we know
 * the prev/next entries already!
 */
static inline void __list_add(struct list_head *new,
                  struct list_head *prev,
                  struct list_head *next)
{
    next->prev = new;
    new->next = next;
    new->prev = prev;
    prev->next = new;
}


/**
 * list_add - add a new entry
 * @new: new entry to be added
 * @head: list head to add it after
 *
 * Insert a new entry after the specified head.
 * This is good for implementing stacks.
 */
static inline void list_add(struct list_head *new, struct list_head *head)
{
    __list_add(new, head, head->next);
}


/**
 * list_add_tail - add a new entry
 * @new: new entry to be added
 * @head: list head to add it before
 *
 * Insert a new entry before the specified head.
 * This is useful for implementing queues.
 */
static inline void list_add_tail(struct list_head *new, struct list_head *head)
{
    __list_add(new, head->prev, head);
}


/*
 * Delete a list entry by making the prev/next entries
 * point to each other.
 *
 * This is only for internal list manipulation where we know
 * the prev/next entries already!
 */
static inline void __list_del(struct list_head * prev, struct list_head * next)
{
    next->prev = prev;
    prev->next = next;
}

static inline void list_del(struct list_head *entry)
{
    __list_del(entry->prev, entry->next);
    entry->next = LIST_POISON1;
    entry->prev = LIST_POISON2;
}

/**
 * list_entry - get the struct for this entry
 * @ptr:    the &struct list_head pointer.
 * @type:   the type of the struct this is embedded in.
 * @member: the name of the list_struct within the struct.
 */
#define list_entry(ptr, type, member) \
    container_of(ptr, type, member)

/**
 * list_first_entry - get the first element from a list
 * @ptr:    the list head to take the element from.
 * @type:   the type of the struct this is embedded in.
 * @member: the name of the list_struct within the struct.
 *
 * Note, that list is expected to be not empty.
 */
#define list_first_entry(ptr, type, member) \
    list_entry((ptr)->next, type, member)


/**
 * list_for_each    -   iterate over a list
 * @pos:    the &struct list_head to use as a loop cursor.
 * @head:   the head for your list.
 */
#define list_for_each(pos, head) \
    for (pos = (head)->next;  pos != (head); \
            pos = pos->next)

/**
 * list_for_each_safe - iterate over a list safe against removal of list entry
 * @pos:    the &struct list_head to use as a loop cursor.
 * @n:      another &struct list_head to use as temporary storage
 * @head:   the head for your list.
 */
#define list_for_each_safe(pos, n, head) \
    for (pos = (head)->next, n = pos->next; pos != (head); \
        pos = n, n = pos->next)





/*
*线程池里所有运行和等待的任务都是一个CThread_worker
*由于所有任务都在链表里，所以是一个链表结构
*/
typedef struct worker
{
    /*回调函数，任务运行时会调用此函数，注意也可声明成其它形式*/
    void *(*process) ( int arg, int i);
    int  arg;/*回调函数的参数*/
    //struct worker *next;

} CThread_worker;

typedef struct worker_list
{ 
    CThread_worker   work;
    list_head        m_list;
}worker_list;


/*线程池结构*/
typedef struct
{
     pthread_mutex_t queue_lock;
     pthread_cond_t queue_ready;

    /*链表结构，线程池中所有等待任务*/
     list_head     queue_head;

    /*是否销毁线程池*/
    int shutdown;
     pthread_t *threadid;
    /*线程池中允许的活动线程数目*/
    int max_thread_num;
    /*当前等待队列的任务数目*/
    int cur_queue_size;

} CThread_pool;


int pool_add_worker (void *(*process) ( int arg, int buff_index), int arg);
void *thread_routine (void *arg);


static CThread_pool *pool = NULL;
void
pool_init (int max_thread_num)
{

     int ret;
     pool = (CThread_pool *) malloc (sizeof (CThread_pool));

     pthread_mutex_init (&(pool->queue_lock), NULL);
     pthread_cond_init (&(pool->queue_ready), NULL);

     INIT_LIST_HEAD(&(pool->queue_head));

     pool->max_thread_num = max_thread_num;
     pool->cur_queue_size = 0;

     pool->shutdown = 0;

     pool->threadid =
         (pthread_t *) malloc (max_thread_num * sizeof (pthread_t));
    int i = 0;
    for (i = 0; i < max_thread_num; i++)
     {
        
        ret =  pthread_create (&(pool->threadid[i]), NULL, &thread_routine,
                 (void*)i);

         if(0 != ret)
         {
             printf("create thread fail  id %d,  error %d \n", i, ret);
         }


     }
}


/*向线程池中加入任务*/
int
pool_add_worker (void *(*process) (int arg, int buff_index),  int fd)
{
    /*构造一个新任务*/
     worker_list *newworker =
         (worker_list *) malloc (sizeof (worker_list));
     newworker->work.process = process;
     newworker->work.arg = fd;
     //newworker->next = NULL;/*别忘置空*/

     pthread_mutex_lock (&(pool->queue_lock));
    /*将任务加入到等待队列中*/


    list_add_tail(&(newworker->m_list), &(pool->queue_head));
 /*    CThread_worker *member = pool->queue_head;
    if (member != NULL)
     {
        while (member->next != NULL)
             member = member->next;
         member->next = newworker;
     }
    else
     {
         pool->queue_head = newworker;
     }

     assert (pool->queue_head != NULL);
*/
     pool->cur_queue_size++;
     pthread_mutex_unlock (&(pool->queue_lock));
    /*好了，等待队列中有任务了，唤醒一个等待线程；
     注意如果所有线程都在忙碌，这句没有任何作用*/
     pthread_cond_signal (&(pool->queue_ready));
    return 0;
}


/*销毁线程池，等待队列中的任务不会再被执行，但是正在运行的线程会一直
把任务运行完后再退出*/
int
pool_destroy ()
{
    list_head *p, *tmp;
    worker_list  *tmpwork;

    if (pool->shutdown)
        return -1;/*防止两次调用*/
     pool->shutdown = 1;

    /*唤醒所有等待线程，线程池要销毁了*/
     pthread_cond_broadcast (&(pool->queue_ready));

    /*阻塞等待线程退出，否则就成僵尸了*/
    int i;
    for (i = 0; i < pool->max_thread_num; i++)
         pthread_join (pool->threadid[i], NULL);
     free (pool->threadid);


    /*销毁等待队列*/
    list_for_each_safe(p,tmp, &(pool->queue_head))
    {
        
        tmpwork = list_entry(p, worker_list, m_list);
        list_del(p);
        free(tmpwork);
        
    }
    
    /* CThread_worker *head = NULL;
    while (pool->queue_head != NULL)
     {
         head = pool->queue_head;
         pool->queue_head = pool->queue_head->next;
         free (head);
     }
     */
    /*条件变量和互斥量也别忘了销毁*/
     pthread_mutex_destroy(&(pool->queue_lock));
     pthread_cond_destroy(&(pool->queue_ready));
    
     free (pool);
    /*销毁后指针置空是个好习惯*/
     pool=NULL;
    return 0;
}


void *
thread_routine (void *arg)
{
     //printf ("starting thread 0x%x\n", pthread_self ());
     //printf("the input para %d \n", ((int)arg));

#ifdef DEBUG_TILERA
    if (tmc_cpus_set_my_cpu(tmc_cpus_find_nth_cpu(&cpus, rank)) < 0)
    {
        printf("tmc_cpus_set_my_cpu() failed.\n");
        tmc_task_die("tmc_cpus_set_my_cpu() failed.");
    }
#endif
     
    while (1)
     {
         pthread_mutex_lock (&(pool->queue_lock));
        /*如果等待队列为0并且不销毁线程池，则处于阻塞状态; 注意
         pthread_cond_wait是一个原子操作，等待前会解锁，唤醒后会加锁*/
        while (pool->cur_queue_size == 0 && !pool->shutdown)
         {
            // printf ("thread 0x%x is waiting\n", pthread_self ());
             pthread_cond_wait (&(pool->queue_ready), &(pool->queue_lock));
         }

        /*线程池要销毁了*/
        if (pool->shutdown)
         {
            /*遇到break,continue,return等跳转语句，千万不要忘记先解锁*/
             pthread_mutex_unlock (&(pool->queue_lock));
             printf ("thread 0x%x will exit\n", pthread_self ());
             pthread_exit (NULL);
         }

         //printf ("thread 0x%x is starting to work\n", pthread_self ());

        /*assert是调试的好帮手*/
         assert (pool->cur_queue_size != 0);
         //assert (pool->queue_head != NULL);
        
        /*等待队列长度减去1，并取出链表中的头元素*/
         pool->cur_queue_size--;
         worker_list *workerlist;

         workerlist = list_first_entry(&(pool->queue_head), worker_list, m_list);
         
         /*delete it from the list*/
         list_del(&(workerlist->m_list));

         //pool->queue_head = worker->next;
         pthread_mutex_unlock (&(pool->queue_lock));

         CThread_worker* worker = &(workerlist->work);

        /*调用回调函数，执行任务*/
         (*(worker->process)) (worker->arg, ((int) arg));
         free (workerlist);
         worker = NULL;
     }
    /*这一句应该是不可达的*/
     pthread_exit (NULL);
}

/*
setnonblocking - 设置句柄为非阻塞方式
*/
int setnonblocking(int sockfd)
{
    if (fcntl(sockfd, F_SETFL, fcntl(sockfd, F_GETFD, 0)|O_NONBLOCK) == -1)
    {
        return -1;
    }
    return 0;
}

/*
handle_message - 处理每个 socket 上的消息收发
*/
void * handle_message(int new_fd, int buff_index)
{
    char* buf = (char*)&buffer[buff_index];
    int len;
    struct epoll_event ev;

    
    /* 开始处理每个新连接上的数据收发 */
    //bzero(buf, MAXBUF + 1);
    /* 接收客户端的消息 */
    while((-1 != (len = recv(new_fd, buf, MAXBUF, 0))) 
           ||((-1 == len) &&(EAGAIN != errno)))
    {
             //perror("recv error ");
            //printf("recv error %d  fd %d\n", errno, new_fd);
            //return NULL;
        
        if (len > 0)
        {
           msgcount++;
            /*printf
            ("%d receive message success   total %d bytes data  msgcount %d\n",
             new_fd,   len, msgcount);*/
         }
        else if(len == 0)
        {
            //printf("the socket %d is closed \n", new_fd);
            epoll_ctl(kdpfd, EPOLL_CTL_DEL, new_fd,&ev);
            close(new_fd);
            connect_count--;
        }
        else
        {
            if (len < 0)
                printf
                (" socket %d receive message fail error code: %d,  error message: '%s'\n",
                 new_fd, errno, strerror(errno));
            
            epoll_ctl(kdpfd, EPOLL_CTL_DEL, new_fd,&ev);
            close(new_fd);
            connect_count--;
            //pthread_mutex_lock (&(curfds_lock));
            //curfds--;
            //pthread_mutex_unlock (&(curfds_lock));
            return NULL;

            
        }

    }
    /* 处理每个新连接上的数据收发结束 */
    return NULL;
}


static void *handle_count(void* arg)
{
    int precount, speed;
    
    while(1)
    {   
        precount = msgcount;
        sleep(5);
        timecount += 5;
        //printf("The tcp connection count is %d\n",count_tcp);
        //printf("The received packets count is %d, time %d\n",msgcount, timecount);
        speed = msgcount - precount ;
        printf("The received speed  is %d, connect %d\n",speed, connect_count);
    }

    return NULL;
}


int main(int argc, char **argv)
{
    int listener, new_fd,  nfds, n, ret;
    socklen_t len;
    struct sockaddr_in my_addr, their_addr;
    unsigned int myport, lisnum;
    struct epoll_event ev;
    struct rlimit rt;

    if(5 != argc)
    {

      printf("Usage: %s <thread_number(0 ~ 200)> <port(0-65535)> <listen queue number>  <IP Address>   \n", argv[0]);
      exit(1);
    
    }

    if(argv[1])
       THREAD_NUMBER = atoi(argv[1]);
       
    if (argv[2])
        myport = atoi(argv[2]);
    else
        myport = 7838;

    if (argv[3])
        lisnum = atoi(argv[3]);
    else
        lisnum = 2;



 #ifdef DEBUG_TILERA
    if (tmc_cpus_get_my_affinity(&cpus) != 0)
    {
        printf("tmc_cpus_get_my_affinity() failed.\n");
        tmc_task_die("tmc_cpus_get_my_affinity() failed.");
    }
    if (tmc_cpus_count(&cpus) < MAX_THREAD)
    {
        printf("\nInsufficient cpus available.\n");
        tmc_task_die("Insufficient cpus available.");
    }
#endif       

   /*threadpool*/
   pool_init (THREAD_NUMBER);/*线程池中最多THREAD_NUMBER个活动线程*/

   pthread_mutex_init (&(curfds_lock), NULL);

        

    /* 设置每个进程允许打开的最大文件数 */
    rt.rlim_max = rt.rlim_cur = MAXEPOLLSIZE;
    if (setrlimit(RLIMIT_NOFILE, &rt) == -1)
    {
        perror("setrlimit");
        exit(1);
    }
    else printf("set the system resource success!\n");

    /* 开启 socket 监听 */
    if ((listener = socket(PF_INET, SOCK_STREAM, 0)) == -1)
    {
        perror("socket");
        exit(1);
    }
    else
        printf("socket create success!n");

    setnonblocking(listener);

    bzero(&my_addr, sizeof(my_addr));
    my_addr.sin_family = PF_INET;
    my_addr.sin_port = htons(myport);
    if (argv[4])
        my_addr.sin_addr.s_addr = inet_addr(argv[4]);
    else
        my_addr.sin_addr.s_addr = INADDR_ANY;

    if (bind
        (listener, (struct sockaddr *) &my_addr, sizeof(struct sockaddr))
        == -1)
    {
        perror("bind");
        exit(1);
    }
    else
        printf("IP address and port bing success!\n");

    if (listen(listener, lisnum) == -1)
    {
        perror("listen");
        exit(1);
    }
    else
        printf("start to work!\n");

    /* 创建 epoll 句柄，把监听 socket 加入到 epoll 集合里 */
    kdpfd = epoll_create(MAXEPOLLSIZE);
    len = sizeof(struct sockaddr_in);
    ev.events = EPOLLIN;
    ev.data.fd = listener;
    if (epoll_ctl(kdpfd, EPOLL_CTL_ADD, listener, &ev) < 0)
    {
        fprintf(stderr, "epoll set insertion error: fd=%d\n", listener);
        return -1;
    }
    else
        printf("listen socket add to epoll success\n");
    if (pthread_create(&thread_count, NULL, &handle_count, NULL) != 0)
    {
#ifdef DEBUG_TILERA      
        tmc_task_die("pthread_create() failed.");
#endif      
    }
    curfds = 1;
    while (1)
    {
        /* 等待有事件发生 */
        nfds = epoll_wait(kdpfd, events, MAXEPOLLSIZE, -1);
        if (nfds == -1)
        {
            perror("epoll_wait");
            break;
        }
        /* 处理所有事件 */
        for (n = 0; n < nfds; ++n)
        {
           // printf("The number of fd %d \n", nfds);
            if (events[n].data.fd == listener)
            {
                new_fd = accept(listener, (struct sockaddr *) &their_addr,
                                &len);
                if (new_fd < 0)
                {
                    perror("accept");
                    continue;
                }
                //else
                   // printf("connect from  %x:%x， allocate socket for %x\n", inet_ntoa(their_addr.sin_addr), ntohs(their_addr.sin_port), new_fd);

                connect_count++;
                setnonblocking(new_fd);
                ev.events = EPOLLIN | EPOLLET;
                ev.data.fd = new_fd;
                if (epoll_ctl(kdpfd, EPOLL_CTL_ADD, new_fd, &ev) < 0)
                {
                    fprintf(stderr, "add socket '%d' to epoll fail %s\n",
                            new_fd, strerror(errno));
                    pool_destroy ();
                    
                    return -1;
                }
                //pthread_mutex_lock (&(curfds_lock));
               // curfds++;
               // pthread_mutex_unlock (&(curfds_lock));

                
            }
            else if(events[n].events & EPOLLIN )
            {
                //printf("socket number %d", events[n].data.fd);
                pool_add_worker (handle_message, events[n].data.fd);
                //ret = handle_message(events[n].data.fd);
               /* if (ret < 1 && errno != 11)
                {
                    epoll_ctl(kdpfd, EPOLL_CTL_DEL, events[n].data.fd,
                              &ev);
                    curfds--;
                }*/

               /*
                ev.data.fd=events[n].data.fd;
                ev.events=EPOLLIN|EPOLLET;
                epoll_ctl(kdpfd,EPOLL_CTL_MOD, events[n].data.fd,&ev);
                */
            }
            else
            {
                printf("other event \n");
            }
        }
    }
    close(listener);

    pool_destroy ();
    return 0;
}