epoll多线程的例子

/*
 ============================================================================
 Name        : epoll_multithreads.c
 Author      : 
 Version     :
 Copyright   : Your copyright notice
 Description : Hello World in C, Ansi-style
 ============================================================================
 */

#include <sys/socket.h>
#include <sys/epoll.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <pthread.h>

#include <errno.h>

/* create an epoll 实例 */
int xs_epoll_create(int __c)
{
  XSOCKET __efd__ = -1;

  __efd__ = epoll_create(__c);
  if(__efd__ < 0){
    xs_dump_error_str(errno);
    return EXIT_ERROR;
  }

  return (__efd__);
}
int xs_epoll_ctl(XSOCKET __efd, int __method, XSOCKET __fd, struct epoll_event* __p)
{
  int __result__ = -1;

  __result__ = (epoll_ctl(__efd, __method, __fd, __p) < 0);
  if(__result__ < 0){
    xs_dump_error_str(errno);
    return EXIT_ERROR;
  }

  return EXIT_OK;
}
int xs_epoll_wait(XSOCKET __efd, struct epoll_event* __ev, int __c, int __tw)
{
  int __num__ = -1;

  __num__ = epoll_wait(__efd, __ev, __c, __tw);
  if(__num__ < 0)
    __num__ = EXIT_ERROR;

  return (__num__);
}
int xs_epoll_close(XSOCKET __f)
{
  return xs_release_socket(__f);
}

/*
  Add fd to epollfd
  Register the target file descriptor fd on the epoll instance referred to by the file descriptor epfd
  and associate the event with the internal file linked to fd.
*/
int xs_epoll_add(int __epollfd, int __fd, int __mod)
{
  xs_epoll_event __xs_ev__;

  __xs_ev__.events = __mod;
  __xs_ev__.data.fd = __fd;

  if(xs_epoll_ctl(__epollfd, EPOLL_CTL_ADD, __fd, &__xs_ev__) < 0)
          return EXIT_ERROR;

  return EXIT_OK;
}

/*
  Remove  (deregister) the target file descriptor fd from the epoll instance referred to by epollfd.
  The event is ignored and can be NULL.
  In kernel versions before 2.6.9, the EPOLL_CTL_DEL operation required a non-NULL pointer in event
  Since Linux 2.6.9, event can be specified as NULL when using EPOLL_CTL_DEL.
*/
int xs_epoll_del(int __epollfd, int __fd)
{
  xs_epoll_event __xs_ev__;

  /* We should better set event not NULL */
  __xs_ev__.events = __xs_ev__.events;
  __xs_ev__.data.fd = __fd;

  if(xs_epoll_ctl(__epollfd, EPOLL_CTL_DEL, __fd, &__xs_ev__) < 0)
          return EXIT_ERROR;

  return EXIT_OK;
}

/* Change the event associated with the target file descriptor __fd. */
int xs_epoll_mode(int __epollfd, int __fd, int __newmod)
{
  xs_epoll_event __xs_ev__;

  __xs_ev__.events = __newmod;
  __xs_ev__.data.fd = __fd;

  if(xs_epoll_ctl(__epollfd, EPOLL_CTL_MOD, __fd, &__xs_ev__) < 0)
          return EXIT_ERROR;

  return EXIT_OK;
}

/*
  xs_epoll_init: creates  an epoll "instance", requesting the kernel to allocate an
  event backing store dimensioned for size descriptors.
  Since Linux 2.6.8, the size argument is unused, but must be greater than zero.
  (The kernel  dynamically  sizes  the  required  data
       structures without needing this initial hint.)
*/
int xs_epoll_init(int size)
{
  int epollfd = -1;

  if(size == 0)
    epollfd = xs_epoll_create(EPOLL_DEFAULT_SIZE);
  else epollfd = xs_epoll_create(size);


  if(epollfd < 0) return EXIT_ERROR;

  xs_logd("epoll create success -> fd : %d", epollfd);

  return epollfd;
}

#define DEFAULT_THREAD_SIZE	10
#define EPOLL_DEFAULT_SIZE	10

typedef struct block_queue
{
  int queue[DEFAULT_THREAD_SIZE];
  long size;
  pthread_cond_t cond;
  pthread_mutex_t mutex;
}block_queue_t;

block_queue_t *bq = NULL;

typedef struct block_queue_param
{
  void* func;
  void* queue;  	/* Point to block queue structure */
} block_queue_param_t;
block_queue_param_t bqp;

#define BUFFER_SIZE 		1024
#include <sys/resource.h>
int g_epoll_fd = -1;
xs_epoll_event xs_ev, xs_events[EPOLL_DEFAULT_SIZE];
int g_epoll_size = EPOLL_DEFAULT_SIZE;

int queue_init( block_queue_t *queue)
{
  if(queue == NULL) {
	  return (-1);
  }

  queue->size = 0;

  pthread_cond_init(&(queue->cond), NULL);
  pthread_mutex_init(&(queue->mutex), NULL);

  return 0;
}

/**
 * @brief 创建block_queue_t对象，并完成初始化
 * @return 指向block_queue_t对象的指针。
 */
block_queue_t *epoll_queue_create()
{
  block_queue_t* queue = malloc(sizeof(block_queue_t));

  assert(queue);

  return ((queue_init(queue) == 0) ? queue : NULL);

}

void network_epoll_loop(void* data) {
	int socket = (int) data;

	char buffer[BUFFER_SIZE];
	memset(buffer, 0, BUFFER_SIZE);

	/* We only send what is received just now */
	int length = recv(socket, buffer, BUFFER_SIZE);
	if (length > 0) {
		send(socket, buffer, strlen(buffer));
	}
}

void *handle_queue(void *param) {
	void(* func)(void*);
	func = ((block_queue_param_t*) param)->func;

	block_queue_t* bque = ((block_queue_param_t*) param)->queue;
	pthread_cond_init(&bque->cond, NULL);
	pthread_mutex_init(&bque->mutex, NULL);

	int fd;
	for (;;) {
		if (pthread_mutex_lock(&bque->mutex) == EXIT_OK) {
			pthread_cond_wait(&bque->cond, &bque->mutex);
			if (bque->size == 0) {
				pthread_mutex_unlock(&bque->mutex);
				continue;
			} else {
				int i;
				fd = bque->queue[0];

				for (i = 0; i < bque->size - 1; ++i)
					bque->queue[i] = bque->queue[i + 1];

				bque->queue[bque->size - 1] = 0;
				bque->size--;
			}
			pthread_mutex_unlock(&bque->mutex);
		}

		func((void *) &fd);
	}
}

int init_threads(void) {
	int i = 0, ret;
	pthread_t child_thread[DEFAULT_THREAD_SIZE];
	pthread_attr_t child_thread_attr[DEFAULT_THREAD_SIZE];

	bqp.func = (void*) network_epoll_loop;
	bqp.queue = (void *) bq;

	for (i = 0; i < DEFAULT_THREAD_SIZE; ++i) {
		ret = pthread_attr_init(&child_thread_attr[i]);
		if (ret != 0) {
			fprintf(stderr, "Error, pthread_attr_init failed, error(%d):%s\n", errno, strerror(errno));
			return 1;
		}

		pthread_attr_setdetachstate(&child_thread_attr[i],
				PTHREAD_CREATE_DETACHED);

		if (pthread_create(&child_thread[i], &child_thread_attr[i],
				handle_queue, (void *)&bqp) < 0) {
			fprintf(stderr, "Error, pthread_create failed, error(%d):%s\n", errno, strerror(errno));
			return 1;
		}
	}
	return 0;
}

int create_and_bind(const char* name, const int port) {

    int listenfd = -1;
    //创建listen socket
    if ((listenfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
        perror("sockfd\n");
        exit(1);
    }

    set_non_block (listenfd);
    setreuseaddr(listenfd);

    struct sockaddr_in local;
    bzero(&local, sizeof(local));
    local.sin_family = AF_INET;
    local.sin_addr.s_addr = htonl(INADDR_ANY);
    local.sin_port = htons(port);

    if (bind(listenfd, (struct sockaddr *) &local, sizeof(local)) < 0) {
        perror("bind\n");
        exit(1);
    }

    return listenfd;
}


int init_server(const char *name, short int port)
{
  int server_socket = create_and_bind(name, port);

  return server_socket;
}

static void xs_insert_queue(block_queue_t *bque, int *fd)
{
  pthread_mutex_lock(&bque->mutex);

  if(bque->size == g_xs_thread_count)
    return;

  bque->queue[bque->size] = *fd;
  bque->size++;
  if(bque->size > g_xs_thread_count)  {
    fprintf(stderr,"Queue size over folow.%ld", bque->size);
    exit (1);
  }

  pthread_cond_signal(&bque->cond);
  pthread_mutex_unlock(&bque->mutex);

}

static inline void xs_handler(void* fd)
{
  printf("handler:fd => %d\n", *(int *)(fd));
  xs_insert_queue(bq, fd);
}

int epoll_entry() {
	int nfds, n;

	int listen_fd = init_server("127.0.0.1", 50437);
	printf("server thread [FD:%d] is ready for ...\n", listen_fd);

	bq = epoll_queue_create();
	if (init_threads() == 0)
		printf("Threads ready to use !");

	int epoll_fd = epoll_init(g_epoll_size);
	epoll_add(g_epoll_fd, g_serv_fd, EPOLLIN | EPOLLET);

	for (;;) {
		struct sockaddr_in local;
		socklen_t length = sizeof(local);
		int client = -1;

		nfds = epoll_wait(g_epoll_fd, xs_events, EPOLL_DEFAULT_SIZE, -1);

		for (n = 0; n < nfds; ++n) {

			if (xs_events[n].data.fd == g_serv_fd) {
				client = xs_net_accept(g_serv_fd, (struct sockaddr *) &local,
						&length);
				if (client < 0) {
					printf("%s", strerror(errno));
					continue;
				} else {
					printf("add socket pool : %d", client);
					set_nonblocking(client);
					xs_epoll_add(g_epoll_fd, client, EPOLLIN | EPOLLOUT
							| EPOLLET);
					client = -1;
				}
			} else {
				/* It's a client fd that needed to process */
				xs_handler((void *) &xs_events[n].data.fd);
			}
		}
	}

	pthread_mutex_destroy(&bq->mutex);
	pthread_cond_destroy(&bq->cond);
	xs_close_socket(g_serv_fd);
	xs_epoll_close(g_epoll_fd);

	xs_free(bq);
	return 0;
}

int main(int argc, char *argv[])
{
  epoll_entry();

  return 0;
}