WorkFlow学习分享:poller与mpoller

一、概括

poller其中封装了epoll,list(内核写法),红黑树，其中list和红黑树是用来管理timefd的，起到定时器的作用，mpoller又是对poller的进一步封装。

由于源码有两千行，不会全部搬上来讲，会根据创建->增删改查->epoll时间循环 等逻辑来梳理

二、poller

2.1、poller的数据结构

有三种数据结构，data,noed,poller；其中node聚合data，poller依赖node

struct poller_data
{
//该宏定义用在epoll响应时，识别时什么操作
#define PD_OP_READ			1
#define PD_OP_WRITE			2
#define PD_OP_LISTEN		3
#define PD_OP_CONNECT		4
#define PD_OP_SSL_READ		PD_OP_READ
#define PD_OP_SSL_WRITE		PD_OP_WRITE
#define PD_OP_SSL_ACCEPT	5
#define PD_OP_SSL_CONNECT	6
#define PD_OP_SSL_SHUTDOWN	7
#define PD_OP_EVENT			8
#define PD_OP_NOTIFY		9
#define PD_OP_TIMER			10
	short operation;//该字段等于上边宏定义的某一个
	unsigned short iovcnt;//writev中的最后一个参数
	int fd;//文件描述符
	union
	{
		SSL *ssl;
		void *(*accept)(const struct sockaddr *, socklen_t, int, void *);
		void *(*event)(void *);
		void *(*notify)(void *, void *);
	};
	void *context;//上下文
	union
	{
		poller_message_t *message;
		struct iovec *write_iov;
		void *result;
	};
};

struct __poller_node
{
	int state;//当前所属的状态
	int error;//错误码
	struct poller_data data;
#pragma pack(1)
	union
	{
		struct list_head list;
		struct rb_node rb;
	};//定时器用(链表+红黑树)
#pragma pack()
	char in_rbtree;//识别该结点是否在树中
	char removed;//识别该结点是否被移除
	int event;
	struct timespec timeout;
	struct __poller_node *res;
};

struct poller_result//该结构用在del中，由node强转过来
{
#define PR_ST_SUCCESS		0
#define PR_ST_FINISHED		1
#define PR_ST_ERROR			2
#define PR_ST_DELETED		3
#define PR_ST_MODIFIED		4
#define PR_ST_STOPPED		5
	int state;
	int error;
	struct poller_data data;
};

struct __poller
{
	size_t max_open_files;//文件描述符最大能开启的数量
	poller_message_t *(*create_message)(void *);
	int (*partial_written)(size_t, void *);
	void (*cb)(struct poller_result *, void *);//回调函数，结点被删除后会调用该函数
	void *ctx;//回调函数的第二个传参

	pthread_t tid;
	int pfd;
	int timerfd;
	int pipe_rd;
	int pipe_wr;
	int stopped;
	struct rb_root timeo_tree;//红黑树根节点
	struct rb_node *tree_first;//红黑树第一个结点
	struct rb_node *tree_last;//红黑树最后一个结点
	struct list_head timeo_list;//有超时机制的链表
	struct list_head no_timeo_list;//没有超时机制的链表
	struct __poller_node **nodes;//该为一个指针数组，用来存poller_node,容量为max_open_files
	pthread_mutex_t mutex;
	char buf[POLLER_BUFSIZE];//大小为262144,该数组在后面会被强转为指针的数组，除与4=65536
};

2.2、poller的创建

poller_t *__poller_create(void **nodes_buf, const struct poller_params *params)

创建时传入的参数有两，nodes_bud即为__poller结构中的**nodes，在传入前已分配好了空间

pararms记录了max_open_files,几个函数的指针以及上下文，创建时会变成poller的

struct poller_params
{
	size_t max_open_files;
	poller_message_t *(*create_message)(void *);
	int (*partial_written)(size_t, void *);
	void (*callback)(struct poller_result *, void *);
	void *context;
};

poller_t *__poller_create(void **nodes_buf, const struct poller_params *params)
{
	poller_t *poller = (poller_t *)malloc(sizeof (poller_t));
	int ret;

	if (!poller)
		return NULL;

	poller->pfd = __poller_create_pfd();//创建epoll的fd
	if (poller->pfd >= 0)
	{
		if (__poller_create_timer(poller) >= 0)
		{
			ret = pthread_mutex_init(&poller->mutex, NULL);
			if (ret == 0)
			{
                //见params的内容赋予poller
				poller->nodes = (struct __poller_node **)nodes_buf;
				poller->max_open_files = params->max_open_files;
				poller->create_message = params->create_message;
				poller->partial_written = params->partial_written;
				poller->cb = params->callback;
				poller->ctx = params->context;

				poller->timeo_tree.rb_node = NULL;
				poller->tree_first = NULL;
				poller->tree_last = NULL;
                //初始化链表
				INIT_LIST_HEAD(&poller->timeo_list);
				INIT_LIST_HEAD(&poller->no_timeo_list);

				poller->stopped = 1;
				return poller;
			}

			errno = ret;
			close(poller->timerfd);
		}

		close(poller->pfd);
	}

	free(poller);
	return NULL;
}

2.2.1、__poller_create_pfd与__poller_create_timer

static inline int __poller_create_pfd()

{

    return epoll_create(1);

}

//创建timefd
static inline int __poller_create_timerfd()
{
	return timerfd_create(CLOCK_MONOTONIC, 0);
}

//将timefd添加到epoll中
static inline int __poller_add_timerfd(int fd, poller_t *poller)
{
	struct epoll_event ev = {
		.events		=	EPOLLIN | EPOLLET,
		.data		=	{
			.ptr	=	NULL
		}
	};
	return epoll_ctl(poller->pfd, EPOLL_CTL_ADD, fd, &ev);
}


static int __poller_create_timer(poller_t *poller)
{
    //创建timefd
	int timerfd = __poller_create_timerfd();
	if (timerfd >= 0)
	{
        //将timefd添加到epoll中
		if (__poller_add_timerfd(timerfd, poller) >= 0)
		{
			poller->timerfd = timerfd;
			return 0;
		}
		close(timerfd);
	}
	return -1;
}

2.3、poller的增删改查

2.3.1、增加结点

1、int poller_add(const struct poller_data *data, int timeout, poller_t *poller)

在插入结点的时候，主要难点在timeout，如果timeout>=0,需要插入到红黑树或者有超时机制的链表在，否则就直接插入普通的链表中

int poller_add(const struct poller_data *data, int timeout, poller_t *poller)
{
	struct __poller_node *res = NULL;
	struct __poller_node *node;
	int need_res;
	int event;

    //查看fd是否超过最大数量
	if ((size_t)data->fd >= poller->max_open_files)
	{
		errno = data->fd < 0 ? EBADF : EMFILE;
		return -1;
	}

    //该函数根据data中的operation来为event赋值，即决定event事件
	need_res = __poller_data_get_event(&event, data);
	if (need_res < 0)
		return -1;

	if (need_res)
	{
		res = (struct __poller_node *)malloc(sizeof (struct __poller_node));
		if (!res)
			return -1;
	}
	node = (struct __poller_node *)malloc(sizeof (struct __poller_node));
	if (node)
	{
        //初始化结点的数据
		node->data = *data;
		node->event = event;
		node->in_rbtree = 0;
		node->removed = 0;
		node->res = res;
        //如果有定时机制，则调用该函数更新node的时间
        //该函数内部是获得系统时间后再加上timeout以此获得计时时间
		if (timeout >= 0)
			__poller_node_set_timeout(timeout, node);

		pthread_mutex_lock(&poller->mutex);
        //先判断当前poller中的结点数组不存在该fd,每个fd都是唯一的
		if (!poller->nodes[data->fd])
		{
            //将该fd添加至epoll中
			if (__poller_add_fd(data->fd, event, node, poller) >= 0)
			{
                //如果需要定时启动,则将其插入到定时链表或红黑树中
                //反之直接插入没有超时机制的链表的链表中
				if (timeout >= 0)
					__poller_insert_node(node, poller);
				else
					list_add_tail(&node->list, &poller->no_timeo_list);

                //将结点纳入结点数组
				poller->nodes[data->fd] = node;
				node = NULL;
			}
		}
		pthread_mutex_unlock(&poller->mutex);
		if (node == NULL)
			return 0;
		free(node);
	}
	free(res);
	return -1;
}

2、static void __poller_insert_node(struct __poller_node *node,poller_t *poller)

该函数用来向红黑树或者超时链表插入结点，插入结点有三种情况

end:最初指向链表中最后一个结点

情况一:如果链表为空，则直接插入,end指针，指向红黑树中的第一个结点，最后 新结点 与 end结点相互比较,如果新节点的时间小于end指向的结点则启动定时器

情况二:当 新结点 的时间大于 最后一个结点 则直接插入并返回

情况三:在 新结点小于链表最后结点的情况下，该结点则插入红黑树中，如果 插入位置是最开始的结点，就意味着最小值出现变动,需要对其判断是否有到时间

也由此达成了O(1)+O(logn)

static void __poller_insert_node(struct __poller_node *node,
								 poller_t *poller)
{
	struct __poller_node *end;

    //链表为双向循环链表，采用的是内核写法，有兴趣可以自行查找
    //获得超时机制链表中最后的poller结点
	end = list_entry(poller->timeo_list.prev, struct __poller_node, list);

	if (list_empty(&poller->timeo_list))
	{
        //情况一:
        //如果链表为空，则直接插入,并且将end指向红黑树中的第一个结点
        //最后 新结点 与 end结点相互比较,如果新节点的时间已到则启动定时器
		list_add(&node->list, &poller->timeo_list);
		end = rb_entry(poller->tree_first, struct __poller_node, rb);
	}
	else if (__timeout_cmp(node, end) >= 0)
	{
        //情况二:
        //当 新结点 的时间大于 最后一个结点 则直接插入并返回
		list_add_tail(&node->list, &poller->timeo_list);
		return;
	}
	else
	{
        //情况三:
        //在 新结点小于链表最后结点的情况下，该结点则插入红黑树中
        //如果 插入位置是最开始的结点，就意味着最小值出现变动
        //需要对其判断是否有到时间
		__poller_tree_insert(node, poller);
        //判断插入位置是不是第一个结点
		if (&node->rb != poller->tree_first)
			return;

		end = list_entry(poller->timeo_list.next, struct __poller_node, list);
	}

	if (!poller->tree_first || __timeout_cmp(node, end) < 0)
		__poller_set_timerfd(poller->timerfd, &node->timeout, poller);
}

3、static void __poller_tree_insert(struct __poller_node *node, poller_t *poller)

该函数用于将新结点插入红黑树，插入的逻辑和排序树一样，较好理解

static void __poller_tree_insert(struct __poller_node *node, poller_t *poller)
{
    //p当前指向的是红黑树的根节点
	struct rb_node **p = &poller->timeo_tree.rb_node;
	struct rb_node *parent = NULL;
	struct __poller_node *entry;

    //当前entry指向的是树中的最后一个结点
	entry = rb_entry(poller->tree_last, struct __poller_node, rb);

    //接下来需要寻找新节点所要插入的位置,逻辑和二叉排序树一样，比较单位是超时时间
	if (!*p)
	{
        //情况一:树没有结点，新结点直接插入成为根节点
		poller->tree_first = &node->rb;
		poller->tree_last = &node->rb;
	}
	else if (__timeout_cmp(node, entry) >= 0)
	{
        //情况二:如果新节点大于最后一个结点，则就插入当前最后结点的右结点
		parent = poller->tree_last;
		p = &parent->rb_right;
		poller->tree_last = &node->rb;
	}
	else
	{
        //情况三:小于最后一个结点，则从根节点开始比较更新p指针的指向
        //逻辑和二叉排序树一样
		do
		{
			parent = *p;
			entry = rb_entry(*p, struct __poller_node, rb);
			if (__timeout_cmp(node, entry) < 0)
				p = &(*p)->rb_left;
			else
				p = &(*p)->rb_right;
		} while (*p);

        //如果找到的位置是第一个结点则更新poller中第一节点指针的指向
		if (p == &poller->tree_first->rb_left)
			poller->tree_first = &node->rb;
	}

    //将结点插入树中，并且调整树的颜色
	node->in_rbtree = 1;
	rb_link_node(&node->rb, parent, p);
	rb_insert_color(&node->rb, &poller->timeo_tree);
}

2.3.2、删除结点

如果一个结点就要执行一次释放，那么影响运行的程度想必会相对较大，该框架下，使用了管道的方法通知epoll有结点需要删除，因此在一定时间点的删除操作会被聚集到一次epoll中，以此优化

也因此在del函数中，仅仅只是更改结点状态和将结点从红黑树或者链表中移除，释放操作则集中在了handle_pipe中

PS：结点删除时会调用回调函数

1、int poller_del(int fd, poller_t *poller)

int poller_del(int fd, poller_t *poller)
{
	struct __poller_node *node;
	if ((size_t)fd >= poller->max_open_files)
	{
		errno = fd < 0 ? EBADF : EMFILE;
		return -1;
	}
	pthread_mutex_lock(&poller->mutex);
	node = poller->nodes[fd];//通过fd获得结点，每个fd是唯一的
	if (node)
	{
		poller->nodes[fd] = NULL;//将该结点从数组中移除

        //结点不一定会在红黑树中，还会在超时链表或者普通链表中
		if (node->in_rbtree)
			__poller_tree_erase(node, poller);
		else
			list_del(&node->list);

        //将fd从epoll中删除
		__poller_del_fd(fd, node->event, poller);

		node->error = 0;
		node->state = PR_ST_DELETED;//当前结点状态是已删除
		if (poller->stopped)
		{
            //如果该poller已停止,则释放掉该结点的res，并且执行回调函数
			free(node->res);
			poller->cb((struct poller_result *)node, poller->ctx);
		}
		else
		{
            //poller正常运行时，就通过管道通知epoll调用handle_pipe
            //所以结点真正的删除发生在handle_pipe中
			node->removed = 1;
			write(poller->pipe_wr, &node, sizeof (void *));
            //将该结点写入管道中
		}
	}
	else
		errno = ENOENT;

	pthread_mutex_unlock(&poller->mutex);
	return -!node;
}

2、static int __poller_handle_pipe(poller_t *poller)

static int __poller_handle_pipe(poller_t *poller)
{
	struct __poller_node **node = (struct __poller_node **)poller->buf;
	int stop = 0;
	int n;
	int i;
    //指针为四个字节(32位下),read的返回值与指针大小相除获得这个时间点下需要删除的结点
	n = read(poller->pipe_rd, node, POLLER_BUFSIZE) / sizeof (void *);
	for (i = 0; i < n; i++)
	{
		if (node[i])
		{
			free(node[i]->res);
            //执行该结点的回调函数
			poller->cb((struct poller_result *)node[i], poller->ctx);
		}
		else
			stop = 1;
	}
	return stop;
}

2.3.3 修改结点

该操作修改后需要将旧结点删除，再把修改后的结点插入进去，难度不高，只是整合add和del的操作

int poller_mod(const struct poller_data *data, int timeout, poller_t *poller)
{
	struct __poller_node *res = NULL;
	struct __poller_node *node;
	struct __poller_node *old;
	int need_res;
	int event;

	if ((size_t)data->fd >= poller->max_open_files)
	{
		errno = data->fd < 0 ? EBADF : EMFILE;
		return -1;
	}
    //获得事件类型
	need_res = __poller_data_get_event(&event, data);
	if (need_res < 0)
		return -1;

	if (need_res)
	{
		res = (struct __poller_node *)malloc(sizeof (struct __poller_node));
		if (!res)
			return -1;
	}
	node = (struct __poller_node *)malloc(sizeof (struct __poller_node));
	if (node)
	{
        //初始化结点信息
		node->data = *data;
		node->event = event;
		node->in_rbtree = 0;
		node->removed = 0;
		node->res = res;
        //需要定时则获得绝对事件，初始化到node中
		if (timeout >= 0)
			__poller_node_set_timeout(timeout, node);
        /
		pthread_mutex_lock(&poller->mutex);
		old = poller->nodes[data->fd];
		if (old)
		{
            //修改结点在epoll中的信息，即epoll_ctl mod
			if (__poller_mod_fd(data->fd, old->event, event, node, poller) >= 0)
			{
                //删除旧结点，即执行del一样的操作
				if (old->in_rbtree)
					__poller_tree_erase(old, poller);
				else
					list_del(&old->list);

				old->error = 0;
				old->state = PR_ST_MODIFIED;
				if (poller->stopped)
				{
					free(old->res);
					poller->cb((struct poller_result *)old, poller->ctx);
				}
				else
				{
					old->removed = 1;
					write(poller->pipe_wr, &old, sizeof (void *));
				}

                //加入新的结点，执行add一样的操作
				if (timeout >= 0)
					__poller_insert_node(node, poller);
				else
					list_add_tail(&node->list, &poller->no_timeo_list);

				poller->nodes[data->fd] = node;
				node = NULL;
			}
		}
		else
			errno = ENOENT;

		pthread_mutex_unlock(&poller->mutex);
		if (node == NULL)
			return 0;

		free(node);
	}

	free(res);
	return -1;
}

2.3.4设置定时和添加定时结点

1、int poller_set_timeout(int fd, int timeout, poller_t *poller)

该函数是修改一个已有的结点的timeout

过程就是修改timeout，将结点从红黑树或链表移除后再插入即可

int poller_set_timeout(int fd, int timeout, poller_t *poller)
{
	struct __poller_node time_node;
	struct __poller_node *node;

	if ((size_t)fd >= poller->max_open_files)
	{
		errno = fd < 0 ? EBADF : EMFILE;
		return -1;
	}

	if (timeout >= 0)//设置绝对时间
		__poller_node_set_timeout(timeout, &time_node);

	pthread_mutex_lock(&poller->mutex);
	node = poller->nodes[fd];
	if (node)
	{
        //将结点从链表或红黑树中移除
		if (node->in_rbtree)
			__poller_tree_erase(node, poller);
		else
			list_del(&node->list);

        //重新插入链表或红红黑树中
		if (timeout >= 0)
		{
			node->timeout = time_node.timeout;
			__poller_insert_node(node, poller);
		}
		else
			list_add_tail(&node->list, &poller->no_timeo_list);
	}
	else
		errno = ENOENT;

	pthread_mutex_unlock(&poller->mutex);
	return -!node;
}

2、int poller_add_timer(const struct timespec *value, void *context,poller_t *poller)

 该函数中的fd无意义，只是申请一个结点插入到红黑树或者链表中，

到时时调用某个函数，传参为context

int poller_add_timer(const struct timespec *value, void *context, poller_t *poller)
{
	struct __poller_node *node;

	node = (struct __poller_node *)malloc(sizeof (struct __poller_node));
	if (node)
	{
		memset(&node->data, 0, sizeof (struct poller_data));
		node->data.operation = PD_OP_TIMER;
		node->data.fd = -1;
		node->data.context = context;
		node->in_rbtree = 0;
		node->removed = 0;
		node->res = NULL;

        //获取绝对时间
		clock_gettime(CLOCK_MONOTONIC, &node->timeout);//获取系统时间
		node->timeout.tv_sec += value->tv_sec;
		node->timeout.tv_nsec += value->tv_nsec;
		if (node->timeout.tv_nsec >= 1000000000)
		{
			node->timeout.tv_nsec -= 1000000000;
			node->timeout.tv_sec++;
		}
        //
		pthread_mutex_lock(&poller->mutex);
		__poller_insert_node(node, poller);//插入结点
		pthread_mutex_unlock(&poller->mutex);
		return 0;
	}
	return -1;
}

3、static void __poller_set_timer(poller_t *poller)

该函数主要用在epoll_wait前，每次等待前都开启一次定时器，以此计时

static void __poller_set_timer(poller_t *poller)
{
	struct __poller_node *node = NULL;
	struct __poller_node *first;
	struct timespec abstime;

	pthread_mutex_lock(&poller->mutex);
    
    //获得超时链表的第一个结点
	if (!list_empty(&poller->timeo_list))
		node = list_entry(poller->timeo_list.next, struct __poller_node, list);

	if (poller->tree_first)
	{
        //获得红黑树中的第一个结点
		first = rb_entry(poller->tree_first, struct __poller_node, rb);
        //如果红黑树的结点比链表的小，即时间比链表的早，则node改为红黑树的结点
		if (!node || __timeout_cmp(first, node) < 0)
			node = first;
	}

	if (node)
		abstime = node->timeout;
	else
	{
		abstime.tv_sec = 0;
		abstime.tv_nsec = 0;
	}
    //开启计时
	__poller_set_timerfd(poller->timerfd, &abstime, poller);
	pthread_mutex_unlock(&poller->mutex);
}

4、static void __poller_handle_timeout(const struct __poller_node *time_node,poller_t *poller)

该函数是定时器到时处理的函数，epoll每次循环一次都会执行该函数

传入的timenode是当前的时间，每次epoll被唤醒都会获取一次系统时间并传入

static void __poller_handle_timeout(const struct __poller_node *time_node,poller_t *poller)
{
	struct __poller_node *node;
	struct list_head *pos, *tmp;
	LIST_HEAD(timeo_list);//一个临时链表，宏定义，意为初始化一个链表

	pthread_mutex_lock(&poller->mutex);
    //遍历超时链表，第一个节点为头结点
	list_for_each_safe(pos, tmp, &poller->timeo_list)
	{
        //获得当前链表结点所在的poller_node
		node = list_entry(pos, struct __poller_node, list);
		if (__timeout_cmp(node, time_node) <= 0)
		{
            //获取时间已经到的结点，从链表中删除并且添加到临时链表中
			if (node->data.fd >= 0)//fd不一定>=0，set_timer函数中为-1
			{
				poller->nodes[node->data.fd] = NULL;
				__poller_del_fd(node->data.fd, node->event, poller);
			}
            //从链表中删除并且添加到临时链表中
			list_move_tail(pos, &timeo_list);
		}
		else
			break;
	}

	if (poller->tree_first)
	{
		while (1)
		{
			node = rb_entry(poller->tree_first, struct __poller_node, rb);
			if (__timeout_cmp(node, time_node) < 0)
			{
                //找到超时结点，从红黑树删除，移入到临时链表
				if (node->data.fd >= 0)
				{
					poller->nodes[node->data.fd] = NULL;
					__poller_del_fd(node->data.fd, node->event, poller);
				}

				poller->tree_first = rb_next(poller->tree_first);
				rb_erase(&node->rb, &poller->timeo_tree);
				list_add_tail(&node->list, &timeo_list);
				if (poller->tree_first)
					continue;

				poller->tree_last = NULL;
			}

			break;
		}
	}
	pthread_mutex_unlock(&poller->mutex);
	while (!list_empty(&timeo_list))
	{
		node = list_entry(timeo_list.next, struct __poller_node, list);
		list_del(&node->list);

		node->error = ETIMEDOUT;
		node->state = PR_ST_ERROR;
		free(node->res);
		poller->cb((struct poller_result *)node, poller->ctx);
        //执行回调函数
	}
}

 2.3.5、poller的开启

1、start

该函数比较简单，主要是打开管道用于处理删除结点和创造epoll的线程

int poller_start(poller_t *poller)
{
	pthread_t tid;
	int ret;
	pthread_mutex_lock(&poller->mutex);
	if (__poller_open_pipe(poller) >= 0)//打开管道
	{
		ret = pthread_create(&tid, NULL, __poller_thread_routine, poller);
		if (ret == 0)
		{
			poller->tid = tid;
			poller->stopped = 0;
		}
		else
		{
			errno = ret;
			close(poller->pipe_wr);
			close(poller->pipe_rd);
		}
	}
	pthread_mutex_unlock(&poller->mutex);
	return -poller->stopped;
}

2、routine

该函数和传统的epoll使用没啥差别，判断然后执行即可。

每个分支的函数不打算在该文章内说明，其函数调用涉及到了结构中的函数指针，该指针指向由上层定义，在该文章说明不方便

static void *__poller_thread_routine(void *arg)
{
	poller_t *poller = (poller_t *)arg;
	__poller_event_t events[POLLER_EVENTS_MAX];
	struct __poller_node time_node;
	struct __poller_node *node;
	int has_pipe_event;
	int nevents;
	int i;

	while (1)
	{
		__poller_set_timer(poller);//开启定时器
		nevents = __poller_wait(events, POLLER_EVENTS_MAX, poller);
		clock_gettime(CLOCK_MONOTONIC, &time_node.timeout);//获得系统时间
		has_pipe_event = 0;
		for (i = 0; i < nevents; i++)
		{
			node = (struct __poller_node *)__poller_event_data(&events[i]);
			if (node > (struct __poller_node *)1)
			{
				switch (node->data.operation)
				{
				case PD_OP_READ:
					__poller_handle_read(node, poller);
					break;
				case PD_OP_WRITE:
					__poller_handle_write(node, poller);
					break;
				case PD_OP_LISTEN:
					__poller_handle_listen(node, poller);
					break;
				case PD_OP_CONNECT:
					__poller_handle_connect(node, poller);
					break;
				case PD_OP_SSL_ACCEPT:
					__poller_handle_ssl_accept(node, poller);
					break;
				case PD_OP_SSL_CONNECT:
					__poller_handle_ssl_connect(node, poller);
					break;
				case PD_OP_SSL_SHUTDOWN:
					__poller_handle_ssl_shutdown(node, poller);
					break;
				case PD_OP_EVENT:
					__poller_handle_event(node, poller);
					break;
				case PD_OP_NOTIFY:
					__poller_handle_notify(node, poller);
					break;
				}
			}
			else if (node == (struct __poller_node *)1)
				has_pipe_event = 1;
		}

		if (has_pipe_event)
		{
			if (__poller_handle_pipe(poller))
				break;
		}

		__poller_handle_timeout(&time_node, poller);
	}

三、mpoller

1、数据结构

struct __mpoller
{
	void **nodes_buf;
	unsigned int nthreads;
	poller_t *poller[1];//可变长数组，根据所需要的数量来分配空间
};

2、create

mpoller_t *mpoller_create(const struct poller_params *params, size_t nthreads)
{
	mpoller_t *mpoller;
	size_t size;

	if (nthreads == 0)
		nthreads = 1;
    
    //获取需要多大的空间，根据线程的数量来
	size = offsetof(mpoller_t, poller) + nthreads * sizeof (void *);
	mpoller = (mpoller_t *)malloc(size);
	if (mpoller)
	{
		mpoller->nthreads = (unsigned int)nthreads;
		if (__mpoller_create(params, mpoller) >= 0)//创建mpoller
			return mpoller;

		free(mpoller);
	}

	return NULL;
}

static int __mpoller_create(const struct poller_params *params,mpoller_t *mpoller)
{
    //存取结点的数组，每个poller共用这一个数组，因为fd是唯一的
	void **nodes_buf = (void **)calloc(params->max_open_files, sizeof (void *));
	unsigned int i;

	if (nodes_buf)
	{
		for (i = 0; i < mpoller->nthreads; i++)
		{
            //创建poller并且用数组储存
			mpoller->poller[i] = __poller_create(nodes_buf, params);
			if (!mpoller->poller[i])
				break;
		}

		if (i == mpoller->nthreads)
		{
			mpoller->nodes_buf = nodes_buf;
			return 0;
		}

		while (i > 0)
			__poller_destroy(mpoller->poller[--i]);

		free(nodes_buf);
	}

	return -1;
}

3、start

int mpoller_start(mpoller_t *mpoller)
{
	size_t i;

	for (i = 0; i < mpoller->nthreads; i++)
	{
		if (poller_start(mpoller->poller[i]) < 0)
			break;
	}
    //开启所有的poller就可以返回了，也由此开启了一部分的异步
	if (i == mpoller->nthreads)
		return 0;

	while (i > 0)
		poller_stop(mpoller->poller[--i]);

	return -1;
}

剩下的函数都是对poller中add，del，mod等的进一步封装，只是多了一行选择poller的代码，感兴趣可自行去观看