libevent代码阅读（15）——epoll复用机制

epollops（eventop类型）对象是用于操作epoll复用机制的结构对象，它内部成员都是回调函数等，这些回调函数用于实现epoll复用机制，定义如下：

// io复用机制——epoll
const struct eventop epollops = {
	// 名字
	"epoll",
	// 初始化
	epoll_init,
	// 添加事件
	epoll_nochangelist_add,
	// 删除事件
	epoll_nochangelist_del,
	// 进入事件多路分发器循环
	epoll_dispatch,
	// 内存释放
	epoll_dealloc,
	// 是否需要重新初始化
	1, /* need reinit */
	// 一些特征
	EV_FEATURE_ET|EV_FEATURE_O1,
	// 额外分配的内存长度
	0
};

epollop结构则是存放epoll模式一些信息的结构，定义如下：

// epoll
struct epollop {
	// epoll事件
	struct epoll_event *events;
	// 事件的数量
	int nevents;
	// 文件描述符
	int epfd;
};

初始化epoll模式：

// epoll初始化
static void *
epoll_init(struct event_base *base)
{
	// epoll的文件描述符
	int epfd;

	// 用于存放epoll的文件描述符以及一些事件
	struct epollop *epollop;

	/* Initialize the kernel queue.  (The size field is ignored since
	 * 2.6.8.) */
	// 创建epoll文件描述符
	// 这里直接使用了底层的系统调用，而不是linux提供的api接口
	if ((epfd = epoll_create(32000)) == -1) {
		if (errno != ENOSYS)
			event_warn("epoll_create");
		return (NULL);
	}
	// 当调用exec时 关闭该描述符
	evutil_make_socket_closeonexec(epfd);

	// 创建epollop结构
	if (!(epollop = mm_calloc(1, sizeof(struct epollop)))) {
		close(epfd);
		return (NULL);
	}

	// 初始化epollop的字段
	epollop->epfd = epfd;


	/* Initialize fields */
	// 初始化epoll事件数组
	epollop->events = mm_calloc(INITIAL_NEVENT, sizeof(struct epoll_event));
	if (epollop->events == NULL) {
		mm_free(epollop);
		close(epfd);
		return (NULL);
	}
	epollop->nevents = INITIAL_NEVENT;

	// 如果设置了使用changelist，那么就使用changelist
	// 使用了changelist可以减少系统调用的次数
	if ((base->flags & EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST) != 0 ||
	    ((base->flags & EVENT_BASE_FLAG_IGNORE_ENV) == 0 &&
		evutil_getenv("EVENT_EPOLL_USE_CHANGELIST") != NULL))
		base->evsel = &epollops_changelist;

	// 信号处理器初始化
	evsig_init(base);

	return (epollop);
}

事件添加：

// 事件添加
static int
epoll_nochangelist_add(struct event_base *base, evutil_socket_t fd,
    short old, short events, void *p)
{
	// 存放旧的事件
	struct event_change ch;
	ch.fd = fd;
	ch.old_events = old;
	ch.read_change = ch.write_change = 0;

	// 写事件
	if (events & EV_WRITE)
		// 记录写变更
		ch.write_change = EV_CHANGE_ADD |
		    (events & EV_ET);

	// 读事件
	if (events & EV_READ)
		// 记录读变更
		ch.read_change = EV_CHANGE_ADD |
		    (events & EV_ET);

	// 应用更改
	return epoll_apply_one_change(base, base->evbase, &ch);
}

epoll_apply_one_change的定义如下：

//可以看出，添加和删除事件的主要工作，都是在epoll_apply_one_change这个函数里面实现的
//某个epoll事件的状改变了
static int
epoll_apply_one_change(struct event_base *base,
    struct epollop *epollop,
    const struct event_change *ch)
{
	// epoll事件
	struct epoll_event epev;
	int op, events = 0;

	/*
	 * 下面就是处理各种操作的组合
	 */

	// 总会进入这里
	if (1) {
		/* The logic here is a little tricky.  If we had no events set
		   on the fd before, we need to set op="ADD" and set
		   events=the events we want to add.  If we had any events set
		   on the fd before, and we want any events to remain on the
		   fd, we need to say op="MOD" and set events=the events we
		   want to remain.  But if we want to delete the last event,
		   we say op="DEL" and set events=the remaining events.  What
		   fun!
		*/

		/* TODO: Turn this into a switch or a table lookup. */

		if ((ch->read_change & EV_CHANGE_ADD) ||
		    (ch->write_change & EV_CHANGE_ADD))
		{
			/* If we are adding anything at all, we'll want to do
			 * either an ADD or a MOD. */
			events = 0;

			// 操作：添加
			op = EPOLL_CTL_ADD;

			/*
			 * 对于读更改
			 */
			// 添加
			if (ch->read_change & EV_CHANGE_ADD)
			{
				events |= EPOLLIN;
			}
			// 删除
			else if (ch->read_change & EV_CHANGE_DEL)
			{
				;
			}
			// 旧的事件包含读
			else if (ch->old_events & EV_READ)
			{
				events |= EPOLLIN;
			}

			/*
			 * 对于写更改
			 */
			// 增加
			if (ch->write_change & EV_CHANGE_ADD)
			{
				events |= EPOLLOUT;
			}
			// 删除
			else if (ch->write_change & EV_CHANGE_DEL)
			{
				;
			}
			// 旧的事件包含写
			else if (ch->old_events & EV_WRITE)
			{
				events |= EPOLLOUT;
			}

			if ((ch->read_change|ch->write_change) & EV_ET)
				events |= EPOLLET;

			/*
			 * 如果包含旧的事件
			 */
			if (ch->old_events)
			{
				/* If MOD fails, we retry as an ADD, and if
				 * ADD fails we will retry as a MOD.  So the
				 * only hard part here is to guess which one
				 * will work.  As a heuristic, we'll try
				 * MOD first if we think there were old
				 * events and ADD if we think there were none.
				 *
				 * We can be wrong about the MOD if the file
				 * has in fact been closed and re-opened.
				 *
				 * We can be wrong about the ADD if the
				 * the fd has been re-created with a dup()
				 * of the same file that it was before.
				 */
				// 那么操作就是修改
				op = EPOLL_CTL_MOD;
			}
		}
		// 删除
		else if ((ch->read_change & EV_CHANGE_DEL) ||
		    (ch->write_change & EV_CHANGE_DEL))
		{
			/* If we're deleting anything, we'll want to do a MOD
			 * or a DEL. */

			// 操作是删除
			op = EPOLL_CTL_DEL;

			/*
			 * 对于读更改
			 */
			if (ch->read_change & EV_CHANGE_DEL)
			{
				if (ch->write_change & EV_CHANGE_DEL)
				{
					events = EPOLLIN|EPOLLOUT;
				}
				else if (ch->old_events & EV_WRITE)
				{

					events = EPOLLOUT;
					op = EPOLL_CTL_MOD;
				}
				else
				{
					events = EPOLLIN;
				}
			}
			/*
			 * 对于写更改
			 */
			else if (ch->write_change & EV_CHANGE_DEL)
			{
				if (ch->old_events & EV_READ)
				{
					events = EPOLLIN;
					op = EPOLL_CTL_MOD;
				}
				else
				{
					events = EPOLLOUT;
				}
			}
		}

		if (!events)
			return 0;

		// 初始化epoll事件
		memset(&epev, 0, sizeof(epev));
		epev.data.fd = ch->fd;
		epev.events = events;

		// 使用epoll_ctl进行变更操作
		if (epoll_ctl(epollop->epfd, op, ch->fd, &epev) == -1)
		{
			if (op == EPOLL_CTL_MOD && errno == ENOENT)
			{
				/* If a MOD operation fails with ENOENT, the
				 * fd was probably closed and re-opened.  We
				 * should retry the operation as an ADD.
				 */
				if (epoll_ctl(epollop->epfd, EPOLL_CTL_ADD, ch->fd, &epev) == -1)
				{
					event_warn("Epoll MOD(%d) on %d retried as ADD; that failed too",
					    (int)epev.events, ch->fd);
					return -1;
				}
				else
				{
					event_debug(("Epoll MOD(%d) on %d retried as ADD; succeeded.",
						(int)epev.events,
						ch->fd));
				}
			}
			else if (op == EPOLL_CTL_ADD && errno == EEXIST)
			{
				/* If an ADD operation fails with EEXIST,
				 * either the operation was redundant (as with a
				 * precautionary add), or we ran into a fun
				 * kernel bug where using dup*() to duplicate the
				 * same file into the same fd gives you the same epitem
				 * rather than a fresh one.  For the second case,
				 * we must retry with MOD. */
				if (epoll_ctl(epollop->epfd, EPOLL_CTL_MOD, ch->fd, &epev) == -1)
				{
					event_warn("Epoll ADD(%d) on %d retried as MOD; that failed too",
					    (int)epev.events, ch->fd);
					return -1;
				}
				else
				{
					event_debug(("Epoll ADD(%d) on %d retried as MOD; succeeded.",
						(int)epev.events,
						ch->fd));
				}
			}
			else if (op == EPOLL_CTL_DEL &&
			    (errno == ENOENT || errno == EBADF ||
				errno == EPERM))
			{
				/* If a delete fails with one of these errors,
				 * that's fine too: we closed the fd before we
				 * got around to calling epoll_dispatch. */
				event_debug(("Epoll DEL(%d) on fd %d gave %s: DEL was unnecessary.",
					(int)epev.events,
					ch->fd,
					strerror(errno)));
			}
			else
			{
				event_warn("Epoll %s(%d) on fd %d failed.  Old events were %d; read change was %d (%s); write change was %d (%s)",
				    epoll_op_to_string(op),
				    (int)epev.events,
				    ch->fd,
				    ch->old_events,
				    ch->read_change,
				    change_to_string(ch->read_change),
				    ch->write_change,
				    change_to_string(ch->write_change));
				return -1;
			}
		}
		else
		{
			event_debug(("Epoll %s(%d) on fd %d okay. [old events were %d; read change was %d; write change was %d]",
				epoll_op_to_string(op),
				(int)epev.events,
				(int)ch->fd,
				ch->old_events,
				ch->read_change,
				ch->write_change));
		}
	}
	return 0;
}

事件等待和分发：

// 事件等待和分发
static int
epoll_dispatch(struct event_base *base, struct timeval *tv)
{
	// epoll的操作集合
	struct epollop *epollop = base->evbase;
	
	// epoll的事件
	struct epoll_event *events = epollop->events;
	
	int i, res;
	long timeout = -1;

	if (tv != NULL) 
	{
		timeout = evutil_tv_to_msec(tv);
		if (timeout < 0 || timeout > MAX_EPOLL_TIMEOUT_MSEC) 
		{
			/* Linux kernels can wait forever if the timeout is
			 * too big; see comment on MAX_EPOLL_TIMEOUT_MSEC. */
			timeout = MAX_EPOLL_TIMEOUT_MSEC;
		}
	}

	// 暂时不用管下面两个函数的调用
	epoll_apply_changes(base);
	event_changelist_remove_all(&base->changelist, base);

	EVBASE_RELEASE_LOCK(base, th_base_lock);

	// 等待事件发生，res存放事件发生的个数
	res = epoll_wait(epollop->epfd, events, epollop->nevents, timeout);

	EVBASE_ACQUIRE_LOCK(base, th_base_lock);

	// 出错
	if (res == -1) 
	{
		if (errno != EINTR) 
		{
			event_warn("epoll_wait");
			return (-1);
		}

		return (0);
	}

	event_debug(("%s: epoll_wait reports %d", __func__, res));
	EVUTIL_ASSERT(res <= epollop->nevents);

	// 遍历所有触发的事件
	for (i = 0; i < res; i++)
	{
		int what = events[i].events;
		short ev = 0;

		if (what & (EPOLLHUP|EPOLLERR))
		{
			// 有错误发生
			ev = EV_READ | EV_WRITE;
		}
		else
		{
			// 可读
			if (what & EPOLLIN)
				ev |= EV_READ;

			// 可写
			if (what & EPOLLOUT)
				ev |= EV_WRITE;
		}

		if (!ev)
			continue;

		// 激活io事件映射表中的事件处理器
		evmap_io_active(base, events[i].data.fd, ev | EV_ET);
	}

	// 如果所有的事件都被触发了，表示事件数组的大小还是太小了，需要扩展事件数组的大小
	if (res == epollop->nevents && epollop->nevents < MAX_NEVENT) 
	{
		/* We used all of the event space this time.  We should
		   be ready for more events next time. */
		int new_nevents = epollop->nevents * 2;
		struct epoll_event *new_events;

		new_events = mm_realloc(epollop->events, new_nevents * sizeof(struct epoll_event));
		
		if (new_events) 
		{
			epollop->events = new_events;
			epollop->nevents = new_nevents;
		}
	}

	return (0);
}