keepalived源码解析 —— socket 超时

vrrp 线程与 main() 主线程一致，首先会在 start_vrrp_child() 创建一个 master，然后在 launch_thread_scheduler() 中对 epoll 事件、双向链表进行循环监控。

此外，vrrp 线程首先会将所有接收 socket，如 ipv4/ipv6 组播地址注册到 master->epoll_fd，回调函数：vrrp_read_dispatcher_thread()，参数：sock：

/* Register VRRP workers threads */
LIST_FOREACH(l, sock, e) {
	if (sock->fd_in != -1)
		/* 调用 epoll_ctl 将套接字 sock->fd_in 注册到 master->epoll_fd */
		sock->thread = thread_add_read_sands(master, vrrp_read_dispatcher_thread,
						       sock, sock->fd_in, vrrp_compute_timer(sock), false);
}

创建对应的 thread，并添加到 master->read rb tree 中：

thread_ref_t
thread_add_read_sands(thread_master_t *m, thread_func_t func, void *arg, int fd, const timeval_t *sands, bool close_on_reload)
{
	...
	...
	...
	thread = thread_new(m);
	thread->type = THREAD_READ;
	thread->master = m;
	thread->func = func;
	thread->arg = arg;
	thread->u.f.fd = fd;
	thread->u.f.close_on_reload = close_on_reload;
	thread->event = event;

	/* Set & flag event */
	__set_bit(THREAD_FL_READ_BIT, &event->flags);
	event->read = thread;
	if (!__test_bit(THREAD_FL_EPOLL_READ_BIT, &event->flags)) {
		/* 调用 epoll_ctl 将套接字 fd 注册到 master->epoll_fd */
		if (thread_event_set(thread) < 0) {
			log_message(LOG_INFO, "scheduler: Cant register read event for fd [%d](%m)", fd);
			thread_add_unuse(m, thread);
			return NULL;
		}
		__set_bit(THREAD_FL_EPOLL_READ_BIT, &event->flags);
	}
	
	/* 超时时间 */
	thread->sands = *sands;
	
	/* 将 thread 放到 m->read rb tree 中 */
	rb_insert_sort_cached(&m->read, thread, n, thread_timer_cmp);
	...
}

然后，vrrp 线程会通过循环调用 launch_thread_scheduler()，对 epoll 事件、双向链表进行监控。

其中，定时器的处理逻辑如下 ：

1、epoll_wait() 捕获到定时器超期后，首先会将其移动到双向链表 ready 中，事件类型：THREAD_READY_READ_FD；

/* READ */
if (ep_ev->events & EPOLLIN) {
	if (!ev->read) {
	log_message(LOG_INFO, "scheduler: No read thread bound on fd:%d (fl:0x%.4X)"
						      , ev->fd, ep_ev->events);
	continue;
	}
	thread_move_ready(m, &m->read, ev->read, THREAD_READY_READ_FD);
	ev->read = NULL;
}

2、process_threads() 从双向链表 ready 中取出节点，并调用定时器回调函数：thread_timerfd_handler()。

if (!shutting_down ||
	((thread->type == THREAD_READY_READ_FD ||
	thread->type == THREAD_READY_WRITE_FD) &&
	(thread->u.f.fd == m->timer_fd ||
	thread->u.f.fd == m->signal_fd
#ifdef _WITH_SNMP_
	|| FD_ISSET(thread->u.f.fd, &m->snmp_fdset)
#endif
							       )) ||
	thread->type == THREAD_CHILD ||
	thread->type == THREAD_CHILD_TIMEOUT ||
	thread->type == THREAD_CHILD_TERMINATED ||
	thread->type == THREAD_TIMER_SHUTDOWN ||
	thread->type == THREAD_TERMINATE) {
		if (thread->func)
			thread_call(thread); /* 执行回调函数 */

3、回调函数 thread_timerfd_handler() 中，遍历 master 中指定的 rb tree，在双向链表 ready 中创建超时任务；

/* 创建超时任务，存入 reay 双向链表 */
thread_rb_move_ready(m, &m->read, THREAD_READ_TIMEOUT);
thread_rb_move_ready(m, &m->write, THREAD_WRITE_TIMEOUT);
thread_rb_move_ready(m, &m->timer, THREAD_READY);
thread_rb_move_ready(m, &m->child, THREAD_CHILD_TIMEOUT);

遍历 rb tree，将超时的 thread 添加到双向链表 ready，事件类型：THREAD_XXX：

/* Move ready thread into ready queue */
static void
thread_rb_move_ready(thread_master_t *m, rb_root_cached_t *root, int type)
{
	thread_t *thread, *thread_tmp;
	
	/* 遍历 rb tree */
	rb_for_each_entry_safe_cached(thread, thread_tmp, root, n) {
		/* 判断 thread 是否超时 */
		if (thread->sands.tv_sec == TIMER_DISABLED || timercmp(&time_now, &thread->sands, <))
			break;

		if (type == THREAD_READ_TIMEOUT)
			thread->event->read = NULL;
		else if (type == THREAD_WRITE_TIMEOUT)
			thread->event->write = NULL;
		thread_move_ready(m, root, thread, type);
	}
}

4、process_threads() 从双向链表 ready 中取出节点，并调用 thread 回调函数。对于 socket，回调函数为 vrrp_read_dispatcher_thread()。
若事件类型为 THREAD_READ_TIMEOUT：读超时，则调用超时处理函数vrrp_dispatcher_read_timeout()：

/* Our read packet dispatcher */
static int
vrrp_read_dispatcher_thread(thread_ref_t thread)
{
	sock_t *sock;
	int fd;

	/* Fetch thread arg */
	sock = THREAD_ARG(thread);

	/* Dispatcher state handler */
	if (thread->type == THREAD_READ_TIMEOUT || sock->fd_in == -1)
		fd = vrrp_dispatcher_read_timeout(sock);
	else
		fd = vrrp_dispatcher_read(sock);/* 接收 vrrp 通告 */

	/* register next dispatcher thread */
	if (fd != -1)
		/* 调用 epoll_ctl 将套接字 sock->fd_in 注册到 master->epoll_fd */
		sock->thread = thread_add_read_sands(thread->master, vrrp_read_dispatcher_thread,
						     sock, fd, vrrp_compute_timer(sock), false);

	return 0;
}