vrrp 线程与 main() 主线程一致,首先会在 start_vrrp_child() 创建一个 master,然后在 launch_thread_scheduler() 中对 epoll 事件、双向链表进行循环监控。
此外,vrrp 线程首先会将所有接收 socket,如 ipv4/ipv6 组播地址注册到 master->epoll_fd,回调函数:vrrp_read_dispatcher_thread(),参数:sock:
/* Register VRRP workers threads */
LIST_FOREACH(l, sock, e) {
if (sock->fd_in != -1)
/* 调用 epoll_ctl 将套接字 sock->fd_in 注册到 master->epoll_fd */
sock->thread = thread_add_read_sands(master, vrrp_read_dispatcher_thread,
sock, sock->fd_in, vrrp_compute_timer(sock), false);
}
创建对应的 thread,并添加到 master->read rb tree 中:
thread_ref_t
thread_add_read_sands(thread_master_t *m, thread_func_t func, void *arg, int fd, const timeval_t *sands, bool close_on_reload)
{
...
...
...
thread = thread_new(m);
thread->type = THREAD_READ;
thread->master = m;
thread->func = func;
thread->arg = arg;
thread->u.f.fd = fd;
thread->u.f.close_on_reload = close_on_reload;
thread->event = event;
/* Set & flag event */
__set_bit(THREAD_FL_READ_BIT, &event->flags);
event->read = thread;
if (!__test_bit(THREAD_FL_EPOLL_READ_BIT, &event->flags)) {
/* 调用 epoll_ctl 将套接字 fd 注册到 master->epoll_fd */
if (thread_event_set(thread) < 0) {
log_message(LOG_INFO, "scheduler: Cant register read event for fd [%d](%m)", fd);
thread_add_unuse(m, thread);
return NULL;
}
__set_bit(THREAD_FL_EPOLL_READ_BIT, &event->flags);
}
/* 超时时间 */
thread->sands = *sands;
/* 将 thread 放到 m->read rb tree 中 */
rb_insert_sort_cached(&m->read, thread, n, thread_timer_cmp);
...
}
然后,vrrp 线程会通过循环调用 launch_thread_scheduler(),对 epoll 事件、双向链表进行监控。
其中,定时器的处理逻辑如下 :
1、epoll_wait() 捕获到定时器超期后,首先会将其移动到双向链表 ready 中,事件类型:THREAD_READY_READ_FD;
/* READ */
if (ep_ev->events & EPOLLIN) {
if (!ev->read) {
log_message(LOG_INFO, "scheduler: No read thread bound on fd:%d (fl:0x%.4X)"
, ev->fd, ep_ev->events);
continue;
}
thread_move_ready(m, &m->read, ev->read, THREAD_READY_READ_FD);
ev->read = NULL;
}
2、process_threads() 从双向链表 ready 中取出节点,并调用定时器回调函数:thread_timerfd_handler()。
if (!shutting_down ||
((thread->type == THREAD_READY_READ_FD ||
thread->type == THREAD_READY_WRITE_FD) &&
(thread->u.f.fd == m->timer_fd ||
thread->u.f.fd == m->signal_fd
#ifdef _WITH_SNMP_
|| FD_ISSET(thread->u.f.fd, &m->snmp_fdset)
#endif
)) ||
thread->type == THREAD_CHILD ||
thread->type == THREAD_CHILD_TIMEOUT ||
thread->type == THREAD_CHILD_TERMINATED ||
thread->type == THREAD_TIMER_SHUTDOWN ||
thread->type == THREAD_TERMINATE) {
if (thread->func)
thread_call(thread); /* 执行回调函数 */
3、回调函数 thread_timerfd_handler() 中,遍历 master 中指定的 rb tree,在双向链表 ready 中创建超时任务;
/* 创建超时任务,存入 reay 双向链表 */
thread_rb_move_ready(m, &m->read, THREAD_READ_TIMEOUT);
thread_rb_move_ready(m, &m->write, THREAD_WRITE_TIMEOUT);
thread_rb_move_ready(m, &m->timer, THREAD_READY);
thread_rb_move_ready(m, &m->child, THREAD_CHILD_TIMEOUT);
遍历 rb tree,将超时的 thread 添加到双向链表 ready,事件类型:THREAD_XXX:
/* Move ready thread into ready queue */
static void
thread_rb_move_ready(thread_master_t *m, rb_root_cached_t *root, int type)
{
thread_t *thread, *thread_tmp;
/* 遍历 rb tree */
rb_for_each_entry_safe_cached(thread, thread_tmp, root, n) {
/* 判断 thread 是否超时 */
if (thread->sands.tv_sec == TIMER_DISABLED || timercmp(&time_now, &thread->sands, <))
break;
if (type == THREAD_READ_TIMEOUT)
thread->event->read = NULL;
else if (type == THREAD_WRITE_TIMEOUT)
thread->event->write = NULL;
thread_move_ready(m, root, thread, type);
}
}
4、process_threads() 从双向链表 ready 中取出节点,并调用 thread 回调函数。对于 socket,回调函数为 vrrp_read_dispatcher_thread()。
若事件类型为 THREAD_READ_TIMEOUT:读超时,则调用超时处理函数vrrp_dispatcher_read_timeout():
/* Our read packet dispatcher */
static int
vrrp_read_dispatcher_thread(thread_ref_t thread)
{
sock_t *sock;
int fd;
/* Fetch thread arg */
sock = THREAD_ARG(thread);
/* Dispatcher state handler */
if (thread->type == THREAD_READ_TIMEOUT || sock->fd_in == -1)
fd = vrrp_dispatcher_read_timeout(sock);
else
fd = vrrp_dispatcher_read(sock);/* 接收 vrrp 通告 */
/* register next dispatcher thread */
if (fd != -1)
/* 调用 epoll_ctl 将套接字 sock->fd_in 注册到 master->epoll_fd */
sock->thread = thread_add_read_sands(thread->master, vrrp_read_dispatcher_thread,
sock, fd, vrrp_compute_timer(sock), false);
return 0;
}