nginx事件模块执行过程

1 基本数据结构

struct ngx_event_s {
    void            *data;
    unsigned         write:1;
    unsigned         accept:1;
    /* used to detect the stale events in kqueue and epoll */
    unsigned         instance:1;
    /*
     * the event was passed or would be passed to a kernel;
     * in aio mode - operation was posted.
     */
    unsigned         active:1;
    unsigned         disabled:1;
    /* the ready event; in aio mode 0 means that no operation can be posted */
    unsigned         ready:1;
    unsigned         oneshot:1;
    /* aio operation is complete */
    unsigned         complete:1;
    unsigned         eof:1;
    unsigned         error:1;
    unsigned         timedout:1;
    unsigned         timer_set:1;
    unsigned         delayed:1;
    unsigned         deferred_accept:1;
    /* the pending eof reported by kqueue, epoll or in aio chain operation */
    unsigned         pending_eof:1;
    unsigned         posted:1;
    unsigned         closed:1;
    /* to test on worker exit */
    unsigned         channel:1;
    unsigned         resolver:1;
    unsigned         cancelable:1;
#if (NGX_HAVE_KQUEUE)
    unsigned         kq_vnode:1;
    /* the pending errno reported by kqueue */
    int              kq_errno;
#endif
    /*
     * kqueue only:
     *   accept:     number of sockets that wait to be accepted
     *   read:       bytes to read when event is ready
     *               or lowat when event is set with NGX_LOWAT_EVENT flag
     *   write:      available space in buffer when event is ready
     *               or lowat when event is set with NGX_LOWAT_EVENT flag
     *
     * epoll with EPOLLRDHUP:
     *   accept:     1 if accept many, 0 otherwise
     *   read:       1 if there can be data to read, 0 otherwise
     *
     * iocp: TODO
     *
     * otherwise:
     *   accept:     1 if accept many, 0 otherwise
     */
#if (NGX_HAVE_KQUEUE) || (NGX_HAVE_IOCP)
    int              available;
#else
    unsigned         available:1;
#endif
    ngx_event_handler_pt  handler;
#if (NGX_HAVE_IOCP)
    ngx_event_ovlp_t ovlp;
#endif
    ngx_uint_t       index;
    ngx_log_t       *log;
    ngx_rbtree_node_t   timer;
    /* the posted queue */
    ngx_queue_t      queue;
#if 0
    /* the threads support */
    /*
     * the event thread context, we store it here
     * if $(CC) does not understand __thread declaration
     * and pthread_getspecific() is too costly
     */
    void            *thr_ctx;
#if (NGX_EVENT_T_PADDING)
    /* event should not cross cache line in SMP */
    uint32_t         padding[NGX_EVENT_T_PADDING];
#endif
#endif
};

struct ngx_listening_s {
    ngx_socket_t        fd;
    struct sockaddr    *sockaddr;
    socklen_t           socklen;    /* size of sockaddr */
    size_t              addr_text_max_len;
    ngx_str_t           addr_text;
    int                 type;
    int                 backlog;
    int                 rcvbuf;
    int                 sndbuf;
#if (NGX_HAVE_KEEPALIVE_TUNABLE)
    int                 keepidle;
    int                 keepintvl;
    int                 keepcnt;
#endif
    /* handler of accepted connection */
    ngx_connection_handler_pt   handler;
    void               *servers;  /* array of ngx_http_in_addr_t, for example */
    ngx_log_t           log;
    ngx_log_t          *logp;
    size_t              pool_size;
    /* should be here because of the AcceptEx() preread */
    size_t              post_accept_buffer_size;
    /* should be here because of the deferred accept */
    ngx_msec_t          post_accept_timeout;
    ngx_listening_t    *previous;
    ngx_connection_t   *connection;
    ngx_uint_t          worker;
    unsigned            open:1;
    unsigned            remain:1;
    unsigned            ignore:1;
    unsigned            bound:1;       /* already bound */
    unsigned            inherited:1;   /* inherited from previous process */
    unsigned            nonblocking_accept:1;
    unsigned            listen:1;
    unsigned            nonblocking:1;
    unsigned            shared:1;    /* shared between threads or processes */
    unsigned            addr_ntop:1;
    unsigned            wildcard:1;
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
    unsigned            ipv6only:1;
#endif
#if (NGX_HAVE_REUSEPORT)
    unsigned            reuseport:1;
    unsigned            add_reuseport:1;
#endif
    unsigned            keepalive:2;
#if (NGX_HAVE_DEFERRED_ACCEPT)
    unsigned            deferred_accept:1;
    unsigned            delete_deferred:1;
    unsigned            add_deferred:1;
#ifdef SO_ACCEPTFILTER
    char               *accept_filter;
#endif
#endif
#if (NGX_HAVE_SETFIB)
    int                 setfib;
#endif
#if (NGX_HAVE_TCP_FASTOPEN)
    int                 fastopen;
#endif

};

struct ngx_connection_s {
    void               *data;
    ngx_event_t        *read;
    ngx_event_t        *write;
    ngx_socket_t        fd;
    ngx_recv_pt         recv;
    ngx_send_pt         send;
    ngx_recv_chain_pt   recv_chain;
    ngx_send_chain_pt   send_chain;
    ngx_listening_t    *listening;
    off_t               sent;
    ngx_log_t          *log;
    ngx_pool_t         *pool;
    int                 type;
    struct sockaddr    *sockaddr;
    socklen_t           socklen;
    ngx_str_t           addr_text;
    ngx_str_t           proxy_protocol_addr;
    in_port_t           proxy_protocol_port;
#if (NGX_SSL)
    ngx_ssl_connection_t  *ssl;
#endif
    struct sockaddr    *local_sockaddr;
    socklen_t           local_socklen;
    ngx_buf_t          *buffer;
    ngx_queue_t         queue;
    ngx_atomic_uint_t   number;
    ngx_uint_t          requests;
    unsigned            buffered:8;
    unsigned            log_error:3;     /* ngx_connection_log_error_e */
    unsigned            timedout:1;
    unsigned            error:1;
    unsigned            destroyed:1;
    unsigned            idle:1;
    unsigned            reusable:1;
    unsigned            close:1;
    unsigned            shared:1;
    unsigned            sendfile:1;
    unsigned            sndlowat:1;
    unsigned            tcp_nodelay:2;   /* ngx_connection_tcp_nodelay_e */
    unsigned            tcp_nopush:2;    /* ngx_connection_tcp_nopush_e */
    unsigned            need_last_buf:1;
#if (NGX_HAVE_AIO_SENDFILE)
    unsigned            busy_count:2;
#endif
#if (NGX_THREADS)
    ngx_thread_task_t  *sendfile_task;
#endif
};

2 ngx事件的执行过程

2.1 nginx初始化套接口过程

在ngx_init_cycle函数中调用函数ngx_open_listening_sockets，完成创建套接口（socket）、绑定套接口（bind）、监听套接口（listen），这些步骤结束后，在cycle的listening.elts所指向的ngx_listening_t结构体指针数组中存储了套接字的描述符信息，有几个监听服务就会在listening.elts中存储几个ngx_listening_t指针类型的元素。

2.2 ngx_single_process_cycle解析

接着进入ngx_single_process_cycle函数，该函数处理事件的主要函数。在介绍该函数之前先看下ngx_event_core_module模块的定义

ngx_module_t  ngx_event_core_module = {
     NGX_MODULE_V1,
     &ngx_event_core_module_ctx,            /* module context */
     ngx_event_core_commands,               /* module directives */
     NGX_EVENT_MODULE,                      /* module type */
     NULL,                                  /* init master */
     ngx_event_module_init,                 /* init module */
     ngx_event_process_init,                /* init process */
     NULL,                                  /* init thread */
     NULL,                                  /* exit thread */
     NULL,                                  /* exit process */
     NULL,                                  /* exit master */
     NGX_MODULE_V1_PADDING
  };
static ngx_command_t  ngx_epoll_commands[] = {     
 { ngx_string("epoll_events"),
        NGX_EVENT_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_num_slot,
       0,
       offsetof(ngx_epoll_conf_t, events),
       NULL },

     { ngx_string("worker_aio_requests"),
       NGX_EVENT_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_num_slot,
       0,
       offsetof(ngx_epoll_conf_t, aio_requests),
       NULL }, 
       ngx_null_command
 };  
 ngx_event_module_t  ngx_epoll_module_ctx = {
     &epoll_name,
     ngx_epoll_create_conf,               /* create configuration */
     ngx_epoll_init_conf,                 /* init configuration */
     {
         ngx_epoll_add_event,             /* add an event */
         ngx_epoll_del_event,             /* delete an  */
         ngx_epoll_add_event,             /* enable an event */
         ngx_epoll_del_event,             /* disable an event */
         ngx_epoll_add_connection,        /* add an connection */
         ngx_epoll_del_connection,        /* delete an connection */
 #if (NGX_HAVE_EVENTFD)
         ngx_epoll_notify,                /* trigger a notify */
 #else
         NULL,                            /* trigger a notify */
 #endif
         ngx_epoll_process_events,        /* process the events */
         ngx_epoll_init,                  /* init the events */
         ngx_epoll_done,                  /* done the events */
     }
 };
 ngx_module_t  ngx_epoll_module = {
     NGX_MODULE_V1,
     &ngx_epoll_module_ctx,               /* module context */
     ngx_epoll_commands,                  /* module directives */
     NGX_EVENT_MODULE,                    /* module type */
     NULL,                                /* init master */
     NULL,                                /* init module */
     NULL,                                /* init process */
     NULL,                                /* init thread */
     NULL,                                /* exit thread */
     NULL,                                /* exit process */
     NULL,                                /* exit master */
     NGX_MODULE_V1_PADDING
 };

在ngx_event_core_module模块定义中有两个回调函数，这两个回调函数就负责初始化套接口

• ngx_event_module_init, /* init module */：在ngx_init_cycle的ngx_init_modules(cycle)函数中调用

• ngx_event_process_init, /* init process */：如果是单进程模式，则在ngx_single_process_cycle(ngx_cycle_t *cycle)函数中调用，如果是多进程模式则在ngx_worker_process_init 中调用。在 ngx_event_process_init中有个重要的功能就是将accept注册到epoll中。

在ngx_single_process_cycle函数的开始位置有一段如下代码，

for (i = 0; cycle->modules[i]; i++) {
        if (cycle->modules[i]->init_process) {
            if (cycle->modules[i]->init_process(cycle) == NGX_ERROR) {
                /* fatal */
                exit(2);
            }
        }
    }

当其执行到ngx_event_core_module模块时就会调用ngx_event_process_init函数，在该函数的732行开始，就会把之前已经绑定的监听套接口添加到epoll中。并将读事件ngx_event_t类型的结构体中的handler回调函数设置为函数ngx_event_accept，该函数将会在epoll中触发监听事件可读时调用，接受请求，并将请求的事件添加到epoll中。

2.3 ngx_process_events_and_timers函数解析

• 在函数中首先调用事件处理函数ngx_epoll_process_events
  在该函数中首先调用epoll_wait获得就绪的事件，此时肯定能获得监听事件（如果有请求的话）在该函数的907行rev->handler(rev);，实际上调用的就是ngx_event_accept函数。
  详细代码如下

for (i = 0; i < events; i++) {
        c = event_list[i].data.ptr;

        instance = (uintptr_t) c & 1;
        c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1);

        rev = c->read;

        if (c->fd == -1 || rev->instance != instance) {

            /*
             * the stale event from a file descriptor
             * that was just closed in this iteration
             */

            ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                           "epoll: stale event %p", c);
            continue;
        }

        revents = event_list[i].events;

        ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                       "epoll: fd:%d ev:%04XD d:%p",
                       c->fd, revents, event_list[i].data.ptr);

        if (revents & (EPOLLERR|EPOLLHUP)) {
            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                           "epoll_wait() error on fd:%d ev:%04XD",
                           c->fd, revents);
        }

#if 0
        if (revents & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) {
            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                          "strange epoll_wait() events fd:%d ev:%04XD",
                          c->fd, revents);
        }
#endif

        if ((revents & (EPOLLERR|EPOLLHUP))
             && (revents & (EPOLLIN|EPOLLOUT)) == 0)
        {
            /*
             * if the error events were returned without EPOLLIN or EPOLLOUT,
             * then add these flags to handle the events at least in one
             * active handler
             */

            revents |= EPOLLIN|EPOLLOUT;
        }

        if ((revents & EPOLLIN) && rev->active) {

#if (NGX_HAVE_EPOLLRDHUP)
            if (revents & EPOLLRDHUP) {
                rev->pending_eof = 1;
            }

            rev->available = 1;
#endif

            rev->ready = 1;

            if (flags & NGX_POST_EVENTS) {
            //如果不是监听事件，而是已经accept或者是已经存在的可读事件则将其加入相应的队列中
                queue = rev->accept ? &ngx_posted_accept_events
                                    : &ngx_posted_events;

                ngx_post_event(rev, queue);

            } else {
            //如果是监听事件则调用ngx_event_accept
                rev->handler(rev);
            }
        }

        wev = c->write;

        if ((revents & EPOLLOUT) && wev->active) {

            if (c->fd == -1 || wev->instance != instance) {

                /*
                 * the stale event from a file descriptor
                 * that was just closed in this iteration
                 */

                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                               "epoll: stale event %p", c);
                continue;
            }

            wev->ready = 1;
#if (NGX_THREADS)
            wev->complete = 1;
#endif

            if (flags & NGX_POST_EVENTS) {
                ngx_post_event(wev, &ngx_posted_events);

            } else {
                wev->handler(wev);
            }
        }
    }

• 处理队列中的事件
  在ngx_epoll_process_events中将accept事件和普通的读写事件分别加入了队列ngx_posted_accept_events，ngx_posted_events中。然后在ngx_process_events_and_timers中分别调用如下代码，处理队列中的事件

ngx_event_process_posted(cycle, &ngx_posted_accept_events);
    if (ngx_accept_mutex_held) {
        ngx_shmtx_unlock(&ngx_accept_mutex);
    }
    if (delta) {
        ngx_event_expire_timers();
    }
    ngx_event_process_posted(cycle, &ngx_posted_events);

说明：
在ngx_epoll_process_events中调用epoll_wait后既可以得到刚建立连接的事件，又可以得到之前已经建立连接的正常读写事件，会把他们放到ngx_posted_accept_events，ngx_posted_events队列中，然后会延迟执行这些队列事件中的回调函数，优先执行刚建立连接的事件，然后执行正常执行的读写事件，这就解决了惊群和负载均衡两个问题。
如果在处理一个事件过程中产生了另一个事件，要求这个新产生的事件稍后执行，就可以把这个新事件加入到post队列中。