memcached源码分析之线程池机制（二）

在上一篇中已分析了memcached线程池的创建流程，由于上篇篇幅较长，因此将memcached线程池中线程的调度流程另立一篇。

先让我们把目光转到主函数中，主线程在调用thread_init函数创建好线程池后，就开始创建监听套接字，memcached支持TCP，UDP，UNIX域套接字，因此相应的要创建三种监听套接字

这里我们只分析TCP listening socket的创建（UDP与TCP的创建采用统一的接口），函数入口为：

1  errno = 0;
2         if (settings.port && server_sockets(settings.port, tcp_transport,
3                                            portnumber_file)) {
4             vperror("failed to listen on TCP port %d", settings.port);
5             exit(EX_OSERR);
6         }

server_sockets函数即为创建TCP listening socket的入口函数。在server_sockets主要调用server_socket函数来实现，

 1 /**
 2  * Create a socket and bind it to a specific port number
 3  * @param interface the interface to bind to
 4  * @param port the port number to bind to
 5  * @param transport the transport protocol (TCP / UDP)
 6  * @param portnumber_file A filepointer to write the port numbers to
 7  *        when they are successfully added to the list of ports we
 8  *        listen on.
 9  */
10 static int server_socket(const char *interface,
11                          int port,
12                          enum network_transport transport,
13                          FILE *portnumber_file)；

server_socket函数实现源码较长，以下只列出部分：

 1 static int server_socket(const char *interface,
 2                          int port,
 3                          enum network_transport transport,
 4                          FILE *portnumber_file) {
 5     int sfd;
 6     struct linger ling = {0, 0};
 7     struct addrinfo *ai;
 8     struct addrinfo *next;
 9     struct addrinfo hints = { .ai_flags = AI_PASSIVE,
10                               .ai_family = AF_UNSPEC };
11     //套接字的创建过程
12     ..................
13     ..................
14         if (IS_UDP(transport)) {
15             int c;
16 
17             for (c = 0; c < settings.num_threads_per_udp; c++) {
18                 /* this is guaranteed to hit all threads because we round-robin */
19                 dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST,
20                                   UDP_READ_BUFFER_SIZE, transport);
21             }
22         } else {
23             if (!(listen_conn_add = conn_new(sfd, conn_listening,
24                                              EV_READ | EV_PERSIST, 1,
25                                              transport, main_base))) {
26                 fprintf(stderr, "failed to create listening connection\n");
27                 exit(EXIT_FAILURE);
28             }
29             listen_conn_add->next = listen_conn;
30             listen_conn = listen_conn_add;
31         }
32     }
33 
34     freeaddrinfo(ai);
35 
36     /* Return zero iff we detected no errors in starting up connections */
37     return success == 0;
38 }

在server_socket中，我们只关注两个函数：

（1）dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST,UDP_READ_BUFFER_SIZE, transport);

当创建的是UDP套接字时，使用这个函数，由于UDP是无连接的，因此直接启动settings.num_threads_per_udp个线程来服务于UDP端口。

（2）listen_conn_add = conn_new(sfd, conn_listening,EV_READ | EV_PERSIST, 1,transport, main_base);

当创建的是TCP套接字时，调用conn_new函数，源码如下：

 1 conn *conn_new(const int sfd, enum conn_states init_state,
 2                 const int event_flags,
 3                 const int read_buffer_size, enum network_transport transport,
 4                 struct event_base *base) {
 5     conn *c = conn_from_freelist();
 6 
 7     ........................//略去部分源码
 8 
 9     event_set(&c->event, sfd, event_flags, event_handler, (void *)c);
10     event_base_set(base, &c->event);
11     c->ev_flags = event_flags;
12 
13     if (event_add(&c->event, 0) == -1) {
14         if (conn_add_to_freelist(c)) {
15             conn_free(c);
16         }
17         perror("event_add");
18         return NULL;
19     }
20 
21     STATS_LOCK();
22     stats.curr_conns++;
23     stats.total_conns++;
24     STATS_UNLOCK();
25 
26     MEMCACHED_CONN_ALLOCATE(c->sfd);
27 
28     return c;
29 }

该函数对套接字设置conn_listening监听事件，回调函数为event_handler，在事件响应函数中调用状态机。

 1 void event_handler(const int fd, const short which, void *arg) {
 2     conn *c;
 3 
 4     c = (conn *)arg;
 5     assert(c != NULL);
 6 
 7     c->which = which;
 8 
 9     /* sanity */
10     if (fd != c->sfd) {
11         if (settings.verbose > 0)
12             fprintf(stderr, "Catastrophic: event fd doesn't match conn fd!\n");
13         conn_close(c);
14         return;
15     }
16 
17     drive_machine(c);
18 
19     /* wait for next event */
20     return;
21 }

memcached中的状态机是memcached运转发动机，它根据链接的不同状态而采取不同的行为，状态枚举如下：

 1 enum conn_states {
 2     conn_listening,  /**< the socket which listens for connections */
 3     conn_new_cmd,    /**< Prepare connection for next command */
 4     conn_waiting,    /**< waiting for a readable socket */
 5     conn_read,       /**< reading in a command line */
 6     conn_parse_cmd,  /**< try to parse a command from the input buffer */
 7     conn_write,      /**< writing out a simple response */
 8     conn_nread,      /**< reading in a fixed number of bytes */
 9     conn_swallow,    /**< swallowing unnecessary bytes w/o storing */
10     conn_closing,    /**< closing this connection */
11     conn_mwrite,     /**< writing out many items sequentially */
12     conn_max_state   /**< Max state value (used for assertion) */
13 };

 状态机的实现函数为drive_machine，由于该函数的源码实现过长，这里只分析对conn_listening状态的响应。

 1 static void drive_machine(conn *c) {
 2     bool stop = false;
 3     int sfd, flags = 1;
 4     socklen_t addrlen;
 5     struct sockaddr_storage addr;
 6     int nreqs = settings.reqs_per_event;
 7     int res;
 8 
 9     assert(c != NULL);
10 
11     while (!stop) {
12 
13         switch(c->state) {
14             //监听套接字发生事件
15         case conn_listening:
16             addrlen = sizeof(addr);
17             if ((sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen)) == -1) {
18                 if (errno == EAGAIN || errno == EWOULDBLOCK) {
19                     /* these are transient, so don't log anything */
20                     stop = true;
21                 } else if (errno == EMFILE) {
22                     if (settings.verbose > 0)
23                         fprintf(stderr, "Too many open connections\n");
24                     accept_new_conns(false);
25                     stop = true;
26                 } else {
27                     perror("accept()");
28                     stop = true;
29                 }
30                 break;
31             }
32             //新套接字设置非阻塞
33             if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 ||
34                 fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) {
35                 perror("setting O_NONBLOCK");
36                 close(sfd);
37                 break;
38             }
39             //调度线程来处理连接
40             dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,
41                                      DATA_BUFFER_SIZE, tcp_transport);
42             stop = true;
43             break;
44 ...........................//略去其他状态的处理
45 }

从源码中我们可以看到，当监听套接字建立新连接时，通过事件响应函数event_handler来触发状态机，再调用dispatch_conn_new调度新线程来处理这个连接的读写事件。

 1 /*
 2  * Dispatches a new connection to another thread. This is only ever called
 3  * from the main thread, either during initialization (for UDP) or because
 4  * of an incoming connection.
 5  */
 6 void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags,
 7                        int read_buffer_size, enum network_transport transport) {
 8     CQ_ITEM *item = cqi_new();
 9     int tid = (last_thread + 1) % settings.num_threads;
10 
11     //以此种方式来取出线程
12     LIBEVENT_THREAD *thread = threads + tid;
13 
14     last_thread = tid;
15 
16     item->sfd = sfd;
17     item->init_state = init_state;
18     item->event_flags = event_flags;
19     item->read_buffer_size = read_buffer_size;
20     item->transport = transport;
21 
22     //将新item放至threads的new_conn_queue队列中
23     cq_push(thread->new_conn_queue, item);
24 
25     MEMCACHED_CONN_DISPATCH(sfd, thread->thread_id);
26     //写一个字节启动新的线程
27     if (write(thread->notify_send_fd, "", 1) != 1) {
28         perror("Writing to thread notify pipe");
29     }
30 }

至此，memcached的线程池调度机制已分析完毕了。