memcached的连接处理

一、相关结构

enum conn_states {
    conn_listening,  /**< the socket which listens for connections */
    conn_new_cmd,    /**< Prepare connection for next command */
    conn_waiting,    /**< waiting for a readable socket */
    conn_read,       /**< reading in a command line */
    conn_parse_cmd,  /**< try to parse a command from the input buffer */
    conn_write,      /**< writing out a simple response */
    conn_nread,      /**< reading in a fixed number of bytes */
    conn_swallow,    /**< swallowing unnecessary bytes w/o storing */
    conn_closing,    /**< closing this connection */
    conn_mwrite,     /**< writing out many items sequentially */
    conn_max_state   /**< Max state value (used for assertion) */
};

二、drive_machine()

当client发起tcp连接（telnet 127.0.0.1 11211）时，event_handler()会调用drive_machine()。之后根据struct conn的当前状态去处理。

1、case conn_listening

调用accept()，之后调用dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,  DATA_BUFFER_SIZE, tcp_transport)，其中sfd是刚刚accept的套接字，conn_new_cmd是设置给sfd的状态。与执行流程一节分析一样，dispatch_conn_new()将该sfd交给某个工作子线程管理。

当client在telnet中键入“set foo 0 0 3”后。

2、case conn_new_cmd（注意此时为工作线程处理）

--nreqs（nreqs = settings.reqs_per_event，本例为20，Maximum number of io to process on each io-event）。

如果nreqs<0，c->thread->stats.conn_yields++（放弃的conn数+1）。如果c->rbytes > 0（该连接的输入缓冲区中已经有数据），很可能不会有新数据到来，那么很可能这个事件永远不会再被signal，这个conn可能就真的被永远放弃了；为了防止这种情况，调用update_event()，将该事件的ev_flags由EV_READ更新为EV_WRITE，这样之后这个事件又被signal，这个连接可以稍后得以处理。本次放弃处理struct conn。

如果nreqs>=0，调用reset_cmd_handler(c)。先设置c->substate = bin_no_state，此时c->item应当为null（c->item  is used to hold an item structure createdafter reading the command line ofset/add/replace commands, but before we finished reading the actual data. The data is read into ITEM_data(item) to avoid extra copying.）。接着调用conn_shrink(c)，作用为Shrinks a connection's buffers if they're too big。

        如果c->rbytes > 0，调用conn_set_state(c, conn_parse_cmd)，否则调用conn_set_state(c, conn_waiting)。

本例中此时c->rbytes = 0，进入conn_waiting状态，继续处理。

3、case conn_waiting

update_event(c, EV_READ | EV_PERSIST)后，conn_set_state(c, conn_read)。然后退出本次处理。

进入conn_read状态。

4、case conn_read

        case conn_read:
            res = IS_UDP(c->transport) ? try_read_udp(c) : try_read_network(c);

            switch (res) {
            case READ_NO_DATA_RECEIVED:
                conn_set_state(c, conn_waiting);
                break;
            case READ_DATA_RECEIVED:
                conn_set_state(c, conn_parse_cmd);
                break;
            case READ_ERROR:
                conn_set_state(c, conn_closing);
                break;
            case READ_MEMORY_ERROR: /* Failed to allocate more memory */
                /* State already set by try_read_network */
                break;
            }
            break;

此时已经读入一定的数据，进入conn_parse_cmd状态，继续处理。

5、case conn_parse_cmd

        case conn_parse_cmd :
            if (try_read_command(c) == 0) {
                /* wee need more data! */
                conn_set_state(c, conn_waiting);
            }

            break;

try_read_command(conn *c)设置c->protocol = binary_prot或ascii_prot，此处使用的是ascii_prot（binary_prot暂不讨论）；最后它调用process_command(c, c->rcurr)。

static void process_command(conn *c, char *command) {  //分解命令行，并相应处理

    token_t tokens[MAX_TOKENS];
    size_t ntokens;
    int comm;

    assert(c != NULL);

    MEMCACHED_PROCESS_COMMAND_START(c->sfd, c->rcurr, c->rbytes);

    if (settings.verbose > 1)
        fprintf(stderr, "<%d %s\n", c->sfd, command);

    /*
     * for commands set/add/replace, we build an item and read the data
     * directly into it, then continue in nread_complete().
     */

    c->msgcurr = 0;
    c->msgused = 0;
    c->iovused = 0;
    if (add_msghdr(c) != 0) {      //Adds a message header to a connection.实际上对struct msghdr *msg = c->msglist + c->msgused进行初始化
                                   //（msg->msg_iov = &c->iov[c->iovused];）。对于UDP其中会调用add_iov(),暂不讨论。
        out_string(c, "SERVER_ERROR out of memory preparing response");
        return;
    }

    ntokens = tokenize_command(command, tokens, MAX_TOKENS);  //只是将一整条命令根据空格分割成数组形式，结尾以length = 0作为结束标志
    if (ntokens >= 3 &&
        ((strcmp(tokens[COMMAND_TOKEN].value, "get") == 0) ||
         (strcmp(tokens[COMMAND_TOKEN].value, "bget") == 0))) {

        process_get_command(c, tokens, ntokens, false);

    } else if ((ntokens == 6 || ntokens == 7) &&
               ((strcmp(tokens[COMMAND_TOKEN].value, "add") == 0 && (comm = NREAD_ADD)) ||
                (strcmp(tokens[COMMAND_TOKEN].value, "set") == 0 && (comm = NREAD_SET)) ||
                (strcmp(tokens[COMMAND_TOKEN].value, "replace") == 0 && (comm = NREAD_REPLACE)) ||
                (strcmp(tokens[COMMAND_TOKEN].value, "prepend") == 0 && (comm = NREAD_PREPEND)) ||
                (strcmp(tokens[COMMAND_TOKEN].value, "append") == 0 && (comm = NREAD_APPEND)) )) {

        process_update_command(c, tokens, ntokens, comm, false);

    } else if ((ntokens == 7 || ntokens == 8) && (strcmp(tokens[COMMAND_TOKEN].value, "cas") == 0 && (comm = NREAD_CAS))) {

        process_update_command(c, tokens, ntokens, comm, true);

    } else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "incr") == 0)) {

        process_arithmetic_command(c, tokens, ntokens, 1);

    } else if (ntokens >= 3 && (strcmp(tokens[COMMAND_TOKEN].value, "gets") == 0)) {

        process_get_command(c, tokens, ntokens, true);

    } else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "decr") == 0)) {

        process_arithmetic_command(c, tokens, ntokens, 0);

    } else if (ntokens >= 3 && ntokens <= 5 && (strcmp(tokens[COMMAND_TOKEN].value, "delete") == 0)) {

        process_delete_command(c, tokens, ntokens);

    } else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "touch") == 0)) {

        process_touch_command(c, tokens, ntokens);

    } else if (ntokens >= 2 && (strcmp(tokens[COMMAND_TOKEN].value, "stats") == 0)) {

        process_stat(c, tokens, ntokens);

    } else if (ntokens >= 2 && ntokens <= 4 && (strcmp(tokens[COMMAND_TOKEN].value, "flush_all") == 0)) {
        time_t exptime = 0;

        set_noreply_maybe(c, tokens, ntokens);

        pthread_mutex_lock(&c->thread->stats.mutex);
        c->thread->stats.flush_cmds++;
        pthread_mutex_unlock(&c->thread->stats.mutex);

        if(ntokens == (c->noreply ? 3 : 2)) {
            settings.oldest_live = current_time - 1;
            item_flush_expired();
            out_string(c, "OK");
            return;
        }

        exptime = strtol(tokens[1].value, NULL, 10);
        if(errno == ERANGE) {
            out_string(c, "CLIENT_ERROR bad command line format");
            return;
        }

        /*
          If exptime is zero realtime() would return zero too, and
          realtime(exptime) - 1 would overflow to the max unsigned
          value.  So we process exptime == 0 the same way we do when
          no delay is given at all.
        */
        if (exptime > 0)
            settings.oldest_live = realtime(exptime) - 1;
        else /* exptime == 0 */
            settings.oldest_live = current_time - 1;
        item_flush_expired();
        out_string(c, "OK");
        return;

    } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "version") == 0)) {

        out_string(c, "VERSION " VERSION);

    } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "quit") == 0)) {

        conn_set_state(c, conn_closing);

    } else if (ntokens > 1 && strcmp(tokens[COMMAND_TOKEN].value, "slabs") == 0) {
        if (ntokens == 5 && strcmp(tokens[COMMAND_TOKEN + 1].value, "reassign") == 0) {
            int src, dst, rv;

            if (settings.slab_reassign == false) {
                out_string(c, "CLIENT_ERROR slab reassignment disabled");
                return;
            }

            src = strtol(tokens[2].value, NULL, 10);
            dst = strtol(tokens[3].value, NULL, 10);

            if (errno == ERANGE) {
                out_string(c, "CLIENT_ERROR bad command line format");
                return;
            }

            rv = slabs_reassign(src, dst);
            switch (rv) {
            case REASSIGN_OK:
                out_string(c, "OK");
                break;
            case REASSIGN_RUNNING:
                out_string(c, "BUSY currently processing reassign request");
                break;
            case REASSIGN_BADCLASS:
                out_string(c, "BADCLASS invalid src or dst class id");
                break;
            case REASSIGN_NOSPARE:
                out_string(c, "NOSPARE source class has no spare pages");
                break;
            case REASSIGN_SRC_DST_SAME:
                out_string(c, "SAME src and dst class are identical");
                break;
            }
            return;
        } else if (ntokens == 4 &&
            (strcmp(tokens[COMMAND_TOKEN + 1].value, "automove") == 0)) {
            process_slabs_automove_command(c, tokens, ntokens);
        } else {
            out_string(c, "ERROR");
        }
    } else if ((ntokens == 3 || ntokens == 4) && (strcmp(tokens[COMMAND_TOKEN].value, "verbosity") == 0)) {
        process_verbosity_command(c, tokens, ntokens);
    } else {
        out_string(c, "ERROR");
    }
    return;
}

本次执行为set命令，故执行process_update_command (c, tokens, ntokens=6, comm=2, handle_cas=false)。其中调用item *it = item_alloc(key, nkey, flags, realtime(exptime), vlen)（数据结构一节讨论）；c->item = it; c->ritem = ITEM_data(it); c->rlbytes = it->nbytes;（即将读入的数据value的长度+2） c->cmd = comm;（此处为set）；conn_set_state(c, conn_nread)进入conn_nread状态。

进入conn_nread，继续处理。由于我们还未敲入value数据，理所当然会退出处理，进入事件循环，在进入事件循环前先update_event(c, EV_READ | EV_PERSIST)。这时我们在client输入“bar”，event_handler()又一次被调用，重新进入conn_nread状态。

6、case conn_nread

当读完c->rlbytes长度的数据后，执行complete_nread(c)。注意，读数据时，数据直接存放在c->ritem= ITEM_data(it)处，这样减少了数据复制次数。complete_nread(c)调用complete_nread_ascii(c)或complete_nread_binary(c)。

static void complete_nread_ascii(conn *c) {
    assert(c != NULL);

    item *it = c->item;
    int comm = c->cmd;
    enum store_item_type ret;

    pthread_mutex_lock(&c->thread->stats.mutex);
    c->thread->stats.slab_stats[it->slabs_clsid].set_cmds++;
    pthread_mutex_unlock(&c->thread->stats.mutex);

    if (strncmp(ITEM_data(it) + it->nbytes - 2, "\r\n", 2) != 0) {
        out_string(c, "CLIENT_ERROR bad data chunk");
    } else {
      ret = store_item(it, comm, c);  //先获得粒度锁，再调用do_store_item(),内部暂不讨论。

      switch (ret) {
      case STORED:
          out_string(c, "STORED");
          break;
      case EXISTS:
          out_string(c, "EXISTS");
          break;
      case NOT_FOUND:
          out_string(c, "NOT_FOUND");
          break;
      case NOT_STORED:
          out_string(c, "NOT_STORED");
          break;
      default:
          out_string(c, "SERVER_ERROR Unhandled storage type.");
      }

    }

    item_remove(c->item);       /* release the c->item reference 应该是在store_item()中，增加了reference，此处减小reference使得最终reference为1。*/
    c->item = 0;
}

此处对out_string(c, "STORED")作进一步的分析：

static void out_string(conn *c, const char *str) {
    size_t len;

    assert(c != NULL);

    if (c->noreply) {
        if (settings.verbose > 1)
            fprintf(stderr, ">%d NOREPLY %s\n", c->sfd, str);
        c->noreply = false;
        conn_set_state(c, conn_new_cmd);
        return;
    }

    if (settings.verbose > 1)
        fprintf(stderr, ">%d %s\n", c->sfd, str);

    /* Nuke a partial output... */
    c->msgcurr = 0;
    c->msgused = 0;
    c->iovused = 0;
    add_msghdr(c);

    len = strlen(str);
    if ((len + 2) > c->wsize) {
        /* ought to be always enough. just fail for simplicity */
        str = "SERVER_ERROR output line too long";
        len = strlen(str);
    }

    memcpy(c->wbuf, str, len);
    memcpy(c->wbuf + len, "\r\n", 2);
    c->wbytes = len + 2;
    c->wcurr = c->wbuf;

    conn_set_state(c, conn_write);
    c->write_and_go = conn_new_cmd;    //设置写完之后的状态。
    return;
}

之后进入conn_write状态，继续处理。

7、case conn_write

        case conn_write:
            /*
             * We want to write out a simple response. If we haven't already,
             * assemble it into a msgbuf list (this will be a single-entry
             * list for TCP or a two-entry list for UDP).
             */
            if (c->iovused == 0 || (IS_UDP(c->transport) && c->iovused == 1)) {
                if (add_iov(c, c->wcurr, c->wbytes) != 0) {   //实际上主要功能为令c使用的(struct msghdr msg)->msg_iov[m->msg_iovlen].iov_base = (void *)(c->wbytes)。
                    if (settings.verbose > 0)                 //即初始化struct msghdr msg的缓存空间。对于udp，需要考虑MTU的问题，具体暂不讨论。
                        fprintf(stderr, "Couldn't build response\n");
                    conn_set_state(c, conn_closing);
                    break;
                }
            }

只要不出错，conn_write和conn_mwrite总是紧接着执行的。进入conn_mwite，继续处理。

8、case conn_mwirte

如果是UDP，需要进一步处理build_udp_headers(c)。然后transmit(c)。transmit()调用sendmsg(c->sfd, &c->msglist[c->msgcurr], 0)来发送数据，每次发送一个msghdr中的数据，并返回TRANSMIT_INCOMPLETE，从而反复调用transmit(c)。一般情况下，最终返回TRANSMIT_COMPLETE。（其他返回情况暂不考虑）

        对于TRANSMIT_COMPLETE，调用conn_set_state(c, c->write_and_go)，注意前面out_string()中设置c->write_and_go = conn_new_cmd，这样又进入了conn_new_cmd状态，可以继续接收新的指令了。

其他状态：

9、case conn_closing

当运行中出错时，常常会进入此状态。

对于UDP，执行conn_cleanup(c)。如果c->item != NULL，item_remove(c->item)；之后检测c->ileft，c->suffixleft，c->write_and_free，c->sasl_conn；对于UDP，最终设置conn_set_state(c, conn_read)。

对于TCP，执行conn_close(c)。首先调用event_del(&c->event)，close(c->sfd)，令allow_new_conns = true（一般情况下此值一直为true）。然后调用conn_cleanup(c)。最后将struct conn放入freeconns数组，或者conn_free(c)（释放c的各个动态空间）。

10、case conn_swallow

在前面conn_read状态中，曾执行过process_update_command()，该函数调用item_alloc()，若返回值为NULL（item都没分配成功，后面的数据又有何用，只能舍弃掉），则向客户端发送错误报告，并在发送完之后进入conn_swallow状态（c->write_and_go = conn_swallow; c->sbytes = vlen;）

实际上就是从客户端读入c->sbytes个字符，并舍弃掉。正常情况下，最终读完c->sbytes个字符，c->sbytes == 0，从而进入conn_new_cmd状态。