Redis源码学习-Master&Slave的命令交互

0. 写在前面

        Version Redis2.2.2

        Redis中可以支持主从结构，本文主要从master和slave的心跳机制出发(PING)，分析redis的命令行交互。

        在Redis中，server为每个连接建立一个redisClient数据对象，来描述对应的连接。其中，redisClient为命令交互设置了缓冲区。querybuf用于存储客户端送过来的命令，buf和reply是用于应答的缓冲。querybuf是在文件事件readQueryFromClient中被填充，每次填充的最大字节数默认为1024B。而应答缓冲区是由addReply()函数填充，并由文件事件sendReplyToClient中发送给客户端。具体数据流如图1所示。MasterPorcess与SlaveProcess进行命令交互。其中，蓝色矩形框代表函数，白色矩形框代表数据，曲线描述数据流，折线描述数据间的从属关系。

图1. Master&Slave交互的数据流(蓝色矩形框代表函数，白色矩形框代表数据，曲线描述数据流，折线描述数据间的从属关系)

1、相关数据结构

typedef struct redisClient {  
  int fd;                 //connect fd  
  ...  
  sds querybuf;           //命令缓冲区,由readQueryFromClient()事件进行填充(sds equals to char*)  
  int argc;               //for command；记录参数个数  
  robj **argv;            //for command；记录命令行参数  
  int reqtype;            //命令解析协议：INLINE or MULTIBULK  
  ...  
  time_t lastinteraction; /* 最近交互时间 */  
  ...  
  list *reply;            //Replay object list  
  /* Response buffer */  
  char buf[REDIS_REPLY_CHUNK_BYTES];  //Reply buffer,由addReply()函数进行填充  
  int bufpos;             //记录buf已填充的长度  
  int sentlen;            //Replay阶段，记录当前buf已发送了多少字节  
} redisClient;  

struct redisServer {  
  ...  
  list *clients;  
  dict *commands;             /* Command table hahs table */  
  ...  
  list *slaves, *monitors;    //Master ： slave链表  
  char neterr[ANET_ERR_LEN];  
  aeEventLoop *el;            //Event list  
  int cronloops;              //ServerCorn 执行次数  
  ...  
  redisClient *master;    //Slave ：记录 master 的连接信息的client  
  int replstate;          //Slave ：当前的状态  
  ...  
};  

struct redisCommand readonlyCommandTable[] = {  
  ...  
  {"sync",syncCommand,1,0,NULL,0,0,0},  
  ...  
  {"ping",pingCommand,1,0,NULL,0,0,0},  
  ...  
}  

2、query的读取和命令的解析

        从图1可以看出，命令交互数据query的读取是在文件事件readQueryFromClient中填充到c->querybuf中。

之后，querybuf由函数processInputBuffer进行命令的解析。命令的解析过程如图2所示。在函数processInputBuffer中，将缓存与querybuf中的所有命令(命令间按\n\r分隔)进行解析。

之后，查询命令hashtabe查找相关命令函数。最后调用相应命令hander执行命令。

图2.querybuf的解析

具体代码分析如下：

void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) {  
    redisClient *c = (redisClient*) privdata;  
    char buf[REDIS_IOBUF_LEN];  
    int nread;  
    REDIS_NOTUSED(el);  
    REDIS_NOTUSED(mask);  
  
    nread = read(fd, buf, REDIS_IOBUF_LEN);  
    ...check...  
    if (nread) {  
        c->querybuf = sdscatlen(c->querybuf,buf,nread);  
        c->lastinteraction = time(NULL);//更新时间戳  
    } else {  
        return;  
    }  
    processInputBuffer(c);//处理client传输过来的数据  
}  
  
void processInputBuffer(redisClient *c) {  
    /* 执行querybub中的所有命令*/  
    while(sdslen(c->querybuf)) {  
        ...check...  
        /*判定命令的解析协议 */  
        if (!c->reqtype) {  
            if (c->querybuf[0] == '*') {  
                c->reqtype = REDIS_REQ_MULTIBULK;  
            } else {  
                c->reqtype = REDIS_REQ_INLINE;//按行解析  
            }  
        }  
  
        if (c->reqtype == REDIS_REQ_INLINE) {  
            /*processInlineBuffer:  
            1. 取出c->querybuf起始端到\r\n位置的字符串，更新c->querybuf 
            2. 将取出的字符串按照“ ”空格进行分段解析，得到命令及其参数 
            格式为： argc，*argv[],其中argv[0]为命令，argv[1~argc-1]为参数*/  
            if (processInlineBuffer(c) != REDIS_OK) break;  
        } else if (c->reqtype == REDIS_REQ_MULTIBULK) {  
            ...  
        }  
  
        /* Multibulk processing could see a <= 0 length. */  
        if (c->argc == 0) {  
            resetClient(c);  
        } else {  
            /* Only reset the client when the command was executed. */  
            if (processCommand(c) == REDIS_OK)  //执行命令  
                resetClient(c);  
        }  
    }  
}  
  
/* If this function gets called we already read a whole 
 * command, argments are in the client argv/argc fields. 
 * processCommand() execute the command or prepare the 
 * server for a bulk read from the client. 
 */  
int processCommand(redisClient *c) {  
    struct redisCommand *cmd;  
  
    ...  
  
    /* Now lookup the command and check ASAP about trivial error conditions 
     * such wrong arity, bad command name and so forth. */  
    cmd = lookupCommand(c->argv[0]->ptr);  
      
    ...check...  
  
    /* Exec the command */  
    if (c->flags & REDIS_MULTI &&  
        cmd->proc != execCommand && cmd->proc != discardCommand &&  
        cmd->proc != multiCommand && cmd->proc != watchCommand)  
    {  
        queueMultiCommand(c,cmd);  
        addReply(c,shared.queued);  
    } else {  
        if (server.vm_enabled && server.vm_max_threads > 0 &&  
            blockClientOnSwappedKeys(c,cmd))   
            return REDIS_ERR;  
        call(c,cmd);    //执行命令  
    }  
    return REDIS_OK;  
}  
  
/* Call() is the core of Redis execution of a command */  
void call(redisClient *c, struct redisCommand *cmd) {  
    long long dirty;  
  
    dirty = server.dirty;  
    cmd->proc(c);        //执行命令  
    dirty = server.dirty-dirty;  
  
    if (server.appendonly && dirty)  
        feedAppendOnlyFile(cmd,c->db->id,c->argv,c->argc);  
    if ((dirty || cmd->flags & REDIS_CMD_FORCE_REPLICATION) &&  
        listLength(server.slaves))  
        replicationFeedSlaves(server.slaves,c->db->id,c->argv,c->argc);  
    if (listLength(server.monitors))  
        replicationFeedMonitors(server.monitors,c->db->id,c->argv,c->argc);  
    server.stat_numcommands++;  
}  

 

3、具体命令的执行（ping命令）

  addReply将相关命令执行结果放入client的reply缓冲区中。reply缓冲区的发送时机是在事件sendReplyToClient中进行。

#define REDIS_STRING 0  
shared.pong = createObject(REDIS_STRING,sdsnew("+PONG\r\n"));  
//{"ping",pingCommand,1,0,NULL,0,0,0}  
void pingCommand(redisClient *c) {  
    addReply(c,shared.pong); //ping的回复是pong，打乒乓，呵呵  
}  
  
//将命令执行的返回结构写入c->buf 或者 c->reply  
void addReply(redisClient *c, robj *obj) {  
    if (_installWriteEvent(c) != REDIS_OK) return;//创建event sendReplyToClient  
    redisAssert(!server.vm_enabled || obj->storage == REDIS_VM_MEMORY);  
  
    /* This is an important place where we can avoid copy-on-write 
     * when there is a saving child running, avoiding touching the 
     * refcount field of the object if it's not needed. 
     * 
     * If the encoding is RAW and there is room in the static buffer 
     * we'll be able to send the object to the client without 
     * messing with its page. */  
    if (obj->encoding == REDIS_ENCODING_RAW) {  
        if (_addReplyToBuffer(c,obj->ptr,sdslen(obj->ptr)) != REDIS_OK)  
            _addReplyObjectToList(c,obj);  
    } else {  
        /* FIXME: convert the long into string and use _addReplyToBuffer() 
         * instead of calling getDecodedObject. As this place in the 
         * code is too performance critical. */  
        obj = getDecodedObject(obj);  
        if (_addReplyToBuffer(c,obj->ptr,sdslen(obj->ptr)) != REDIS_OK)  
            _addReplyObjectToList(c,obj);  
        decrRefCount(obj);  
    }  
}  

4、 reply缓冲区数据的发送

将c->buf 和 c->reply中的数据发送到客户端(slave or master)。

在每次文件事件中发送所有的reply缓冲区中的数据。

void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) {  
    redisClient *c = privdata;  
    int nwritten = 0, totwritten = 0, objlen;  
    robj *o;  
    REDIS_NOTUSED(el);  
    REDIS_NOTUSED(mask);  
  
    while(c->bufpos > 0 || listLength(c->reply)) {  
        if (c->bufpos > 0) {  
            //发送c->buf中的数据  
            if (c->flags & REDIS_MASTER) {  
                /* Don't reply to a master */  
                nwritten = c->bufpos - c->sentlen;  
            } else {  
                nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen);  
                if (nwritten <= 0) break;  
            }  
            c->sentlen += nwritten;  
            totwritten += nwritten;  
  
            /* If the buffer was sent, set bufpos to zero to continue with 
             * the remainder of the reply. */  
            if (c->sentlen == c->bufpos) {  
                c->bufpos = 0;  
                c->sentlen = 0;  
            }  
        } else {  
            //发送c->reply中的数据  
            o = listNodeValue(listFirst(c->reply));  
            objlen = sdslen(o->ptr);  
  
            if (objlen == 0) {  
                listDelNode(c->reply,listFirst(c->reply));  
                continue;  
            }  
  
            if (c->flags & REDIS_MASTER) {  
                /* Don't reply to a master */  
                nwritten = objlen - c->sentlen;  
            } else {  
                nwritten = write(fd, ((char*)o->ptr)+c->sentlen,objlen-c->sentlen);  
                if (nwritten <= 0) break;  
            }  
            c->sentlen += nwritten;  
            totwritten += nwritten;  
  
            /* If we fully sent the object on head go to the next one */  
            if (c->sentlen == objlen) {  
                listDelNode(c->reply,listFirst(c->reply));  
                c->sentlen = 0;  
            }  
        }  
        /* Note that we avoid to send more thank REDIS_MAX_WRITE_PER_EVENT 
         * bytes, in a single threaded server it's a good idea to serve 
         * other clients as well, even if a very large request comes from 
         * super fast link that is always able to accept data (in real world 
         * scenario think about 'KEYS *' against the loopback interfae) */  
        if (totwritten > REDIS_MAX_WRITE_PER_EVENT) break;  
    }  
    ...check...  
    if (totwritten > 0) c->lastinteraction = time(NULL);  
    /*reply数据全部发送完毕后，要关闭该文件的写事件。 
    该事件重新开启的时机为addReply()>>_installWriteEvent(c)(见3小节)*/  
    if (listLength(c->reply) == 0) {  
        c->sentlen = 0;  
        aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE);  
  
        /* Close connection after entire reply has been sent. */  
        if (c->flags & REDIS_CLOSE_AFTER_REPLY) freeClient(c);  
    }  
}  

5、总结

命令行交互过程中：

1）为每个连接有相应的数据进行描述（redisClient），这样便于连接的管理。

2）命令行交互中，引入命令缓冲区querybuf，这样可以延时处理命令，这在事件轮询机制中，是至关重要的。