redis之ziplist

号外号外，新建Redis交流讨论群：332160890，欢迎加入！！

类型介绍

ziplist和zipmap一样，同样也是为了节省内存而设计的，这个还是一个双向的链表设计；它能够保存字符串和整型数据，这里面的整型数据是真实的整型而不是一系列的字符串，对于列表的基本操作无非是push和pop，这两个操作的时间复杂度都是0(1)，但是这里面的链表因为是连续内存，因此无论是push和pop都伴随着内存大小的调整，因此真实的复杂度还会跟内存的使用量有关；
ziplist的结构：
zlbytes->zltail->zllen->entry->entry->zlend
zlbytes:ziplist占用的内存大小
zltail:最后一个entry相对于ziplist的首地址的偏移量
zllen:ziplist中entry的数目
entry:数据存储的一个item,是一个struct结构
zlend:结尾标记，也是0XFF
编码类型：
00pppppp -> 1 byte->用来表示小于等于63个字节的字符串
01ppppppqqqqqqqq->2 byte->用来表示小于等于16383个字节的字符串
10______ppppppppqqqqqqqqrrrrrrrrssssssss->5 byte->用来表示大于等于16383个字节的字符串
11000000->1 byte->用来表示后面是一个16位的整型数据
11010000->1 byte->用来表示后面是一个32位的整型数据
11100000->1 byte->用来表示后面是一个64位的整型数据
11110000->1 byte->用来表示后面是一个24位的整型数据（真是为了节省内存不折手段，哈哈）
11111110->1 byte->用来表示后面是一个8位的整型数据
1111xxxx->1 byte->这里面是xxxx来描述一个数字(也是为了内存使用拼了，一个字节也不舍得给部分小数字来用)

代码分析

自定义类型

typedef struct zlentry {
    unsigned int prevrawlensize, prevrawlen;
    unsigned int lensize, len;
    unsigned int headersize;
    unsigned char encoding;
    unsigned char *p;
} zlentry;

变量介绍

函数介绍

zipIntSize，根据当前编码类型，返回该整型的字节数；

static unsigned int zipIntSize(unsigned char encoding) {
    switch(encoding) {
    case ZIP_INT_8B:  return 1;  //8bit => 11111110
    case ZIP_INT_16B: return 2;  //16bit => 11000000
    case ZIP_INT_24B: return 3;  //24bit => 11110000
    case ZIP_INT_32B: return 4;  //32bit => 11010000
    case ZIP_INT_64B: return 8;  //64bit => 11100000
    default: return 0; /* 4 bit immediate */ //这个是指那个4位的整型数据，不占用新的内存 4bit => 1111xxxx
    }
    assert(NULL);
    return 0;
}

zipEncodeLength，根据类型将rawlen保存到指针p指向的内存空间，如果p为空，返回该rawlen存储所需要的字节数；

static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, unsigned int rawlen) {
    unsigned char len = 1, buf[5];

    if (ZIP_IS_STR(encoding)) {  //判断是否是字符串编码
        /* Although encoding is given it may not be set for strings,
         * so we determine it here using the raw length. */
        if (rawlen <= 0x3f) {  //长度小于63个字节
            if (!p) return len;  //一个字节保存该rawlen，00pppppp
            buf[0] = ZIP_STR_06B | rawlen;
        } else if (rawlen <= 0x3fff) {  //小于等于16383个字节
            len += 1; //两个字节保存，01ppppppqqqqqqqq
            if (!p) return len;
            buf[0] = ZIP_STR_14B | ((rawlen >> 8) & 0x3f);  //保存高位
            buf[1] = rawlen & 0xff;   //保存低位
        } else { //大于16383
            len += 4; //5个字节
            if (!p) return len;
            buf[0] = ZIP_STR_32B;
            buf[1] = (rawlen >> 24) & 0xff;
            buf[2] = (rawlen >> 16) & 0xff;
            buf[3] = (rawlen >> 8) & 0xff;
            buf[4] = rawlen & 0xff;
        }
    } else {  //数字，只会占据一个，用于保存编码类型
        /* Implies integer encoding, so length is always 1. */
        if (!p) return len;
        buf[0] = encoding;
    }

    /* Store this length at p */
    memcpy(p,buf,len);
    return len;
}

zipPrevEncodeLength，将len编码到指针p指向的空间，如果p为空，则返回编码所需的字节数；

static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) {
    if (p == NULL) { //如果指针为空，只返回需要的数目
        return (len < ZIP_BIGLEN) ? 1 : sizeof(len)+1;
    } else {
        if (len < ZIP_BIGLEN) {
            p[0] = len;
            return 1;
        } else {
            p[0] = ZIP_BIGLEN;
            memcpy(p+1,&len,sizeof(len));
            memrev32ifbe(p+1); //这是当长度信息超过最大限制时，需要编码
            return 1+sizeof(len);
        }
    }
}

ziplistNew，常见一个ziplist;

unsigned char *ziplistNew(void) {
    unsigned int bytes = ZIPLIST_HEADER_SIZE+1;  //这是一个ziplist的最小字节数，分别是占用字节数+偏移量+entry数+结尾标记
    unsigned char *zl = zmalloc(bytes); //分配内存
    ZIPLIST_BYTES(zl) = intrev32ifbe(bytes); //将占用字节数写入到内存
    ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(ZIPLIST_HEADER_SIZE);  //将结尾偏移量写入到内存
    ZIPLIST_LENGTH(zl) = 0;  //将entry写入到内存
    zl[bytes-1] = ZIP_END;  //写入结尾标记位
    return zl;  //返回ziplist
}

ziplistResize，动态调整ziplist的大小，这个函数里面不需要考虑调整后内存空间的数据，一般是用来填充的，或者删减

static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
    zl = zrealloc(zl,len);  //调整大小，可能重新分配内存空间
    ZIPLIST_BYTES(zl) = intrev32ifbe(len);  //重新设置存储空间
    zl[len-1] = ZIP_END;  //将末尾标志位写入
    return zl;  //返回调整后的ziplist
}

__ziplistCascadeUpdate，

static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
    size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), rawlen, rawlensize;   //当前占用空间
    size_t offset, noffset, extra;
    unsigned char *np;
    zlentry cur, next;

    while (p[0] != ZIP_END) { //还没有到结尾
        zipEntry(p, &cur);  //返回当前指针指向的entry信息
        rawlen = cur.headersize + cur.len;
        rawlensize = zipPrevEncodeLength(NULL,rawlen);

        /* Abort if there is no next entry. */
        if (p[rawlen] == ZIP_END) break;
        zipEntry(p+rawlen, &next);

        /* Abort when "prevlen" has not changed. */
        if (next.prevrawlen == rawlen) break;

        if (next.prevrawlensize < rawlensize) {
            /* The "prevlen" field of "next" needs more bytes to hold
             * the raw length of "cur". */
            offset = p-zl;
            extra = rawlensize-next.prevrawlensize;
            zl = ziplistResize(zl,curlen+extra);
            p = zl+offset;

            /* Current pointer and offset for next element. */
            np = p+rawlen;
            noffset = np-zl;

            /* Update tail offset when next element is not the tail element. */
            if ((zl+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))) != np) {
                ZIPLIST_TAIL_OFFSET(zl) =
                    intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+extra);
            }

            /* Move the tail to the back. */
            memmove(np+rawlensize,
                np+next.prevrawlensize,
                curlen-noffset-next.prevrawlensize-1);
            zipPrevEncodeLength(np,rawlen);

            /* Advance the cursor */
            p += rawlen;
            curlen += extra;
        } else {
            if (next.prevrawlensize > rawlensize) {
                /* This would result in shrinking, which we want to avoid.
                 * So, set "rawlen" in the available bytes. */
                zipPrevEncodeLengthForceLarge(p+rawlen,rawlen);
            } else {
                zipPrevEncodeLength(p+rawlen,rawlen);
            }

            /* Stop here, as the raw length of "next" has not changed. */
            break;
        }
    }
    return zl;
}

__ziplistDelete, 删除从p开始的num个entry;

static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) {
    unsigned int i, totlen, deleted = 0;
    size_t offset;
    int nextdiff = 0;
    zlentry first, tail;

    zipEntry(p, &first); //获取当前p指向的entry
    for (i = 0; p[0] != ZIP_END && i < num; i++) {
        p += zipRawEntryLength(p);  //跳过当前指向entry
        deleted++;  //会被删除的entry计数，因为p后面的entry数目不一定会有num个
    }

    totlen = p-first.p;  //这个是跳过若干个entry后与起始位置p之间的距离，这里面也就是上面说到，也可能一个没跳过
    if (totlen > 0) {  
        if (p[0] != ZIP_END) {  //确实跳过了，且当前p指向的不是链表尾端
            /* Storing `prevrawlen` in this entry may increase or decrease the
             * number of bytes required compare to the current `prevrawlen`.
             * There always is room to store this, because it was previously
             * stored by an entry that is now being deleted. */
            //这时候就需要将first中存储的前一个entry的长度赋给当前p中存储的prevrawlen中，这时候存储的大小可能不太一样，需要变动
            nextdiff = zipPrevLenByteDiff(p,first.prevrawlen);  //这个是比较p指针中存储prevrawlen需要的字节数与当前自己使用的字节数的差异，可能是正数也可能是负数，也可能是0
            p -= nextdiff;  //根据情况，调整出内存空间 
            zipPrevEncodeLength(p,first.prevrawlen);  //将该prevrawlen保存到p指向的内存空间

            /* Update offset for tail */
            ZIPLIST_TAIL_OFFSET(zl) =
                intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))-totlen);  //更新zl中的tail到偏移量，如果当前的entry就是最后一个entry，那么tail的偏移量减去totlen就可以了

            /* When the tail contains more than one entry, we need to take
             * "nextdiff" in account as well. Otherwise, a change in the
             * size of prevlen doesn't have an effect on the *tail* offset. */
            zipEntry(p, &tail);  //获取待删除的最后一个entry
            if (p[tail.headersize+tail.len] != ZIP_END) { //但是如果不是最后一个
                ZIPLIST_TAIL_OFFSET(zl) =
                   intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);  //这时候就需要减去或者加上变化的偏移
            }

            /* Move tail to the front of the ziplist */
            memmove(first.p,p,
                intrev32ifbe(ZIPLIST_BYTES(zl))-(p-zl)-1);  //移动即可，将当前的entry移到first开始
        } else {
            /* The entire tail was deleted. No need to move memory. */
            //如果到达末尾
            ZIPLIST_TAIL_OFFSET(zl) =
                intrev32ifbe((first.p-zl)-first.prevrawlen);  //则tail的偏移量应该是之前的那个entry，所以需要移到前面去
        }

        /* Resize and update length */
        offset = first.p-zl;
        zl = ziplistResize(zl, intrev32ifbe(ZIPLIST_BYTES(zl))-totlen+nextdiff);  //内存操作，需要调整大小
        ZIPLIST_INCR_LENGTH(zl,-deleted);  //减少保存entry数目的字段，deleted才是真实删除的数目
        p = zl+offset;

        /* When nextdiff != 0, the raw length of the next entry has changed, so
         * we need to cascade the update throughout the ziplist */
        //如果entry占用字节数变了，那么该ziplist所有entry内部也就全都变化了，如果没变，内部不用变动，这个函数貌似有点问题，如果说移动了内存，但是貌似被移动的entry中的p没有做更改？？？待确认！！！
        if (nextdiff != 0)
            zl = __ziplistCascadeUpdate(zl,p);
    }
    return zl;
}

__ziplistInsert，在p的位置插入一个item;

static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
    size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen;
    unsigned int prevlensize, prevlen = 0;
    size_t offset;
    int nextdiff = 0;
    unsigned char encoding = 0;
    long long value = 123456789; /* initialized to avoid warning. Using a value
                                    that is easy to see if for some reason
                                    we use it uninitialized. */
    zlentry tail;

    /* Find out prevlen for the entry that is inserted. */
    //待会待插入的entry的前一个entry的长度
    if (p[0] != ZIP_END) {
        //如果p不是链表末尾，获取prevlen
        ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
    } else {
        //是队列末尾的，就需要找到结尾的entry
        unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl);
        if (ptail[0] != ZIP_END) { //如果结尾的entry不是链表末尾的话，获取该entry的长度，否则只能是0了，也就是该插入的entry将会是list的第一个entry
            prevlen = zipRawEntryLength(ptail);
        }
    }

    /* See if the entry can be encoded */
    //看看entry是否可以被编码
    if (zipTryEncoding(s,slen,&value,&encoding)) {
        /* 'encoding' is set to the appropriate integer encoding */
        reqlen = zipIntSize(encoding);  //编码后的长度，注意这个长度只是占用字节信息的长度
    } else {
        /* 'encoding' is untouched, however zipEncodeLength will use the
         * string length to figure out how to encode it. */
        reqlen = slen;  //要求长度
    }
    /* We need space for both the length of the previous entry and
     * the length of the payload. */
    reqlen += zipPrevEncodeLength(NULL,prevlen);  //加上reqlen的长度
    reqlen += zipEncodeLength(NULL,encoding,slen);  //加上当前数据的长度

    /* When the insert position is not equal to the tail, we need to
     * make sure that the next entry can hold this entry's length in
     * its prevlen field. */
    nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0; //如果当前数据的长度与后面entry能够保存的长度不一致，需要修改后面的长度，这个比较只会在不是list末尾的时候进行

    /* Store offset because a realloc may change the address of zl. */
    offset = p-zl; //待插入entry相对于链表头的偏移量
    zl = ziplistResize(zl,curlen+reqlen+nextdiff);
    p = zl+offset;

    /* Apply memory move when necessary and update tail offset. */
    if (p[0] != ZIP_END) {  //不是插到队尾需要移动内存
        /* Subtract one because of the ZIP_END bytes */
        memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);

        /* Encode this entry's raw length in the next entry. */
        zipPrevEncodeLength(p+reqlen,reqlen);

        /* Update offset for tail */
        ZIPLIST_TAIL_OFFSET(zl) =
            intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+reqlen);

        /* When the tail contains more than one entry, we need to take
         * "nextdiff" in account as well. Otherwise, a change in the
         * size of prevlen doesn't have an effect on the *tail* offset. */
        zipEntry(p+reqlen, &tail);
        if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {
            ZIPLIST_TAIL_OFFSET(zl) =
                intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);  //这里面跟上面函数类似，都是要调整末尾偏移量
        }
    } else {
        /* This element will be the new tail. */
        //如果就是插到末尾，那就直接计算偏移量即可，不再需要移动内存
        ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(p-zl);
    }

    /* When nextdiff != 0, the raw length of the next entry has changed, so
     * we need to cascade the update throughout the ziplist */
    if (nextdiff != 0) {
        offset = p-zl;
        zl = __ziplistCascadeUpdate(zl,p+reqlen);
        p = zl+offset;
    }

    /* Write the entry */
    p += zipPrevEncodeLength(p,prevlen);
    p += zipEncodeLength(p,encoding,slen);
    if (ZIP_IS_STR(encoding)) {
        memcpy(p,s,slen);
    } else {
        zipSaveInteger(p,value,encoding);
    } //真正写入entry内部数据 
    ZIPLIST_INCR_LENGTH(zl,1);
    return zl;
}

ziplistPush，push一个新的数据；

unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
    unsigned char *p;
    p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);  //判断是加入前向链表还是反向链表
    return __ziplistInsert(zl,p,s,slen);  //调用insert内部函数
}

ziplistIndex，根据index找到对应的entry地址；

unsigned char *ziplistIndex(unsigned char *zl, int index) {
    unsigned char *p;
    unsigned int prevlensize, prevlen = 0;
    if (index < 0) {  //小于0，表示反向寻找
        index = (-index)-1;
        p = ZIPLIST_ENTRY_TAIL(zl);  //找到最后一个entry
        if (p[0] != ZIP_END) {
            ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
            while (prevlen > 0 && index--) { //第一个entry的prevlen为0，可能存在list的长度小于index的情况
                p -= prevlen;
                ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
            }
        }
    } else {
        p = ZIPLIST_ENTRY_HEAD(zl);
        while (p[0] != ZIP_END && index--) {
            p += zipRawEntryLength(p); //同上，可能存在到达末尾的情况
        }
    }
    //如果到了结尾，或者是该索引比list的长度大，返回NULL
    return (p[0] == ZIP_END || index > 0) ? NULL : p;
}

ziplistNext，返回p之后的那个entry地址

unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
    ((void) zl);

    /* "p" could be equal to ZIP_END, caused by ziplistDelete,
     * and we should return NULL. Otherwise, we should return NULL
     * when the *next* element is ZIP_END (there is no next entry). */
    if (p[0] == ZIP_END) { //如果指向链表尾部，后面没有entry，返回NULL
        return NULL;
    }

    p += zipRawEntryLength(p); //否则根据当前entry的长度，跳过当前entry 
    if (p[0] == ZIP_END) {
        return NULL; //这里也就是说当链表只有这一个entry,返回NULL
    }

    return p;
}

ziplistPrev，返回p之前的那个entry地址；

unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
    unsigned int prevlensize, prevlen = 0;

    /* Iterating backwards from ZIP_END should return the tail. When "p" is
     * equal to the first element of the list, we're already at the head,
     * and should return NULL. */
    if (p[0] == ZIP_END) { //如果当前指向的是队列末尾
        p = ZIPLIST_ENTRY_TAIL(zl); //找到队尾entry 
        return (p[0] == ZIP_END) ? NULL : p; //如果队列为空返回NULL，否则返回队尾entry
    } else if (p == ZIPLIST_ENTRY_HEAD(zl)) { 
        return NULL; //如果当前指向的是链表头，返回NULL，因为没有前面的entry
    } else {
        //这是链表中，只需要获取前面那个entry长度，然后偏移过去就行
        ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
        assert(prevlen > 0);
        return p-prevlen;
    }
}

ziplistGet，获取p指向的entry，结果根据市字符串还是数字存入到对应的指针中；

unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) {
    zlentry entry;
    if (p == NULL || p[0] == ZIP_END) return 0; //如果当前指针为空或者指向的是链表尾部，都直接返回
    if (sstr) *sstr = NULL;  //初始化

    zipEntry(p, &entry);  //获取p指向的entry
    if (ZIP_IS_STR(entry.encoding)) { //判断是不是字符串编码
        if (sstr) {
            *slen = entry.len;
            *sstr = p+entry.headersize; //是的话，记录下字符串长度以及字符串其实地址
        }
    } else {
        if (sval) { //数字的话，获取该数字内容
            *sval =  zipLoadInteger(p+entry.headersize,entry.encoding);
        }
    }
    return 1;  //成功获取返回1
}

ziplistCompare，比较p指向的entry和slen长度的sstr字符串，一样的话返回1；

unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
    zlentry entry;
    unsigned char sencoding;
    long long zval, sval;
    if (p[0] == ZIP_END) return 0; //没有entry直接返回0

    zipEntry(p, &entry); 
    if (ZIP_IS_STR(entry.encoding)) {
        /* Raw compare */
        if (entry.len == slen) {
            return memcmp(p+entry.headersize,sstr,slen) == 0; //如果是字符串的话，比较字符串内容
        } else {
            return 0;
        }
    } else {
        /* Try to compare encoded values. Don't compare encoding because
         * different implementations may encoded integers differently. */
        if (zipTryEncoding(sstr,slen,&sval,&sencoding)) { //首先需要根据编码类型，然后重新再进行编码后比较
          zval = zipLoadInteger(p+entry.headersize,entry.encoding);
          return zval == sval; //如果是数字的话，解析长度信息，直接比较
        }
    }
    return 0;
}

ziplistFind，寻找到指向与当前entry相等的entry指针，skip表示每次比较厚跳过的entry数目；

unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip) {
    int skipcnt = 0;
    unsigned char vencoding = 0;
    long long vll = 0;

    while (p[0] != ZIP_END) { //只要没到链表尾部一直比较
        unsigned int prevlensize, encoding, lensize, len;
        unsigned char *q;

        ZIP_DECODE_PREVLENSIZE(p, prevlensize);
        ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len);
        q = p + prevlensize + lensize;

        if (skipcnt == 0) {
            /* Compare current entry with specified entry */
            if (ZIP_IS_STR(encoding)) {
                if (len == vlen && memcmp(q, vstr, vlen) == 0) {
                    return p;
                }
            } else {
                /* Find out if the searched field can be encoded. Note that
                 * we do it only the first time, once done vencoding is set
                 * to non-zero and vll is set to the integer value. */
                if (vencoding == 0) {
                    if (!zipTryEncoding(vstr, vlen, &vll, &vencoding)) {
                        /* If the entry can't be encoded we set it to
                         * UCHAR_MAX so that we don't retry again the next
                         * time. */
                        vencoding = UCHAR_MAX;
                    }
                    /* Must be non-zero by now */
                    assert(vencoding);
                }

                /* Compare current entry with specified entry, do it only
                 * if vencoding != UCHAR_MAX because if there is no encoding
                 * possible for the field it can't be a valid integer. */
                if (vencoding != UCHAR_MAX) {
                    long long ll = zipLoadInteger(q, encoding);
                    if (ll == vll) {
                        return p;
                    }
                }
            }
            //比较字符串或者数字，找到了直接返回退出
            /* Reset skip count */
            skipcnt = skip; 
        } else { //如果指定了跳跃的数目，需要一直跳跃
            /* Skip entry */
            skipcnt--;
        }

        /* Move to next entry */
        p = q + len;
    }

    return NULL; //没找到返回NULL
}

ziplistLen，返回链表的entry数目；

unsigned int ziplistLen(unsigned char *zl) {
    unsigned int len = 0;  //32位
    if (intrev16ifbe(ZIPLIST_LENGTH(zl)) < UINT16_MAX) {
        len = intrev16ifbe(ZIPLIST_LENGTH(zl));
    } else {
        unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
        while (*p != ZIP_END) {
            p += zipRawEntryLength(p);
            len++;
        }

        /* Re-store length if small enough */
        if (len < UINT16_MAX) ZIPLIST_LENGTH(zl) = intrev16ifbe(len);  //重置长度信息
    }
    return len;
}