号外号外,新建Redis交流讨论群:332160890,欢迎加入!!
类型介绍
ziplist和zipmap一样,同样也是为了节省内存而设计的,这个还是一个双向的链表设计;它能够保存字符串和整型数据,这里面的整型数据是真实的整型而不是一系列的字符串,对于列表的基本操作无非是push和pop,这两个操作的时间复杂度都是0(1),但是这里面的链表因为是连续内存,因此无论是push和pop都伴随着内存大小的调整,因此真实的复杂度还会跟内存的使用量有关;
ziplist的结构:
zlbytes->zltail->zllen->entry->entry->zlend
zlbytes:ziplist占用的内存大小
zltail:最后一个entry相对于ziplist的首地址的偏移量
zllen:ziplist中entry的数目
entry:数据存储的一个item,是一个struct结构
zlend:结尾标记,也是0XFF
编码类型:
00pppppp -> 1 byte->用来表示小于等于63个字节的字符串
01ppppppqqqqqqqq->2 byte->用来表示小于等于16383个字节的字符串
10______ppppppppqqqqqqqqrrrrrrrrssssssss->5 byte->用来表示大于等于16383个字节的字符串
11000000->1 byte->用来表示后面是一个16位的整型数据
11010000->1 byte->用来表示后面是一个32位的整型数据
11100000->1 byte->用来表示后面是一个64位的整型数据
11110000->1 byte->用来表示后面是一个24位的整型数据(真是为了节省内存不折手段,哈哈)
11111110->1 byte->用来表示后面是一个8位的整型数据
1111xxxx->1 byte->这里面是xxxx来描述一个数字(也是为了内存使用拼了,一个字节也不舍得给部分小数字来用)
代码分析
自定义类型
typedef struct zlentry {
unsigned int prevrawlensize, prevrawlen;
unsigned int lensize, len;
unsigned int headersize;
unsigned char encoding;
unsigned char *p;
} zlentry;
变量介绍
函数介绍
zipIntSize,根据当前编码类型,返回该整型的字节数;
static unsigned int zipIntSize(unsigned char encoding) {
switch(encoding) {
case ZIP_INT_8B: return 1; //8bit => 11111110
case ZIP_INT_16B: return 2; //16bit => 11000000
case ZIP_INT_24B: return 3; //24bit => 11110000
case ZIP_INT_32B: return 4; //32bit => 11010000
case ZIP_INT_64B: return 8; //64bit => 11100000
default: return 0; /* 4 bit immediate */ //这个是指那个4位的整型数据,不占用新的内存 4bit => 1111xxxx
}
assert(NULL);
return 0;
}
zipEncodeLength,根据类型将rawlen保存到指针p指向的内存空间,如果p为空,返回该rawlen存储所需要的字节数;
static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, unsigned int rawlen) {
unsigned char len = 1, buf[5];
if (ZIP_IS_STR(encoding)) { //判断是否是字符串编码
/* Although encoding is given it may not be set for strings,
* so we determine it here using the raw length. */
if (rawlen <= 0x3f) { //长度小于63个字节
if (!p) return len; //一个字节保存该rawlen,00pppppp
buf[0] = ZIP_STR_06B | rawlen;
} else if (rawlen <= 0x3fff) { //小于等于16383个字节
len += 1; //两个字节保存,01ppppppqqqqqqqq
if (!p) return len;
buf[0] = ZIP_STR_14B | ((rawlen >> 8) & 0x3f); //保存高位
buf[1] = rawlen & 0xff; //保存低位
} else { //大于16383
len += 4; //5个字节
if (!p) return len;
buf[0] = ZIP_STR_32B;
buf[1] = (rawlen >> 24) & 0xff;
buf[2] = (rawlen >> 16) & 0xff;
buf[3] = (rawlen >> 8) & 0xff;
buf[4] = rawlen & 0xff;
}
} else { //数字,只会占据一个,用于保存编码类型
/* Implies integer encoding, so length is always 1. */
if (!p) return len;
buf[0] = encoding;
}
/* Store this length at p */
memcpy(p,buf,len);
return len;
}
zipPrevEncodeLength,将len编码到指针p指向的空间,如果p为空,则返回编码所需的字节数;
static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) {
if (p == NULL) { //如果指针为空,只返回需要的数目
return (len < ZIP_BIGLEN) ? 1 : sizeof(len)+1;
} else {
if (len < ZIP_BIGLEN) {
p[0] = len;
return 1;
} else {
p[0] = ZIP_BIGLEN;
memcpy(p+1,&len,sizeof(len));
memrev32ifbe(p+1); //这是当长度信息超过最大限制时,需要编码
return 1+sizeof(len);
}
}
}
ziplistNew,常见一个ziplist;
unsigned char *ziplistNew(void) {
unsigned int bytes = ZIPLIST_HEADER_SIZE+1; //这是一个ziplist的最小字节数,分别是占用字节数+偏移量+entry数+结尾标记
unsigned char *zl = zmalloc(bytes); //分配内存
ZIPLIST_BYTES(zl) = intrev32ifbe(bytes); //将占用字节数写入到内存
ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(ZIPLIST_HEADER_SIZE); //将结尾偏移量写入到内存
ZIPLIST_LENGTH(zl) = 0; //将entry写入到内存
zl[bytes-1] = ZIP_END; //写入结尾标记位
return zl; //返回ziplist
}
ziplistResize,动态调整ziplist的大小,这个函数里面不需要考虑调整后内存空间的数据,一般是用来填充的,或者删减
static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
zl = zrealloc(zl,len); //调整大小,可能重新分配内存空间
ZIPLIST_BYTES(zl) = intrev32ifbe(len); //重新设置存储空间
zl[len-1] = ZIP_END; //将末尾标志位写入
return zl; //返回调整后的ziplist
}
__ziplistCascadeUpdate,
static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), rawlen, rawlensize; //当前占用空间
size_t offset, noffset, extra;
unsigned char *np;
zlentry cur, next;
while (p[0] != ZIP_END) { //还没有到结尾
zipEntry(p, &cur); //返回当前指针指向的entry信息
rawlen = cur.headersize + cur.len;
rawlensize = zipPrevEncodeLength(NULL,rawlen);
/* Abort if there is no next entry. */
if (p[rawlen] == ZIP_END) break;
zipEntry(p+rawlen, &next);
/* Abort when "prevlen" has not changed. */
if (next.prevrawlen == rawlen) break;
if (next.prevrawlensize < rawlensize) {
/* The "prevlen" field of "next" needs more bytes to hold
* the raw length of "cur". */
offset = p-zl;
extra = rawlensize-next.prevrawlensize;
zl = ziplistResize(zl,curlen+extra);
p = zl+offset;
/* Current pointer and offset for next element. */
np = p+rawlen;
noffset = np-zl;
/* Update tail offset when next element is not the tail element. */
if ((zl+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))) != np) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+extra);
}
/* Move the tail to the back. */
memmove(np+rawlensize,
np+next.prevrawlensize,
curlen-noffset-next.prevrawlensize-1);
zipPrevEncodeLength(np,rawlen);
/* Advance the cursor */
p += rawlen;
curlen += extra;
} else {
if (next.prevrawlensize > rawlensize) {
/* This would result in shrinking, which we want to avoid.
* So, set "rawlen" in the available bytes. */
zipPrevEncodeLengthForceLarge(p+rawlen,rawlen);
} else {
zipPrevEncodeLength(p+rawlen,rawlen);
}
/* Stop here, as the raw length of "next" has not changed. */
break;
}
}
return zl;
}
__ziplistDelete, 删除从p开始的num个entry;
static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) {
unsigned int i, totlen, deleted = 0;
size_t offset;
int nextdiff = 0;
zlentry first, tail;
zipEntry(p, &first); //获取当前p指向的entry
for (i = 0; p[0] != ZIP_END && i < num; i++) {
p += zipRawEntryLength(p); //跳过当前指向entry
deleted++; //会被删除的entry计数,因为p后面的entry数目不一定会有num个
}
totlen = p-first.p; //这个是跳过若干个entry后与起始位置p之间的距离,这里面也就是上面说到,也可能一个没跳过
if (totlen > 0) {
if (p[0] != ZIP_END) { //确实跳过了,且当前p指向的不是链表尾端
/* Storing `prevrawlen` in this entry may increase or decrease the
* number of bytes required compare to the current `prevrawlen`.
* There always is room to store this, because it was previously
* stored by an entry that is now being deleted. */
//这时候就需要将first中存储的前一个entry的长度赋给当前p中存储的prevrawlen中,这时候存储的大小可能不太一样,需要变动
nextdiff = zipPrevLenByteDiff(p,first.prevrawlen); //这个是比较p指针中存储prevrawlen需要的字节数与当前自己使用的字节数的差异,可能是正数也可能是负数,也可能是0
p -= nextdiff; //根据情况,调整出内存空间
zipPrevEncodeLength(p,first.prevrawlen); //将该prevrawlen保存到p指向的内存空间
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))-totlen); //更新zl中的tail到偏移量,如果当前的entry就是最后一个entry,那么tail的偏移量减去totlen就可以了
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
zipEntry(p, &tail); //获取待删除的最后一个entry
if (p[tail.headersize+tail.len] != ZIP_END) { //但是如果不是最后一个
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff); //这时候就需要减去或者加上变化的偏移
}
/* Move tail to the front of the ziplist */
memmove(first.p,p,
intrev32ifbe(ZIPLIST_BYTES(zl))-(p-zl)-1); //移动即可,将当前的entry移到first开始
} else {
/* The entire tail was deleted. No need to move memory. */
//如果到达末尾
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe((first.p-zl)-first.prevrawlen); //则tail的偏移量应该是之前的那个entry,所以需要移到前面去
}
/* Resize and update length */
offset = first.p-zl;
zl = ziplistResize(zl, intrev32ifbe(ZIPLIST_BYTES(zl))-totlen+nextdiff); //内存操作,需要调整大小
ZIPLIST_INCR_LENGTH(zl,-deleted); //减少保存entry数目的字段,deleted才是真实删除的数目
p = zl+offset;
/* When nextdiff != 0, the raw length of the next entry has changed, so
* we need to cascade the update throughout the ziplist */
//如果entry占用字节数变了,那么该ziplist所有entry内部也就全都变化了,如果没变,内部不用变动,这个函数貌似有点问题,如果说移动了内存,但是貌似被移动的entry中的p没有做更改???待确认!!!
if (nextdiff != 0)
zl = __ziplistCascadeUpdate(zl,p);
}
return zl;
}
__ziplistInsert,在p的位置插入一个item;
static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen;
unsigned int prevlensize, prevlen = 0;
size_t offset;
int nextdiff = 0;
unsigned char encoding = 0;
long long value = 123456789; /* initialized to avoid warning. Using a value
that is easy to see if for some reason
we use it uninitialized. */
zlentry tail;
/* Find out prevlen for the entry that is inserted. */
//待会待插入的entry的前一个entry的长度
if (p[0] != ZIP_END) {
//如果p不是链表末尾,获取prevlen
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
} else {
//是队列末尾的,就需要找到结尾的entry
unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl);
if (ptail[0] != ZIP_END) { //如果结尾的entry不是链表末尾的话,获取该entry的长度,否则只能是0了,也就是该插入的entry将会是list的第一个entry
prevlen = zipRawEntryLength(ptail);
}
}
/* See if the entry can be encoded */
//看看entry是否可以被编码
if (zipTryEncoding(s,slen,&value,&encoding)) {
/* 'encoding' is set to the appropriate integer encoding */
reqlen = zipIntSize(encoding); //编码后的长度,注意这个长度只是占用字节信息的长度
} else {
/* 'encoding' is untouched, however zipEncodeLength will use the
* string length to figure out how to encode it. */
reqlen = slen; //要求长度
}
/* We need space for both the length of the previous entry and
* the length of the payload. */
reqlen += zipPrevEncodeLength(NULL,prevlen); //加上reqlen的长度
reqlen += zipEncodeLength(NULL,encoding,slen); //加上当前数据的长度
/* When the insert position is not equal to the tail, we need to
* make sure that the next entry can hold this entry's length in
* its prevlen field. */
nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0; //如果当前数据的长度与后面entry能够保存的长度不一致,需要修改后面的长度,这个比较只会在不是list末尾的时候进行
/* Store offset because a realloc may change the address of zl. */
offset = p-zl; //待插入entry相对于链表头的偏移量
zl = ziplistResize(zl,curlen+reqlen+nextdiff);
p = zl+offset;
/* Apply memory move when necessary and update tail offset. */
if (p[0] != ZIP_END) { //不是插到队尾需要移动内存
/* Subtract one because of the ZIP_END bytes */
memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);
/* Encode this entry's raw length in the next entry. */
zipPrevEncodeLength(p+reqlen,reqlen);
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+reqlen);
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
zipEntry(p+reqlen, &tail);
if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff); //这里面跟上面函数类似,都是要调整末尾偏移量
}
} else {
/* This element will be the new tail. */
//如果就是插到末尾,那就直接计算偏移量即可,不再需要移动内存
ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(p-zl);
}
/* When nextdiff != 0, the raw length of the next entry has changed, so
* we need to cascade the update throughout the ziplist */
if (nextdiff != 0) {
offset = p-zl;
zl = __ziplistCascadeUpdate(zl,p+reqlen);
p = zl+offset;
}
/* Write the entry */
p += zipPrevEncodeLength(p,prevlen);
p += zipEncodeLength(p,encoding,slen);
if (ZIP_IS_STR(encoding)) {
memcpy(p,s,slen);
} else {
zipSaveInteger(p,value,encoding);
} //真正写入entry内部数据
ZIPLIST_INCR_LENGTH(zl,1);
return zl;
}
ziplistPush,push一个新的数据;
unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
unsigned char *p;
p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl); //判断是加入前向链表还是反向链表
return __ziplistInsert(zl,p,s,slen); //调用insert内部函数
}
ziplistIndex,根据index找到对应的entry地址;
unsigned char *ziplistIndex(unsigned char *zl, int index) {
unsigned char *p;
unsigned int prevlensize, prevlen = 0;
if (index < 0) { //小于0,表示反向寻找
index = (-index)-1;
p = ZIPLIST_ENTRY_TAIL(zl); //找到最后一个entry
if (p[0] != ZIP_END) {
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
while (prevlen > 0 && index--) { //第一个entry的prevlen为0,可能存在list的长度小于index的情况
p -= prevlen;
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
}
}
} else {
p = ZIPLIST_ENTRY_HEAD(zl);
while (p[0] != ZIP_END && index--) {
p += zipRawEntryLength(p); //同上,可能存在到达末尾的情况
}
}
//如果到了结尾,或者是该索引比list的长度大,返回NULL
return (p[0] == ZIP_END || index > 0) ? NULL : p;
}
ziplistNext,返回p之后的那个entry地址
unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
((void) zl);
/* "p" could be equal to ZIP_END, caused by ziplistDelete,
* and we should return NULL. Otherwise, we should return NULL
* when the *next* element is ZIP_END (there is no next entry). */
if (p[0] == ZIP_END) { //如果指向链表尾部,后面没有entry,返回NULL
return NULL;
}
p += zipRawEntryLength(p); //否则根据当前entry的长度,跳过当前entry
if (p[0] == ZIP_END) {
return NULL; //这里也就是说当链表只有这一个entry,返回NULL
}
return p;
}
ziplistPrev,返回p之前的那个entry地址;
unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
unsigned int prevlensize, prevlen = 0;
/* Iterating backwards from ZIP_END should return the tail. When "p" is
* equal to the first element of the list, we're already at the head,
* and should return NULL. */
if (p[0] == ZIP_END) { //如果当前指向的是队列末尾
p = ZIPLIST_ENTRY_TAIL(zl); //找到队尾entry
return (p[0] == ZIP_END) ? NULL : p; //如果队列为空返回NULL,否则返回队尾entry
} else if (p == ZIPLIST_ENTRY_HEAD(zl)) {
return NULL; //如果当前指向的是链表头,返回NULL,因为没有前面的entry
} else {
//这是链表中,只需要获取前面那个entry长度,然后偏移过去就行
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
assert(prevlen > 0);
return p-prevlen;
}
}
ziplistGet,获取p指向的entry,结果根据市字符串还是数字存入到对应的指针中;
unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) {
zlentry entry;
if (p == NULL || p[0] == ZIP_END) return 0; //如果当前指针为空或者指向的是链表尾部,都直接返回
if (sstr) *sstr = NULL; //初始化
zipEntry(p, &entry); //获取p指向的entry
if (ZIP_IS_STR(entry.encoding)) { //判断是不是字符串编码
if (sstr) {
*slen = entry.len;
*sstr = p+entry.headersize; //是的话,记录下字符串长度以及字符串其实地址
}
} else {
if (sval) { //数字的话,获取该数字内容
*sval = zipLoadInteger(p+entry.headersize,entry.encoding);
}
}
return 1; //成功获取返回1
}
ziplistCompare,比较p指向的entry和slen长度的sstr字符串,一样的话返回1;
unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
zlentry entry;
unsigned char sencoding;
long long zval, sval;
if (p[0] == ZIP_END) return 0; //没有entry直接返回0
zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) {
/* Raw compare */
if (entry.len == slen) {
return memcmp(p+entry.headersize,sstr,slen) == 0; //如果是字符串的话,比较字符串内容
} else {
return 0;
}
} else {
/* Try to compare encoded values. Don't compare encoding because
* different implementations may encoded integers differently. */
if (zipTryEncoding(sstr,slen,&sval,&sencoding)) { //首先需要根据编码类型,然后重新再进行编码后比较
zval = zipLoadInteger(p+entry.headersize,entry.encoding);
return zval == sval; //如果是数字的话,解析长度信息,直接比较
}
}
return 0;
}
ziplistFind,寻找到指向与当前entry相等的entry指针,skip表示每次比较厚跳过的entry数目;
unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip) {
int skipcnt = 0;
unsigned char vencoding = 0;
long long vll = 0;
while (p[0] != ZIP_END) { //只要没到链表尾部一直比较
unsigned int prevlensize, encoding, lensize, len;
unsigned char *q;
ZIP_DECODE_PREVLENSIZE(p, prevlensize);
ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len);
q = p + prevlensize + lensize;
if (skipcnt == 0) {
/* Compare current entry with specified entry */
if (ZIP_IS_STR(encoding)) {
if (len == vlen && memcmp(q, vstr, vlen) == 0) {
return p;
}
} else {
/* Find out if the searched field can be encoded. Note that
* we do it only the first time, once done vencoding is set
* to non-zero and vll is set to the integer value. */
if (vencoding == 0) {
if (!zipTryEncoding(vstr, vlen, &vll, &vencoding)) {
/* If the entry can't be encoded we set it to
* UCHAR_MAX so that we don't retry again the next
* time. */
vencoding = UCHAR_MAX;
}
/* Must be non-zero by now */
assert(vencoding);
}
/* Compare current entry with specified entry, do it only
* if vencoding != UCHAR_MAX because if there is no encoding
* possible for the field it can't be a valid integer. */
if (vencoding != UCHAR_MAX) {
long long ll = zipLoadInteger(q, encoding);
if (ll == vll) {
return p;
}
}
}
//比较字符串或者数字,找到了直接返回退出
/* Reset skip count */
skipcnt = skip;
} else { //如果指定了跳跃的数目,需要一直跳跃
/* Skip entry */
skipcnt--;
}
/* Move to next entry */
p = q + len;
}
return NULL; //没找到返回NULL
}
ziplistLen,返回链表的entry数目;
unsigned int ziplistLen(unsigned char *zl) {
unsigned int len = 0; //32位
if (intrev16ifbe(ZIPLIST_LENGTH(zl)) < UINT16_MAX) {
len = intrev16ifbe(ZIPLIST_LENGTH(zl));
} else {
unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
while (*p != ZIP_END) {
p += zipRawEntryLength(p);
len++;
}
/* Re-store length if small enough */
if (len < UINT16_MAX) ZIPLIST_LENGTH(zl) = intrev16ifbe(len); //重置长度信息
}
return len;
}