基础对象(redisObject)
- 数据结构
#define LRU_BITS 24
typedef struct redisObject {
unsigned type:4;
unsigned encoding:4;
unsigned lru:LRU_BITS; /* LRU time (relative to global lru_clock) or
* LFU data (least significant 8 bits frequency
* and most significant 16 bits access time). */
int refcount;
void *ptr;
} robj;
- 基本字段详解
在redisObject数据结构中，采用了C中的位域来节省内存，位域操作非常方便。
需要注意的就是他的移植性,比如某些嵌入式中1个字节不是8位，还有比如你的类型跨两个字节了，这样话可能会导致不可预期的后果，因此适当字节填充是必要的。当然不要被这两个问题吓着，毕竟位域要比位运算好操作多。
时间计算：获取时间的函数属于系统调用，比较耗费资源，因此redis采用缓存的方式，在定时任务中定期更新见serverCron函数。
int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
//....
/* Update the time cache. */
updateCachedTime();
//....
}
需要注意的是在lruclock中使用了atomicGet是因为可能在别的线程可能也会用到该时间，如集群状态。
字符串
数据结构
- 数据结构
redis中字符串是分三种类型的。首先我们回顾一下对象类型
char *strEncoding(int encoding) {
switch(encoding) {
case OBJ_ENCODING_RAW: return "raw";
case OBJ_ENCODING_INT: return "int";
case OBJ_ENCODING_HT: return "hashtable";
case OBJ_ENCODING_QUICKLIST: return "quicklist";
case OBJ_ENCODING_ZIPLIST: return "ziplist";
case OBJ_ENCODING_INTSET: return "intset";
case OBJ_ENCODING_SKIPLIST: return "skiplist";
case OBJ_ENCODING_EMBSTR: return "embstr";
default: return "unknown";
}
}
由上可见字符串分为"raw"和"embstr"，其实还有一种就是int.
/* Try to encode a string object in order to save space */
robj *tryObjectEncoding(robj *o) {
long value;
sds s = o->ptr;
size_t len;
/* Make sure this is a string object, the only type we encode
* in this function. Other types use encoded memory efficient
* representations but are handled by the commands implementing
* the type. */
serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
/* We try some specialized encoding only for objects that are
* RAW or EMBSTR encoded, in other words objects that are still
* in represented by an actually array of chars. */
if (!sdsEncodedObject(o)) return o;
/* It's not safe to encode shared objects: shared objects can be shared
* everywhere in the "object space" of Redis and may end in places where
* they are not handled. We handle them only as values in the keyspace. */
if (o->refcount > 1) return o;
/* Check if we can represent this string as a long integer.
* Note that we are sure that a string larger than 20 chars is not
* representable as a 32 nor 64 bit integer. */
len = sdslen(s);
if (len <= 20 && string2l(s,len,&value)) {
/* This object is encodable as a long. Try to use a shared object.
* Note that we avoid using shared integers when maxmemory is used
* because every object needs to have a private LRU field for the LRU
* algorithm to work well. */
if ((server.maxmemory == 0 ||
!(server.maxmemory_policy & MAXMEMORY_FLAG_NO_SHARED_INTEGERS)) &&
value >= 0 &&
value < OBJ_SHARED_INTEGERS)
{
decrRefCount(o);
incrRefCount(shared.integers[value]);
return shared.integers[value];
} else {
if (o->encoding == OBJ_ENCODING_RAW) sdsfree(o->ptr);
o->encoding = OBJ_ENCODING_INT;
o->ptr = (void*) value;
return o;
}
}
/* If the string is small and is still RAW encoded,
* try the EMBSTR encoding which is more efficient.
* In this representation the object and the SDS string are allocated
* in the same chunk of memory to save space and cache misses. */
if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT) {
robj *emb;
if (o->encoding == OBJ_ENCODING_EMBSTR) return o;
emb = createEmbeddedStringObject(s,sdslen(s));
decrRefCount(o);
return emb;
}
/* We can't encode the object...
*
* Do the last try, and at least optimize the SDS string inside
* the string object to require little space, in case there
* is more than 10% of free space at the end of the SDS string.
*
* We do that only for relatively large strings as this branch
* is only entered if the length of the string is greater than
* OBJ_ENCODING_EMBSTR_SIZE_LIMIT. */
if (o->encoding == OBJ_ENCODING_RAW &&
sdsavail(s) > len/10)
{
o->ptr = sdsRemoveFreeSpace(o->ptr);
}
/* Return the original object. */
return o;
}
可以使用命令 object encoding xxx 查看
embstr与raw分析
| 类型 | 特点 | 优点 | 缺点 |
|---|---|---|---|
| embstr | 1.只分配一次内存空间，因此robj和sds是连续的；2.只读；3.Embstr字符串需要修改时，会转成raw，之后一直为raw | 1.创建和删除只需要一次； 2.寻找速度快 | 1.重分配涉及到robj和sds整个对象，因此embstr是只读的 |
| raw | 1.robj和sds非连续； 2.可修改 |
#define OBJ_ENCODING_EMBSTR_SIZE_LIMIT 44
robj *createStringObject(const char *ptr, size_t len) {
if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT)
return createEmbeddedStringObject(ptr,len);
else
return createRawStringObject(ptr,len);
}
适用场景
- 缓存功能：mysql存储，redis做缓存
- 计数器：如点赞次数，视频播放次数
- 限流：见基于redis的分布式限流
public static final String DISTRIBUTE_LIMIT_SCRIPT = "local key = KEYS[1] \n"
+ "local limit = tonumber(ARGV[1]) \n"
+ "local current = tonumber(redis.call('get',key) or '0') \n"
+ " \n"
+ "if (limit < current + 1) then \n"
+ " return 0 \n"
+ "else \n"
+ " redis.call('incrby',key,'1') \n"
+ " redis.call('expire',key,'2') \n"
+ " return 1 \n"
+ "end \n";
/*
* 1表示通过，0表示为限流
* */
private static boolean tryAcquire(ResourceLimiterCfg apiRateLimiter,Cluster redisClient){
Long distributeLimitResult=1L;
try{
String key= RedisCachePrefix.DISTRIBUTE_LIMIT_PRE+
(StringUtils.isNotEmpty(apiRateLimiter.getPrefix())?apiRateLimiter.getPrefix()+SplitCharacter.SPLIT_COLON.key:"")+apiRateLimiter.getResource()+SplitCharacter.SPLIT_COLON.key+System.currentTimeMillis()/1000;
if(distributeLimit==null){
distributeLimit = redisClient.scriptLoad(RedisLuaScript.DISTRIBUTE_LIMIT_SCRIPT);
}
distributeLimitResult = (Long) redisClient.evalsha(distributeLimit, Collections.singletonList(key),
Collections.singletonList(apiRateLimiter.getPerSecCount().toString()), true, ScriptOutputType.INTEGER);
log.info("tryAcquire key={},distributeLimitResult={} ",key,distributeLimitResult );
}catch (Exception e){
log.error("tryAcquire={}",e);
e.printStackTrace();
}finally {
return distributeLimitResult.intValue()==1;
}
}
注意点
- redis的字符串存储分为三种：整数(共享int和非共享)、embstr和raw；
- redis的字符串约定其长度不能大于512M
static int checkStringLength(client *c, long long size) {
if (size > 512*1024*1024) {
addReplyError(c,"string exceeds maximum allowed size (512MB)");
return C_ERR;
}
return C_OK;
}
列表
数据结构
- 数据结构
新版redis中的list实际上只有quicklist一种类型，而且是一个双向链表，因此我们来具体分析一下：
void listTypePush(robj *subject, robj *value, int where) {
if (subject->encoding == OBJ_ENCODING_QUICKLIST) {
int pos = (where == LIST_HEAD) ? QUICKLIST_HEAD : QUICKLIST_TAIL;
value = getDecodedObject(value);
size_t len = sdslen(value->ptr);
quicklistPush(subject->ptr, value->ptr, len, pos);
decrRefCount(value);
} else {
serverPanic("Unknown list encoding");
}
}
/* quicklistNode is a 32 byte struct describing a ziplist for a quicklist.
* We use bit fields keep the quicklistNode at 32 bytes.
* count: 16 bits, max 65536 (max zl bytes is 65k, so max count actually < 32k).
* encoding: 2 bits, RAW=1, LZF=2.
* container: 2 bits, NONE=1, ZIPLIST=2.
* recompress: 1 bit, bool, true if node is temporarry decompressed for usage.
* attempted_compress: 1 bit, boolean, used for verifying during testing.
* extra: 10 bits, free for future use; pads out the remainder of 32 bits */
typedef struct quicklistNode {
//双向链表的前节点
struct quicklistNode *prev;
//双向链表的后节点
struct quicklistNode *next;
///不设置压缩数据参数recompress时指向一个ziplist结构
//设置压缩数据参数recompress指向quicklistLZF结构
unsigned char *zl;
//压缩列表ziplist的总长度
unsigned int sz; /* ziplist size in bytes */
//每个ziplist中entry总个数
unsigned int count : 16; /* count of items in ziplist */
//表示是否采用了LZF压缩算法压缩quicklist节点，1表示压缩过，2表示没压缩，占2 bits长度
unsigned int encoding : 2; /* RAW==1 or LZF==2 */
//表示是否启用ziplist来进行存储
unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */
//记录该几点之前是否被压缩过
unsigned int recompress : 1; /* was this node previous compressed? */
//测试是使用
unsigned int attempted_compress : 1; /* node can't compress; too small */
//额外扩展位，占10bits长度
unsigned int extra : 10; /* more bits to steal for future usage */
} quicklistNode;
/*
当指定使用lzf压缩算法压缩ziplist的entry节点时，
quicklistNode结构的zl成员指向quicklistLZF结构
quicklistLZF is a 4+N byte struct holding 'sz' followed by 'compressed'.
* 'sz' is byte length of 'compressed' field.
* 'compressed' is LZF data with total (compressed) length 'sz'
* NOTE: uncompressed length is stored in quicklistNode->sz.
* When quicklistNode->zl is compressed, node->zl points to a quicklistLZF */
typedef struct quicklistLZF {
表示被LZF算法压缩后的ziplist的大小
unsigned int sz; /* LZF size in bytes*/
//压缩之后的数据，柔性数组
char compressed[];
} quicklistLZF;
/* quicklist is a 40 byte struct (on 64-bit systems) describing a quicklist.
* 'count' is the number of total entries.
* 'len' is the number of quicklist nodes.
* 'compress' is: -1 if compression disabled, otherwise it's the number
* of quicklistNodes to leave uncompressed at ends of quicklist.
* 'fill' is the user-requested (or default) fill factor. */
typedef struct quicklist {
//链表表头
quicklistNode *head;
//链表表尾
quicklistNode *tail;
//所有quicklistnode节点中所有的entry个数
unsigned long count; /* total count of all entries in all ziplists */
//quicklistnode节点个数，也就是quicklist的长度
unsigned long len; /* number of quicklistNodes */
//单个节点的填充因子，也就是ziplist的大小
int fill : 16; /* fill factor for individual nodes */
//保存压缩程度值，配置文件设定，占16bits，0表示不压缩
unsigned int compress : 16; /* depth of end nodes not to compress;0=off */
} quicklist;
//quicklist中quicklistNode的entry结构
typedef struct quicklistEntry {
//指向所属的quicklist指针
const quicklist *quicklist;
//指向所属的quicklistNode节点的指针
quicklistNode *node;
//指向当前ziplist结构的指针
unsigned char *zi;
//指向当前ziplist结构的字符串value成员
unsigned char *value;
//指向当前ziplist结构的整型value成员
long long longval;
//当前ziplist结构的字节数
unsigned int sz;
//保存相对ziplist的偏移量
int offset;
} quicklistEntry;
数据结构分析
- 双向链表，插入和删除效率高，但是对内存不友好，而且需要存取额外的前后两个指针
- 数组为代表的连续内存，插入和删除时间复杂度高，但是对内存友好(局部性原理)，因此综合了两个，催生了quicklist数据结构，其实在C++的stl中deque也是这种设计模式。
思考：如何设置来均衡链表长度和数组(ziplist)为代表的比例呢？我们来考虑一下两者极端情况，如果链表长度为1，那么就退化成一个ziplist了，此时可能会因为找不到一块很大的连续内存而导致OOM；如果ziplist为1，那么就退化成双向链表了，此时对内存是非常不友好的，而且会造成大量的内存碎片，影响性能？那怎么办呢？且看后面具体分析。
- 具体字段分析
- fill： 16bit，控制着ziplist大小，存放
list-max-ziplist-size参数的值
list-max-ziplist-size -2
可以看出，正值表示每个quicklist上ziplist的entry个数。
//在原来节点的基础上又需要新添加一个
REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
const int fill, const size_t sz) {
//...
//这里这里，count表示当前entry的总个数
else if ((int)node->count < fill)
return 1;
//....
}
- compress: 16bit，节点压缩深度设置，存放
list-compress-depth参数的值
由于当数据很多时，最容易被访问的很可能是两端数据，中间的数据被访问的频率比较低（访问起来性能也很低）。如果应用场景符合这个特点，那么list还提供了一个选项，能够把中间的数据节点进行压缩，从而进一步节省内存空间。见配置参数list-compress-depth。
list-compress-depth 0
注意他是node级别的，而不是entry级别的，但是node被压缩时，整个ziplist都会被压缩。
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-nDGx2dN6-1601027186000)(assets/1589279350909.png)]
0: 是个特殊值，表示都不压缩。这是Redis的默认值。
1: 表示quicklist两端各有1个节点不压缩，中间的节点压缩。
2: 表示quicklist两端各有2个节点不压缩，中间的节点压缩。
3: 表示quicklist两端各有3个节点不压缩，中间的节点压缩。
依此类推…
由此可知，头尾节点是不会进行压缩的，因为经常用到他呀。而对于压缩这是采用的是LZF(具体自己去看)。
-
recompress
执行lindex等操作后会对节点进行暂时解压，recompress表示之后某个时刻再进行压缩。
比较重要的操作
- 插入分为头部和尾部
void quicklistPush(quicklist *quicklist, void *value, const size_t sz,
int where) {
if (where == QUICKLIST_HEAD) {
quicklistPushHead(quicklist, value, sz);
} else if (where == QUICKLIST_TAIL) {
quicklistPushTail(quicklist, value, sz);
}
}
/* Add new entry to head node of quicklist.
*
* Returns 0 if used existing head.
* Returns 1 if new head created. */
int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
quicklistNode *orig_head = quicklist->head;
if (likely(
_quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) {
quicklist->head->zl =
ziplistPush(quicklist->head->zl, value, sz, ZIPLIST_HEAD);
quicklistNodeUpdateSz(quicklist->head);
} else {
quicklistNode *node = quicklistCreateNode();
node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_HEAD);
quicklistNodeUpdateSz(node);
_quicklistInsertNodeBefore(quicklist, quicklist->head, node);
}
quicklist->count++;
quicklist->head->count++;
return (orig_head != quicklist->head);
}
/* Add new entry to tail node of quicklist.
*
* Returns 0 if used existing tail.
* Returns 1 if new tail created. */
int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) {
quicklistNode *orig_tail = quicklist->tail;
if (likely(
_quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) {
quicklist->tail->zl =
ziplistPush(quicklist->tail->zl, value, sz, ZIPLIST_TAIL);
quicklistNodeUpdateSz(quicklist->tail);
} else {
quicklistNode *node = quicklistCreateNode();
node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_TAIL);
quicklistNodeUpdateSz(node);
_quicklistInsertNodeAfter(quicklist, quicklist->tail, node);
}
quicklist->count++;
quicklist->tail->count++;
return (orig_tail != quicklist->tail);
}
REDIS_STATIC void __quicklistInsertNode(quicklist *quicklist,
quicklistNode *old_node,
quicklistNode *new_node, int after) {
if (after) {
new_node->prev = old_node;
if (old_node) {
new_node->next = old_node->next;
if (old_node->next)
old_node->next->prev = new_node;
old_node->next = new_node;
}
if (quicklist->tail == old_node)
quicklist->tail = new_node;
} else {
new_node->next = old_node;
if (old_node) {
new_node->prev = old_node->prev;
if (old_node->prev)
old_node->prev->next = new_node;
old_node->prev = new_node;
}
if (quicklist->head == old_node)
quicklist->head = new_node;
}
/* If this insert creates the only element so far, initialize head/tail. */
if (quicklist->len == 0) {
quicklist->head = quicklist->tail = new_node;
}
if (old_node)
quicklistCompress(quicklist, old_node);
quicklist->len++;
}
在__quicklistInsertNode中由于支持按下标插入，所以操作起来可能复杂点，而且还需要考虑是否需要压缩。
- _quicklistNodeAllowInsert判断是否能直接插入
/* Optimization levels for size-based filling */
static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
//参数sz表示插入对象的总的字节数
REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
const int fill, const size_t sz) {
if (unlikely(!node))
return 0;
int ziplist_overhead;
/* size of previous offset
下一个对象存储前一个对象的信息
*/
if (sz < 254)
ziplist_overhead = 1;
else
ziplist_overhead = 5;
/* size of forward offset
当前对象的长度信息
*/
if (sz < 64)
ziplist_overhead += 1;
else if (likely(sz < 16384))
ziplist_overhead += 2;
else
ziplist_overhead += 5;
//如果是整数的话，其实超过了。不考虑连锁更新
/* new_sz overestimates if 'sz' encodes to an integer type */
//这个算出来的是最大值
unsigned int new_sz = node->sz + sz + ziplist_overhead;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)))
return 1;
//fill为正数时单个节点不能超过8kb
else if (!sizeMeetsSafetyLimit(new_sz))
return 0;
//整数判断
else if ((int)node->count < fill)
return 1;
else
return 0;
}
REDIS_STATIC int
_quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz,
const int fill) {
if (fill >= 0)
return 0;
size_t offset = (-fill) - 1;
//判断是否越界
if (offset < (sizeof(optimization_level) / sizeof(*optimization_level))) {
//满足该偏移量下的max 字节设置，那么就返回1
if (sz <= optimization_level[offset]) {
return 1;
} else {
return 0;
}
} else {
return 0;
}
}
- 节点压缩
REDIS_STATIC int __quicklistCompressNode(quicklistNode *node) { // 压缩节点
#ifdef REDIS_TEST
node->attempted_compress = 1;
#endif
/* Don't bother compressing small values */
if (node->sz < MIN_COMPRESS_BYTES) // 小于48字节不进行压缩
return 0;
quicklistLZF *lzf = zmalloc(sizeof(*lzf) + node->sz);
/* Cancel if compression fails or doesn't compress small enough */
if (((lzf->sz = lzf_compress(node->zl, node->sz, lzf->compressed,
node->sz)) == 0) ||
lzf->sz + MIN_COMPRESS_IMPROVE >= node->sz) { // 如果压缩失败或压缩后节省的空间不到8字节放弃压缩
/* lzf_compress aborts/rejects compression if value not compressable. */
zfree(lzf);
return 0;
}
lzf = zrealloc(lzf, sizeof(*lzf) + lzf->sz); // 重新分配内存
zfree(node->zl); // 释放原有节点
node->zl = (unsigned char *)lzf; // 将压缩节点赋值给node
node->encoding = QUICKLIST_NODE_ENCODING_LZF; // 记录编码
node->recompress = 0;
return 1;
}
###列表一些特性
- 阻塞和非阻塞
List的部分操作支持阻塞和非阻塞。重点看阻塞——当所有给定的key中不存在，或者key中包含的是空列表，那么 BLPOP 或 BLPOP 命令将会被阻塞连接，直到另一个client对这些key中执行 [LR]PUSH 命令将一个新数据出现在任意key的列表中，那么这个命令会解除调用BLPOP 或 BLPOP 命令的client的阻塞状态。
其实实现很简单，判断是否超时了，或者有数据了，否则不返回就行了。
BLPOP key [key ...] timeout
BRPOP key [key ...] timeout
BRPOPLPUSH source destination timeout
使用场景
-
常见数据结构
- lpush+lpop=Stack（栈）
- lpush+rpop=Queue（队列）
- lpush+ltrim=Capped Collection（有限集合）
- lpush+brpop=Message Queue（消息队列）
-
文章列表
可以使用组合的方式文章用hash存储，而文章列表则使用list存储
1899
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
