redis之数据淘汰策略（一）

一、简介

redis是内存数据库，全部数据都存放在内存中，但是内存的大小也是有限制的，不能无限使用，所以redis提供了相应的策略。
前提是配置了最大内存限制 maxmemory <bytes>

• redis2.0.0之前的版本
  内存使用超过配置限制时，写失败，读正常
• redis2.0.0版本开始提供了vm功能
  vm功能介绍
  当内存使用超过配置限制时，并且使能vm功能，能将剔除一些数据交换到磁盘文件中
  正常读写，数据不丢失，但因为读写文件，性能低，受限磁盘空间大小
• redis2.2.0开始提供了5种数据淘汰策略
  默认策略为volatile-lru

# volatile-lru -> remove the key with an expire set using an LRU algorithm
# allkeys-lru -> remove any key accordingly to the LRU algorithm
# volatile-random -> remove a random key with an expire set
# allkeys-random -> remove a random key, any key
# volatile-ttl -> remove the key with the nearest expire time (minor TTL)
# noeviction -> don't expire at all, just return an error on write operations
#
# The default is:
#
# maxmemory-policy volatile-lru

redis3.0.0开始默认策略为noeviction，不进行数据淘汰

# The default is:
#
# maxmemory-policy noeviction

• redis 4.0.0新增2种淘汰策略

# volatile-lfu -> Evict using approximated LFU among the keys with an expire set.
# allkeys-lfu -> Evict any key using approximated LFU.

lfu是基于lru进行改进的一种策略。

二、代码

redis2.2.0

/* Redis maxmemory strategies */
#define REDIS_MAXMEMORY_VOLATILE_LRU 0
#define REDIS_MAXMEMORY_VOLATILE_TTL 1
#define REDIS_MAXMEMORY_VOLATILE_RANDOM 2
#define REDIS_MAXMEMORY_ALLKEYS_LRU 3
#define REDIS_MAXMEMORY_ALLKEYS_RANDOM 4
#define REDIS_MAXMEMORY_NO_EVICTION 5

2.1 默认配置

void initServerConfig() {
...
	server.maxmemory = 0;
    server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU;
    server.maxmemory_samples = 3;
...
}

2.2 配置加载解析

...
else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) {
server.maxmemory = memtoll(argv[1],NULL);
} else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) {
    if (!strcasecmp(argv[1],"volatile-lru")) {
        server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU;
    } else if (!strcasecmp(argv[1],"volatile-random")) {
        server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_RANDOM;
    } else if (!strcasecmp(argv[1],"volatile-ttl")) {
        server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_TTL;
    } else if (!strcasecmp(argv[1],"allkeys-lru")) {
        server.maxmemory_policy = REDIS_MAXMEMORY_ALLKEYS_LRU;
    } else if (!strcasecmp(argv[1],"allkeys-random")) {
        server.maxmemory_policy = REDIS_MAXMEMORY_ALLKEYS_RANDOM;
    } else if (!strcasecmp(argv[1],"noeviction")) {
        server.maxmemory_policy = REDIS_MAXMEMORY_NO_EVICTION;
    } else {
        err = "Invalid maxmemory policy";
        goto loaderr;
    }
} else if (!strcasecmp(argv[0],"maxmemory-samples") && argc == 2) {
    server.maxmemory_samples = atoi(argv[1]);
    if (server.maxmemory_samples <= 0) {
        err = "maxmemory-samples must be 1 or greater";
        goto loaderr;
    }
}
...

2.3 具体淘汰策略实现

void freeMemoryIfNeeded(void) {
    /* Remove keys accordingly to the active policy as long as we are
     * over the memory limit. */
    if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION) return;

    while (server.maxmemory && zmalloc_used_memory() > server.maxmemory) {
        int j, k, freed = 0;

        for (j = 0; j < server.dbnum; j++) {
            long bestval = 0; /* just to prevent warning */
            sds bestkey = NULL;
            struct dictEntry *de;
            redisDb *db = server.db+j;
            dict *dict;

            if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
                server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM)
            {
                dict = server.db[j].dict;
            } else {
                dict = server.db[j].expires;
            }
            if (dictSize(dict) == 0) continue;

            /* volatile-random and allkeys-random policy */
            if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM ||
                server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_RANDOM)
            {
                de = dictGetRandomKey(dict);
                bestkey = dictGetEntryKey(de);
            }

            /* volatile-lru and allkeys-lru policy */
            else if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
                server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
            {
                for (k = 0; k < server.maxmemory_samples; k++) {
                    sds thiskey;
                    long thisval;
                    robj *o;

                    de = dictGetRandomKey(dict);
                    thiskey = dictGetEntryKey(de);
                    /* When policy is volatile-lru we need an additonal lookup
                     * to locate the real key, as dict is set to db->expires. */
                    if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
                        de = dictFind(db->dict, thiskey);
                    o = dictGetEntryVal(de);
                    thisval = estimateObjectIdleTime(o);

                    /* Higher idle time is better candidate for deletion */
                    if (bestkey == NULL || thisval > bestval) {
                        bestkey = thiskey;
                        bestval = thisval;
                    }
                }
            }

            /* volatile-ttl */
            else if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_TTL) {
                for (k = 0; k < server.maxmemory_samples; k++) {
                    sds thiskey;
                    long thisval;

                    de = dictGetRandomKey(dict);
                    thiskey = dictGetEntryKey(de);
                    thisval = (long) dictGetEntryVal(de);

                    /* Expire sooner (minor expire unix timestamp) is better
                     * candidate for deletion */
                    if (bestkey == NULL || thisval < bestval) {
                        bestkey = thiskey;
                        bestval = thisval;
                    }
                }
            }

            /* Finally remove the selected key. */
            if (bestkey) {
                robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
                propagateExpire(db,keyobj);
                dbDelete(db,keyobj);
                server.stat_evictedkeys++;
                decrRefCount(keyobj);
                freed++;
            }
        }
        if (!freed) return; /* nothing to free... */
    }
}

• allkeys-*开头的策略是针对整个redis数据库的数据
• volatile-*开头的策略只针对设置了过期信息的key
• ttl的针对设置了过期key的剩余ttl进行筛选淘汰
• random则随机筛选
• lru则根据最后访问时间进行筛选

三、淘汰策略

3.1 随机策略

随机的从整个数据集（allkeys-random）或则从有过期时间的数据集（volatile-random）中筛选需要删除的数据

• 首先确定筛选数据集

if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
    server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM)
{
    dict = server.db[j].dict; //整个数据集
} else {
    dict = server.db[j].expires; //会过期的数据集
}

• 随机筛选一个key

 /* volatile-random and allkeys-random policy */
if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM ||
    server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_RANDOM)
{
    de = dictGetRandomKey(dict);
    bestkey = dictGetEntryKey(de);
}

• 删除淘汰的key

if (bestkey) {
    robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
    propagateExpire(db,keyobj);//为aof和从服务器生成del命令
    dbDelete(db,keyobj);//删除key以及对应的value
    server.stat_evictedkeys++;
    decrRefCount(keyobj);
    freed++;
}

3.2 lru策略

#define REDIS_LRU_CLOCK_MAX ((1<<21)-1) /* Max value of obj->lru */
#define REDIS_LRU_CLOCK_RESOLUTION 10 /* LRU clock resolution in seconds */
typedef struct redisObject {
    unsigned type:4;
    unsigned storage:2;     /* REDIS_VM_MEMORY or REDIS_VM_SWAPPING */
    unsigned encoding:4;
    unsigned lru:22;        /* lru time (relative to server.lruclock) */
    int refcount;
    void *ptr;
    /* VM fields are only allocated if VM is active, otherwise the
     * object allocation function will just allocate
     * sizeof(redisObjct) minus sizeof(redisObjectVM), so using
     * Redis without VM active will not have any overhead. */
} robj;

redis每个key，value都是一个redisObject对象，其中有一个22bit表示lru。

• 每个对象创建时，将系统时间server.lruclock赋值给对象中的lru字段

robj *createObject(int type, void *ptr) {
    robj *o = zmalloc(sizeof(*o));
    o->type = type;
    o->encoding = REDIS_ENCODING_RAW;
    o->ptr = ptr;
    o->refcount = 1;

    /* Set the LRU to the current lruclock (minutes resolution).
     * We do this regardless of the fact VM is active as LRU is also
     * used for the maxmemory directive when Redis is used as cache.
     *
     * Note that this code may run in the context of an I/O thread
     * and accessing server.lruclock in theory is an error
     * (no locks). But in practice this is safe, and even if we read
     * garbage Redis will not fail. */
    o->lru = server.lruclock;
    /* The following is only needed if VM is active, but since the conditional
     * is probably more costly than initializing the field it's better to
     * have every field properly initialized anyway. */
    o->storage = REDIS_VM_MEMORY;
    return o;
}

• 每次访问时更新lru

robj *lookupKey(redisDb *db, robj *key) {
    dictEntry *de = dictFind(db->dict,key->ptr);
    if (de) {
        robj *val = dictGetEntryVal(de);

        /* Update the access time for the aging algorithm.
         * Don't do it if we have a saving child, as this will trigger
         * a copy on write madness. */
        if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1)
            val->lru = server.lruclock;
            
			...
        }
        server.stat_keyspace_hits++;
        return val;
    } else {
        server.stat_keyspace_misses++;
        return NULL;
    }
}

• 系统时间的lru的更新
  每1毫秒调用一次的定时任务，其中将进行lru的更新
  大概有1.5年的时间范围,虽然不长，但也够用了

aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL);

int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
...
/* We have just 22 bits per object for LRU information.
 * So we use an (eventually wrapping) LRU clock with 10 seconds resolution.
 * 2^22 bits with 10 seconds resoluton is more or less 1.5 years.
 *
 * Note that even if this will wrap after 1.5 years it's not a problem,
 * everything will still work but just some object will appear younger
 * to Redis. But for this to happen a given object should never be touched
 * for 1.5 years.
 *
 * Note that you can change the resolution altering the
 * REDIS_LRU_CLOCK_RESOLUTION define.
 */
    updateLRUClock();
...
}

void updateLRUClock(void) {
    server.lruclock = (time(NULL)/REDIS_LRU_CLOCK_RESOLUTION) &
                                                REDIS_LRU_CLOCK_MAX;
}

• 首先确定筛选数据集

if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
    server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM)
{
    dict = server.db[j].dict;
} else {
    dict = server.db[j].expires;
}

• 计算访问时间，筛选最久未被访问key
  随机抽样samples个key，计算lru，从这samples个中淘汰最久未访问的

/* volatile-lru and allkeys-lru policy */
else if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
    server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
{
    for (k = 0; k < server.maxmemory_samples; k++) { //抽样个数，配置获得
        sds thiskey;
        long thisval;
        robj *o;

        de = dictGetRandomKey(dict);
        thiskey = dictGetEntryKey(de);
        /* When policy is volatile-lru we need an additonal lookup
         * to locate the real key, as dict is set to db->expires. */
        if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU) //如果是从过期集合中筛选，需要转换到对应原始数据对象
            de = dictFind(db->dict, thiskey);
        o = dictGetEntryVal(de);
        thisval = estimateObjectIdleTime(o); //计算未访问时间

        /* Higher idle time is better candidate for deletion */
        if (bestkey == NULL || thisval > bestval) {
            bestkey = thiskey;
            bestval = thisval;
        }
    }
}

/* Given an object returns the min number of seconds the object was never
 * requested, using an approximated LRU algorithm. */
unsigned long estimateObjectIdleTime(robj *o) {
    if (server.lruclock >= o->lru) {
        return (server.lruclock - o->lru) * REDIS_LRU_CLOCK_RESOLUTION;
    } else {
        return ((REDIS_LRU_CLOCK_MAX - o->lru) + server.lruclock) *
                    REDIS_LRU_CLOCK_RESOLUTION;
    }
}

• 删除淘汰的key

/* Finally remove the selected key. */
if (bestkey) {
    robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
    propagateExpire(db,keyobj);
    dbDelete(db,keyobj);
    server.stat_evictedkeys++;
    decrRefCount(keyobj);
    freed++;
}

3.3 ttl策略

对于过期key，server.db[j].expires中key对应的值只是一个简单的ttl整数值，ttl策略则选择ttl最小的值（即最先过期的key）进行淘汰。

• 首先确定筛选数据集
  ttl只针对有过期时间的key

if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
    server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM)
{
    dict = server.db[j].dict;
} else {
    dict = server.db[j].expires;
}

• 从数据集中随机筛选samples个key，然后选取ttl值最小的一个key

/* volatile-ttl */
else if(server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_TTL) {
    for (k = 0; k < server.maxmemory_samples; k++) {
        sds thiskey;
        long thisval;

        de = dictGetRandomKey(dict);
        thiskey = dictGetEntryKey(de);
        thisval = (long) dictGetEntryVal(de);

        /* Expire sooner (minor expire unix timestamp) is better
         * candidate for deletion */
        if (bestkey == NULL || thisval < bestval) {
            bestkey = thiskey;
            bestval = thisval;
        }
    }
}

• 删除淘汰的key

/* Finally remove the selected key. */
if (bestkey) {
    robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
    propagateExpire(db,keyobj);
    dbDelete(db,keyobj);
    server.stat_evictedkeys++;
    decrRefCount(keyobj);
    freed++;
}

根据代码可以看出，整个大的逻辑都是随机筛选一些key,再根据具体的策略进行进一步的筛选。很可能会将热数据淘汰出去。