redis数据迁移python实现

数据迁移要扫描redis的全部key
使用next_cursor,this_keys = from_redis.scan(cursor=next_cursor, match=None, count=100)来扫描全表
然后在对一个个的key,读源库,写新的库
写新库时,根据业务对key做了些小改动,吧db添加到了key的开头

这里还根据业务做了key类型的统计,以便最终可以看到多少key被扫描,多少key被迁移

from_redis
to_redis

def add_one(the_dict,the_key):
    if the_key in the_dict:
        the_dict[the_key] = the_dict[the_key] + 1
    else:
        the_dict[the_key] = 1
        
def move_one_key(k,db,stat_dict,redis_from,redis_to):
    add_one(stat_dict,'cnt')
    data_type = redis_from.type(k)
    # 判断key的类型 string\hash\list\set
    new_k = '{}.{}'.format(db,k)
    if data_type == 'string':
        v = redis_from.get(k)
        t = redis_from.ttl(k)
        redis_to.set(new_k, v)
        if int(t) > 0:
            redis_to.expire(new_k,t)
        add_one(stat_dict,'scnt')
    elif data_type == 'list':
        values = redis_from.lrange(k, 0, -1)
        t = redis_from.ttl(k)           
        redis_to.lpush(new_k, values)
        if int(t) > 0:
            redis_to.expire(new_k,t)
        add_one(stat_dict,'lcnt')
    elif data_type == 'set':
        values = redis_from.smembers(k)
        t = redis_from.ttl(k)
        redis_to.sadd(new_k, values)
        if int(new_k) > 0:
            redis_to.expire(new_k,t)
        add_one(stat_dict,'setcnt')
    elif data_type == 'hash':
        add_one(stat_dict,'hcnt')
        keys = redis_from.hkeys(k)                        
        for key in keys:
            value = redis_from.hget(k, key)
            t = redis_from.ttl(k)
            redis_to.hset(new_k, key, value)
            if int(t) > 0:
                redis_to.expire(new_k,t)
    elif data_type == 'zset':
        add_one(stat_dict,'zsetcnt')
        i,v_list = redis_from.zscan(k)
        #print(k)
        t = redis_from.ttl(k)
        for v in v_list:
            redis_to.zadd(new_k,{v[0]:v[1]})
        if int(t) > 0:
            redis_to.expire(new_k,t)
    else:
        add_one(stat_dict,'unkonwcnt')
        print('not known type',data_type,k)
        unkonwcnt = unkonwcnt+1
        ttl=redis_from.ttl(k)
        file_object = open(f'./{db}_unkonw_type_key.txt','a+')
        file_object.write(data_type+','+k+',ttl:{}'.format(ttl))
        file_object.write('\n')
        file_object.close( )

def scan_all_keys(db,from_redis,to_redis,move=False):
    key_stat = {}
    move_stat = {}
    key_type1_re = re.compile(r'^[\w]{48,50}$')
    key_type2_re = re.compile(r'^[\w]{48,64}\.type2$')
    key_type3_re = re.compile(r'^type3\.[\w]{32}$')
    key_type4_re = re.compile(r'^type4\.[\w]{48,50}$')
    key_type5_re = re.compile(r'^[\w]{48,50}\.type5$')
    key_type6_re =re.compile(r'^[\w]{48,50}\..*\.type6$')
    next_cursor = 0
    while True:
        next_cursor,this_keys =from_redis.scan(cursor=next_cursor, match=None, count=100)
        for the_key in this_keys:
            if the_key.startswith('begin1:'):
                add_one(key_stat,'begin1')
                continue
            elif the_key.endswith('.end1'):
                add_one(key_stat,'.end1')
            elif the_key.endswith("')") and the_key.startswith("('"):
                add_one(key_stat,'.quter')
            elif key_type1_re .match(the_key):
                add_one(key_stat,'type1')
            elif key_type2_re .match(the_key):
                add_one(key_stat,'type2')
            elif key_type3_re.match(the_key):
                add_one(key_stat,'type3')
            else:
                #print(the_key)
                add_one(key_stat,'other_keys')
            #不跳出for的就移到新的kedis去
            if move:
                the_thread = threading.Thread(target=move_one_key,args=(the_key,db,move_stat,from_redis,to_redis))
                the_thread.run()
                #move_one_key(the_key,db,move_stat,from_redis,to_redis)
        if next_cursor == 0:
            return key_stat,move_stat

def mv_db(db,from_redis ,to_redis ,move=False):
    key_stat,move_stat = scan_all_keys(db,from_redis,to_redis,move=move)
    print(key_stat)
    print(move_stat)