redis集群scan_利用scan迁移部分单点redis数据到RedisCluster

#!/usr/bin/env python3.7.4#-*- coding:utf-8 -*-#--------------------#File: rds2rdsCluster.py#Project: 将单点redis的数据迁移到cluster#Author: Aven#Last Modified: 2020-07-09 14:20:29#安装依赖命令如下:#pip3 install redis-py-cluster#pip3 install redis#--------------------

importsysimportredisfrom rediscluster importRedisClusterimportloggingfrom datetime importdatetime as dtfrom pathlib importPath#init log

logger =logging.getLogger()

formatter= logging.Formatter(fmt="%(asctime)s %(levelname)s - %(message)s", datefmt="[%Y/%m/%d %H:%M:%S]")

logger.setLevel(logging.INFO)#file log

base_path = Path(__file__).parent

log_dir= base_path / 'logs'log_dir.mkdir(parents=True, exist_ok=True)

filename= dt.strftime(dt.now(), '%Y-%m-%d.log')

handler= logging.FileHandler(log_dir/filename, 'a', 'utf-8')

handler.setFormatter(formatter)

logger.addHandler(handler)#console log

console =logging.StreamHandler()

console.setLevel(logging.INFO)

console.setFormatter(formatter)

logger.addHandler(console)

sourceReidsHost='192.168.1.16'sourceReidsPort=6379sourceReidsDatabase=0

sourceReidsPassword=None

sourceRedis= redis.StrictRedis(host=sourceReidsHost, port=sourceReidsPort, db=sourceReidsDatabase,password=sourceReidsPassword)

targetRedisClusterNodes=[

{'host': '192.168.1.23', 'port': 7001},

{'host': '192.168.1.23', 'port': 7002},

{'host': '192.168.1.23', 'port': 7003}

]

targetRedisClusterPassword= '123456'targetRedisCluster= RedisCluster(startup_nodes=targetRedisClusterNodes, decode_responses=True, password=targetRedisClusterPassword)

count=0

patterns=[

(r'hkey1:*','hash'),

(r'hkey2:*','hash'),

(r'skey3:*','string')

]#默认过期时间1个月

defaultRedisTimeoutSeconds=30*24*60*60

defcopyData(kind,keys):if "hash"==kind:returncopyHashData(keys)elif "string"==kind:returncopyStringData(keys)defcopyStringData(keys):for key inkeys:

data=sourceRedis.get(key)

targetRedisCluster.setex(key,defaultRedisTimeoutSeconds,data)

logging.info('copy the key :{0},type:string'.format(key))defcopyHashData(keys):for key inkeys:

data=sourceRedis.hgetall(key)

targetRedisCluster.hmset(key,data)

targetRedisCluster.expire(key,defaultRedisTimeoutSeconds)

logging.info('copy the key :{0},type:hash'.format(key))

logging.info ("Begin Scan From Redis({})..".format(sourceReidsHost))

cur=0

keys=[]for pattern inpatterns:while (1==1):

count=count+1a, b= sourceRedis.scan(cursor=cur, match=pattern[0], count=1000)

cur=aif(cur==0):#logging.info('the key pattern:{} has None Data '.format(pattern))

break

if(len(b)>0):#keys.append(b)

keys=keys+b#copy the data to redisCluster

copyData(pattern[1],b)

logging.info ('Scaned Over! copied {0} keys'.format(len(keys)))