#!/usr/bin/env python3.7.4#-*- coding:utf-8 -*-#--------------------#File: rds2rdsCluster.py#Project: 将单点redis的数据迁移到cluster#Author: Aven#Last Modified: 2020-07-09 14:20:29#安装依赖命令如下:#pip3 install redis-py-cluster#pip3 install redis#--------------------
importsysimportredisfrom rediscluster importRedisClusterimportloggingfrom datetime importdatetime as dtfrom pathlib importPath#init log
logger =logging.getLogger()
formatter= logging.Formatter(fmt="%(asctime)s %(levelname)s - %(message)s", datefmt="[%Y/%m/%d %H:%M:%S]")
logger.setLevel(logging.INFO)#file log
base_path = Path(__file__).parent
log_dir= base_path / 'logs'log_dir.mkdir(parents=True, exist_ok=True)
filename= dt.strftime(dt.now(), '%Y-%m-%d.log')
handler= logging.FileHandler(log_dir/filename, 'a', 'utf-8')
handler.setFormatter(formatter)
logger.addHandler(handler)#console log
console =logging.StreamHandler()
console.setLevel(logging.INFO)
console.setFormatter(formatter)
logger.addHandler(console)
sourceReidsHost='192.168.1.16'sourceReidsPort=6379sourceReidsDatabase=0
sourceReidsPassword=None
sourceRedis= redis.StrictRedis(host=sourceReidsHost, port=sourceReidsPort, db=sourceReidsDatabase,password=sourceReidsPassword)
targetRedisClusterNodes=[
{'host': '192.168.1.23', 'port': 7001},
{'host': '192.168.1.23', 'port': 7002},
{'host': '192.168.1.23', 'port': 7003}
]
targetRedisClusterPassword= '123456'targetRedisCluster= RedisCluster(startup_nodes=targetRedisClusterNodes, decode_responses=True, password=targetRedisClusterPassword)
count=0
patterns=[
(r'hkey1:*','hash'),
(r'hkey2:*','hash'),
(r'skey3:*','string')
]#默认过期时间1个月
defaultRedisTimeoutSeconds=30*24*60*60
defcopyData(kind,keys):if "hash"==kind:returncopyHashData(keys)elif "string"==kind:returncopyStringData(keys)defcopyStringData(keys):for key inkeys:
data=sourceRedis.get(key)
targetRedisCluster.setex(key,defaultRedisTimeoutSeconds,data)
logging.info('copy the key :{0},type:string'.format(key))defcopyHashData(keys):for key inkeys:
data=sourceRedis.hgetall(key)
targetRedisCluster.hmset(key,data)
targetRedisCluster.expire(key,defaultRedisTimeoutSeconds)
logging.info('copy the key :{0},type:hash'.format(key))
logging.info ("Begin Scan From Redis({})..".format(sourceReidsHost))
cur=0
keys=[]for pattern inpatterns:while (1==1):
count=count+1a, b= sourceRedis.scan(cursor=cur, match=pattern[0], count=1000)
cur=aif(cur==0):#logging.info('the key pattern:{} has None Data '.format(pattern))
break
if(len(b)>0):#keys.append(b)
keys=keys+b#copy the data to redisCluster
copyData(pattern[1],b)
logging.info ('Scaned Over! copied {0} keys'.format(len(keys)))