通过 python 脚本迁移 Redis 数据

背景

• 需求：需要将的 Redis 数据迁移由云厂商 A 迁移至云厂商 B
• 问题：云版本的 Redis 版本不支持 SYNC、MIGRATE、BGSAVE 等命令，使得许多工具用不了（如 redis-port）

思路

• （1）从 Redis A 获取所有 key，依次导出数据
• （2）将导出的数据加载到 Redis B

实现

示例：将 127.0.0.1:6379 数据迁移至 127.0.0.1:6380

导出脚本 export.py

from optparse import OptionParser
import os
import redis
import logging
import sys

mylogger = None
fdata = None


def new_logger(
        logger_name='AppName',
        level=logging.INFO,
        to_file=True,
        log_file_name="app.log",
        format='[%(asctime)s] %(filename)s:%(lineno)d %(message)s'):
    if to_file:
        handler = logging.FileHandler(log_file_name)
    else:
        handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(logging.Formatter(format))

    logger = logging.getLogger(logger_name)
    logger.addHandler(handler)
    logger.setLevel(level)

    return logger


def gen_redis_proto(*args):
    '''Write out the string in redis protocol so it can be replayed back later'''
    proto = '*{0}\\r\\n'.format(len(args))
    for arg in args:
        proto += '${0}\\r\\n'.format(len(arg))
        proto += '{0}\\r\\n'.format(arg)

    return proto


# 只取出指定前缀的 key
def match_key(key: str):
    return key.startswith('normal')

def extract(options, fd, logg: logging.Logger):
    src_r = redis.StrictRedis(host=options.redis_source_url, port=options.redis_source_port, db=options.redis_source_db)
    all_keys = src_r.keys('*')
    for key in all_keys:
        key_type = ''
        arr = []
        try:
            key = key.decode('utf8')
            if not match_key(key):
                continue

            key_type = src_r.type(key).decode('utf8')
            if key_type == 'hash':
                arr.append('HMSET')
                arr.append(key)
                for k, v in src_r.hgetall(key).items():
                    arr.append(k.decode('utf8'))
                    arr.append(v.decode('utf8'))
            elif key_type == 'string':
                arr.append('SET')
                arr.append(key)
                arr.append(src_r.get(key).decode('utf8'))
            elif key_type == 'set':
                arr.append('SADD')
                arr.append(key)
                arr.extend([v.decode('utf8') for v in src_r.smembers(key)])
            elif key_type == 'list':
                arr.append('LPUSH')
                arr.append(key)
                arr.extend([v.decode('utf8') for v in src_r.lrange(key, 0, -1)])
            elif key_type == 'zset':
                arr.append('ZADD')
                arr.append(key)
                for member, score in src_r.zrange(key, 0, -1, withscores=True):
                    arr.append(str(score))
                    arr.append(member.decode('utf8'))
            else:
                # TODO 其它的数据类型
                logg.error('Unsupported key type detected: {}, key: {}'.format(key_type, key))
                continue

            fd.write(gen_redis_proto(*arr) + "\n")

			# 设置 ttl
            ttl = src_r.ttl(key)
            if ttl != -1:
                fd.write(gen_redis_proto(*['EXPIRE', key, str(ttl)]) + "\n")
        except Exception as e:
            logg.error('Unsupported key type detected: {}, key: {}, error: {}'.format(key_type, key, e.__str__()))


if __name__ == '__main__':
    parser = OptionParser()

    parser.add_option('-s', '--redis-source-url',
                      action='store',
                      dest='redis_source_url',
                      help='The url of the source redis which is to be cloned [required]')
    parser.add_option('-p', '--redis-source-port',
                      action='store',
                      dest='redis_source_port',
                      default=6379,
                      type=int,
                      help='The port of the source redis which is to be cloned [required, \
                            default: 6379]')
    parser.add_option('-n', '--redis-source-db',
                      action='store',
                      dest='redis_source_db',
                      default=0,
                      type=int,
                      help='The db num of the source redis[required, default: 0]')

    parser.add_option('-o', '--redis-data-output-file-name',
                      action='store',
                      dest='redis_data_output_file_name',
                      default='redis.data',
                      type=str,
                      help='The output file name of the source redis data[required, default: redis.data]')

    parser.add_option('-l', '--log-file-name',
                      action='store',
                      dest='log_file_name',
                      default='app.log',
                      type=str,
                      help='The log file name[required, default: app.log]')

    (options, args) = parser.parse_args()

    if not (options.redis_source_url and options.redis_source_port):
        parser.error('redis-source-url, redis-source-port are required arguments. Please see help')

    data_path = options.redis_data_output_file_name
    if os.path.exists(data_path):
        os.remove(data_path)

    mylogger = new_logger(to_file=True, level=logging.ERROR, log_file_name=options.log_file_name)
    with open(data_path, 'a+') as fd:
        extract(options, fd, mylogger)

导出数据

$ python export.py -s 127.0.0.1 -p 6379

$ head redis.data
*5\r\n$5\r\nLPUSH\r\n$11\r\nnormalQueue\r\n$3\r\nccc\r\n$2\r\nbb\r\n$3\r\naaa\r\n
*8\r\n$4\r\nSADD\r\n$9\r\nnormalSet\r\n$2\r\ndd\r\n$2\r\nbb\r\n$2\r\ncc\r\n$2\r\nee\r\n$2\r\naa\r\n$2\r\nff\r\n
*3\r\n$6\r\nEXPIRE\r\n$9\r\nnormalSet\r\n$4\r\n1728\r\n
*6\r\n$5\r\nHMSET\r\n$10\r\nnormalHash\r\n$2\r\nk1\r\n$2\r\nv1\r\n$2\r\nk2\r\n$2\r\nv2\r\n
*3\r\n$3\r\nSET\r\n$9\r\nnormalStr\r\n$3\r\nvvv\r\n

导入脚本 load.sh

#!/bin/sh

while read -r line
do
    nohup printf "%b" "$line"| redis-cli -p 6380 --pipe >> load-std.log 2>> load-err.log &
done < $1

导入数据

sh load.sh redis.data

参考

• https://stackoverflow.com/questions/44288974/sync-with-master-failed-err-unknown-command-sync
• https://gist.github.com/jimmyislive/0efd7a6a1c7f7afd73e8#file-clone_redis-py