Redis 数据使用备份恢复

起因 ：由于之前公司介绍说，Redis只是做缓存，并未涉及贮存部分
于 2020-4-24晚21:50左右 清理缓存导致，业务部分链接失效，最终到Redis头上
最新的Redis备份是 2020-4-24 凌晨2:30 的备份数据，（发现问题已经是26号）

目前的情况是 ： Redis 已经存在了 缓存及贮存数据包含2020-4-24 晚21:50 - 2020-4-26 的 数据

方案确定：使用备份在新的Redis实例中恢复数据，使用脚本进行对现有Redis进行插入操作，（切记是插入操作，不是覆盖）。
1、使用备份对新实例导入数据，并查看。
2、按需恢复，贮存数据在Redis DB3 库中贮存，恢复到生产的Redis DB3 上面，* 插入恢复 *
3、实现步奏：shell or Python

推荐使用Python

shell 实现以下存在缺陷不识别特殊字符

如果有兴趣可以尝试下

复制一个库的部分key

#!/bin/bash
src_ip=redis-01
src_port=6379
src_db=4
src_pw='1234'

dest_ip=redis-02
dest_port=6379
dest_db=5
desc_pw='1234'
 
#要遍历的key
k=(test ws we)

for loop in ${k[*]}
do
    redis-cli -h $src_ip -p $src_port -a $src_pw -n $src_db --raw dump $loop | perl -pe 'chomp if eof' | redis-cli -h $dest_ip -p $dest_port -a $desc_pw -n $dest_db -x restore $loop 0
    echo "The value is: $loop"
done

复制全部一个库的全部key

#!/bin/bash
src_ip=Redis-01         # 备份 Redis
src_port=6379
src_db=3
src_pw='1234'

dest_ip=Redis-02        # 目标 Redis
dest_port=6379
dest_db=3
desc_pw='1234'

redis-cli -h $src_ip -p $src_port  -a $src_pw  -n $src_db keys "*" | while read key
do
    redis-cli -h $src_ip -p $src_port -a $src_pw  -n $src_db --raw dump $key | perl -pe 'chomp if eof' | redis-cli -h $dest_ip -p $dest_port -a $desc_pw -n $dest_db -x restore $key 0
    echo "migrate key $key"
done

Python 实现

import redis
import time
import sys
import getopt


class RedisCopy:
    """A class for copying keys from one server to another.
    """

    #some key prefix for this script
    mprefix = 'mig:'
    keylistprefix = 'keylist:'
    hkeylistprefix = 'havekeylist:'

    # numbers of keys to copy on each iteration
    limit = 10000

    def __init__(self, source, target, dbs, spass, tpass):
        self.source = source
        self.target = target
        self.dbs = dbs
        self.spass = spass
        self.tpass = tpass

    def save_keylists(self, prefix="*"):
        """Function to save the keys' names of the source redis server into a list for later usage.
        """

        for db in self.dbs:
            db = int(db[0])
            servername = self.source['host'] + ":" + str(
                self.source['port']) + ":" + str(db)
            #get redis handle for server-db
            r = redis.StrictRedis(
                host=self.source['host'], port=self.source['port'], db=db, password=self.spass)

            #returns the number of keys in the current database
            dbsize = r.dbsize()
            #check whether we already have the list, if not get it
            hkl = r.get(self.mprefix + self.hkeylistprefix + servername)
            if hkl is None or int(hkl) != 1:
                print ("Saving the keys in %s to temp keylist...\n" % servername)
                moved = 0
                r.delete(self.mprefix + self.keylistprefix + servername)
                #returns a list of keys matching pattern
                for key in r.keys(prefix):
                    moved += 1
                    #push values onto the tail of the list name
                    r.rpush(self.mprefix + self.keylistprefix + servername, key)
                    if moved % self.limit == 0:
                        print  ("%d keys of %s inserted in temp keylist at %s...\n" % (moved, servername, time.strftime("%Y-%m-%d %I:%M:%S")))
                #set the value at key name to value
                r.set(self.mprefix + self.hkeylistprefix + servername, 1)
            print ("ALL %d keys of %s already inserted to temp keylist ...\n\n" % (dbsize - 1, servername))

    def copy_db(self, limit=None):
        """Function to copy all the keys from the source into the new target.
        - limit : optional numbers of keys to copy per run
        """

        #set the limit per run
        try:
            limit = int(limit)
        except (ValueError, TypeError):
            limit = None

        if limit is not None:
            self.limit = limit

        for db in self.dbs:
            servername = self.source['host'] + ":" + str(
                self.source['port']) + ":" + db[0]
            print ("Processing keys copying of server %s at %s...\n" % (
                servername, time.strftime("%Y-%m-%d %I:%M:%S")))
            #get redis handle for current source server-db
            r = redis.StrictRedis(
                host=self.source['host'], port=self.source['port'], db=int(db[0]), password=self.spass)
            moved = 0
            # dbsize without run key, keylist key, havekeylist key, firstrun key
            dbsize = r.dbsize() - 4
            #get keys already moved
            #return the value at key name, or None if the key doesn’t exist
            keymoved = r.get(self.mprefix + "keymoved:" + servername)
            keymoved = 0 if keymoved is None else int(keymoved)
            #check if we already have all keys copied for current source server-db
            if dbsize < keymoved:
                print ("ALL %d keys from %s have already been copied.\n" % (
                    dbsize, servername))
                continue

            print ("Started copy of %s keys from %d to %d at %s...\n" % (servername, keymoved, dbsize, time.strftime("%Y-%m-%d %I:%M:%S")))

            #get redis handle for corresponding target server-db
            rr = redis.StrictRedis(
                host=self.target['host'], port=self.target['port'], db=int(db[1]), password=self.tpass)

            #max index for lrange
            newkeymoved = keymoved + \
                self.limit if dbsize > keymoved + self.limit else dbsize

            #return a slice of the list name between position start and end
            for key in r.lrange(self.mprefix + self.keylistprefix + servername, keymoved, newkeymoved):
                #get key type
                ktype = r.type(key).decode('utf-8')
                # key = key.decode('utf-8')
                #if undefined type go to next key
                if ktype == 'none':
                    continue

                #save key to target server-db
                if ktype == 'string':
                    if key == self.mprefix + "run":
                        continue
                    rr.set(key, r.get(key))
                elif ktype == 'hash':
                    rr.hmset(key, r.hgetall(key))
                elif ktype == 'list':
                    if key == self.mprefix + "keylist:" + servername:
                        continue
                    #value = r.lrange(key, 0, -1)
                    #rr.rpush(key, *value)
                    for k in r.lrange(key, 0, -1):
                        rr.rpush(key, k)
                elif ktype == 'set':
                    #value = r.smembers(key)
                    #rr.sadd(key, *value)
                    for k in r.smembers(key):
                        rr.sadd(key, k)
                elif ktype == 'zset':
                    #value = r.zrange(key, 0, -1, withscores=True)
                    #rr.zadd(key, **dict(value))
                    for k, v in r.zrange(key, 0, -1, withscores=True):
                        rr.zadd(key, v, k)

                # Handle keys with an expire time set
                kttl = r.ttl(key)
                kttl = -1 if kttl is None else int(kttl)
                if kttl != -1:
                    rr.expire(key, kttl)

                moved += 1

                if moved % 10000 == 0:
                    print ("%d keys have been copied on %s at %s...\n" % (
                        moved, servername, time.strftime("%Y-%m-%d %I:%M:%S")))

            r.set(self.mprefix + "keymoved:" + servername, newkeymoved)
            print ("%d keys have been copied on %s at %s\n" % (
                newkeymoved, servername, time.strftime("%Y-%m-%d %I:%M:%S")))

    def flush_target(self):
        """Function to flush the target server.
        """
        for db in self.dbs:
            servername = self.target['host'] + ":" + str(
                self.target['port']) + ":" + db[1]
            print ("Flushing server %s at %s...\n" % (
                servername, time.strftime("%Y-%m-%d %I:%M:%S")))
            rr = redis.StrictRedis(
                host=self.target['host'], port=self.target['port'], db=int(db[1]), password=tpass)
            #delete all keys in the current database
            rr.flushdb()
            print ("Flushed server %s at %s...\n" % (
                servername, time.strftime("%Y-%m-%d %I:%M:%S")))

    def clean(self):
        """Function to clean all variables, temp lists created previously by the script.
        """

        print ("Cleaning all temp variables...\n")
        for db in self.dbs:
            servername = self.source['host'] + ":" + str(
                self.source['port']) + ":" + db[0]
            r = redis.StrictRedis(
                host=self.source['host'], port=self.source['port'], db=int(db[0]), password=spass)
            r.delete(self.mprefix + "keymoved:" + servername)
            r.delete(self.mprefix + self.keylistprefix + servername)
            r.delete(self.mprefix + self.hkeylistprefix + servername)
            r.delete(self.mprefix + "firstrun")
            r.delete(self.mprefix + "run")
        print ("Done.\n")


def main(source, target, databases, spass, tpass, limit=None, clean=False, flush=False, prefix="*"):
    #getting source and target
    if (source == target):
        exit('The 2 servers adresses are the same.')
    so = source.split(':')
    if len(so) == 2:
        source_server = {'host': so[0], 'port': int(so[1])}
    else:
        exit('Supplied source address is wrong.')

    sn = target.split(':')
    if len(sn) == 2:
        target_server = {'host': sn[0], 'port': int(sn[1])}
    else:
        exit('Supplied target address is wrong.')

    #getting the dbs
    dbs = [k.split(':') for k in databases.split(',')]
    if len(dbs) < 1:
        exit('Supplied list of db is wrong.')

    try:
        r = redis.StrictRedis(
            host=source_server['host'], port=source_server['port'], db=int(dbs[0][0]), password=spass)
    except AttributeError as e:
        exit('Please this script requires redis-py >= 2.4.10, your current version is :' + redis.__version__)

    mig = RedisCopy(source_server, target_server, dbs, spass, tpass)

    if clean == False:
        #check if script already running
        run = r.get(mig.mprefix + "run")
        if run is not None and int(run) == 1:
            exit('another process already running the script')

        r.set(mig.mprefix + "run", 1)
        mig.save_keylists(prefix)

        firstrun = r.get(mig.mprefix + "firstrun")
        firstrun = 0 if firstrun is None else int(firstrun)
        if firstrun == 0:
            if flush:
                mig.flush_target()
            r.set(mig.mprefix + "firstrun", 1)

        mig.copy_db(limit)
        # setting script completion flag
        r.set(mig.mprefix + "run", 0)

    else:
        mig.clean()

def usage():
    print (__doc__)


if __name__ == "__main__":
    clean = False
    flush = False
    prefix = "*"
    spass = tpass = None

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hl:s:t:d:fp:", ["help", "limit=", "source=", "target=", \
                                                                  "databases=", "clean", "flush", "prefix=", "spass=", "tpass="])
    except getopt.GetoptError:
        usage()
        sys.exit(2)

    for opt, arg in opts:

        if opt in ("-h", "--help"):
            usage()
            sys.exit()
        elif opt == "--clean":
            clean = True
        elif opt in ("-l", "--limit"):
            limit = arg
        elif opt in ("-s", "--source"):
            source = arg
        elif opt in ("-t", "--target"):
            target = arg
        elif opt in ("-d", "--databases"):
            databases = arg
        elif opt in ("-f", "--flush"):
            flush = True
        elif opt in ("-p", "--prefix"):
            prefix = arg
        elif opt in ("--spass"):
            spass = arg
        elif opt in ("--tpass"):
            tpass = arg

    try:
        limit = int(limit)
    except (NameError, TypeError, ValueError):
        limit = None

    try:
        main(source, target, databases, spass, tpass, limit, clean, flush, prefix)
    except NameError as e:
        usage()

python copy.py --source=redis-01:6379 --target=redis-02:6379 --databases=3:3 --spass="Pps@123456"  --tpass='Pps@1234'