查询Redis未设置超时时间KEY

需求

1. 查询redis中ttl=-1的key
2. 输出到文档中

代码 (python)

# encoding: utf-8

import redis
import argparse
import time
import sys
import os

# """
# author: Cyberpunch
# time: 2020-03-15 22:57:03
# func: 获取数据库中没有设置ttl的 key
# useing: python RedisScanTTL-2.py -host {域名} -port {端口} -password {密码} -db {分库列表，ex： 0,1,2}
# """


class ShowProcess:
    # """
    # 显示处理进度的类
    # 调用该类相关函数即可实现处理进度的显示
    # """
    i = 0  # 当前的处理进度
    max_steps = 0  # 总共需要处理的次数
    max_arrow = 50  # 进度条的长度

    # 初始化函数，需要知道总共的处理次数
    def __init__(self, max_steps):
        self.max_steps = max_steps
        self.i = 0

    # 显示函数，根据当前的处理进度i显示进度
    # 效果为[>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>]100.00%
    def show_process(self, i=None):
        if i is not None:
            self.i = i
        else:
            self.i += 1
        num_arrow = int(self.i * self.max_arrow / self.max_steps)  # 计算显示多少个'>'
        num_line = self.max_arrow - num_arrow  # 计算显示多少个'-'
        percent = self.i * 100.0 / self.max_steps  # 计算完成进度，格式为xx.xx%
        process_bar = '[' + '>' * num_arrow + ' ' * num_line + ']' \
                      + '%.2f' % percent + '%' + '\r'  # 带输出的字符串，'\r'表示不换行回到最左边
        sys.stdout.write(process_bar)  # 这两句打印字符到终端
        sys.stdout.flush()

    def close(self, words='done'):
        print
        ''
        print
        words
        self.i = 0


def check_ttl(redis_conn, no_ttl_file, dbindex):
    start_time = time.time()
    no_ttl_num = 0
    keys_num = redis_conn.dbsize()
    print
    "there are {num} keys in db {index} ".format(num=keys_num, index=dbindex)
    process_bar = ShowProcess(keys_num)
    with open(no_ttl_file, 'a') as f:

        for key in redis_conn.scan_iter(count=1000):
            process_bar.show_process()
            if redis_conn.ttl(key) == -1:
                no_ttl_num += 1
                if no_ttl_num < 1000:
                    f.write(key + '\n')
            else:
                continue

    process_bar.close()
    print
    "cost time(s):", time.time() - start_time
    print
    "no ttl keys number:", no_ttl_num
    print
    "we write keys with no ttl to the file: %s" % no_ttl_file


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-password', type=str, dest='password', action='store', help='password of redis ')
    parser.add_argument('-port', type=int, dest='port', action='store', help='port of redis ')
    parser.add_argument('-db', type=str, dest='db_list', action='store', default=0,
                        help='ex : -d all / -d 1,2,3,4 ')
    parser.add_argument('-host', type=str, dest='host', action='store', default='127.0.0.1',
                        help='host of redis ')

    args = parser.parse_args()
    port = args.port
    password = args.password
    host = args.host
    # 当前路径
    path = os.getcwd()
    # db列表
    if args.db_list == 'all':
        db_list = [i for i in xrange(0, 16)]
    else:
        db_list = [int(i) for i in args.db_list.split(',')]

    for index in db_list:
        try:
            pool = redis.ConnectionPool(host=host, port=port, password=password, db=index)
            r = redis.StrictRedis(connection_pool=pool)
        except redis.exceptions.ConnectionError as e:
            print
            e
        else:
            no_ttl_keys_file = "{path}/{host}_{port}_{db}_no_ttl_keys.txt".format(path=path, host=host, port=port,
                                                                                  db=index)
            check_ttl(r, no_ttl_keys_file, index)


if __name__ == '__main__':
    main()

代码（Shell）

• 会创建历史记录文件
• 可以中断继续

db_host={你的redis host}
db_port={你的redis port}
db_pwd={你的redis password}

async_count=200

function checkTTL() {
   host=$1
   port=$2
   pwd=$3
   key=$4

   ttl_result=$(redis-cli -h $host -p $port -a $pwd TTL $key)
   if [[ $ttl_result == -1 ]]; then
   	echo $key >>./tmp/no_ttl_scan.log
   fi
}

function fixTTL() {
   host=$1
   port=$2
   pwd=$3
   key=$4

   ttl_result=$(redis-cli -h $host -p $port -a $pwd TTL $key)
   if [[ $ttl_result == -1 ]]; then
   	new_ttl=$(($RANDOM + 2592000))
   	reuslt=$(redis-cli -h $host -p $port -a $pwd EXPIRE $key $new_ttl)
   	echo "$key | $new_ttl | $reuslt" >>./tmp/no_ttl_fix.log
   fi
}

# 历史记录文件夹
if [ ! -d "./tmp" ]; then
   mkdir ./tmp
fi

# info中获取key总数
if [ ! -f ./tmp/KEYSPACE ]; then
   redis-cli -h $db_host -p $db_port -a $db_pwd INFO KEYSPACE >./tmp/KEYSPACE
   echo "INFO KEYSPACE重新查询"
fi
keys_count=$(cat ./tmp/KEYSPACE | grep 'keys' | awk '{t=$0;gsub( ".*keys=|,expires.*" ,"" ,t );print t}' | awk -FS '{sum+=$1} END {print sum}')
echo "KEY总数[$keys_count]"

# count列表统计count总数
if [ -f "./tmp/count_list" ]; then
   cat "./tmp/count_list" | awk -FS '{sum+=$1} END {print sum}' >>./tmp/count_total
else
   echo "0" >>./tmp/count_total
fi

# 历史scan游标
new_cursor=$(sed -n '1p' ./tmp/scan_tmp)
if [[ -n $new_cursor ]]; then
   echo "从历史游标[$new_cursor]开始"
else
   echo "从全新游标[0]开始"
   new_cursor=0
fi

# 循环开始
while :; do
   {

   	if [[ ! "$new_cursor" =~ ^[0-9]+$ ]]; then
   		echo "当前游标异常[$new_cursor], 退出"
   		exit
   	fi

   	echo "当前游标[$new_cursor], 每次查询[$async_count]个"
   	redis-cli -h $db_host -p $db_port -a $db_pwd SCAN $new_cursor COUNT $async_count >./tmp/scan_tmp
   	new_cursor=$(sed -n '1p' ./tmp/scan_tmp)

   	if [[ $new_cursor == 0 ]]; then
   		echo "当前游标[$new_cursor], 退出"
   		break 2
   	fi

   	sed -n '2,$p' ./tmp/scan_tmp >./tmp/key_tmp
   	cat ./tmp/key_tmp | while read line; do
   		if [[ $1 == "-f" ]]; then
   			fixTTL $db_host $db_port $db_pwd $line &
   		else
   			checkTTL $db_host $db_port $db_pwd $line &
   		fi
   	done
   	wait

   	echo $async_count >>./tmp/count_list

   	total=$(($(sed -n '$p' ./tmp/count_total) + $async_count))

   	echo $total >>./tmp/count_total

   	echo "当前已经扫描[$total|$keys_count]"

   	sleep 0.2
   }
done