redis去重

import hashlib
import redis

"""
利用redis的集合不允许添加重复元素来进行去重
"""

import configparser

config = configparser.ConfigParser()
config.read('db_conf')
host = config['redis']['host']
port = config.getint('redis', 'port')
db = config.getint('redis', 'db')
password = config.get('redis', 'password')


def redis_init():
    """
    初始化redis
    :param parasecname:
    :return: redis连接池
    """
    pool = redis.ConnectionPool(host=host, port=port, db=db, password=password)
    # r = redis.Redis(connection_pool=pool)
    r = redis.StrictRedis(connection_pool=pool)
    return pool, r


def sha1(x):
    sha1obj = hashlib.sha1()
    sha1obj.update(x.encode('utf-8'))
    hash_value = sha1obj.hexdigest()
    return hash_value


def rem_redis(r, check_str, set_name):
    hash_value = sha1(check_str)
    r.srem(set_name, hash_value)


def add_redis(r, check_str, set_name):
    """
    将check_str加入redis连接r的set_name集合中
    :param r:redis连接
    :param check_str:被添加的字符串
    :param set_name:项目所使用的集合名称，建议如下格式：”projectname:task_remove_repeate“
    :return:
    """
    hash_value = sha1(check_str)
    r.sadd(set_name, hash_value)


def is_new(r, check_str, set_name):
    """
    check_str是否为set_name的新元素 新元素返回1 重复元素返回0
    :param r:
    :param check_str:
    :param set_name:
    :return:
    """
    hash_value = sha1(check_str)#加密，节省空间
    return 0 if r.sismember(set_name, hash_value) else 1


def redis_close(pool):
    """
    释放redis连接池
    :param pool:
    :return:
    """
    pool.disconnect()