读取本地文件以字典的形式存入redis

存入redis

from redis import Redis
import json


client = Redis(host='', db=1)

with open("court.txt", "a", encoding='utf-8')as f:
    f.write(json.dumps(data, ensure_ascii=False))
    f.write("\n")

with open("court.txt", "r", encoding="utf-8") as f:
    court_set = set(f.readlines())

for fydw in court_set:
    fydw = fydw.strip()
    client.sadd("court", fydw)
    

注意：redis3.0以上的版本不支持将字典作为键保存，redis==2.10.6

读取数据

def run():
    spider = Spider()
    while True:
        result = client.spop('court')
        if result:
            court_dict = eval(result.decode())
            fydw = court_dict["fydw"]
            pages = court_dict["pages"]
            spider.get_page(fydw, pages)
        else:
            break
      
def main():
    thread_list = []
    try:
        for i in range(10):
            t = threading.Thread(target=run, args=())
            thread_list.append(t)
        for i in range(10):
            thread_list[i].start()
        for i in range(10):
            thread_list[i].join()
    except Exception as e:
        logger.error(e)

注意：读取数据注意编码问题