爬虫 裁判文书网爬取part2

继续上篇：

receive_data.py

import time
import requests

from method.save_data import RedisPara
from method.wenshu_method import ParseJs, GenerateFormRequest, ParseDetail, GenerateRequest, Parse_Indexjs
from static.docid import getkey, decode_docid


class ReceiveData(object):
    def __init__(self):
        self.temp_count = 0
        self.page = 0
        self.wzws_cid = ''
        self.proxy = ''
        self.User_Agent = ''
        self.proxies = {}
        self.vl5x = ''
        self.guid = ''
        self.vjkl5 = ''
        self.docid = ''
        self.param = '案件类型:刑事案件,文书类型:判决书'  # 请求列表页的查询条件
        self.p = ''  # 用来判断参数是否存在

        # 需要保存的参数
        self.sync = 0
        self.title = ''
        self.court = ''
        self.pdate = ''
        self.writ = ''
        self.reason = ''
        self.docid = ''

    def get_cookies(self):
        # 调用这个方法刷新cookie参数
        r = RedisPara()
        cookies = r.get_para()
        if cookies:
            self.wzws_cid = cookies.get('wzws_cid')
            self.proxy = cookies.get('pr')
            self.User_Agent = cookies.get('ua')
            self.proxies = {"http": "http://{}".format(self.proxy)}
            self.vjkl5 = cookies.get('vjkl5')
            context = ParseJs()
            self.vl5x, self.guid = context.get_key_para(self.vjkl5)
        return cookies

    def listpage_2(self):
        cookies = self.get_cookies()

        if not cookies:
            return None

        gen_obj = GenerateFormRequest(self.param, self.page, self.vl5x, self.guid, self.proxy, self.User_Agent,self.wzws_cid, self.vjkl5)
        try:
            resp = gen_obj.get_request()
        except Exception as e:
            resp =None
            print('错误5',e)

        return resp

    def parse_listpage_2(self):
        resp = self.listpage_2()
        if not resp:
            return None

        # 提取json中的datalist
        context = ParseDetail(resp.text)
        datalist = context.parse_list_data()
        RunEval = datalist[0]['RunEval']
        key = getkey(RunEval).encode('utf-8')

        for case in data