爬虫requests请求自定义类——（实现自动更换ip，自动错误重试，判断返回内容是否为目标内容）

import json
import random
import requests
from time import sleep


# 请求出错自动重试
# 自动切换代理
class RequestsHelper:

    def __init__(self):
        self.__retry_count = 3
        self.__switch_ip_count = 3
        self.__proxies = self.__get_new_ip()

    def __get_new_ip(self):
        url = 'http://route.xiongmaodaili.com/xiongmao-web/api/glip?secret=82af34f3189d1b3b4be072357341fc08&orderNo=GL20230711173724NIfZ0w5e&count=1&isTxt=1&proxyType=1'
        try:
            response = requests.get(url)
            result = response.content.decode()
            ip = result.replace('\r', '').replace('\n', '')
            print("获取一个新IP：", ip)
            return {"http": "http://" + ip, "https": "http://" + ip}
        except:
            return None

    def get(self, url: str, headers: dict = None, params: dict = None, cookies: dict = None):
        if headers is None:
            headers = {}
        if params is None:
            params = {}

        for i in range(self.__switch_ip_count):
            for j in range(self.__retry_count):
                try:
                    response = requests.get(url, headers=headers, params=params, proxies=self.__proxies, timeout=10, cookies=cookies)
                    # 有时候请求并没有发生异常 但是请求的结果不包含想要的数据 也需要做判断
                    # 以中国审判信息流程公开网为案例 有两种异常情况
                    # 1. 返回一个字典{"pageCheck":true,"pageNum":1}
                    # 2. 返回一个html页面
                    # 先判断是不是json格式 不是会抛出异常 如果是json格式 则判断是否包含想要的数据
                    result = json.loads(response.text)
                    if result['data'] == 'nulll':
                        print(f"返回的数据为空, 请求次数{j}, 重新请求")
                        continue
                    else:
                        return response

                except Exception as e:
                    print(e)
                    sleep(random.randint(1, 2))
            sleep(2)
            self.__proxies = self.__get_new_ip()
        return None

    def post(self, url: str, headers: dict = None, params: dict = None, data: dict = None, json_1: dict = None,
             cookies: dict = None):
        if headers is None:
            headers = {}
        if params is None:
            params = {}
        if data is None:
            data = {}
        if json_1 is None:
            json_1 = {}
        if cookies is None:
            cookies = {}

        for i in range(self.__switch_ip_count):
            for j in range(self.__retry_count):
                try:
                    response = requests.post(url, headers=headers, params=params, data=data, json=json_1, cookies=cookies,
                                            proxies=self.__proxies, timeout=10)
                    # 有时候请求并没有发生异常 但是请求的结果不包含想要的数据 也需要做判断
                    # 以中国审判信息流程公开网为案例 有两种异常情况
                    # 1. 返回一个字典{"pageCheck":true,"pageNum":1}
                    # 2. 返回一个html页面
                    # 先判断是不是json格式 不是会抛出异常 如果是json格式 则判断是否包含想要的数据
                    result = json.loads(response.text)
                    '''如果data为空 重新请求'''
                    if result['data'] is None:
                        print(f"返回的数据为空, 请求次数{j}, 重新请求")
                        continue
                    else:
                        return response

                except Exception as e:
                    print(e)
                    sleep(random.randint(1, 3))
            sleep(5)
            self.__proxies = self.__get_new_ip()
        return None


# if __name__ == '__main__':
#     requests_helper = RequestsHelper()
#     response = requests_helper.post()
#     if response:
#         print(response.json())
#     else:
#         print("请求失败")