简单爬虫函数的封装（Python）

def downloadPage2(url, req_num=10, charset='utf-8', user_agent=None):
    """
    下载网页信息,user-Agent的设置
    处理了500-600的错误
    限制了错误处理的上限次数
    网页的编码设置
    :param url: 请求的url连接
    :return: 返回值
    """
    if not user_agent:
        user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)\
            AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'
    headers = {'User-Agent': user_agent}
    req = request.Request(url, headers=headers)
    try:
        response = request.urlopen(req)
        with open('test.html', 'wb') as f:
            f.write(response.read())
    except error.HTTPError as e:
        # 服务器错误
        print(e.code)
        html = None
        if req_num > 0:
            if hasattr(e, 'code') and 500 <= e.code <=