python学习——爬虫request异常代码

一、网上摘录的https://blog.csdn.net/xunkhun/article/details/79349621 

def getHTMLText(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

r.raise_for_status()会跑出异常，没有异常则是None。抓取一切异常

二、自己用的，也是抄袭这位博文（https://blog.csdn.net/polyhedronx/article/details/81589196）

def get_text_from_url(url):
    # 从链接中获取网页文本
    try:
        response = requests.get(url, headers=headers, timeout=30)
        if response.status_code == 200:
            return response.text
        else:
            print('get page failed.', response.status_code)
            return None
    except (ConnectionError, TimeoutError) as e:
        print('crawling failed', url, e)
        return None