网络爬虫的实际运用

def get_one_page(url,offset):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'}
        response = requests.get(url=url, headers=headers,params={"offset":offset})
        if response.status_code==200:#如果返回的响应状态值为200，说明请求成功
            return response.text
            pass
        else:
            return None
            pass
        pass
    except RequestException as e:
        return None
        pass
    pass

使用正则表达式提取网站数据，yields生成器，生成器可迭代

def parse_one_page(html):
    pattern = '<dd>.*?board-index.*?">(\d+)</i>.*?data-src="(.*?)".*?/>.*?movie-item-info.*?title="(.*?)".*?star">' + \
              '(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>'
    # re.S匹配多行
    regex = re.compile(pattern, re.S)
    items = regex.findall(html)
    for item in items:
        yield {
            'index': item[0],
            'thumb': get_large_thumb(item[1]),
            'title': item[2],
            'actors': item[3].strip()[3:],
            'release_time': get_release_time(item[4].strip()[5:]),
            'area': get_release_area(item[4].strip()[5:]),
            'score': item[5] + item[6]
        }
        pass
    pass

之后通过该爬虫爬取该网站的信息。

https://blog.csdn.net/qq_42866234/article/details/82660031