requests+re库爬取豆瓣图书TOP250

最新推荐文章于 2024-10-13 20:33:56 发布

看破说不破

最新推荐文章于 2024-10-13 20:33:56 发布

阅读量310

点赞数

文章标签： python 正则表达式 httpwebrequest

本文链接：https://blog.csdn.net/con_learn/article/details/105973555

版权

import requests
import re
import time

# 通用代码框架
def getHTMLText(url):
    headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
     'Cookies':'ll="118209"; bid=2L02TY_GX4U; __utma=30149280.1282056899.1583758170.1583758170.1583758170.1; __utmc=30149280; __utmz=30149280.1583758170.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; gr_user_id=383b4b10-d648-4a2b-8353-5cdb486d7c79; '
    }
    try:
        r = requests.get(url, headers=headers, timeout=30)
        r.raise_for_status()  # 如果状态不是200，引发HTTPError异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

def parse_one_page(html):
    pattern = re.compile('<table.*?pl2.*?title="(.*?)".*?pl.*?>(.*?)</p>.*?rating_nums.*?>(.*?)</span>.*?</table>',
                         re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            'title':item[0].strip(),
            'author': item[1].split('/')[0].strip(),
            'score': item[2].strip()
        }

def main(start):
    url = 'https://book.douban.com/top250?start=' + str(start)
    html = getHTMLText(url)
    for item in parse_one_page(html):
        print(item)

if __name__ == '__main__':
    for i in range(10):
        main(start = i * 25)
        time.sleep(1)