爬虫经典案例之爬取豆瓣读书Top250的书籍信息

# 导入相应模块
from bs4 import BeautifulSoup
from random import randint
import requests, time
import pandas as pd

# 模拟浏览器信息
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'}

#将所有页面的url创建为列表
url_list = ['https://book.douban.com/top250']
base_url = 'https://book.douban.com/top250?start={start}'
for start in range(25, 226, 25):
    url_list.append(base_url.format(start=start))

# 创建一个空列表，用于保存书籍信息
book_info = []
# 循环访问页面url来获取书籍信息
for url in url_list:
    print("开始爬取数据，", url)
    # 每爬取一页信息，随机停顿1-2秒
    time.sleep(randint(1, 3))
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, features='html.parser')
    book_itmes = soup.find_all('tr', class_='item')
    for book_itme in book_itmes:
    	# 书籍名称
        book_title = book_itme.find('div', class_='pl2').find('a').get('title')
        # 出版信息，为了美观和统一，替换中文括号为英文括号
        publish_info = book_itme.find('p', class_='pl').string.replace('【', '[').replace('】', ']').replace('（丹麦）','[丹麦] ').strip()
        # 评分
        rating_num = book_itme.find('span', class_='rating_nums').string.strip()
        # 投票人数
        people = book_itme.find('div', class_='star clearfix').find('span', class_='pl').string.replace('\n','').replace('\r', '').replace('(', '').replace(')', '').replace('人评价', '').strip()
        # 将以上信息汇总成字典
        mock_data = {'书名': book_title, '出版信息': publish_info, '评分': rating_num, '投票人数': people}
        # 将每本书的字典信息添加到列表
        book_info.append(mock_data)
print("数据爬取完成，准备写入文件book.xlsx中")
df = pd.DataFrame(book_info, columns=['书名', '出版信息', '评分', '投票人数'])
file_path = './book.xlsx'
df.to_excel(file_path, index=False)
print("数据写入完成，请查看book.xlsx文件")