# 导入相应模块
from bs4 import BeautifulSoup
from random import randint
import requests, time
import pandas as pd
# 模拟浏览器信息
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'}
#将所有页面的url创建为列表
url_list = ['https://book.douban.com/top250']
base_url = 'https://book.douban.com/top250?start={start}'
for start in range(25, 226, 25):
url_list.append(base_url.format(start=start))
# 创建一个空列表,用于保存书籍信息
book_info = []
# 循环访问页面url来获取书籍信息
for url in url_list:
print("开始爬取数据,", url)
# 每爬取一页信息,随机停顿1-2秒
time.sleep(randint(1, 3))
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, features='html.parser')
book_itmes = soup.find_all('tr', class_='item')
for book_itme in book_itmes:
# 书籍名称
book_title = book_itme.find('div', class_='pl2').find('a').get('title')
# 出版信息,为了美观和统一,替换中文括号为英文括号
publish_info = book_itme.find('p', class_='pl').string.replace('【', '[').replace('】', ']').replace('(丹麦)','[丹麦] ').strip()
# 评分
rating_num = book_itme.find('span', class_='rating_nums').string.strip()
# 投票人数
people = book_itme.find('div', class_='star clearfix').find('span', class_='pl').string.replace('\n','').replace('\r', '').replace('(', '').replace(')', '').replace('人评价', '').strip()
# 将以上信息汇总成字典
mock_data = {'书名': book_title, '出版信息': publish_info, '评分': rating_num, '投票人数': people}
# 将每本书的字典信息添加到列表
book_info.append(mock_data)
print("数据爬取完成,准备写入文件book.xlsx中")
df = pd.DataFrame(book_info, columns=['书名', '出版信息', '评分', '投票人数'])
file_path = './book.xlsx'
df.to_excel(file_path, index=False)
print("数据写入完成,请查看book.xlsx文件")
爬虫经典案例之爬取豆瓣读书Top250的书籍信息
于 2024-06-18 11:18:45 首次发布