Python 爬虫 豆瓣电影

"""
分析榜单中数据来源于 那一个网络请求
https://movie.douban.com/chart
"""
# 1. 引入相关模块
import requests
from bs4 import BeautifulSoup
import pymongo

headers = {
	# 代表使用谷歌浏览器 发起请求
	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
}
# 2. 发起请求  headers 代表请求头
# 解决状态码为 418
res = requests.get("https://movie.douban.com/chart", headers=headers)
# 3. 在请求结果数据中 解析榜单
soup = BeautifulSoup(res.text, parser="html.parser")
movies = soup.find_all("tr", attrs={"class": "item"})

items = []
for movie in movies:
	# 4. 解析每一部电影数据
	movie_soup = BeautifulSoup(str(movie), parser='html.parser')
	main_img = movie_soup.find("img")
	title = movie_soup.find("span", attrs={"style": "font-size:13px;"})
	desc = movie_soup.find("p", attrs={"class": "pl"})
	rate = movie_soup.find("div", attrs={"class": "star"})
	rate_value = rate.find("span", attrs={"class": "rating_nums"})
	rate_num = rate.find("span", attrs={"class": "pl"})

	items.append({
		"main_img": main_img.attrs['src'],
		"title": title.text,
		"desc": desc.text,
		"rate_value": float(rate_value.text) if rate_value else 0,
		"rate_num": float(rate_num.text[1:-4]) if rate_num.text != "(暂无评分)" else 0
	})

# 5. 将数据保存到mongodb
# 构建客户端
client = pymongo.MongoClient()
# 创建数据库
db = client.get_database("豆瓣")
# 创建集合
collection = db.get_collection("电影榜单")
# 插入数据
collection.insert_many(items)
# 关闭数据
client.close()