今天开始学爬虫

import requests
from bs4 import BeautifulSoup
import pandas as pd

base_url = "https://movie.douban.com/top250"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"}


def fetch_page_data(url):
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, "html.parser")
    movie_items = soup.find_all('div', class_='item')

    data = []
    for item in movie_items:
        rank = item.find('em').text
        title = item.find('span', class_='title').text
        link = item.find('a')['href']
        image = item.find('img')['src']
        rating = item.find('span', class_='rating_num').text
        num_reviews = item.find('div', class_='star').find_all('span')[-1].text[:-3]
        summary = item.find('span', class_='inq').text if item.find('span', class_='inq') else ""

        # 获取导演信息
        director = item.find('p').text.split('\n')[1].strip().split(' ')[1]

        data.append({
            'Rank': rank,
            'Title': title,
            'Link': link,
            'Image': image,
            'Rating': rating,
            'Number of Reviews': num_reviews,
            'Summary': summary,
            'Director': director
        })
    return data


all_data = []
for i in range(0, 250, 25):
    url = f"{base_url}?start={i}&filter="
    all_data.extend(fetch_page_data(url))

df = pd.DataFrame(all_data)
df.to_excel("Douban_Top_250_Movies.xlsx", index=False)

print("Data has been saved to Douban_Top_250_Movies.xlsx")

import pandas as pd
import matplotlib.pyplot as plt

# 加载数据
df = pd.read_excel("Douban_Top_250_Movies.xlsx", engine='openpyxl')

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

# 查看数据规模及字段的数据类型
print("数据规模:", df.shape)
print("字段数据类型:\n", df.dtypes)

# 查看字段的统计信息
print("统计信息:\n", df.describe())

# 删除缺失值记录
clean_df = df.dropna()
print("处理后记录数:", clean_df.shape[0])

# 保存清洗后的数据
clean_df.to_excel("Douban_Top_250_Movies_Clean.xlsx", index=False)

# 可视化1: 任选10部电影名称评分柱状图
plt.figure(figsize=(10, 5))
sample_movies = clean_df.sample(10)
plt.bar(sample_movies['Title'], sample_movies['Rating'], color='pink')
plt.xlabel('电影名称')
plt.ylabel('评分')
plt.title('XX - 10部电影评分柱状图')
plt.xticks(rotation=45)
plt.tight_layout()
plt.savefig('Visualization_1.png')
plt.show()

# 可视化2: 任选4位导演上榜电影数量柱状图
directors = clean_df['Director'].value_counts().index[:4]
director_counts = clean_df['Director'].value_counts().values[:4]

plt.figure(figsize=(10, 5))
plt.bar(directors, director_counts, color='green')
plt.xlabel('导演')
plt.ylabel('上榜电影数量')
plt.title('从聪202209305 - 导演上榜电影数量柱状图')
plt.tight_layout()
plt.savefig('Visualization_2.png')
plt.show()

# 可视化3: 10部电影评论人数趋势图
sample_movies = clean_df.sample(10)
plt.figure(figsize=(10, 5))
plt.plot(sample_movies['Title'], sample_movies['Number of Reviews'], marker='o', linestyle='-', color='orange')
plt.xlabel('电影名称')
plt.ylabel('评论人数')
plt.title('XX - 10部电影评论人数趋势图')
plt.xticks(rotation=45)
plt.tight_layout()
plt.savefig('Visualization_3.png')
plt.show()