import requests from bs4 import BeautifulSoup import pandas as pd base_url = "https://movie.douban.com/top250" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"} def fetch_page_data(url): response = requests.get(url, headers=headers) soup = BeautifulSoup(response.text, "html.parser") movie_items = soup.find_all('div', class_='item') data = [] for item in movie_items: rank = item.find('em').text title = item.find('span', class_='title').text link = item.find('a')['href'] image = item.find('img')['src'] rating = item.find('span', class_='rating_num').text num_reviews = item.find('div', class_='star').find_all('span')[-1].text[:-3] summary = item.find('span', class_='inq').text if item.find('span', class_='inq') else "" # 获取导演信息 director = item.find('p').text.split('\n')[1].strip().split(' ')[1] data.append({ 'Rank': rank, 'Title': title, 'Link': link, 'Image': image, 'Rating': rating, 'Number of Reviews': num_reviews, 'Summary': summary, 'Director': director }) return data all_data = [] for i in range(0, 250, 25): url = f"{base_url}?start={i}&filter=" all_data.extend(fetch_page_data(url)) df = pd.DataFrame(all_data) df.to_excel("Douban_Top_250_Movies.xlsx", index=False) print("Data has been saved to Douban_Top_250_Movies.xlsx")
import pandas as pd import matplotlib.pyplot as plt # 加载数据 df = pd.read_excel("Douban_Top_250_Movies.xlsx", engine='openpyxl') plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['axes.unicode_minus'] = False # 查看数据规模及字段的数据类型 print("数据规模:", df.shape) print("字段数据类型:\n", df.dtypes) # 查看字段的统计信息 print("统计信息:\n", df.describe()) # 删除缺失值记录 clean_df = df.dropna() print("处理后记录数:", clean_df.shape[0]) # 保存清洗后的数据 clean_df.to_excel("Douban_Top_250_Movies_Clean.xlsx", index=False) # 可视化1: 任选10部电影名称评分柱状图 plt.figure(figsize=(10, 5)) sample_movies = clean_df.sample(10) plt.bar(sample_movies['Title'], sample_movies['Rating'], color='pink') plt.xlabel('电影名称') plt.ylabel('评分') plt.title('XX - 10部电影评分柱状图') plt.xticks(rotation=45) plt.tight_layout() plt.savefig('Visualization_1.png') plt.show() # 可视化2: 任选4位导演上榜电影数量柱状图 directors = clean_df['Director'].value_counts().index[:4] director_counts = clean_df['Director'].value_counts().values[:4] plt.figure(figsize=(10, 5)) plt.bar(directors, director_counts, color='green') plt.xlabel('导演') plt.ylabel('上榜电影数量') plt.title('从聪202209305 - 导演上榜电影数量柱状图') plt.tight_layout() plt.savefig('Visualization_2.png') plt.show() # 可视化3: 10部电影评论人数趋势图 sample_movies = clean_df.sample(10) plt.figure(figsize=(10, 5)) plt.plot(sample_movies['Title'], sample_movies['Number of Reviews'], marker='o', linestyle='-', color='orange') plt.xlabel('电影名称') plt.ylabel('评论人数') plt.title('XX - 10部电影评论人数趋势图') plt.xticks(rotation=45) plt.tight_layout() plt.savefig('Visualization_3.png') plt.show()