使用python抓取豆瓣top250电影数据进行分析
抓取豆瓣Top250电影数据的链接和电影名称
代码如下:
import urllib.request as urlrequest
from bs4 import BeautifulSoup
import re
import csv,codecs
top250_url ='https://movie.douban.com/top250?start={}&filter='
movie_name='名称'
movie_assess='评价人数'
movie_score='评分'
movie_url ='链接'
movie_intro='介绍'
movie_num =0
#print('{} {} {} {} {}'.format(movie_name,movie_assess,movie_score,movie_url,movie_intro))
with open('top250_movie.csv','w',encoding='utf8') as outputfile:
#outputfile.write(codecs.BOM_UTF8)
writer = csv.writer(outputfile)
#writer.writerow(["movie_num","movie_name","movie_assess","movie_score","movie_url","movie_intro"])
outputfile.write("movie_num#movie_name#movie_year#movie_country#movie_type#movie_director#movie_assess#movie_score#movie_url#movie_intro\n")
for list in range(10):
movies_content = urlrequest.urlopen(top250_url.format(list*25)).read()
movies_html = movies_content.decode('utf8')
moviessoup = BeautifulSoup(movies_html,'html.parser')
all_list = moviessoup.find_all(class_='item')
#print(all_list)
for item in all_list:
item_data=item.find(class_='pic')
movie_url = item_data.find('a')['href']
movie_name = item_data.find('img')['alt']
item_info = item.find(class_='star')
info = item.find('div', attrs={
'class': 'star'})
#find_all 将star标签中的所有span 存入一个列表中
movie_assess =info.find_all('span')[3].get_text()[:-3]
movie_score = item_info.find('span',attrs={
'class':'rating_num'}).get_text()
try:
movie_intro = item.find(class_='quote').find(class_='inq').get_text()
except Exception as e