数据分析与挖掘案例之使用python抓取豆瓣top250电影数据进行分析

使用python抓取豆瓣top250电影数据进行分析

抓取豆瓣Top250电影数据的链接和电影名称

代码如下：

import urllib.request as urlrequest
from bs4 import BeautifulSoup
import re
import csv,codecs 

top250_url ='https://movie.douban.com/top250?start={}&filter='
movie_name='名称'
movie_assess='评价人数'
movie_score='评分'
movie_url ='链接'
movie_intro='介绍'
movie_num =0

#print('{} {} {} {} {}'.format(movie_name,movie_assess,movie_score,movie_url,movie_intro))
with open('top250_movie.csv','w',encoding='utf8') as outputfile:
    #outputfile.write(codecs.BOM_UTF8)
    writer = csv.writer(outputfile)
    #writer.writerow(["movie_num","movie_name","movie_assess","movie_score","movie_url","movie_intro"])
    outputfile.write("movie_num#movie_name#movie_year#movie_country#movie_type#movie_director#movie_assess#movie_score#movie_url#movie_intro\n")
    for list in range(10):
             movies_content = urlrequest.urlopen(top250_url.format(list*25)).read()
             movies_html = movies_content.decode('utf8')
             moviessoup = BeautifulSoup(movies_html,'html.parser')
             all_list = moviessoup.find_all(class_='item')
             #print(all_list)
             for item in all_list:
                 item_data=item.find(class_='pic')
                 movie_url = item_data.find('a')['href']
                 movie_name = item_data.find('img')['alt']
                 item_info = item.find(class_='star')
                 info = item.find('div', attrs={
   'class': 'star'})
                #find_all 将star标签中的所有span 存入一个列表中
                 movie_assess =info.find_all('span')[3].get_text()[:-3]
                 movie_score = item_info.find('span',attrs={
   'class':'rating_num'}).get_text()
                 try:
                     movie_intro = item.find(class_='quote').find(class_='inq').get_text()
                 except Exception as e:
                     movie_intro='None'  
                        
                 movie_num =movie_num+1