豆瓣top250 爬虫 python 保存excel

1.安装相关的包

# 安装相关包
# beatifulSoup4是一个著名的爬虫框架
!pip install bs4
!pip install openpyxl
import bs4
import sys
from bs4 import BeautifulSoup
import re
import urllib
import openpyxl
import requests
from pandas import Series,DataFrame
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
from pylab import mpl

2.函数代码块

#相关定义
def init():
    url1='https://movie.douban.com/top250?start='
    path='D:\python learning material\豆瓣电影Top250.xlsx'
    namechina=[]
    nameother=[]
    picturelink=[]
    rate=[]
    people=[]
    quote=[]
    info=[]
    movielink=[]


#爬取网页代码
def askURL(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}
        response = requests.get(url, headers=headers)
        html = response.text
    except requests.exceptions.HTTPError as e:
        print(e)
    return html

#解析数据 存放到列表中
#a是每次爬取出来的网页代码
def getdata(a):
    soup=bs4.BeautifulSoup(a)
    item=soup.findAll(name='div',attrs={'class':'item'})
    for i in item:
        i=str(i)
        #print(i)
        movie=bs4.BeautifulSoup(i)
        #名字title标签中存放了两种，需要区别
        name=movie.findAll(name='span',attrs={'class':'title'})
        if(len(name)==2):
            namechina.append(str(name[0]).replace('</span>','').replace('<span class="title">',''))
            nameother.append(str(name[1]).replace('</span>','').replace('<span class="title">','').replace('\xa0/\xa0',''))
        else:
            namechina.append(str(name[0]).replace('</span>','').replace('<span class="title">',''))
            nameother.append('') 

        movielink.append(re.findall('<a href="(.*?)">',i))
        picturelink.append(re.findall('<img.*src="(.*?)"',i))
        rate.append(re.findall('<span class="rating_num" property="v:average">(.*?)</span>',i))
        people.append(re.findall('<span>(.*?)人评价</span>',i))
        qu=movie.findAll(name='span',attrs={'class':'inq'})
        quote.append(str(qu).replace('<span class="inq">','').replace('</span>',''))
        informa=movie.findAll(name='p',attrs={'class':''})
        info.append(str(informa).replace('<p class="">\n                            ','').replace('\xa0\xa0\xa0',' ')
                .replace('<br/>\n                            ','').replace('\xa0/\xa0','').replace('\n                        </p>]',''))
    return namechina,nameother,picturelink,rate,people,quote,info,movielink
    
#清除之前运行的数据   
def cleardata():
    namechina.clear()
    nameother.clear()
    picturelink.clear()
    rate.clear()
    people.clear()
    quote.clear()
    info.clear()
    movielink.clear()
    
    
#采用 openpyxl保存数据
def savedata(path,namechina,nameother,picturelink,rate,people,quote,info,movielink):
    book=openpyxl.Workbook()
    sheet = book.create_sheet("豆瓣电影Top250")
    col=('电影详情链接','图片链接','影片中文名','影片外国名', '评分','评价数','概况','相关信息')
    sheet.append(col)  #添加列头
    for i in range(0,len(picturelink)):
        sheet.cell(row = (i+2),column = 1,value =str(movielink[i])) 
        sheet.cell(row=(i+2),column=2,value=str(picturelink[i]))
        sheet.cell(row=(i+2),column=3,value=str(namechina[i]))
        sheet.cell(row=(i+2),column=4,value=str(nameother[i]))
        sheet.cell(row=(i+2),column=5,value=str(rate[i]))
        sheet.cell(row=(i+2),column=6,value=str(people[i]))
        sheet.cell(row=(i+2),column=7,value=str(quote[i]))
        sheet.cell(row=(i+2),column=8,value=str(info[i]))
        book.save(path) #保存
    

#展示排名前10
def show_topten():
    wb = load_workbook(filename = 'D:\python learning material\豆瓣电影Top250.xlsx')#加载文件
    sheet_ranges = wb['豆瓣电影Top250']#选择表
    df = DataFrame(sheet_ranges.values)#形成dataframe
    topten=df.head(11)
    print(topten)  

#绘图
def dataanalyze():
    mpl.rcParams['font.sans-serif'] = ['SimHei']#正常显示中文
    ratings = list(df[4].values[1:])
    matplotlib.rcParams['font.size']=10
    plt.figure(figsize=(10, 10))
    plt.subplot(2,1,1)
    plt.scatter(ratings,range(1,251))
    plt.xlabel('评分')
    plt.ylabel('排名')
    plt.gca().invert_yaxis()
    plt.subplot(2,1,2)
    plt.hist(ratings,bins=15)
    
    
def main():  
    cleardata()
    for i in range(0,250,25):
        url=url1+str(i)
        content=askURL(url)
        getdata(content)
    savedata(path,namechina,nameother,picturelink,rate,people,quote,info,movielink)

3.运行

init()
main()
show_topten()
dataanalyze()

运行结果