#需求:对top250进行requests爬取,并清洗数据后制作柱状图,折线图等
#定义函数。爬取top250的信息
#影片详情链接、图片链接、、影片概况和相关内容
from bs4 import BeautifulSoup
import requests
import re
import openpyxl
import time
def get_data():
wb = openpyxl.Workbook()
sheet = wb.active
sheet.append(['中文标题', '英文标题', '发行时间', '电影评分', '评价人数','影片详细链接','图片链接','影片概况','导演名称'])
wb.save('data.xlsx')
for m in range(0,10):
print(f'正在爬取第{m+1}页内容')
nus=m*25
URL=f"https://movie.douban.com/top250?start={nus}&filter="#这里的网址并未变化-第一页网址
Headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; an-ES; rv:1.9.0.20) Gecko/9663-05-06 15:44:33 Firefox/3.8'}
response = requests.get(url=URL, headers=Headers)
if response.status_code!=200:
print(f"爬取失败,状态码是{response.status_code}")
else:
response.encoding = 'utf-8'
soup=BeautifulSoup(response.text,'html.parser')
label_numbers=soup.select('#content>div>div.article>ol>li')
for label_number in range(1,len(label_numbers)+1):
a1234=f'#content > div > div.article > ol > li:nth-child({label_number}) '
#中文标题
label1=soup.select(a1234+'> div > div.info > div.hd > a > span:nth-child(1)')
#英文标题
label2 = soup.select(a1234+ f'> div > div.info > div.hd > a > span:nth-child(2)')
english_label=label2[0].text.replace("/", "").strip()
#导演,主演,发行时间,发行地区,剧情归属标签需要二次处理#发行时间很好切出来,其他的没法切
label4 = soup.select(a1234+'> div > div.info > div.bd > p:nth-child(1)')
# 发行时间
start_time=re.findall(r'\d{4}',label4[0].text)
#电影评分
label5 = soup.select(f'#content > div > div.article > ol > li:nth-child({label_number}) > div > div.info > div.bd > div > span.rating_num')
#评价人数
label6 = soup.select(f'#content > div > div.article > ol > li:nth-child({label_number}) > div > div.info > div.bd > div > span:nth-child(4)')
man_number=label6[0].text.rstrip('人评价')
#影片详细链接
label7 = soup.select(f'#content > div > div.article > ol > li:nth-child({label_number}) > div > div.info > div.hd > a')
#图片链接
label8 = soup.select(f'#content > div > div.article > ol > li:nth-child({label_number}) > div > div.pic > a > img')
#详细内容
label10 = soup.select(
f'#content > div > div.article > ol > li:nth-child({label_number}) > div > div.info > div.bd > p:nth-child(1)')
#对详细内容进行清洗,筛选出导演中文名
pattern = r'导演: ([\u4e00-\u9fa5]+)'
directors = re.findall(pattern, label10[0].text)
if directors==[]:
director='空'
else:
director=directors[0]
# #写入元素
write_excel(label1[0].text,english_label,
start_time[0],label5[0].text,man_number
,label7[0]['href'],label8[0]['src'],label10[0].text,director)
time.sleep(1)
def write_excel(a,b,c,d,e,f,g,h,i):
wb=openpyxl.load_workbook('data.xlsx')
sheet=wb.active
sheet.append([a,b,c,d,e,f,g,h,i])
wb.save('data.xlsx')
if __name__ == '__main__':
get_data()