爬取豆瓣top250网页

#mocie.py

import requests
from bs4 import BeautifulSoup
from pandas import DataFrame

'''
    最后成功提取了
    '电影排名','电影名','上映时间','导演','主演','电影类型','电影评分','评价人数','电影链接'
    最后将结果输出到了 豆瓣电影Top250.xlsx 
    但是还存在问题：就是提取语言和制片国家/地区时，出现没有selector的情况。
    要解决该问题可能需要xpath
'''

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}


start_num = [i for i in range(0, 226, 25)]

list_url_mv = []  # 所有电影的URL

for start in start_num:
    url = 'https://movie.douban.com/top250?start={}&filter='.format(start)
    print('正在处理url：', url)

    response = requests.get(url=url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')

    url_mv_list = soup.select('#content > div > div.article > ol > li > div > div.info > div.hd > a')
    # print(url_mv_list)
    for index_url in range(len(url_mv_list)):
        url_mv = url_mv_list[index_url]['href']
        list_url_mv.append(url_mv)
        # print(url_mv)


# 对每部电影进行处理
def loading_mv(url, number):
    list_mv = []
    print('-----正在处理第{}部电影-----'.format(number + 1))
    list_mv.append(number + 1)  # 排名

    # 解析网页
    response_mv = requests.get(url=url, headers=headers)
    soup_mv = BeautifulSoup(response_mv.text, 'html.parser')

    # 爬取电影名
    mv_name = soup_mv.find_all('span', attrs={'property': 'v:itemreviewed'})  # 电影名
    mv_name = mv_name[0].get_text()
    list_mv.append(mv_name)
    # print(mv_name)

    # 爬取电影的上映时间
    mv_year = soup_mv.select('span.year')  # 电影上映时间
    mv_year = mv_year[0].get_text()[1:5]
    list_mv.append(mv_year)
    # print(mv_year)

    # 爬取导演信息
    list_mv_director = []  # 导演
    mv_director = soup_mv.find_all('a', attrs={'rel': "v:directedBy"})
    for director in mv_director:
        list_mv_director.append(director.get_text())
    string_director = '/'.join(list_mv_director)  # 重新定义格式
    list_mv.append(string_director)
    # print(list_mv_director)

    # 爬取主演信息
    list_mv_star = []  # 主演
    mv_star = soup_mv.find_all('a', attrs={'rel': 'v:starring'})
    if mv_star == []:  # 在第210部时没有主演
        list_mv.append(None)
    else:
        mv_star = mv_star[0].get_text().strip().split('/')
        mv_first_star = mv_star[0].split(':')
        list_mv_star.append(mv_first_star[-1].strip())
        del mv_star[0]  # 去除'主演'字段
        for star in mv_star:
            list_mv_star.append(star.strip())
        string = '/'.join(list_mv_star)  # 重新定义格式
        list_mv.append(string)

    # 爬取电影类型
    list_mv_type = []  # 电影类型
    mv_type = soup_mv.find_all('span', attrs={'property': 'v:genre'})
    for type in mv_type:
        list_mv_type.append(type.get_text())
    string_type = '/'.join(list_mv_type)
    list_mv.append(string_type)
    # print(list_mv_type)

    # 爬取电影评分
    mv_score = soup_mv.select('strong.ll.rating_num')  # 评分
    mv_score = mv_score[0].get_text()
    list_mv.append(mv_score)

    # 爬取评价人数
    mv_evaluation_num = soup_mv.select('a.rating_people')  # 评价人数
    mv_evaluation_num = mv_evaluation_num[0].get_text().strip()
    list_mv.append(mv_evaluation_num)

    # 爬取剧情简介
    mv_plot = soup_mv.find_all('span', attrs={"class": "all hidden"})  # 剧情简介
    if mv_plot == []:
        list_mv.append(None)
    else:
        string_plot = mv_plot[0].get_text().strip().split()
        new_string_plot = ' '.join(string_plot)
        list_mv.append(new_string_plot)

    # 加入电影网址
    list_mv.append(url)
    #图片链接
    mv_pictures_url = soup_mv.find('a', class_='nbgnbg')
    if mv_pictures_url:
        img_tag = mv_pictures_url.find('img')
        if img_tag:
            mv_pictures_url = img_tag['src']
        else:
            mv_pictures_url = None
    else:
        mv_pictures_url = None

    list_mv.append(mv_pictures_url)
    return list_mv


# url1 = 'https://movie.douban.com/subject/1292052/'
# url2 = 'https://movie.douban.com/subject/26430107/'      # 210部
# a = loading_mv(url1,1)
# # b = loading_mv(url2,210)
# # list_all_mv.append(a)
# # list_all_mv.append(b)


list_all_mv = []

dict_mv_info = {}
for number in range(len(list_url_mv)):
    mv_info = loading_mv(list_url_mv[number], number)
    list_all_mv.append(mv_info)
print('-----运行结束-----')

pd = DataFrame(list_all_mv, columns=['电影排名', '电影名', '上映时间', '导演', '主演', '电影类型', '电影评分', '评价人数', '电影简介', '电影链接','图片链接'])
# print(pd)

pd.to_excel(r'D:\111\豆瓣电影Top250.xlsx')