Python爬取《悲伤逆流成河》猫眼信息

1.猫眼电影短评接口

http://maoyan.com/films/1217236
我们直接访问这个，在web端只能看到最热的10条短评，那怎么获取到所有短评呢？
(1) 访问上面的链接，按下F12，然后点击图片上的图标，把浏览模式(响应式设计模式，火狐快捷键Ctrl+Shift+M)改为手机模式，刷新页面。


(2)换用谷歌浏览器，F12下进行上面操作，加载完毕后下拉短评，页面继续加载，找到含有offset和startTime的加载条，发现它的Response中包含我们想要的数据，为json格式。

2.获取短评

(1)简单分析

通过上面分析

Request URL: http://m.maoyan.com/mmdb/comments/movie/1217236.json?v=yes&offset=0&startTime=0%2021%3A09%3A31
Request Method: GET

下滑了几次次，我发现了下面规律：
分析上面数据变化，可以大致猜测出：offset表示该接口显示评论开始位置，每个页面15条，比如：15，则显示15-30这中间的15条评论； startTime表示当前评论的时间，固定格式（2018-10-06）。
另外接口最后的%2021%3A09%3A31是不变的。
(2)代码获取

import requests
from fake_useragent import UserAgent
import json
'''
小编准备的python学习资料，加群：1136201545 即可免费获取！
'''
headers = {
        "User-Agent": UserAgent(verify_ssl=False).random,
        "Host":"m.maoyan.com",
        "Referer":"http://m.maoyan.com/movie/1217236/comments?_v_=yes"
    }
# 猫眼电影短评接口
offset = 0
# 电影是2018.9.21上映的
startTime = '2018-09-21'
comment_api = 'http://m.maoyan.com/mmdb/comments/movie/1217236.json?_v_=yes&offset={0}&startTime={1}%2021%3A09%3A31'.format(offset,startTime)
# 发送get请求
response_comment = requests.get(comment_api,headers = headers)
json_comment = response_comment.text
json_comment = json.loads(json_comment)
print(json_comment)

返回数据：
(3)数据简单介绍
(4)数据提取

# 获取数据并存储
def get_data(self,json_comment):
    json_response = json_comment["cmts"]  # 列表
    list_info = []
    for data in json_response:
        cityName = data["cityName"]
        content = data["content"]
        if "gender" in data:
            gender = data["gender"]
        else:
            gender = 0
        nickName = data["nickName"]
        userLevel = data["userLevel"]
        score = data["score"]
        list_one = [self.time,nickName,gender,cityName,userLevel,score,content]
        list_info.append(list_one)
    self.file_do(list_info)

3.存储数据

# 存储文件
def file_do(list_info):
    # 获取文件大小
    file_size = os.path.getsize(r'G:\maoyan\maoyan.csv')
    if file_size == 0:
        # 表头
        name = ['评论日期', '评论者昵称', '性别', '所在城市','猫眼等级','评分','评论内容']
        # 建立DataFrame对象
        file_test = pd.DataFrame(columns=name, data=list_info)
        # 数据写入
        file_test.to_csv(r'G:\maoyan\maoyan.csv', encoding='gbk', index=False)
    else:
        with open(r'G:\maoyan\maoyan.csv', 'a+', newline='') as file_test:
            # 追加到文件后面
            writer = csv.writer(file_test)
            # 写入文件
            writer.writerows(list_info)

数据显示

数据分析可视化

1.提取数据

代码：

def read_csv():
    content = ''
    # 读取文件内容
    with open(r'G:\maoyan\maoyan.csv', 'r', encoding='utf_8_sig', newline='') as file_test:
        # 读文件
        reader = csv.reader(file_test)
        i = 0
        for row in reader:
            if i != 0:
                time.append(row[0])
                nickName.append(row[1])
                gender.append(row[2])
                cityName.append(row[3])
                userLevel.append(row[4])
                score.append(row[5])
                content = content + row[6]
                # print(row)
            i = i + 1
        print('一共有：' + str(i - 1) + '条数据')
        return content

一共有：15195条数据

评论者性别分布可视化

# 评论者性别分布可视化
def sex_distribution(gender):
    # print(gender)
    from pyecharts import Pie
    list_num = []
    list_num.append(gender.count('0')) # 未知
    list_num.append(gender.count('1')) # 男
    list_num.append(gender.count('2')) # 女
    attr = ["其他","男","女"]
    pie = Pie("性别饼图")
    pie.add("", attr, list_num, is_label_show=True)
    pie.render("H:\PyCoding\spider_maoyan\picture\sex_pie.html")

评论者所在城市分布可视化

# 评论者所在城市分布可视化
def city_distribution(cityName):
    city_list = list(set(cityName))
    city_dict = {city_list[i]:0 for i in range(len(city_list))}
    for i in range(len(city_list)):
        city_dict[city_list[i]] = cityName.count(city_list[i])
    # 根据数量(字典的键值)排序
    sort_dict = sorted(city_dict.items(), key=lambda d: d[1], reverse=True)
    city_name = []
    city_num = []
    for i in range(len(sort_dict)):
        city_name.append(sort_dict[i][0])
        city_num.append(sort_dict[i][1])

    import random
    from pyecharts import Bar
    bar = Bar("评论者城市分布")
    bar.add("", city_name, city_num, is_label_show=True, is_datazoom_show=True)
    bar.render("H:\PyCoding\spider_maoyan\picture\city_bar.html")

# 地图可视化
def render_city(cities):

每日评论总数可视化分析

def time_num_visualization(time):
    from pyecharts import Line
    time_list = list(set(time))
    time_dict = {time_list[i]: 0 for i in range(len(time_list))}
    time_num = []
    for i in range(len(time_list)):
        time_dict[time_list[i]] = time.count(time_list[i])
    # 根据数量(字典的键值)排序
    sort_dict = sorted(time_dict.items(), key=lambda d: d[0], reverse=False)
    time_name = []
    time_num = []
    print(sort_dict)
    for i in range(len(sort_dict)):
        time_name.append(sort_dict[i][0])
        time_num.append(sort_dict[i][1])

    line = Line("评论数量日期折线图")
    line.add(
        "日期-评论数",
        time_name,
        time_num,
        is_fill=True,
        area_color="#000",
        area_opacity=0.3,
        is_smooth=True,
    )
    line.render("H:\PyCoding\spider_maoyan\picture\c_num_line.html")

评论者猫眼等级、评分可视化

def level_score_visualization(userLevel,score):
    from pyecharts import Pie
    userLevel_list = list(set(userLevel))
    userLevel_num = []
    for i in range(len(userLevel_list)):
        userLevel_num.append(userLevel.count(userLevel_list[i]))

    score_list = list(set(score))
    score_num = []
    for i in range(len(score_list)):
        score_num.append(score.count(score_list[i]))

    pie01 = Pie("等级环状饼图", title_pos='center', width=900)
    pie01.add(
        "等级",
        userLevel_list,
        userLevel_num,
        radius=[40, 75],
        label_text_color=None,
        is_label_show=True,
        legend_orient="vertical",
        legend_pos="left",
    )
    pie01.render("H:\PyCoding\spider_maoyan\picture\level_pie.html")
    pie02 = Pie("评分玫瑰饼图", title_pos='center', width=900)
    pie02.add(
        "评分",
        score_list,
        score_num,
        center=[50, 50],
        is_random=True,
        radius=[30, 75],
        rosetype="area",
        is_legend_show=False,
        is_label_show=True,
    )
    pie02.render("H:\PyCoding\spider_maoyan\picture\score_pie.html")