Python数据分析《黑客帝国》-一切都不是偶然

1、《黑客帝国》背景介绍

每个人的身上都潜藏这完美灵性世界的碎片，他们死后体内的“灵性碎片”便会脱离肉体束缚，重返宇宙之外那个的至善至美的世界，成为永恒。每个人都是身处黑暗宇宙的“异乡人”，他们的家乡在这个宇宙之外。
总体来说反向很不错豆瓣评分8.9分，四星以上评分占了90%以上，具体到每位观众如何呢？接下来就跟着老司机一起来一探究竟。

2、数据爬取

import requests
from lxml import etree
import time
import random
headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
           'Accept-Encoding': 'gzip',
           'Accept-Language': 'zh-CN,zh;q=0.9',
           'Connection': 'keep-alive',
           'Cookie': '_ga=GA1.2.557027477.1504366471; _vwo_uuid_v2=938EB395FEDF0DA9E9D90650DE9042F0|e0a43720af285a1e9923074545b60ff1; gr_user_id=d044fe6a-3a16-4f08-ad14-cc71033b6767; __utmv=30149280.16469; douban-fav-remind=1; bid=uZG6zNr6IdM; __yadk_uid=l4ZlS88dSS6QGaomSBmqi5XqNAaUQv5r; viewed="21477429_24703171_24746415_10769749_3288908_5377669_2201479_27055214_30203973_1088812"; acw_tc=2760823015682940060847273e162d65a7cec22e493c6aa10f371b90c5086c; ll="118092"; trc_cookie_storage=taboola%2520global%253Auser-id%3Da1a11787-6229-4357-a25a-b869f8653388-tuctb0db9e; ap_v=0,6.0; __utmc=30149280; __utmc=223695111; dbcl2="164698173:AM/bSYaNPzk"; ck=oih5; push_doumail_num=0; push_noty_num=0; _ck_desktop_mode=; vmode=; _gid=GA1.2.2054464412.1568389839; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1568390107%2C%22https%3A%2F%2Fm.douban.com%2Fmovie%2Fsubject%2F1291843%2Fcomments%3Fsort%3Dnew_score%26start%3D50%22%5D; _pk_ses.100001.4cf6=*; __utma=30149280.557027477.1504366471.1568384207.1568390107.31; __utmb=30149280.0.10.1568390107; __utmz=30149280.1568390107.31.27.utmcsr=m.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/movie/subject/1291843/comments; __utma=223695111.557027477.1504366471.1568384207.1568390107.30; __utmb=223695111.0.10.1568390107; __utmz=223695111.1568390107.30.27.utmcsr=m.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/movie/subject/1291843/comments; _pk_id.100001.4cf6=b6270075d3f0f249.1516866774.32.1568391082.1568386453.',
           'Host': 'movie.douban.com',
           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}
if __name__ == '__main__':
    # 0、20、40、60、……490
    fp = open('./hacks.csv',mode='w',encoding='utf-8')
    fp.write('user\tcomment\tstar\tvote\tdate\n')
    url = 'https://movie.douban.com/subject/1291843/comments?start=%d&limit=20&sort=new_score&status=P'
    for i in range(0,26):
        if i == 25:
            url_comments = url%(490)
        else:
            url_comments = url%(i*20)
        response = requests.get(url=url_comments,headers = headers)
        response.encoding = 'utf-8'
        tree = etree.HTML(response.text)
        comments = tree.xpath('//div[@id="comments"]/div[@class="comment-item"]')
        for c in comments:
            user = c.xpath('.//div[@class="avatar"]/a/@title')[0]
            comment = c.xpath('.//span[@class="short"]/text()')[0].replace('\t', '')
            try:
                star = c.xpath('.//span[contains(@class,"allstar")]/@title')[0]
            except:
                star = 'none'
            t = c.xpath('.//span[@class="comment-time "]/@title')[0]
            votes = c.xpath('.//span[@class="votes"]/text()')[0]
            fp.write('%s\t%s\t%s\t%s\t%s\n'%(user,comment,star,votes,t))
        print('第%d页数据保成功%d'%(i+1,len(comments)))
        time.sleep(random.uniform(1,5))
    fp.close()

3、描述统计分析（评价星级）

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
if __name__ == '__main__':
    data = pd.read_csv('./hacks.csv',sep = '\t')
    # 将爬虫获取数据评价等级化
    m = {'none':'0 stars','18':'2 stars','还行':'3 stars','推荐':'4 stars','力荐':'5 stars'}
    data['star'] = data['star'].map(m)
    bar_data = data['star'].value_counts()
    print(bar_data)
    plt.bar(x = np.arange(0,5),height = bar_data,width=0.5)
    plt.xticks(np.arange(0,5),labels = [i[1] for i in m.items()][::-1])
    plt.show()
    percent = data['star'].value_counts(normalize=True)
    plt.pie(percent,labels=[i[1] for i in m.items()][::-1],autopct = '%0.2f%%')
    plt.show()

整体来看，四星五星评价巨多，说明大家对于《黑客帝国》整体评价非常好！

4、情感分析

from snownlp import SnowNLP
import pandas as pd
def sentiment(content):
    s = SnowNLP(str(content))
    return s.sentiments
if __name__ == '__main__':
    data = pd.read_csv('./hacks.csv',sep = '\t')
    data['sentiment'] = data.comment.apply(sentiment)
    data.sort_values('sentiment',ascending=False,inplace=True)
    data.to_csv('./hacks2.csv',sep='\t',index=False)
    # 正面评价，在jupyter中显示效果会好
    print(data.iloc[:10])
    # 负面评价
    print(data.iloc[-10:])

先来看看正面评价，sentiment表示情感得分

看上去正面评价基本都是高星级评分的观众给出的，有理有据，令人信服。再来看看负面评价

这里的负面评价，言语偏少，但是像《黑客帝国》这样的电影，纵使我们认为给了负面评价，发现负面评价，也是比较‘正面的’，这就是大片的魅力。

分词关键字统计词云

通过jieba进行分词，根基TF-IDF算法提取关键词，代码及部分关键词如下

import pandas as pd
import jieba
from jieba import analyse
import wordcloud
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
if __name__ == '__main__':
    data = pd.read_csv('./hacks.csv',sep = '\t')
    texts = ';'.join([str(c) for c in data.comment.tolist()])
    cut_text = ' '.join(jieba.cut(texts))
    keywords = jieba.analyse.extract_tags(cut_text,topK=500,withWeight=True,allowPOS=['ns', 'n', 'vn', 'v','nr','a','e'])
    words = pd.DataFrame(keywords,columns = ['词语','重要性'])
    # 关键字
    plt.barh(range(0,20),width = words['重要性'][:20][::-1])
    plt.ylabel('Importance')
    plt.yticks(range(0,20),labels=words['词语'][:20][::-1],fontproperties = 'KaiTi')
    plt.show()
    # 词云
    hack_mask = np.array(Image.open('./hack.jpg'))
    text_cloud = dict(keywords)
    cloud = wordcloud.WordCloud(width=500,font_path='./simkai.ttf',height=333,background_color='white',mask = hack_mask,max_words=500,max_font_size=150)
    plt.figure(figsize=(12,12))
    word_cloud = cloud.generate_from_frequencies(text_cloud)
    plt.imshow(word_cloud)
    plt.show()

关键字

分词词云

百度网盘代码下载百度网盘
提取码：0qag