肖申克救赎电影评论之关联分析

1、加载模块

# 导入相关库
import pandas as pd
import numpy as np
import jieba
import re
import string
from pyecharts import options as opts
from pyecharts.charts import WordCloud
from collections import Counter

2、导入数据并进行数据预处理

1、导入数据

data = pd.read_excel('评论.xlsx')

2、进行空行处理

data = data[data['正文'].notnull()]   #去除空行

3、文本规范化处理

# 加载停用词

with open("stop_words.utf8", encoding="utf8") as f:
    stopword_list = f.readlines()
    stopword_list = [word.strip() for word in stopword_list]   # 去掉停用词后面的空格符


# 分词，并去掉词前后的空格
def tokenize_text(text):
    tokens = jieba.cut(text)
    tokens = [token.strip() for token in tokens]
    return tokens

# 去掉停用词
def remove_stopwords(text):
    tokens = tokenize_text(text)
    filtered_tokens = [token for token in tokens if token not in stopword_list]
    filtered_text = ''.join(filtered_tokens)
    return filtered_text

# 数据规范化
def normalize_corpus(corpus):
    normalized_corpus = []
    text = remove_stopwords(corpus)
    normalized_corpus.append(text)
    return normalized_corpus


text = data['正文']
# 清洗
text_he = ' '.join(text)     # 将每条句子合起来
# 获取中文词语
pattern = '[\u4e00-\u9fa5]'
zhong_ls = re.findall(pattern, text_he)
# 合并
zhong = ''.join(zhong_ls)
text_he = normalize_corpus(zhong)
text_ls = jieba.lcut(text_he[0])
text_ls[:10]    # 显示前10条

3、进行数据可视化（词云图）

c = (
    WordCloud()
    .add(series_name="热点分析", data_pair=list(Counter(text_ls).items()), word_size_range=[6, 66])
    .set_global_opts(
        title_opts=opts.TitleOpts(
            title="评论分析", 
        ),
        tooltip_opts=opts.TooltipOpts(is_show=True),
    )
    
    )
c.render("ciyuntu.html")

4、统计每个词出现的次数

words = {}   # 用来存储每个词在每一句中出现的次数
text_ls = list(set(text_ls))   # 去重
for k in text:
    for i in text_ls:
        if i in k:
            words[i] = words.get(i, 0) + 1

print(words)

5、统计输入词与其他词共同出现的次数

# 计算共同出现的次数
def together(name):
    part_word = {}
    for i in text:
        for j in word:
            if (name in i) and (j in i):
                part_word[(name, j)] = part_word.get((name, j), 0) + 1
    return part_word
part_word = together('结局')     # 在这里写入要分析的词

6、统计要分析词与其他词单独出现的次数

alone_word = {}
for i in part_word.keys():
    a = word[i[0]] - part_word[i]    # 单词出现的总次数 - 共同出现的次数，即为这个单词单独出现的次数
    b = word[i[1]] - part_word[i]
    alone_word[i] = (a, b)
alone_word  

7、输出与输入词关联最大的前十的词

l = list(pmi_dict.items())
l.sort(key= lambda x: x[1], reverse= True)
print(l[:10])