Python_文本分析_困惑度计算

这篇博文介绍一个困惑度的神仙方法，困惑度是用来确定最佳主题数一种方式。

• 本代码使用4折交叉验证
• LDA里面有两个参数需要根据自己的数据等进行设定

import csv
import datetime
import re
import pandas as pd
import numpy as np
import jieba
import matplotlib.pyplot as plt
import jieba.posseg as jp, jieba
import gensim
from snownlp import seg
from snownlp import SnowNLP
from snownlp import sentiment
from gensim import corpora, models
from gensim.models import CoherenceModel
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
import warnings
warnings.filterwarnings("ignore")

# Load data
comment = pd.read_csv(r"good.csv", header = 0, index_col = False, engine='python',encoding = 'utf-8')
csv_data = comment[[(len(str(x)) > 100) for x in comment['segment']]]
print(csv_data.shape)

# 设置四折交叉验证
kf = KFold(n_splits = 4, shuffle=False) 

perplexity_train_all = []
perplexity_test_all = []
    
for train_index , test_index in kf.split(list(csv_data['segment'].values)):  
    train = [ csv_data.iloc[x,7] for x in train_index]
    test = [ csv_data.iloc[x,7] for x in test_index]
    
    vectoriser = CountVectorizer(stop_words = 'english', max_features=1000)
    doc_train = vectoriser.fit_transform(train)
    features = vectoriser.get_feature_names()
    doc_test = vectoriser.fit_transform(test)

    perplexity_train = []
    perplexity_test = []
    
    # 这两个参数可以根据自己的需要设定
    alpha = 0.005
    beta = 0.1

    # 进行迭代的topic可以适当增加，这里只用10以内进行计算
    for topics in range(1, 10):
        # Fit LDA to the data 
        LDA = LatentDirichletAllocation(n_components = topics, doc_topic_prior = alpha, topic_word_prior = beta, max_iter=300, learning_method='batch')
        
        # 用来监督处理进程
        news_lda = LDA.fit(doc_train)
        perplexity_train.append(news_lda.perplexity(doc_train))
        perplexity_test.append(news_lda.perplexity(doc_test))
        print(topics, end = '   ')
        
    perplexity_train_all.append(perplexity_train)
    perplexity_test_all.append(perplexity_test)

根据上面代码计算出的困惑度，可以进行如下作图，也在这里展示出来，只作出数据中的两折交叉验证的数据及其对应的均值，仅供参考。

# 求均值
perplexity = [(perplexity_train_all[0][i]+perplexity_train_all[1][i])/2 for i in range(9)]

plt.plot(range(1,10),perplexity_train_all[0], label='prep_1', color='r') 
plt.plot(range(1,10),perplexity_train_all[1], label='prep_2', color='b') 
plt.plot(range(1,10),perplexity,label='perp_ave',color='r',marker='o', markerfacecolor='blue',markersize=5) 

plt.xlabel("Num Topics")
plt.ylabel("Perplexity")
plt.legend(loc='best')

plt.savefig(r"C:\Users\lenovo\Desktop\cm.jpeg",dpi = 600)