python LDA主题模型

准备数据

使用路透社新闻数据的一个子集：R8，包含8类新闻。
本文直接读取清洗后的R8，清洗内容包含：去掉特殊字符，标点符号，停用词和低频词，且英文文本不需要分词。

doc_list = []
f = open('R8.clean.txt', 'r')
lines = f.readlines()
for line in lines:
    doc_list.append(line.strip())
f.close()

print(doc_list[0])

champion products approves stock split champion products inc said board directors approved two one stock split common shares shareholders record april company also said board voted recommend shareholders annual meeting april increase authorized capital stock five mln mln shares reuter

LDA主题模型准备

import gensim
from gensim import corpora
from gensim.models.ldamodel import LdaModel
from gensim.corpora import Dictionary

# 文本需要拆成单词，每一个文本用单词列表表示，才可输入后面的字典函数
texts = [doc.split(' ') for doc in doc_list]
print(texts[0])

# 生成id_word 字典,每个单词对应一个id
dictionary = Dictionary(texts)

# 对字典中的单词进行筛选
dictionary.filter_extremes(no_below=1,
                          no_above=0.5, # 去掉出现在50%以上文章中的单词
                          keep_n=3000) # 去前3000个高频单词

# 将文档转化成词袋(记录文档中有哪些单词以及每个词出现的次数)
corpus = [dictionary.doc2bow(text) for text in texts]

# 模型训练
lda_model = LdaModel(corpus=corpus,
                      id2word=dictionary,
                      num_topics=8,# 主题数量
                      passes=20,# 类似于在机器学习中常见的epoch，也就是训练了多少轮
                      random_state=10)# 一个随机状态对象或生成一个随机状态对象的种子。用于再现性。(保持每次模型训练的一致性)

topic_words = lda_model.print_topics(num_topics=8, num_words=50) # 输出这个模型的各个主题下的主题词
print(topic_words[0]) # 展示第一个主题的主题词

['champion', 'products', 'approves', 'stock', 'split', 'champion', 'products', 'inc', 'said', 'board', 'directors', 'approved', 'two', 'one', 'stock', 'split', 'common', 'shares', 'shareholders', 'record', 'april', 'company', 'also', 'said', 'board', 'voted', 'recommend', 'shareholders', 'annual', 'meeting', 'april', 'increase', 'authorized', 'capital', 'stock', 'five', 'mln', 'mln', 'shares', 'reuter']
(0, '0.066*"billion" + 0.038*"pct" + 0.024*"year" + 0.019*"p" + 0.016*"profit" + 0.015*"stg" + 0.014*"group" + 0.013*"ltd" + 0.012*"plc" + 0.011*"profits" + 0.010*"company" + 0.009*"marks" + 0.009*"l" + 0.009*"u" + 0.008*"francs" + 0.008*"tax" + 0.008*"k" + 0.008*"last" + 0.007*"turnover" + 0.007*"rose" + 0.006*"net" + 0.006*"total" + 0.006*"sales" + 0.005*"rise" + 0.005*"statement" + 0.005*"kong" + 0.005*"dividend" + 0.005*"cents" + 0.005*"hong" + 0.004*"january" + 0.004*"ag" + 0.004*"february" + 0.004*"issue" + 0.004*"owned" + 0.004*"fell" + 0.004*"business" + 0.004*"pre" + 0.004*"operating" + 0.004*"new" + 0.004*"yen" + 0.004*"capital" + 0.004*"making" + 0.004*"five" + 0.004*"surplus" + 0.004*"figures" + 0.004*"parent" + 0.004*"interest" + 0.004*"guilders" + 0.004*"pretax" + 0.004*"increase"')

可视化(notebook中才可显示)

import pyLDAvis
import matplotlib.pyplot as plt
import pyLDAvis.gensim
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model,corpus,dictionary)
vis