文本分析 词云 特征值 LDT主题模型

import pandas as pd
import numpy as np
import numpy
import os
from sklearn.feature_extraction.text import TfidfVectorizer
import jieba
data=pd.read_csv("煤炭文本1.csv",encoding="gbk")
data=data.dropna()
text=data.正文.values.tolist()
content=[]
for line in text:
    word=jieba.lcut(line)
    if len(word)>1 and word !='\r\n':
        content.append(word)
df_content=pd.DataFrame({'content':content})     

stopwords=pd.read_csv("停用词表.txt",index_col=False,sep='\t',quoting=3,names=['stopword'],encoding="utf-8")   
def drop_stopwords(text,stopwords):
    content_clean = []
    all_words=[]
    for line in text:
        line_clean=[]
        for word in line:
           if word in stopwords or word==" ":
                continue
           line_clean.append(word)
           all_words.append(str(word))
        content_clean.append(line_clean)
    return content_clean,all_words
contents=df_content.content.values.tolist()
stopwords=stopwords.stopword.values.tolist()
contents_clean,all_words=drop_stopwords(contents,stopwords)
  
df_content=pd.DataFrame({'contents_clean':contents_clean})    
df_all_words=pd.DataFrame({'all_words':all_words}) 

words_count=df_all_words.groupby(by=["all_words"])["all_words"].agg([("count",np.size)])
words_count=words_count.reset_index().sort_values(by=["count"],ascending=False)


# #词云生成
# from wordcloud import WordCloud
# import matplotlib.pyplot as plt
# %matplotlib inline
# import matplotlib
# matplotlib.rcParams['figure.figsize']=(10.0,5.0)

# wordcloud=WordCloud(font_path=r"C:\\Windows\\Fonts\\msyh.ttc",background_color="white",max_font_size=80)
# word_frequence={x[0]:x[1] for x in words_count.head(100).values}
# wordcloud=wordcloud.fit_words(word_frequence)
# plt.imshow(wordcloud)

# #简单特征提取
# import jieba.analyse
# index=100
# print(data["正文"][index])
# content_str="".join(text[index])
# print(" ".join(jieba.analyse.extract_tags(content_str,topK=10,withWeight=(False))))#topK关键词个数

#LDA主题模型
from gensim import corpora,models,similarities
import gensim

dic=corpora.Dictionary(contents_clean)
corpus=[dic.doc2bow(sentence) for sentence in contents_clean]
lda=gensim.models.ldamodel.LdaModel(corpus=corpus,id2word=dic,num_topics=20)#主题个数
print(lda.print_topic(1,topn=5))

for topic in lda.print_topics(num_topics=20,num_words=5):
    print(topic[1])