import pandas as pd
import numpy as np
import numpy
import os
from sklearn.feature_extraction.text import TfidfVectorizer
import jieba
data=pd.read_csv("煤炭文本1.csv",encoding="gbk")
data=data.dropna()
text=data.正文.values.tolist()
content=[]
for line in text:
word=jieba.lcut(line)
if len(word)>1 and word !='\r\n':
content.append(word)
df_content=pd.DataFrame({'content':content})
stopwords=pd.read_csv("停用词表.txt",index_col=False,sep='\t',quoting=3,names=['stopword'],encoding="utf-8")
def drop_stopwords(text,stopwords):
content_clean = []
all_words=[]
for line in text:
line_clean=[]
for word in line:
if word in stopwords or word==" ":
continue
line_clean.append(word)
all_words.append(str(word))
content_clean.append(line_clean)
return content_clean,all_words
contents=df_content.content.values.tolist()
stopwords=stopwords.stopword.values.tolist()
contents_clean,all_words=drop_stopwords(contents,stopwords)
df_content=pd.DataFrame({'contents_clean':contents_clean})
df_all_words=pd.DataFrame({'all_words':all_words})
words_count=df_all_words.groupby(by=["all_words"])["all_words"].agg([("count",np.size)])
words_count=words_count.reset_index().sort_values(by=["count"],ascending=False)
# #词云生成
# from wordcloud import WordCloud
# import matplotlib.pyplot as plt
# %matplotlib inline
# import matplotlib
# matplotlib.rcParams['figure.figsize']=(10.0,5.0)
# wordcloud=WordCloud(font_path=r"C:\\Windows\\Fonts\\msyh.ttc",background_color="white",max_font_size=80)
# word_frequence={x[0]:x[1] for x in words_count.head(100).values}
# wordcloud=wordcloud.fit_words(word_frequence)
# plt.imshow(wordcloud)
# #简单特征提取
# import jieba.analyse
# index=100
# print(data["正文"][index])
# content_str="".join(text[index])
# print(" ".join(jieba.analyse.extract_tags(content_str,topK=10,withWeight=(False))))#topK关键词个数
#LDA主题模型
from gensim import corpora,models,similarities
import gensim
dic=corpora.Dictionary(contents_clean)
corpus=[dic.doc2bow(sentence) for sentence in contents_clean]
lda=gensim.models.ldamodel.LdaModel(corpus=corpus,id2word=dic,num_topics=20)#主题个数
print(lda.print_topic(1,topn=5))
for topic in lda.print_topics(num_topics=20,num_words=5):
print(topic[1])
文本分析 词云 特征值 LDT主题模型
最新推荐文章于 2022-07-30 11:36:11 发布