日常代码分享

import jieba
from gensim.corpora.dictionary import Dictionary
from sklearn.feature_extraction.text import TfidfVectorizer

docs=["新型互联网大数据技术研究"
"大数据采集技术与应用方法"
"一种互联网技术研究方法"
"计算机系统的分析与设计技术"
]

stoplist=open('stopword.txt','r',encoding="utf-8").readlines()
stoplist = set(w.strip() for w in stoplist)

jieba.load_userdict("userdict.txt")

texts=[]
for d in docs:
doc=[]
for w in list(jieba.cut(d,cut_all=False)):
if len(w)>1 and w not in stoplist:
doc.append(w)
texts.append(doc)

dictionary = Dictionary(texts)
dictionary.filter_extremes(no_below=2,no_above=1.0,keep_n=10)
d=dict(dictionary.items())
docwords=set(d.values())
print("维度词汇是：",docwords)

corpus=[]
for text in texts:
d=""
for w in text:
if w in docwords:
d=d+w+""
corpus.append(d)

vectorizer = TfidfVectorizer
tfidf = vectorizer.fit_transform(corpus)
words = vectorizer.get_feature_names()
for i in range(len(corpus)):
print('----Document %d----'%(i))
for j in range(len(words)):
print(words[j],tfidf[i,j])