from math import log10
import numpy as np
# docList is the corpus with each element a doc, each doc is a list of words
def tfidf(docList):
docNum = len(docList)
docList = [i.split(' ') for i in docList]
term_idf = dict()
for doc in docList:
# set(doc) 得到每篇文档的词,不包含重复, 即可统计该词在不同文档的出现次数
for term in set(doc):
if term not in term_idf:
term_idf[term] = 1.0
else:
term_idf[term] += 1.0
# IDF 统计词语的逆文档频率
for term in term_idf:
# log10 10为底数
term_idf[term] = log10(docNum / term_idf[term])
print('all word num = ', len(term_idf))
# term_tfidf 总词典
term_tfidf = dict()
doc_id = 0
for doc in docList:
term_tfidf[doc_id] = dict()
# 每个文档的词频统计
term_tf = dict()
for term in doc:
if term not in term_tf:
TF-IDF算法实现,稀疏矩阵的转化
最新推荐文章于 2024-06-15 10:15:20 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)