复现
"""
Created on Mon May 31 14:27:13 2021
@author: K43
"""
from collections import defaultdict
import collections
import math
import operator
import xlrd
import jieba
import scipy.io
import jieba.analyse
"""
函数说明:创建数据样本
Returns:
dataset - 实验样本切分的词条
classVec - 类别标签向量
"""
def loadDataSet():
dataset = [ ['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
['stop', 'posting', 'stupid', 'worthless', 'garbage'],
['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
['quit', 'buying', 'worthless', 'dog', 'food', 'stupid'] ]
classVec = [0, 1, 0, 1, 0, 1]
return dataset, classVec
def readexcel(rPath):
workbook = xlrd.open_workbook(rPath)
sheet = workbook.sheet_by_index(1)
re = []
for i in range(1, sheet.nrows):
l=str(sheet.cell_value(rowx = i, colx = 1))
l = list(jieba.lcut(l))
re.append(l)
return re
"""
函数说明:特征选择TF-IDF算法
Parameters:
list_words:词列表
Returns:
dict_feature_select:特征选择词字典
"""
def feature_select(list_words):
doc_frequency=defaultdict(int)
for word_list in list_words:
for i in word_list:
doc_frequency[i]+=1
word_tf={}
for i in doc_frequency:
word_tf[i]=doc_frequency[i]/sum(doc_frequency.values())
doc_num=len(list_words)
word_idf={}
word_doc=defaultdict(int)
for i in doc_frequency:
for j in list_words:
if i in j:
word_doc[i]+=1
for i in doc_frequency:
word_idf[i]=math.log(doc_num/(word_doc[i]+1))
word_tf_idf={}
for i in doc_frequency:
word_tf_idf[i]=word_tf[i]*word_idf[i]
dict_feature_select=sorted(word_tf_idf.items(),key=operator.itemgetter(1),reverse=True)
return dict_feature_select
if __name__=='__main__':
rPath = r'C:\Users\K43\Desktop\xppcode\热词统计.xlsx'
wPath = r'C:\Users\K43\Desktop\xppcode\TF-IDF.mat'
ylist=readexcel(rPath)
print(ylist)
features=feature_select(ylist)
scipy.io.savemat(wPath, mdict={'TF-IDF': features})
print(features)