机器学习-文本聚类实例-kmeans
import os
import gensim
import jieba
from gensim.models.doc2vec import Doc2Vec
from sklearn.cluster import KMeans
TaggededDocument = gensim.models.doc2vec.TaggedDocument
# 定义两个函数读取和保存文件
# 保存至文件
def savefile(savepath, content, encode):
fp = open(savepath, "w", encoding=encode)
fp.write(content)
fp.close()
# 读文件
def readfile(path, encode):
content = None
try:
fp = open(path, "r", encoding=encode)
content = fp.read()
fp.close()
except UnicodeDecodeError:
print("Error: 文件读取失败")
else:
return content
stop_words_path = '/Users/FengZhen/Desktop/accumulate/机器学习/kmeans聚类/中文停用词表.txt'
origin_text_path = '/Users/FengZhen/Desktop/accumulate/机器学习/kmeans聚类/测试文本集/'
cut_combine_path = '/Users/FengZhen/Desktop/accumulate/机器学习/kmeans聚类/cut_combine.txt'
corpus_text_path = '/Users/FengZhen/Desktop/accumulate/机器学习/kmeans聚类/corpus.txt'
result_text_path = '/Users/FengZhen/Desktop/accumulate/机器学习/kmeans聚类/own_claasify.txt'
# 分词-去停用词-保存
def segment():
stop_words = readfile(stop_words_path,'UTF-8')
fileList = os.listdir(origin_text_path)
save = open(cut_combine_path, 'w')
for file in fileList:
if not file.startswith("."):
content_result = ''
content = readfile(origin_text_path + file, 'GBK')
content_words = jieba.cut(content)
for content_word in content_words:
if content_word not in stop_words:
content_result = content_result + " " + content_word
save.write(content_result.replace('\r','').replace('\n',''))
save.write('\n')
def get_datasest():
with open(cut_combine_path, 'r') as cf:
docs = cf.readlines()
print
len(docs)
x_train = []
for i, text in enumerate(docs):
word_list = text.split(' ')
l = len(word_list)
word_list[l - 1] = word_list[l - 1].strip()
# 训练模型前,先将语料整理成规定的形式,这里用到TaggedDocument模型
# 输入输出内容都为 词袋 + tag列表
document = TaggededDocument(word_list, tags=[i])
x_train.append(document)
return x_train
def train(x_train, size=200):
# 初始化训练模型的参数,再保存训练结果以释放内存
# 提供x_train可初始化, min_cout 忽略总频率低于这个的所有单词, window 预测的词与上下文词之间最大的距离, 用于预测 size 特征向量的维数 negative 接受杂质的个数 worker 工作模块数
model_dm = Doc2Vec(x_train, min_count=1, window=3, size=size, sample=1e-3, negative=5, workers=4)
# corpus_count是文件个数 epochs 训练次数
model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=100)
# 保存模型训练结果,释放内存空间,后续可用load加载
model_dm.save(corpus_text_path)
return model_dm
def cluster(x_train):
infered_vectors_list = []
print
"load doc2vec model..."
# 加载训练的模型 model_dm输出类似Doc2Vec(dm/m,d500,n5,w3,s0.001,t4)
model_dm = Doc2Vec.load(corpus_text_path)
print
"load train vectors..."
i = 0
for text, label in x_train:
vector = model_dm.infer_vector(text)
infered_vectors_list.append(vector)
i += 1
print
"train kmean model..."
kmean_model = KMeans(n_clusters=15)
kmean_model.fit(infered_vectors_list)
labels = kmean_model.predict(infered_vectors_list[0:100])
print(labels)
cluster_centers = kmean_model.cluster_centers_
with open(result_text_path, 'w') as wf:
i = 0
while i < len(x_train):
string = ""
text = x_train[i][0]
for word in text:
string = string + word
string = string + '\t'
string = string + str(labels[i])
string = string + '\n'
wf.write(string)
i = i + 1
return cluster_centers
if __name__ == '__main__':
segment()
x_train = get_datasest()
model_dm = train(x_train)
cluster_centers = cluster(x_train)