调用gensim中的word2vec

"""
功能：测试gensim使用，处理中文语料
"""

from gensim.models import word2vec
import logging
from gensim.models import Word2Vec
from seaborn import heatmap
from matplotlib import pyplot
import numpy as np

# 主程序
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
sentences = word2vec.Text8Corpus(u"D:\APP\PycharmProjects\mnistProject1\Word2Vec\庆余年.txt_cut.txt")  # 加载语料
print(sentences)
model = word2vec.Word2Vec(sentences)  # 默认window=5

# 计算两个词的相似度/相关程度
y1 = model.wv.similarity(u"范闲", u"范建")
print(u"【范闲】和【范建】的相似度为：", y1)
print("--------\n")

# 计算某个词的相关词列表
y2 = model.wv.most_similar(u"范闲", topn=20)  # 20个最相关的
print(u"和【范闲】最相关的词有：\n")
for item in y2:
    print(item[0], item[1])
print("--------\n")

# 寻找对应关系
print(u"书-不错，质量-")
y3 = model.wv.most_similar([u'质量', u'不错'], [u'书'], topn=3)
for item in y3:
    print(item[0], item[1])
print("--------\n")

# 寻找不合群的词
y4 = model.wv.doesnt_match(u"范闲 王启年 林婉儿 庆帝".split())
print(u"不合群的词：", y4)
print("--------\n")

# 保存模型，以便重用
# model.save(u"庆余年.model")
# 对应的加载方式
model_2 = word2vec.Word2Vec.load("庆余年.model")

# # 以一种C语言可以解析的形式存储词向量
# model.wv.save_word2vec_format(u"庆余年.model.bin", binary=True)
# # 对应的加载方式
# # model_3 = word2vec.Word2Vec.load_word2vec_format("text8.model.bin", binary=True)

# 词向量
print('词向量维度：', model.wv.vectors.shape)
print(model.wv.vectors)

# 提取范闲的词向量
# word = '范闲'
# vec = model.wv.word_vec(word)
# print('词向量长度：', vec.shape)
# print('词向量：\n', vec)


np.random.seed(0)

vectors = model.wv.vectors[:10]

words = model.wv.index_to_key[:20]
print(words)
print(vectors)
heatmap(vectors, center=np.max(vectors))
pyplot.show()


# # 捏造数据
# from random import choice
# ls_of_ls = [['芝士', '酸奶', '蛋糕', '巧克力', '做', '吃'],
#             ['文本', '数据', '挖掘', '分析', '做', '玩'],
#             ['佛山', '广州', '南海', '天河', '吃', '玩']]
# ls_of_words = []  # 存放分词列表（假设是jieba.lcut后得到的）的列表
# for i in range(2500):
#     ls = choice(ls_of_ls)
#     ls_of_words.append([choice(ls) for _ in range(9, 15)])
#
# # 建模训练
# from gensim.models import Word2Vec
# model = Word2Vec(ls_of_words, size=3, window=7)

# 词向量聚类（基于密度）
from sklearn.cluster import DBSCAN
vectors = model.wv.vectors[:20]
labels = DBSCAN(eps=0.24, min_samples=3).fit(vectors).labels_
print(labels)
# 词向量可视化
import matplotlib
from mpl_toolkits import mplot3d
import matplotlib.pyplot as mp
mp.rcParams['font.sans-serif'] = ['SimHei']  # 显示中文
matplotlib.rcParams['axes.unicode_minus'] = False  # 显示负号
fig = mp.figure()
ax = mplot3d.Axes3D(fig)  # 创建3d坐标轴
colors = ['red', 'blue', 'green', 'black']
for word, vector, label in zip(model.wv.index_to_key[:20], vectors, labels):
    ax.scatter(vector[0], vector[1], vector[2], c=colors[label], s=500, alpha=0.4)
    ax.text(vector[0], vector[1], vector[2], word, ha='center', va='center')
mp.show()



if __name__ == "__main__":
    pass