- 🍨 本文为🔗365天深度学习训练营 中的学习记录博客
- 🍖 原作者:K同学啊
概述
word2vec是词嵌入( word embedding) 的一种, 是将人类文字符号转化为数值形式,或者说——嵌入到一个数学空间里,这种嵌入方式,就叫词嵌入(word embedding),而 Word2vec,就是词嵌入( word embedding) 的一种。
Word2Vec是一种用于学习词嵌入(word embeddings)的技术,其原理基于分布式假设:在大规模文本语料库中,上下文相似的词往往具有相似的语义。Word2Vec通过神经网络模型来学习单词的密集向量表示,使得相似的词在向量空间中距离更近,捕捉了单词之间的语义关系和语法信息。
code
import jieba
import jieba.analyse
jieba.suggest_freq('沙瑞金', True) # 加入一些词,使得jieba分词准确率更高
jieba.suggest_freq('田国富', True)
jieba.suggest_freq('高育良', True)
jieba.suggest_freq('侯亮平', True)
jieba.suggest_freq('钟小艾', True)
jieba.suggest_freq('陈岩石', True)
jieba.suggest_freq('欧阳菁', True)
jieba.suggest_freq('易学习', True)
jieba.suggest_freq('王大路', True)
jieba.suggest_freq('蔡成功', True)
jieba.suggest_freq('孙连城', True)
jieba.suggest_freq('季昌明', True)
jieba.suggest_freq('丁义珍', True)
jieba.suggest_freq('郑西坡', True)
jieba.suggest_freq('赵东来', True)
jieba.suggest_freq('高小琴', True)
jieba.suggest_freq('赵瑞龙', True)
jieba.suggest_freq('林华华', True)
jieba.suggest_freq('陆亦可', True)
jieba.suggest_freq('刘新建', True)
jieba.suggest_freq('刘庆祝', True)
jieba.suggest_freq('赵德汉', True)
with open('./data/in_the_name_of_people.txt') as f:
result_cut = []
lines = f.readlines()
for line in lines:
result_cut.append(list(jieba.cut(line)))
f.close()
Building prefix dict from the default dictionary …
Loading model from cache C:\Users\dlt_t\AppData\Local\Temp\jieba.cache
Loading model cost 0.984 seconds.
Prefix dict has been built successfully.
stopwords_list = [',', '。', '\n', '\u3000', ',', ':', '?', '!', '…']
def remove_stopwords(ls):
return [word for word in ls if word not in stopwords_list]
result_stop = [remove_stopwords(x) for x in result_cut if remove_stopwords(x)]
print(result_stop[100:103])
[[’ ', ‘像是’, ‘为’, ‘他’, ‘的’, ‘思路’, ‘做’, ‘注解’, ‘赵德汉’, ‘咀嚼’, ‘着’, ‘自由’, ‘时光’, ‘里’, ‘的’, ‘最后’, ‘一碗’, ‘炸酱面’, ‘抱怨’, ‘说’, ‘你们’, ‘反贪’, ‘总局’, ‘抓’, ‘贪官’, ‘怎么’, ‘抓到’, ‘我’, ‘这儿’, ‘来’, ‘了’, ‘哎’, ‘有’, ‘几个’, ‘贪官’, ‘住’, ‘这种’, ‘地方’, ‘七层’, ‘老楼’, ‘连’, ‘个’, ‘电梯’, ‘都’, ‘没有’, ‘要是’, ‘贪官’, ‘都’, ‘这’, ‘样子’, ‘老百姓’, ‘得’, ‘放鞭炮’, ‘庆贺’, ‘了’, ‘他’, ‘的’, ‘声音’, ‘被’, ‘面条’, ‘堵’, ‘在’, ‘嗓子眼’, ‘有些’, ‘呜呜’, ‘噜’, ‘噜’, ‘的’, ’ ‘], [’ ', ‘是’, ‘是’, ‘老赵’, ‘瞧’, ‘你’, ‘多’, ‘简朴’, ‘啊’, ‘一碗’, ‘炸酱面’, ‘就’, ‘对付’, ‘一顿’, ‘晚饭’, ’ ‘], [’ ', ‘赵德汉’, ‘吃’, ‘得’, ‘有滋有味’, ‘农民’, ‘的’, ‘儿子’, ‘嘛’, ‘好’, ‘这’, ‘一口’, ’ ']]
from gensim.models import Word2Vec
model = Word2Vec(result_stop, # 用于训练的语料数据
vector_size=100, # 特征向量的维度
window=5, # 一个句子中当前单词和被预测单词的最大距离
min_count=1) # 词频少于min_count次数的单词会被丢弃,默认5
print(model.wv.similarity('沙瑞金','季昌明'))
print(model.wv.similarity('沙瑞金','田国富'))
0.99952054
0.99929386
# 选出最相似的5个词
for e in model.wv.most_similar(positive=['沙瑞金'], topn=5):
print(e)
(‘侯亮平’, 0.9997656941413879)
(‘陈岩石’, 0.9997247457504272)
(‘赵立春’, 0.9996910691261292)
(‘这位’, 0.99966961145401)
(‘孙连城’, 0.9996566772460938)
odd_word = model.wv.doesnt_match(['苹果', '香蕉', '橙子', '书'])
print(f'在这组词汇中不匹配的词汇:{odd_word}')
在这组词汇中不匹配的词汇:书
word_frequency = model.wv.get_vecattr('沙瑞金', 'count')
print(f'沙瑞金:{word_frequency}')
沙瑞金:353