- 🍨 本文为🔗365天深度学习训练营 中的学习记录博客
- 🍖 原作者:K同学啊 | 接辅导、项目定制
- 🚀 文章来源:K同学的学习圈子
1.安装Gensim库
pip install --upgrade gensim
2.对原始语料分词
import jieba
import jieba.analyse
jieba.suggest_freq('沙瑞金',True)#加入一些词,使得jieba分词准确率更高
jieba.suggest_freq('田国富',True)
jieba.suggest_freq('高育良',True)
jieba.suggest_freq('侯亮平',True)
jieba.suggest_freq('钟小艾',True)
jieba.suggest_freq('陈岩石',True)
jieba.suggest_freq('欧阳菁',True)
jieba.suggest_freq('易学习',True)
jieba.suggest_freq('王大路',True)
jieba.suggest_freq('蔡成功',True)
jieba.suggest_freq('孙连城',True)
jieba.suggest_freq('季昌明',True)
jieba.suggest_freq('丁义珍',True)
jieba.suggest_freq('郑西坡',True)
jieba.suggest_freq('赵东来',True)
jieba.suggest_freq('高小琴',True)
jieba.suggest_freq('赵瑞龙',True)
jieba.suggest_freq('林华华',True)
jieba.suggest_freq('陆亦可',True)
jieba.suggest_freq('刘新建',True)
jieba.suggest_freq('刘庆祝',True)
jieba.suggest_freq('赵德汉',True)
with open('./data/in_the_name_of_people.txt', encoding='utf-8')as f:
result_cut = []
lines = f.readlines()
for line in lines:
result_cut.append(list(jieba.cut(line)))
f.close()
#添加自定义停用词
stopwords_list = [",","。","\n","\u3000","",":","!","?","…"]
def remove_stopwords(ls): #去除停用词
return [word for word in ls if word not in stopwords_list]
result_stop=[remove_stopwords(x)for x in result_cut if remove_stopwords(x)]
print(result_stop[100:103])
输出:
[[’ ', ‘像是’, ‘为’, ‘他’, ‘的’, ‘思路’, ‘做’, ‘注解’, ‘,’, ‘赵德汉’, ‘咀嚼’, ‘着’, ‘自由’, ‘时光’, ‘里’, ‘的’, ‘最后’, ‘一碗’, ‘炸酱面’, ‘,’, ‘抱怨’, ‘说’, ‘:’, ‘你们’, ‘反贪’, ‘总局’, ‘抓’, ‘贪官’, ‘怎么’, ‘抓到’, ‘我’, ‘这儿’, ‘来’, ‘了’, ‘?’, ‘哎’, ‘,’, ‘有’, ‘几个’, ‘贪官’, ‘住’, ‘这种’, ‘地方’, ‘?’, ‘七层’, ‘老楼’, ‘,’, ‘连’, ‘个’, ‘电梯’, ‘都’, ‘没有’, ‘,’, ‘要是’, ‘贪官’, ‘都’, ‘这’, ‘样子’, ‘,’, ‘老百姓’, ‘得’, ‘放鞭炮’, ‘庆贺’, ‘了’, ‘!’, ‘他’, ‘的’, ‘声音’, ‘被’, ‘面条’, ‘堵’, ‘在’, ‘嗓子眼’, ‘,’, ‘有些’, ‘呜呜’, ‘噜’, ‘噜’, ‘的’, ’ ‘], [’ ', ‘是’, ‘,’, ‘是’, ‘,’, ‘老赵’, ‘,’, ‘瞧’, ‘你’, ‘多’, ‘简朴’, ‘啊’, ‘,’, ‘一碗’, ‘炸酱面’, ‘就’, ‘对付’, ‘一顿’, ‘晚饭’, ’ ‘], [’ ', ‘赵德汉’, ‘吃’, ‘得’, ‘有滋有味’, ‘:’, ‘农民’, ‘的’, ‘儿子’, ‘嘛’, ‘,’, ‘好’, ‘这’, ‘一口’, ’ ']]
3.训练Word2Vec模型
from gensim.models import Word2Vec
model =Word2Vec(result_stop,#用于训练的语料数据
vector_size=100,#是指特征向量的维度,默认为100。一个句子中当前单词和被预测单词的最大距离。
window=5,min_count=1)#可以对字典做截断.词频少于min_count次数的单词会被丢弃掉,
#计算两个词的相似度
print(model.wv.similarity('沙瑞金','季昌明'))
print(model.wv.similarity('沙瑞金','田国富'))
#选出最相似的5个词
for e in model.wv.most_similar(positive=['沙瑞金'],topn=5):print(e[0],e[1])
odd_word =model.wv.doesnt_match(["苹果","香蕉","橙子","书"])
print(f"在这组词汇中不匹配的词汇:{odd_word}")
word_frequency = model.wv.get_vecattr("沙瑞金", "count")
print(f"沙瑞金: {word_frequency}")
打印出结果:
0.9992292
0.9993775
祁同伟 0.9995201741252625
高育良 0.9994417985196838
肖钢玉 0.9994159936904907
田国富 0.999377429485321
一下 0.99935382604599
在这组词汇中不匹配的词汇:书
沙瑞金: 353