word2vec简介及词向量构建

$$word2vec简介及词向量构建$$

#导入包
import pandas as pd
import numpy as np
from gensim.models import Word2Vec

#读取数据集
train = pd.read_csv('./cnews/train.tsv',sep='\t',header=None,names=['label','content'])
val = pd.read_csv('./cnews/dev.tsv',sep='\t',header=None,names=['label','content'])
test = pd.read_csv('./cnews/test.tsv',sep='\t',header=None,names=['label','content'])

train.head()

jieba pkuseg

import jieba
def content_cut(x):
    x = jieba.lcut(x)
    x = " ".join(x)
    return x

train['content'] = train['content'].map(lambda x: content_cut(x))
val['content'] = val['content'].map(lambda x: content_cut(x))
test['content'] = test['content'].map(lambda x: content_cut(x))

df = pd.concat([train,val,test],axis=0)

df.head()

训练word2vec

sentences = [document.split(' ') for document in df['content'].values]

model = Word2Vec(sentences=sentences,
     size=200,#维度
     alpha=0.025, #默认
     window=5, #默认
     min_count=2,#2，3
     sample=0.001,#
     seed=2018, #
     workers=11, #线程
     min_alpha=0.0001, 
     sg=0, #cbow
     hs=0, #负采样
     negative=5,#负采样个数
     ns_exponent=0.75, 
     cbow_mean=1,#求和再取平均
     iter=10, #10到20
     compute_loss =True
     )

## 保存word2vec

model.save("./word2vec/word2vec_word_200")

model = Word2Vec.load("./word2vec/word2vec_word_200")

model.vo

#查看单词的向量
model.wv['816903'].shape

help(model.most_similar)

#和这个单词最相似的单词
model.most_similar("",topn=20)

#计算两个单词之间相似性
model.wv.similarity("816903","1226448")

#看词表
model.wv.vocab.keys()

#迭代模型

sentences_next=[]
for document in test['word_seg'].tolist():
    sentences_next.append(document.split(" "))

model.train(sentences=sentences_next, total_examples=model.corpus_count,  epochs=model.iter) 
model.save()