word2vec
我首先用word2vec计算词向量,对于每条文档的向量表示方法,使用词向量总和表示。
首先把印尼语和中文新闻所有的文本都放在一个txt中。
with open("yinni_cut2.txt",'a',encoding='utf-8') as f:
for i in range(yindi.shape[0]):
text=""
for j in range(3):
if pd.isna(yindi.iloc[i,j])==False:
# print(yindi.iloc[i,j])
text+=(yindi.iloc[i,j]+" ")
new_text = jieba.cut(text, cut_all=False)
str_out = ' '.join(new_text).replace(',', '').replace('。', '').replace('?', '').replace('!', '') \
.replace('“', '').replace('”', '').replace(':', '').replace('…', '').replace('(', '').replace(')', '') \
.replace('—', '').replace('《', '').replace('》', '').replace('、', '').replace('‘', '') \
.replace('’', '')
f.write(str_out+'\n')
读文件生成词向量模型并保存
from gensim.models import word2vec
import gensim
import logging
save_model_file='word2vec2.model'
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
sentences = word2vec.Text8Corpus("yinni_cut2.txt") # 加载语料
model = gensim.models.Word2Vec(sentences,min_count=1,sg=1,vector_size=100,window=5) # 训练skip-gram模型; 默认window=5
model.save(save_model_file)
model.wv.save_word2vec_format(save_model_file + "2.bin", binary=True)
读取模型
load2 =Word2Vec.load("word2vec2.model")
load2.wv.get_vector("陆续")
计算余弦相似度
import numpy as np
#需要去除停用词才可达到效果!
def vector_similarity(s1, s2):
def sentence_vector(s):
words = list(jieba.cut(s,cut_all=False))
#words = jieba.analyse.extract_tags(s,allowPOS=('n','nr','nr1','nr2','nrj','nrf','ns','nsf','nt','nz','nl','ng','nrfg'))
ba = []
k=0
for i in range(len(words)):
if len(words[i])<=1:
ba.append(words[i])
words=list(set(words)-set(ba))
words=list(set(words)-set(stopwords))
v = np.zeros(100)
for word in words:
try:
v += load2.wv.get_vector(word)
except:
k+=1
print(word)
v /= (len(words)-k)
return v
v1, v2 = sentence_vector(s1), sentence_vector(s2)
return np.dot(v1, v2) / (np.linalg.norm(v1) *np.linalg.norm(v2))
最后的效果不是很好,虽然依然能匹配到相似中文新闻,但是相似度都在0.999。所以只能转tfidf。
tfidf
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
#将文本中的词语转换为词频矩阵,矩阵元素a[i][j] 表示j词在i类文本下的词频
vectorizer = CountVectorizer(stop_words=stopwords)
#计算每一个词语的TF-IDF权值
tf_idf_transformer = TfidfTransformer()
#计算每一个词语出现的次数#将文本转换为词频并计算tf-idf;fit_transform()方法用于计算每一个词语出现的次数
X = vectorizer.fit_transform(allcut)
tf_idf = tf_idf_transformer.fit_transform(X)
#将tf-idf矩阵抽取出来,元素a[i][j]表示j词在i类文本中的tf-idf权重
x_train_weight = tf_idf.toarray()
print('输出X_train文本变量:')
print(x_train_weight)
用tfidf值构建一个文本的向量。最终的结果正常点。