文本张量的表示方法之word2vec

'''
Description: word2vec
Autor: 365JHWZGo
Date: 2021-12-01 16:27:34
LastEditors: 365JHWZGo
LastEditTime: 2021-12-01 17:02:31
'''

前期准备

下载fasttext

pip install fasttext

import fasttext

# 1.获取训练数据
# 从新闻截取英文
# 2.训练词向量
model = fasttext.train_unsupervised('./demoText.txt')

#获取指定单词的词向量
MawkishTensor = model.get_word_vector("mawkish")

# 3.模型超参数设置
model = fasttext.train_unsupervised('./demoText.txt',"cbow",dim=80,epoch=10,lr=0.1,thread=8)

# 4.模型效果检验
appleNeighbors = model.get_nearest_neighbors("apple")
print(appleNeighbors)

#由于数据量太少，误差很大，没有得到结果
# 5.模型的保存与重加载

model.save_model("demoText.bin")

model1 = fasttext.load_model("demoText.bin")
print(model1.get_word_vector('apple'))

运行结果

model：

MawkishTensor

[-2.5558758e-03  7.0714066e-04  1.3326192e-03 -1.1297366e-04
 -1.0115254e-03  1.3817545e-03  2.1902670e-03 -1.9726966e-04
  3.5115163e-04 -1.0150258e-03  1.8033127e-03  4.3908608e-04
  9.4841578e-04 -5.6133978e-04  3.2306842e-05 -5.1400409e-04
 -1.9743524e-03 -1.1904427e-03  1.3882502e-04 -2.3718565e-05
 -1.7377484e-03 -8.4501057e-04  1.4572268e-03 -1.2398418e-04
  8.3771284e-04 -1.1136065e-03 -5.0031557e-04  8.1253861e-04
  1.3606325e-03  1.3178245e-04 -1.4802934e-04 -7.2319806e-04
 -2.6084835e-04 -1.5604346e-04  2.5253687e-03 -1.0863495e-04
  ·············  ·············  ·············  ·············
  1.4098656e-03 -1.9964598e-04  2.9838810e-04 -1.6173745e-03
 -2.0243111e-04 -7.4366649e-04 -1.7261954e-03  2.2205994e-03
  1.1768306e-03 -2.1802648e-03 -2.8712628e-04 -1.2993505e-03]

model = fasttext.train_unsupervised(’./demoText.txt’,“cbow”,dim=80,epoch=10,lr=0.1,thread=8)

[-5.30393329e-04  5.00563532e-04 -7.30612082e-05 -1.14947643e-05
  3.33502801e-04  1.07924956e-04 -8.70969452e-05  6.67623608e-05
  3.45882931e-04  5.85770176e-04 -8.40409048e-05 -2.23085604e-04
····························     ·······················  ······································
  3.03002918e-04 -6.02676650e-04  5.35927829e-04 -1.33444933e-04
 -3.03110486e-04 -5.33382045e-05 -1.60771218e-04 -3.70099384e-04
  1.30162138e-04  3.50929942e-04  6.11778378e-05 -7.36467657e-04
  5.47022617e-04 -6.17561862e-04  8.01715883e-04  1.23664242e-04
  2.06744560e-04  2.38146913e-05 -7.11010798e-05 -2.94125755e-04
  3.40379454e-04 -4.92578372e-04 -3.77209013e-04 -3.99547134e-04
 -1.54289519e-04  2.68595875e-04 -6.81898382e-04 -4.46360908e-04
 -3.97199823e-04  3.09181691e-04 -1.16018113e-04  1.10858877e-04
  8.58363346e-05 -3.09071736e-04  4.46455524e-04 -1.31989393e-04
  3.77790799e-04  8.79662330e-05 -1.06547641e-05  2.30477541e-04
 -3.62061197e-04  6.11312280e-04 -7.49468745e-05 -9.96475341e-04]