fromgensim.models import Word2Vec
sys.argv.append('train_200000_split.utf8') sys.argv.append('ner_training_word2vec_200001.model') sys.argv.append('ner_training_word2vec_200001.vector')
input_file, output_model_file, output_vector_file = sys.argv[1:4] sentences=[] for line in open(input_file, 'r', encoding='utf-8'): sentences.append(line.split(' '))
model = Word2Vec(sentences, size=128, window=5, min_count=5, workers=multiprocessing.cpu_count()) # trim unneeded model memory = use(much) less RAM #model.init_sims(replace=True) model.save(output_model_file) model.save_word2vec_format(output_vector_file, binary=False) end_time = time.time() print("used time : %d s" % (end_time - start_time))
#使用word2vec w2vModel = Word2Vec.load(word2vec_model_file) print(word2vec_model_file) print(w2vModel.layer1_size) embeddingDim = w2vModel.layer1_size #vector_size embeddingUnknown = [0 for i in range(embeddingDim)] embeddingWeights = np.zeros((vocabSize + 3, embeddingDim)) for word, index in vocab.items(): if word in w2vModel: e = w2vModel[word] print(word) else: print('not in model:',word) e = embeddingUnknown # print(e) embeddingWeights[index, :] = e