解决word2vec内存爆炸问题

解决word2vec内存爆炸

1.方法：生成器

import gensim
from gensim.models import KeyedVectors
from gensim.models import Word2Vec

class Sentences(object):
    def __init__(self, corpus):
        self.corpus = corpus

    def __iter__(self):
        with open(self.corpus,'r',encoding='utf-8') as f:
            for line in open(f):
                yield line.split()

def runTraining(input_file,output_file):
    sentences = Sentences(input_file)
    model = gensim.models.Word2Vec(sentences, size=200)
    model.wv.save_word2vec_format(output_file,binary=False)
    model=KeyedVectors.load_word2vec_format(file,binary=False)
    return model