参见:http://www.52nlp.cn/%E4%B8%AD%E8%8B%B1%E6%96%87%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E8%AF%AD%E6%96%99%E4%B8%8A%E7%9A%84word2vec%E5%AE%9E%E9%AA%8C
1 install python3 opencc
brew install python3 opencc
2 download zhwiki-latest-pages-articles2.xml.bz2
3 抽取xml为文本文字
python process.py zhwiki-latest-pages-articles2.xml.bz2 test.txt
4 optionalopencc -i test.txt -o test_simple.txt -c t2s.json5from test_simple.txt (test.txt) to testjieba.txt by TestJieba.pyimport jieba import jieba.analyse import jieba.posseg as pseg import codecs,sys def cut_words(sentence): #print sentence return " ".join(jieba.cut(sentence)).encode('utf-8') f=codecs.open('test.txt','r',encoding="utf8") target = codecs.open("testjieba.txt", 'w',encoding="utf8") print ('open files') line_num=1 line = f.readline() while line: print('---- processing ', line_num, ' article----------------') line_seg = " ".join(jieba.cut(line)) target.writelines(line_seg) line_num = line_num + 1 line = f.readline() f.close() target.close() exit() while line: curr = [] for oneline in line: #print(oneline) curr.append(oneline) after_cut = map(cut_words, curr) target.writelines(after_cut) print ('saved',line_num,'articles') exit() line = f.readline1() f.close() target.close() # python Testjieba.py
5
python word2vec_model.py testjieba.txt wiki.zh.text.model wiki.zh.text.vectorimport logging import os.path import sys import multiprocessing from gensim.corpora import WikiCorpus from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence if __name__ == '__main__': program = os.path.basename(sys.argv[0]) logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s') logging.root.setLevel(level=logging.INFO) logger.info("running %s" % ' '.join(sys.argv)) # check and process input arguments if len(sys.argv) < 4: print (globals()['__doc__'] % locals()) sys.exit(1) inp, outp1, outp2 = sys.argv[1:4] model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count()) model.save(outp1) model.model.wv.save_word2vec_format(outp2, binary=False)#opencc -i test.txt -o test_simple.txt -c t2s.json#python word2vec_model.py testjieba.txt wiki.zh.text.model wiki.zh.text.vector