word2vec训练中文模型

首先需要一份比较大的中文语料数据，可以考虑中文的维基百科（也可以试试搜狗的新闻语料库）。中文维基百科的打包文件地址为 

https://dumps.wikimedia.org/zhwiki/latest/zhwiki-latest-pages-articles.xml.bz2

中文维基百科的数据不是太大，xml的压缩文件大约1G左右。首先用 process_wiki_data.py处理这个XML压缩文件，执行：python process_wiki_data.py zhwiki-latest-pages-articles.xml.bz2 wiki.zh.text




1. #!/usr/bin/env python
2. # -*- coding: utf-8 -*-
3. # process_wiki_data.py 用于解析XML，将XML的wiki数据转换为text格式
5. import logging
6. import os.path
7. import sys
9. from gensim.corpora import WikiCorpus
11. if __name__ == '__main__':
12. program = os.path.basename(sys.argv[0])
13. logger = logging.getLogger(program)
15. logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
16. logging.root.setLevel(level=logging.INFO)
17. logger.info("running %s" % ' '.join(sys.argv))
19. # check and process input arguments
20. if len(sys.argv) < 3:
21. print (globals()['__doc__'] % locals())
22. sys.exit(1)
23. inp, outp = sys.argv[1:3]
24. space = " "
25. i = 0
27. output = open(outp, 'w')
28. wiki = WikiCorpus(inp, lemmatize=False, dictionary={})
29. for text in wiki.get_texts():
30. output.write(space.join(text) + "\n")
31. i = i + 1
32. if (i % 10000 == 0):
33. logger.info("Saved " + str(i) + " articles")
35. output.close()
36. logger.info("Finished Saved " + str(i) + " articles")
37. 

38. Python用jieba完成分词，生成分词文件wiki.zh.text.seg 

39. 分词文件：seg.py
40. import jieba
    import jieba.analyse
    import jieba.posseg as pseg
    import codecs,sys
    def cut_words(sentence):
        #print sentence
    return " ".join(jieba.cut(sentence)).encode('utf-8')
    f=codecs.open('wiki.zh.text','r',encoding="utf8")
    target = codecs.open("wiki.zh.text.seg", 'w',encoding="utf8")
    print ('open files')
    line_num=1
    line = f.readline()
    while line:
        print('---- processing ', line_num, 'article----------------')
        line_seg = " ".join(jieba.cut(line))
        target.writelines(line_seg)
        line_num = line_num + 1
        line = f.readline()
    f.close()
    target.close()
    exit()
    while line:
        curr = []
        for oneline in line:
            #print(oneline)
            curr.append(oneline)
        after_cut = map(cut_words, curr)
        target.writelines(after_cut)
        print ('saved ',line_num,' articles')
        exit()
        line = f.readline1()
    f.close()
    target.close()
    
41. 运行 python seg.py
42. 接着用word2vec工具训练： 
    python train_word2vec_model.py wiki.zh.text.seg wiki.zh.text.model wiki.zh.text.vector
    
43. 1. #!/usr/bin/env python
    2. # -*- coding: utf-8 -*-
    3. # train_word2vec_model.py用于训练模型
    5. import logging
    6. import os.path
    7. import sys
    8. import multiprocessing
    10. from gensim.corpora import WikiCorpus
    11. from gensim.models import Word2Vec
    12. from gensim.models.word2vec import LineSentence
    14. if __name__ == '__main__':
    15. program = os.path.basename(sys.argv[0])
    16. logger = logging.getLogger(program)
    18. logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
    19. logging.root.setLevel(level=logging.INFO)
    20. logger.info("running %s" % ' '.join(sys.argv))
    22. # check and process input arguments
    23. if len(sys.argv) < 4:
    24. print globals()['__doc__'] % locals()
    25. sys.exit(1)
    26. inp, outp1, outp2 = sys.argv[1:4]
    28. model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5,
    29. workers=multiprocessing.cpu_count())
    31. # trim unneeded model memory = use(much) less RAM
    32. #model.init_sims(replace=True)
    33. model.save(outp1)
    34. model.save_word2vec_format(outp2, binary=False)

    测试模型效果：

    1. In [1]: import gensim
    3. In [2]: model = gensim.models.Word2Vec.load("wiki.zh.text.model")
    5. In [3]: model.most_similar(u"足球")
    6. Out[3]:
    7. [(u'\u8054\u8d5b', 0.6553816199302673),
    8. (u'\u7532\u7ea7', 0.6530429720878601),
    9. (u'\u7bee\u7403', 0.5967546701431274),
    10. (u'\u4ff1\u4e50\u90e8', 0.5872289538383484),
    11. (u'\u4e59\u7ea7', 0.5840631723403931),
    12. (u'\u8db3\u7403\u961f', 0.5560152530670166),
    13. (u'\u4e9a\u8db3\u8054', 0.5308005809783936),
    14. (u'allsvenskan', 0.5249762535095215),
    15. (u'\u4ee3\u8868\u961f', 0.5214947462081909),
    16. (u'\u7532\u7ec4', 0.5177896022796631)]
    18. In [4]: result = model.most_similar(u"足球")
    20. In [5]: for e in result:
    21. print e[0], e[1]
    22. ....:
    23. 联赛 0.65538161993
    24. 甲级 0.653042972088
    25. 篮球 0.596754670143
    26. 俱乐部 0.587228953838
    27. 乙级 0.58406317234
    28. 足球队 0.556015253067
    29. 亚足联 0.530800580978
    30. allsvenskan 0.52497625351
    31. 代表队 0.521494746208
    32. 甲组 0.51778960228
    34. In [6]: result = model.most_similar(u"男人")
    36. In [7]: for e in result:
    37. print e[0], e[1]
    38. ....:
    39. 女人 0.77537125349
    40. 家伙 0.617369174957
    41. 妈妈 0.567102909088
    42. 漂亮 0.560832381248
    43. 잘했어 0.540875017643
    44. 谎言 0.538448691368
    45. 爸爸 0.53660941124
    46. 傻瓜 0.535608053207
    47. 예쁘다 0.535151124001
    48. mc刘 0.529670000076