Windows下中文维基百科语料上的Word2Vec

本文是参照原文：https://zhuanlan.zhihu.com/p/28643244之后发现一些问题，对操作过程记录。
实验环境： Windows 10 pro （64位）
所用代码为Python3*
用到的python包有：gensim, jieba

1.下载中文维基百科数据与预处理
首先下载数据。下载地址：https://dumps.wikimedia.org/zhwiki/latest/zhwiki-latest-pages-articles.xml.bz2
首先将xml的wiki数据转换为text格式，通过下面这个脚本(process_wiki.py)实现：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Pan Yang (panyangnlp@gmail.com)
# Copyrigh 2017

from __future__ import print_function

import logging
import os.path
import six
import sys
import io
from gensim.corpora import WikiCorpus

sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') #改变标准输出的默认编码


if __name__ == '__main__':
    program = os.path.basename(sys.argv[0])
    logger = logging.getLogger(program)

    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s" % ' '.join(sys.argv))

    # check and process input arguments
    if len(sys.argv) != 3:
        print("Using: python process_wiki.py enwiki.xxx.xml.bz2 wiki.en.text")
        sys.exit(1)
    inp, outp = sys.argv[1:3]
    space = " "
    i = 0

    output = open(outp, 'w', encoding="utf-8")
    wiki = WikiCorpus(inp, lemmatize=False, dictionary={})
    for text in wiki.get_texts():
        if six.PY3:
            output.write(bytes(' '.join(text), 'utf-8').decode('utf-8') + '\n')
        #   ###another method###
        #    output.write(
        #            space.join(map(lambda x:x.decode("utf-8"), text)) + '\n')
        else:
            output.write(space.join(text) + "\n")
        i = i + 1
        if (i % 10000 == 0):
            logger.info("Saved " + str(i) + " articles")

    output.close()
    logger.info("Finished Saved " + str(i) + " articles")

执行： python process_wiki.py zhwiki-latest-pages-articles.xml.bz2 wiki.zh.text
安装opencc（参照：https://www.twblogs.net/a/5b8bfc4d2b717718832f9739/zh-cn），然后将wiki.zh.text中的繁体字转化为简体字：

opencc -i wiki.zh.text -o wiki.zh.text.sim -c D:\opencc-1.0.4\share\opencc\t2s.json   #这里-c后面写t2s.json的路径

用jieba分词
运行fenci.py得到分词后的结果文本。

##!/usr/bin/env python
## coding=utf-8
import jieba
import sys
import importlib

filePath='wiki.zh.text.sim'
fileSegWordDonePath ='wiki.zh.text.jian.seg.utf-8'
# read the file by line
fileTrainRead = []
#fileTestRead = []
with open(filePath,encoding="utf-8") as fileTrainRaw:
    for line in fileTrainRaw:
        fileTrainRead.append(line)


# define this function to print a list with Chinese
def PrintListChinese(list):
    for i in range(len(list)):
        print(list[i]),
# segment word with jieba
fileTrainSeg=[]
for i in range(len(fileTrainRead)):
    importlib.reload(sys)
    fileTrainSeg.append([' '.join(list(jieba.cut(fileTrainRead[i][9:-11],cut_all=False)))])
    if i % 100 == 0 :
        print (i)
        PrintListChinese(fileTrainSeg[i])

# to test the segment result
PrintListChinese(fileTrainSeg[10])

# save the result
with open(fileSegWordDonePath,'wb') as fW:
    for i in range(len(fileTrainSeg)):
        fW.write(fileTrainSeg[i][0].encode('utf-8'))
        fW.write('\n'.encode('utf-8'))

2.使用gensim训练Word2Vec
执行：python train_word2vec_model.py wiki.zh.text.jian.seg.utf-8 wiki.zh.text.model wiki.zh.text.vector
脚本train_word2vec_model.py如下：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Pan Yang (panyangnlp@gmail.com)
# Copyright 2017

from __future__ import print_function

import logging
import os
import sys
import multiprocessing

from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

if __name__ == '__main__':
    program = os.path.basename(sys.argv[0])
    logger = logging.getLogger(program)

    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s" % ' '.join(sys.argv))

    # check and process input arguments
    if len(sys.argv) < 4:
        print("Useing: python train_word2vec_model.py input_text "
              "output_gensim_model output_word_vector")
        sys.exit(1)
    inp, outp1, outp2 = sys.argv[1:4]

    model = Word2Vec(LineSentence(inp), size=200, window=5, min_count=5,
                     workers=multiprocessing.cpu_count())

    model.save(outp1)
    model.wv.save_word2vec_format(outp2, binary=False)

训练好的中文维基百科word2vec模型“wiki.zh.text.vector"的效果：

In [1]: import gensim

In [2]: model = gensim.models.Word2Vec.load("wiki.zh.text.model")

In [3]: model.most_similar(u"足球")
Out[3]: 
[(u'\u8054\u8d5b', 0.6553816199302673),
 (u'\u7532\u7ea7', 0.6530429720878601),
 (u'\u7bee\u7403', 0.5967546701431274),
 (u'\u4ff1\u4e50\u90e8', 0.5872289538383484),
 (u'\u4e59\u7ea7', 0.5840631723403931),
 (u'\u8db3\u7403\u961f', 0.5560152530670166),
 (u'\u4e9a\u8db3\u8054', 0.5308005809783936),
 (u'allsvenskan', 0.5249762535095215),
 (u'\u4ee3\u8868\u961f', 0.5214947462081909),
 (u'\u7532\u7ec4', 0.5177896022796631)]

In [4]: result = model.most_similar(u"足球")

In [5]: for e in result:
    print e[0], e[1]
   ....:     
联赛 0.65538161993
甲级 0.653042972088
篮球 0.596754670143
俱乐部 0.587228953838
乙级 0.58406317234
足球队 0.556015253067
亚足联 0.530800580978
allsvenskan 0.52497625351
代表队 0.521494746208
甲组 0.51778960228

训练得到的gensim Word2Vec 模型可用作Pre train embedding Layer 例如：

'''set network structure'''
    embedding = gensim.models.Word2Vec.load("wiki.zh.text.model")

    encoder_decoder = Sequential()
    encoder_decoder.add(AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=tar_maxlen, input_shape=(input_maxlen, hidden_dim)))
    encoder_decoder.add(Activation('softmax'))
    encoder_decoder.compile(loss='categorical_crossentropy', optimizer="RMSprop")
    '''train model'''    

    train_batch = 100
    train_list_length=int(len(tar_list)/train_batch)
    for epoch in range(0,800):
        for i in range(0,train_batch-1):
            tars_train=one_hot(tar_list[train_list_length*i:train_list_length*(i+1)], word_to_idx, idx_to_word, tar_maxlen, vocab_size)
            word = get_Embedding(embedding,inputs_train[train_list_length*i:train_list_length*(i+1)],input_maxlen)
            encoder_decoder.fit(word, tars_train, batch_size=10, epochs=1, verbose=1)
        tars_train=one_hot(tar_list[train_list_length*(train_batch-1):], word_to_idx, idx_to_word, tar_maxlen, vocab_size)
        word = get_Embedding(embedding,inputs_train[train_list_length*(train_batch-1):],input_maxlen)
        encoder_decoder.fit(word, tars_train, batch_size=10, epochs=1, verbose=1)
        print(epoch)
    print('Train complete...')

用于得到样本Embedding vector的函数如下：

def get_Embedding(embedding,inputs_train,input_maxlen):
    embedding_matrix = []
    for sentence in inputs_train:
        embedding_sentence = []
        for word in sentence:
            try:
                embedding_sentence.append(embedding[word])
            except Exception as err:
                embedding_sentence.append(embedding["unkown"])              
        embedding_matrix.append(embedding_sentence)
    return np.array(embedding_matrix)