Word2vec训练中文维基百科

1.取得中文维基百科数据，本实验用的数据是zhwiki-20180320-pages-articles-multistream.xml.bz2

也可以前往维基百科数据获取下载最新的数据。（请挑选以pages-articles.xml.bz2为结尾的文档）

2.利用wiki_to_txt.py将xml文档转换成txt文档

# coding=utf-8
## 后台运行 python3.5 wiki_to_txt.py zhwiki-20180320-pages-articles-multistream.xml.bz2
import logging ##记录数据日志
import sys ##sys模块提供了一系列有关Python运行环境的变量和函数

from gensim.corpora import WikiCorpus

def main():
    if len(sys.argv) !=2: ##获取当前正在执行的命令行参数的参数列表(list)
        print("Usage:Python3" + sys.argv[0] + "wiki_data_path") ##sys.argv[0]是当前程序名
        exit()
    logging.basicConfig(format = '%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
    wiki_corpus = WikiCorpus(sys.argv[1],dictionary={})
    texts_num = 0

    with open("wiki_texts.txt",'w',encoding='utf-8') as output:
        for text in wiki_corpus.get_texts():
            output.write(''.join(text) + '\n')
            texts_num += 1
            if texts_num % 10000 ==0:
                logging.info("已处理 %d 篇文章" % texts_num)


if __name__ == "__main__":
    main()

后台运行：python3.5  wiki_to_txt.py  zhwiki-20180320-pages-articles-multistream.xml.bz2

3.繁转简

利用opencc工具进行转换

在ubuntu上安装opencc：sudo apt-get install opencc

利用命令行：

opencc -i wiki_texts.txt -o wiki_zh_tw.txt  -c zht2zhs.ini

4.利用jieba分词除去停用词

pip3  install jieba

利用segment.py

# coding=utf-8


import jieba
import logging

def main():
    logging.basicConfig(format='%(asctime)s : %(levelname)s :%(message)s', level=logging.INFO)

    #jieba custom setting
    jieba.set_dictionary('jieba_dict/dict.txt.big')

    #load stopwords set
    stopword_set = set()
    with open('jieba_dict/stopwords.txt', 'r', encoding='utf-8') as stopwords:
        for stopword in stopwords:
            stopword_set.add(stopword.strip('\n'))

    output = open('wiki_seg.txt', 'w',encoding='utf-8')
    with open('wiki_zh_tw.txt','r',encoding='utf-8')as content:
        for texts_num,line in enumerate(content):
            line = line.strip('\n')
            words = jieba.cut(line, cut_all=False)
            for word in words:
                if word not in stopword_set:
                    output.write(word + '')
            output.write('\n')
            if (texts_num +1) % 10000 ==0:
                logging.info("已完成 %d 行的断词" % (texts_num + 1))
    output.close()
if __name__ == '__main__':
    main()

通过命令行：python3 segment.py

5.使用gensim库中的word2vec进行训练

pip3 install gensim

利用train.py训练模型

# coding=utf-8

import logging
from gensim.models import word2vec

def main():

    logging.basicConfig(format('%(asctimes)s : %(levelname)s : %(message)s',level=logging.INFO))
    sentences = word2vec.LineSentence("wiki_seg.txt")
    model = word2vec.Word2Vec(sentences,size=250)


    #保存模型
    model.save("word2vec.model")

    #模型读取方式
    # model = word2vec.Word2vec.load("your_model_name")

if __name__ == '__main__':
    main()

命令行运行:python3 train.py