1.取得中文维基百科数据,本实验用的数据是zhwiki-20180320-pages-articles-multistream.xml.bz2
也可以前往维基百科数据获取下载最新的数据。(请挑选以pages-articles.xml.bz2为结尾的文档)
2.利用wiki_to_txt.py将xml文档转换成txt文档
# coding=utf-8
## 后台运行 python3.5 wiki_to_txt.py zhwiki-20180320-pages-articles-multistream.xml.bz2
import logging ##记录数据日志
import sys ##sys模块提供了一系列有关Python运行环境的变量和函数
from gensim.corpora import WikiCorpus
def main():
if len(sys.argv) !=2: ##获取当前正在执行的命令行参数的参数列表(list)
print("Usage:Python3" + sys.argv[0] + "wiki_data_path") ##sys.argv[0]是当前程序名
exit()
logging.basicConfig(format = '%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
wiki_corpus = WikiCorpus(sys.argv[1],dictionary={})
texts_num = 0
with open("wiki_texts.txt",'w',encoding='utf-8') as output:
for text in wiki_corpus.get_texts():
output.write(''.join(text) + '\n')
texts_num += 1
if texts_num % 10000 ==0:
logging.info("已处理 %d 篇文章" % texts_num)
if __name__ == "__main__":
main()
后台运行:python3.5 wiki_to_txt.py zhwiki-20180320-pages-articles-multistream.xml.bz2
3.繁转简
利用opencc工具进行转换
在ubuntu上安装opencc:sudo apt-get install opencc
利用命令行:
opencc -i wiki_texts.txt -o wiki_zh_tw.txt -c zht2zhs.ini
4.利用jieba分词除去停用词
pip3 install jieba
利用segment.py
# coding=utf-8
import jieba
import logging
def main():
logging.basicConfig(format='%(asctime)s : %(levelname)s :%(message)s', level=logging.INFO)
#jieba custom setting
jieba.set_dictionary('jieba_dict/dict.txt.big')
#load stopwords set
stopword_set = set()
with open('jieba_dict/stopwords.txt', 'r', encoding='utf-8') as stopwords:
for stopword in stopwords:
stopword_set.add(stopword.strip('\n'))
output = open('wiki_seg.txt', 'w',encoding='utf-8')
with open('wiki_zh_tw.txt','r',encoding='utf-8')as content:
for texts_num,line in enumerate(content):
line = line.strip('\n')
words = jieba.cut(line, cut_all=False)
for word in words:
if word not in stopword_set:
output.write(word + '')
output.write('\n')
if (texts_num +1) % 10000 ==0:
logging.info("已完成 %d 行的断词" % (texts_num + 1))
output.close()
if __name__ == '__main__':
main()
通过命令行:python3 segment.py
5.使用gensim库中的word2vec进行训练
pip3 install gensim
利用train.py训练模型
# coding=utf-8
import logging
from gensim.models import word2vec
def main():
logging.basicConfig(format('%(asctimes)s : %(levelname)s : %(message)s',level=logging.INFO))
sentences = word2vec.LineSentence("wiki_seg.txt")
model = word2vec.Word2Vec(sentences,size=250)
#保存模型
model.save("word2vec.model")
#模型读取方式
# model = word2vec.Word2vec.load("your_model_name")
if __name__ == '__main__':
main()
命令行运行:python3 train.py