1.下载维基百科中文语料
http://www.52nlp.cn/中英文维基百科语料上的Word2Vec实验中下载中文维基百科数据,也可从中文数据的下载地址是:https://dumps.wikimedia.org/zhwiki/latest/zhwiki-latest-pages-articles.xml.bz2。
2.xml的wiki数据转换为text格式
参考http://www.mamicode.com/info-detail-1699780.html这篇博客
import logging
import os.path
import sys
from gensim.corpora import WikiCorpus
if __name__ == '__main__':
program = os.path.basename(sys.argv[0])
logger = logging.getLogger(program)
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
logging.root.setLevel(level=logging.INFO)
logger.info("running %s" % ' '.join(sys.argv))
# check and process input arguments
if len(sys.argv) < 3:
print (globals()['__doc__'] % locals())
sys.exit(1)
inp, outp = sys.argv[1:3]
space = b''
i = 0
output = open(outp, 'w',encoding='utf-8')
wiki = WikiCorpus(inp, lemmatize=False, dictionary={})
for text in wiki.get_texts():
s=space.join(text)
s=s.decode('utf8') + "\n"
output.write(s)
i = i + 1
if (i % 10000 == 0):
logger.info("Saved " + str(i) + " articles")
output.close()
logger.info("Finished Saved " + str(i) + " articles")
然后打开cmd,输入python G:\维基百科语料\process_wiki.py G:\维基百科语料\zhwiki-latest-pages-articles.xml.bz2 G:\维基百科语料\wiki.zh.text