接wiki数据处理章
1.我曾经把从wiki下载下来的压缩包解压缩过,大概7个g左右,经过处理后,txt文件1个g左右。直接文本编辑器是打不开的,可以用python进行一行展示:
import codecs,sys
f = codecs.open('C:/Users/Shirley/Desktop/python/wiki_zh_1.txt','r',encoding="utf8")
line = f.readline()
print(line)
结果如下:
2.接下来要对这一坨数据进行分词,jieba是个很好的选择。
分词代码:
import jieba
import jieba.analyse
import jieba.posseg as pseg
import codecs,sys
def cut_words(sentence):
return " ".join(jieba.cut(sentence)).encode('utf8')
# f = codecs.open('C:/Users/Shirley/Desktop/python/wiki_zh_1.txt','r',encoding="utf8")
f = codecs.open('wiki_zh_1.txt','r',encoding="utf8")
# line = f.readline()
# print(line)
# target = codecs.open('C:/Users/Shirley/Desktop/python/wiki_zh_1.txt/wiki_cut_words.txt','w',encoding="utf8")
target = codecs.open('wiki_cut_words.txt','w',encoding="utf8")
print ("open files")
line_num =1
line =f.readline()
while line:
print("处理数据:",line_num)
line_seg = " ".join(jieba.cut(line))
target.writelines(line_seg)
line_num = line_num+1
line = f.readline()
f.close()
target.close()
exit()