# -*- coding:utf8 -*- import os import jieba def splitSentence(inputFile): fin = open(inputFile, 'r') #以读的方式打开文件 global fout #以写得方式打开文件 #print fin global stop for eachLine in fin: #print eachLine line = eachLine.strip()#.decode('utf-8', 'ignore') #去除每行首尾可能出现的空格,并转为Unicode进行处理 line=line.strip('\n') #去掉多余空行 wordList = list(jieba.cut(line)) #用结巴分词,对每行内容进行分词 #wordList = list(jieba.cut_for_search(line)) outStr = '' for word in wordList:# if len(word)>1: if not word in stop: outStr += word outStr += ' ' fout.write(outStr.strip().encode('utf-8')) #将分词好的结果写入到输出文件 fout.write('\n') fin.close() #path=r'/media/软件/zhuomian/VARandLDAr/train' #r'D:/zhuomian/VARandLDA/train' path='/home/xdj/train' fns=[os.path.join(root,fn) for root,dirs,files in os.walk(path) for fn in files] stop = [line.strip().decode('utf-8', 'ignore') for line in open('/home/xdj/chstop.txt').readlines()] fout = open('myOutput.txt', 'w') fout.write('%d' %len(fns)+'\n') for f in fns: splitSentence(f) #splitSentence('/home/xdj/train/C3-Art/C3-Art1459.txt', 'myOutput.txt') print(len(fns)) fout.close()