python 文本处理

# -*- coding: utf-8 -*-
#-----------------------------------------------------------------------------------------------------------------------
__Author__ = 'assasin'
__DateTime__ = '2020/1/4 20:44'
#-----------------------------------------------------------------------------------------------------------------------
'''
多条文本信息处理
'''
from stop_words import readFile,seg_doc
import os,time

class loadFolder(object):
    def __init__(self,parent_path):
        self.parent_path = parent_path

    def __iter__(self):
        for file in os.listdir(self.parent_path):
            file_abspath = os.path.join(self.parent_path,file)
            if os.path.isdir(file_abspath):
                yield file_abspath  # return



class loadFiles(object):
    def __init__(self,parent_path):
        self.parent_path = parent_path

    def __iter__(self):
        folders = loadFolder(self.parent_path)
        for folder in folders:      # level1
            catg = folder.split(os.sep)[-1]
            for file in os.listdir(folder):  # level2
                file_path = os.path.join(folder,file)
                if os.path.isfile(file_path):
                    this_file = open(file_path,'rb')
                    content = this_file.read().decode('utf8')
                    yield catg,content
                    this_file.close()



if __name__ == '__main__':
    start_time = time.time()

    filepath = os.path.abspath(r'../shell/datas')
    files = loadFiles(filepath)
    n = 5 # 表示抽样率

    for i,msg in enumerate(files):
        if i % n == 0:
            catg = msg[0]
            file = msg[1]
            file = seg_doc(file)
            if int(i % n) % 20 == 0:
                print('{t} *** {i} \t docs has been deal '.format(i=i,
                                                                  t=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())))


    end_time = time.time()

    print('total cost time : %.2f' % (end_time - start_time) + 's')