python 统一去重txt文件中的行列内容

# -*- coding: utf-8 -*-

"""

Created on Tue Oct 11 09:16:48 2022

@author: Chester

"""

import os

pth = r'\test\use'

file = 'error.log'

pthfile = os.path.join(pth,file)

trgfile = os.path.join(pth,'error.log')

#print(pthfile)

#print(trgfile)

def rm_dup():

    pth = r'\test\use'

    trgpth = r'\test\use\update'

    for file in os.listdir(pth):

        pthfile = os.path.join(pth,file)

        trgfile = os.path.join(trgpth,file)

        f_read=open(pthfile,'r',encoding='utf-8') 

        f_write=open(trgfile,'w',encoding='utf-8') 

        data=set()

        for a in [a.strip('\n') for a in list(f_read)]:

            if a not in data and 'LINE' not in a and a.split():

                f_write.write(a+'\n')

                data.add(a)

    f_read.close()

    f_write.close()

rm_dup()

print('完成')