txt文档里筛选出重复数据，并保存到新的txt文档

txt文档里筛选出重复数据，并保存到新的txt文档

input_file = r'D:\pythonXangmu\quchong\input_file.txt' #原始文档
#output_file = 'output.txt'#重复内容记录文档
output_file = r'D:\pythonXangmu\quchong\output.txt'#绝对路径，解决报错找不到文件或文件夹
with open(input_file, 'r', encoding='utf-8') as file:
    content = file.readlines()
print('content',content)
unique_lines = set()#存储唯一的行数据，是列表
duplicate_lines = []#存储重复的行，是列表

#筛选出每行重复数据
for line in content:
    if line in unique_lines:
        duplicate_lines.append(line)
    else:
        unique_lines.add(line)
        

with open(output_file, 'w', encoding='utf-8') as file:
    for line in duplicate_lines:
        file.write(line)
    for line2 in   duplicate_lines:
        line2 =line2
        print('line2',line2)
print('unique_lines',unique_lines)
print('duplicate_lines',duplicate_lines)

打印：
content [‘1200001\n’, ‘1233331\n’, ‘1244441\n’, ‘0000121\n’, ‘1200001\n’, ‘1233331\n’, ‘1233331\n’]
line2 1200001

line2 1233331

line2 1233331

unique_lines {‘1233331\n’, ‘1244441\n’, ‘0000121\n’, ‘1200001\n’}
duplicate_lines [‘1200001\n’, ‘1233331\n’, ‘1233331\n’]

input_file.txt

output.txt