Python-去除txt文档中重复的关键词（语句）

去除关键词前的txt文档

key_word.txt

和除去相同关键词新建的txt文档：

key_word_new.txt

# _*_ coding: utf-8 _*_
__author__ = 'gyx'
__date__ = '2020/3/23 14:05'

import jieba
import pandas as pd
import os
from collections import Counter

txtPath = 'key_word.txt'
txtPath_new = 'key_word_new.txt'
#不加首行： data = pd.read_csv(txtPath, encoding='utf-8')
data = pd.read_csv(txtPath, encoding='utf-8',header=None)
with open(txtPath_new, 'a+', encoding='utf-8') as f:
    for line in data.values:
        counts = {}
        words = jieba.lcut(line[0])
        data_key = dict(Counter([i for i in words]))
        print('去除前：', data_key)
        data_keys_new = [k for k in data_key.keys()]
        print('去除后：', data_keys_new)
        line_new="".join(data_keys_new)
        print(line_new)
        f.write(str(line_new))
        f.write('\n')

去除前和去除后：