csv文件合并、去重

import pandas as pd
import glob
 
outputfile=r'G:\shiyabn\LZQ\1108_check\hebing.csv'
csv_list = glob.glob(r'G:\shiyabn\LZQ\1108_check\*.csv')

print('共发现%s个CSV文件'% len(csv_list))
print('正在处理')

def hebing(csv_list):
    for inputfile in csv_list:
        f=open(inputfile)
        data=pd.read_csv(f,header=None)
        data.to_csv(outputfile,mode='a',header=False,index=False)
    print('完成合并')
    
def quchong(file):
    df = pd.read_csv(file,header=None)
    print(df[df.duplicated()]) #打印重复的行
    datalist = df.drop_duplicates()
    datalist.to_csv(file)
    print('完成去重')
 
if __name__ == '__main__':
    hebing()
    quchong(outputfile)

参考：
参考1
参考2