python pandas 批量合并相同列名的csv文件

 python 实现过程文件批量合并

import pandas as pd
import os

def getFile(path):
    dirlst= []
    dirs = os.listdir(path)
    for i in dirs:
        file = os.path.splitext(i)[0]
        if file.startswith('爬虫_'):
            filename = path + '\\' + file+'.csv'
            dirlst.append(filename)
    return dirlst

    
def hebing(csv_list, outputfile):
    for inputfile in csv_list:
        f = open(inputfile,encoding='utf8')
        data = pd.read_csv(f,encoding='utf8')
        data.to_csv(outputfile, mode='a', index=False,encoding='utf8')
    print('完成合并')

def quchong(file):
    df = pd.read_csv(file, header=None)
    datalist = df.drop_duplicates()
    datalist.to_csv('D:\\pycharmproject\\result_new.csv', index=False, header=False)
    print('完成去重')
    
if __name__ == '__main__':
    path = r'F:\wk\del'
    dirlst = getFile(path)
    output_csv_path = 'D:\\pycharmproject\\result.csv'
    hebing(dirlst, output_csv_path)
    quchong(output_csv_path)