python 实现过程文件批量合并
import pandas as pd
import os
def getFile(path):
dirlst= []
dirs = os.listdir(path)
for i in dirs:
file = os.path.splitext(i)[0]
if file.startswith('爬虫_'):
filename = path + '\\' + file+'.csv'
dirlst.append(filename)
return dirlst
def hebing(csv_list, outputfile):
for inputfile in csv_list:
f = open(inputfile,encoding='utf8')
data = pd.read_csv(f,encoding='utf8')
data.to_csv(outputfile, mode='a', index=False,encoding='utf8')
print('完成合并')
def quchong(file):
df = pd.read_csv(file, header=None)
datalist = df.drop_duplicates()
datalist.to_csv('D:\\pycharmproject\\result_new.csv', index=False, header=False)
print('完成去重')
if __name__ == '__main__':
path = r'F:\wk\del'
dirlst = getFile(path)
output_csv_path = 'D:\\pycharmproject\\result.csv'
hebing(dirlst, output_csv_path)
quchong(output_csv_path)