python获取指定文件夹下的csv文件,进行文件的合并,并去除两文件中重复的行
import pandas as pd
def mergerF(data_dir,csv_list,save_dir,save_file):
print(u'total%s CSV files'% len(csv_list))
for i in csv_list:
fr = open(data_dir + i,'r').read()
with open(save_dir+save_file,'a') as f:
f.write(fr)
print(u'complete merger!')
#drop duplicates after merging
def drop_dup(save_dir,save_file):
df = pd.read_csv(save_dir+save_file,header=0)
datalist = df.drop_duplicates()
datalist.to_csv(save_dir+save_file)
data_dir ='./data/'#The path of the file to be merged
out_dir='./data/merger/'# save path after merging
csv_list = ['a.csv','b.csv'] # file name to be merged
save_file = 'mergered.csv'# file name for after merging
mergerF(data_dir,csv_list,save_dir,save_file)
drop_dup(save_dir,save_file)