import sys
from job.base.JobBase import ExitCode
import job.base.ClientUtil as util
import traceback
import os
def insert_table(tablename):
try:
deletesql='delete from hermes_ads.'+tablename
util.execSql(deletesql,locals())
with open('D:/'+tablename+'.csv','r') as fp:
i=0
j=0
line_list=[]
sql = 'insert into hermes_ads.' + tablename + '({0}) values ({1})'
for line in fp:
if i == 0:
sql='insert into hermes_ads.' + tablename + '('+line+') values ({})'
else :
linedata="','".join(line.split(','))
# print("('"+linedata+"')"
line_list.append("('"+linedata+"')")
if j>=5000:
linedata = ','.join (line_list)
sql2=sql.format(linedata)
util.exceSql(sql2,locals())
j=0
line_list = []
j+=1
i+=1
linedata = ','.join(line_list)
sql2 = sql.format(linedata)
util.exceSql(sql2, locals())
except:
traceback.print_exc()
if __name__=='__main__':
for root,dirs,files in os.walk('D:/',topdown=False):
if root =='D:/':
filenames = files
tables = [tablename.split('.')[0] for tablename in filenames if '.csv' in tablename]
# print(tables)
for tablename in tables:
insert_table(tablename)
python将csv文件插入kudu表
最新推荐文章于 2024-04-14 16:19:54 发布