将csv文件插入hive(Python代码)

from pyhive import presto
import pandas as pd
import numpy as np
import os
import time

#连接presto
conn = presto.connect(protocol = 'https', host = 'host.com', port = 8443, username = 'username', password = 'password')
cursor = conn.cursor()

#数据读取
wd = 'C:/Users/desktop'  #改变默认路径,可以省略此步骤直接读取绝对路径
os.chdir(wd)
file_name = pd.read_csv("file.csv",header = None)

#建表
sql = '''
create table table_name
(
id bigint, 
uid bigint,
order_id bigint,
aa varchar(125),
bb bigint
)
WITH (format = 'ORC')'''
cursor.execute(sql)
cursor.fetchall()

#插入数据
sql = 'insert into table_name'
for i in range(file_name.iloc[:,0].size):
    sql = sql + str(tuple(file_name.iloc[i])) + ','
sql = sql[:-1]
start = time.perf_counter()
cursor.execute(sql)
cursor.fetchall()
conn.commit()
end = time.perf_counter()
print('use time is %6.3f' % (end-start))

https://blog.csdn.net/longshenlmj/article/details/51702343?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522164420627516780255281795%2522%252C%2522scm%2522%253A%252220140713.130102334…%2522%257D&request_id=164420627516780255281795&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2allsobaiduend~default-1-51702343.pc_search_insert_ulrmf&utm_term=with+%28format+%3D+orc%29&spm=1018.2226.3001.4187

  • 2
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值