python连接mysql数据库
用pip安装好pymysql库,python版本 3.6,连接数据库
import pymysql
import xlwt
import datetime
from pyhive import hive
conn = pymysql.connect(host='********', user = '***', password = '****', database = '****', charset = 'utf8')
cursor = conn.cursor()
执行sql,获取数据,进行逻辑操作
res = cursor.execute(sql)
data = cursor.fetchall()
for item in data:
count = count + 1
node_id = str(item[0])
result = str(item[2])
result = ''
column = 0
将处理好的数据存入文本中
注意点:存入hive的数据字段为空的要填写为null,字段数量需要对上
with open('test.txt', 'a') as f:
f.write(result + '\n')
存入hive表中
load data local inpath '/root/****.txt' overwrite into table dwd.**** partition(pt='20210209');