import pymysql
import re
def to_hive_type(type):
if re.match(r'varchar|char|text|tinytext|longtext|blo',type,flags=0):
return 'string'
elif re.match(r'timestamp|date|bigint',type,flags=0):
return 'bigint'
elif re.match(r'int|tinyint|smallint',type,flags=0):
return 'int'
elif re.match(r'double|float',type,flags=0):
return 'double'
else:
return type
def process(hosts,user,password,port,tables):
for host in hosts:
mysql = pymysql.connect(host=host, user=user, password=password, port=port)
cursor = mysql.cursor()
for dbtbs in tables:
db_name = dbtbs.split('.')[0]
tb_name = dbtbs.split('.')[1]
# print("【{}】【{}】【{}】【{}】".format(dbtbs, db_name, tb_name, host))
sql = "select TABLE_NAME,TABLE_COMMENT from information_schema.tables where TABLE_SCHEMA = '{}' and TABLE_NAME = '{}' ".format(db_name,tb_name)
cursor.execute(sql)
result = cursor.fetchall()
count = len(result)
if count > 0:
print("---该表{}.{}在这台{}服务器上---".format(db_name,tb_name,host))
create_sql = "SHOW FULL columns FROM `{}`.`{}` ".format(db_name,tb_name)
cursor.execute(create_sql)
db_name = "{}_ods".format(db_name).replace('-','_')
tb_name = "ods_{}".format(tb_name).replace('-','_')
print("\r\n -------------{}.{}---hive 建表开始-------------- ".format(db_name,tb_name))
print("drop table if exists {}.{} ;".format(db_name,tb_name))
print("create external table if not exists {}.{} (".format(db_name, tb_name))
cou = 1
for rs in cursor.fetchall():
clomun = str(rs[0]).lower()
clo_type = to_hive_type(rs[1])
clo_comment = "COMMENT '{}' ".format(str(rs[8]).replace('\r\n|\r|\n',''))
if cou == 1 :
print(clomun,clo_type,clo_comment)
else:
print(',',clomun, clo_type,clo_comment)
cou += 1
print(") \r\ncomment '{}' \r\npartitioned by (dt string, area_set string) \r\nstored as parquet \r\nlocation '/apps/data/warehouse/{}/{}'\r\n;".format(result[0][1],db_name, tb_name))
print("-------------{}.{}---hive 建表结束--------------\r\n ".format(db_name,tb_name))
# 6.关闭查询
cursor.close()
# 关闭数据库
mysql.close()
if __name__ == '__main__':
tables = ["×××××"]
hosts = ["×××××"]
user = '×××××'
password = "×××××"
port =
process(hosts, user, password, port, tables)
python3 读取mysql库生成hive建表语句
最新推荐文章于 2024-04-07 17:42:21 发布