python3 读取mysql库生成hive建表语句

import pymysql
import re

def to_hive_type(type):
    if re.match(r'varchar|char|text|tinytext|longtext|blo',type,flags=0):
        return 'string'
    elif re.match(r'timestamp|date|bigint',type,flags=0):
        return 'bigint'
    elif re.match(r'int|tinyint|smallint',type,flags=0):
        return 'int'
    elif re.match(r'double|float',type,flags=0):
        return 'double'
    else:
        return type

def process(hosts,user,password,port,tables):
    for host in hosts:
        mysql = pymysql.connect(host=host, user=user, password=password, port=port)
        cursor = mysql.cursor()
        for dbtbs in tables:
            db_name = dbtbs.split('.')[0]
            tb_name = dbtbs.split('.')[1]
            # print("【{}】【{}】【{}】【{}】".format(dbtbs, db_name, tb_name, host))
            sql = "select TABLE_NAME,TABLE_COMMENT from information_schema.tables where TABLE_SCHEMA = '{}' and TABLE_NAME = '{}' ".format(db_name,tb_name)
            cursor.execute(sql)
            result = cursor.fetchall()
            count = len(result)
            if count > 0:
                print("---该表{}.{}在这台{}服务器上---".format(db_name,tb_name,host))
                create_sql = "SHOW FULL columns FROM   `{}`.`{}` ".format(db_name,tb_name)
                cursor.execute(create_sql)
                db_name = "{}_ods".format(db_name).replace('-','_')
                tb_name = "ods_{}".format(tb_name).replace('-','_')
                print("\r\n -------------{}.{}---hive 建表开始-------------- ".format(db_name,tb_name))
                print("drop table if exists {}.{} ;".format(db_name,tb_name))
                print("create external table  if not  exists  {}.{} (".format(db_name, tb_name))
                cou = 1
                for rs in cursor.fetchall():
                    clomun = str(rs[0]).lower()
                    clo_type = to_hive_type(rs[1])
                    clo_comment = "COMMENT '{}' ".format(str(rs[8]).replace('\r\n|\r|\n',''))
                    if cou == 1 :
                        print(clomun,clo_type,clo_comment)
                    else:
                        print(',',clomun, clo_type,clo_comment)
                    cou += 1
                print(") \r\ncomment '{}' \r\npartitioned by (dt string, area_set string) \r\nstored as parquet \r\nlocation '/apps/data/warehouse/{}/{}'\r\n;".format(result[0][1],db_name, tb_name))
                print("-------------{}.{}---hive 建表结束--------------\r\n  ".format(db_name,tb_name))
        # 6.关闭查询
        cursor.close()
        # 关闭数据库
        mysql.close()

if __name__ == '__main__':
    tables = ["×××××"]
    hosts = ["×××××"]
    user = '×××××'
    password = "×××××"
    port = 
    process(hosts, user, password, port, tables)
  • 0
    点赞
  • 4
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值