python连接hive

环境 python 2.7.18

pyhive

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
https://www.cnblogs.com/hider/p/16279747.html
# Liunx系统
pip install sasl
pip install thrift
pip install thrift-sasl
pip install PyHive

pip install pandas

# Windows系统会出现莫名其妙的报错
"""

from pyhive import hive
import pandas as pd

# def read_jdbc(host, port, database: str, table: str, query_sql: str) -> DataFrame:
def read_jdbc(host, port, database, table, query_sql) :
    # 1、连接hive服务端
    conn = hive.Connection(
        host=host, port=10000, database=database)
    cursor = conn.cursor()
    # logger.info('connect hive successfully.')

    # 2、执行hive sql
    cursor.execute(query_sql)
    # logger.info('query hive table successfully.')

    # 3、返回pandas.dataframe
    table_len = len(table) + 1
    columns = [col[0] for col in cursor.description]
    col = list(map(lambda x: x[table_len:], columns))
    result = cursor.fetchall()
    # 关闭连接 释放资源
    cursor.close()
    conn.close()

    return pd.DataFrame(result, columns=col)


if __name__ == "__main__":
    # print(__name__)
    host = '***'
    port = '10000'
    database = 'tdxdb'
    table = ''
    query_sql = 'select * from tdxdb.t_enterprise limit 2'
    
    rs = read_jdbc(host, port, database, table, query_sql)
    print(rs)

impala

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
pip install pure-sasl
pip install thrift_sasl==0.2.1 --no-deps
pip install thrift==0.9.3
pip install thriftpy
pip install impyla

# Windows系统会出现莫名其妙的报错
"""

from impala.dbapi import connect

 
conn = connect(host='***', port=10000, auth_mechanism='PLAIN', user='hdfs', database='default')
cursor = conn.cursor()
cursor.execute('show tables')
for row in cursor:
    print(row)

# 关闭连接
cursor.close()
conn.close()

HiveServer2 

# 使用 Python 代码来连接 HiveServer2
https://www.tencentcloud.com/zh/document/product/1026/31148
pip install pyhs2

暂未成功
    compilation terminated.
    error: command 'gcc' failed with exit status 1
    ----------------------------------------
ERROR: Command errored out with exit status 1: /usr/bin/python -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-0YDrAQ/sasl/setup.py'"'"'; __file__='"'"'/tmp/pip-install-0YDrAQ/sasl/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record /tmp/pip-record-ukxUok/install-record.txt --single-version-externally-managed --compile --install-headers /usr/local/python2.7.18/include/python2.7/sasl Check the logs for full command output.