Kerberos 认证的Hive Java/Spark Scala Api
配置文件:
#加载kerberos 配置文件,如果不加载,会默认在本地:c:/windows/krb5.ini 或者集群 /etc/krb5.conf 寻找
kerberos.conf = C:/Users/user/Desktop/keytable/krb5.conf
#Hive JDBC连接参数
hive.driver = org.apache.hive.jdbc.HiveDriver
kerberos.hive2.url = jdbc:hive2://cdh21:10000/;principal=hive/cdh21@HADOOP.COM
Java 版本连接Hive2查询
import com.demo.getconf.GetConfigures;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
* @Auther: w
* @Date: 2018/12/12 13:59
* @Description: kerbers 认证的hive2
*/
public class ReadJdbcKerberosHive2 {
private static final Logger log = Logger.getLogger(ReadJdbcKerberosHive2.class);
static {
try {
Class.forName(GetConfigures.getValueWithKey("hive.driver")); //hive-jdbc pom
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
authKrb5ToHive2();
Connection connection = null;
ResultSet rs = null;
PreparedStatement ps = null;
String query_sql = "select * from tmp.hivetest";
try {
connection = DriverManager.getConnection(GetConfigures.getValueWithKey("kerberos.hive2.url"));
ps = connection.prepareStatement("select * from tmp.hivetest");
rs = ps.executeQuery();
while (rs.next()) {
System.out.println(rs.getString(1));
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static void authKrb5ToHive2() {
// 加载kerberos配置信息,这边也可以不加载,因为会默认在C:\Windows文件夹里面寻找 krb5.ini 配置文件
System.setProperty("java.security.krb5.conf", GetConfigures.getValueWithKey("kerberos.conf"));
Configuration conf = new Configuration();
// 加载kerberos认证信息 生成的数据 hive.keytab 可以通过生成的方式进行下载 具体生成看看笔记
UserGroupInformation.setConfiguration(conf);
try {
UserGroupInformation.loginUserFromKeytab(GetConfigures.getValueWithKey("kerberos.hdfs.user21"), GetConfigures.getValueWithKey("kerberos.hdfs.keytab21"));
} catch (IOException e) {
log.error("load kerberos hdfs or keytab faile : cause by " + e.getCause());
e.printStackTrace();
System.exit(-1);
}
}
}
Scala 版本API
Scala版本API使用spark进行hive读取,所以需要把集群上面用到的配置文件:hdfs-site.xml、croe-site.xml、hive-site.xml、yarn-site.xml等,都放到resources文件夹下面
import com.demo.getconf.GetConfigures
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.security.UserGroupInformation
import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
object ReadKerberosHive {
def main(args: Array[String]): Unit = {
authKrb5ToHive()
val spark = SparkSession.builder().appName("ReadKerberosHive").master("local[3]")
.enableHiveSupport()
.getOrCreate()
val df: DataFrame = spark.sql("select name,phone from tmp.hivetest")
df.show()
df.rdd.map(f => {
println(f(0) + " " + f(1))
}).foreach(f => f)
}
def authKrb5ToHive() {
val conf: Configuration = new Configuration()
// 加载kerberos配置信息,这边也可以不加载,因为会默认在C:\Windows文件夹里面寻找 krb5.ini 配置文件
System.setProperty("java.security.krb5.conf", GetConfigures.getValueWithKey("kerberos.conf"))
UserGroupInformation.setConfiguration(conf)
UserGroupInformation.loginUserFromKeytab(GetConfigures.getValueWithKey("kerberos.hdfs.user21"), GetConfigures.getValueWithKey("kerberos.hdfs.keytab21"))
}
}