环境准备
cdh集群,开启kerberos认证,需要通过spark访问hbase
本地idea开发
//是否需要kerberos认证
val IS_KERBEROS = PropertiesUtils.getBoolean("hbase.kerberos")
//krb5.conf路径
val KRB5_CONF_PATH = PropertiesUtils.getString("java.security.krb5.conf")
//hbase.keytab路径
val KEYTAB_PATH = PropertiesUtils.getString("hbase.kerberos.keytab.path")
//一般都是hbase/xxx@XXX.COM
val KERBEROS_USER = PropertiesUtils.getString("hbase.kerberos.user")
//hbase-site.xml路径
val HBASE_SITE_FILE = PropertiesUtils.getString("hbase.site.file")
//core-site.xml路径
val CORE_SITE_FILE = PropertiesUtils.getString("core.site.file")
//hdfs-site.xml路径
val HDFS_SITE_FILE = PropertiesUtils.getString("hdfs.site.file")
def login(conf: Configuration): Option[UserGroupInformation] = {
//kerberos
try {
System.setProperty("java.security.krb5.conf", KRB5_CONF_PATH)
conf.set("hadoop.security.authentication", "Kerberos")
// 这个hbase.keytab也是从远程服务器上copy下来的, 里面存储的是密码相关信息
// 这样我们就不需要交互式输入密码了
conf.set("keytab.file", KEYTAB_PATH)
// 这个可以理解成用户名信息,也就是Principal
conf.set("kerberos.principal", KERBEROS_USER)
UserGroupInformation.setConfiguration(conf)
val ugi: UserGroupInformation = UserGroupInformation.loginUserFromKeytabAndReturnUGI(KERBEROS_USER, KEYTAB_PATH)
Some(ugi)
}catch {
case e: IOException =>
logger.error(s"login hbase from keytab error,Cause:${e}")
None
}
}
def getConnection(conf: Configuration): Connection = {
if (IS_KERBEROS) {
conf.addResource(new Path(CORE_SITE_FILE))
conf.addResource(new Path(HDFS_SITE_FILE))
conf.addResource(new Path(HBASE_SITE_FILE))
} else {
conf.set(HConstants.ZOOKEEPER_QUORUM,QUORUM)
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,PORT)
conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT,ZNODE)
}
ConnectionFactory.createConnection(conf)
}
def getData(tableName: String,get: Get): Option[JSONObject] = {
val conf: Configuration = HBaseConfiguration.create()
if (IS_KERBEROS) {
val ugi: UserGroupInformation = login(conf).get
ugi.doAs(new PrivilegedExceptionAction[Option[JSONObject]] {
override def run(): Option[JSONObject] = getHBaseData(tableName, get, conf)
})
} else {
getHBaseData(tableName, get, conf)
}
}
获取链接后,后续进行业务处理
生产环境测试
报错1
kerberos认证失败,登陆异常
给每个yarn节点的hbase.keytab文件更改权限
chmod 666 /opt/hbase.keytab
报错2
遇到Insufficient permissions for user 'hive'异常,hive用户没有权限读取hbase表数据
打开hbase shell,输入
grant 'hive','RWXCA'
后续再研究使用hbase用户或者其他用户认证kerberos后,反而报hive用户没有权限读取hbase表数据异常。
参考
https://blog.csdn.net/blackice1015/article/details/49422855
https://blog.csdn.net/fanchw/article/details/108770529