93.Livy向Kerberos环境的CDH集群提交作业

93.1 演示环境介绍

• 集群未启用Kerberos
• CM和CDH版本：5.13.1
• Livy版本：0.4

93.2 操作演示

• 将作业运行的jar包上传到HDFS目录
• 准备访问集群的keytab及集群的krb5.conf文件
  • krb5.conf配置文件，获取方式：文件在CDH集群中将KDC服务器上的/etc/目录

# Configuration snippets may be placed in this directory as well
includedir /etc/krb5.conf.d/

[logging]
default = FILE:/var/log/krb5libs.log
kdc = FILE:/var/log/krbSkdc.log
admin_server = FILE:/var/Log/kadnind.Log

[libdefaults]
dns_lookup_realm = false
ticket_lifetime = 24h
renew_lifetine = 7d
forvardable = true
rdns = false
defauIt_realm = FAYSON.COM
#default_ccache_name = KEYRING:persistent:(uid}

[rea lms]
FAYSON.COM = {
kdc = ip-168-31-16-68.ap-southeast-1.compute. internal
adnin_server = ip-168-31-16-68.ap-southeast-1.compute,internal
1
[domain_realm]
·ap-southeast-1.compute.internal = FAYSON.COM
ap-southeast-1.compute.internal = FAYSON.COM

• 在KDC所在服务器执行如下命令生成fayson.keytab文件

[root@ip-168-31-16-68 ~]# kadmin.local -q "xst -norandkey -k /root/fayson.keytab fayson@FAYSON.COM"

• JAAS文件login-yarn.conf 如下：
  • Client与KBHttpUtils代码中SpnegoHttpURLConnection spnego = new SpnegoHttpURLConnection(“Client”)一致

Client {
    com.sun.security.auth.module.Krb5LoginModule required
    storeKey=true
    useKeyTab=true
    debug=true
    keyTab="/Volumes/Transcend/keytab/fayson.keytab"
    principal="fayson@FAYSON.COM";
};

• pom文件如下

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>cdh-project</artifactId>
        <groupId>com.cloudera</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>
    <artifactId>livy-demo</artifactId>
    <packaging>jar</packaging>
    <name>livy-demo</name>
    <url>http://maven.apache.org</url>
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
    <repositories>
        <repository>
            <id>maven2</id>
            <url>http://repository.jspresso.org/maven2/</url>
            <name>Maven2 Repositories</name>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>
    </repositories>
    <dependencies>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.4</version>
        </dependency>
        <dependency>
            <groupId>net.sourceforge.spnego</groupId>
            <artifactId>spnego</artifactId>
            <version>7.0</version>
        </dependency>
    </dependencies>
</project>

• Kerberos的HTTP请求工具类（KBHttpUtils.java）

package com.cloudera.utils;
import net.sourceforge.spnego.SpnegoHttpURLConnection;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Map;
/**
 * package: com.cloudera.utils
 * describe: 访问Kerberos环境的Http工具类
 * creat_user: Fayson
 * email: xxxx
 * creat_date: 2018/2/12
 * creat_time: 下午4:57
 * 公众号：碧茂大数据
 */
public class KBHttpUtils {
    /**
     * HttpGET请求
     * @param url
     * @param headers
     * @return
     */
    public static String getAccess(String url, Map<String,String> headers) {
        StringBuilder sb = new StringBuilder();
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        InputStream in = null;
        try {
            final SpnegoHttpURLConnection spnego = new SpnegoHttpURLConnection("Client");
            spnego.setRequestMethod("GET");
            if(headers != null && headers.size() > 0){
                headers.forEach((K,V)->spnego.setRequestProperty(K,V));
            }
            spnego.connect(new URL(url),bos);
            in = spnego.getInputStream();
            byte[] b = new byte[1024];
            int len ;
            while ((len = in.read(b)) > 0) {
                sb.append(new String(b, 0, len));
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (bos != null) {
                try {
                    bos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        System.out.println("Result:" + sb.toString());
        return sb.toString();
    }
    /**
     * HttpDelete请求
     * @param url
     * @param headers
     * @return
     */
    public  static String deleteAccess(String url, Map<String,String> headers) {
        StringBuilder sb = new StringBuilder();
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        InputStream in = null;
        try {
            final SpnegoHttpURLConnection spnego = new SpnegoHttpURLConnection("Client");
            spnego.setRequestMethod("DELETE");
            if(headers != null && headers.size() > 0){
                headers.forEach((K,V)->spnego.setRequestProperty(K,V));
            }
            spnego.connect(new URL(url),bos);
            in = spnego.getInputStream();
            byte[] b = new byte[1024];
            int len ;
            while ((len = in.read(b)) > 0) {
                sb.append(new String(b, 0, len));
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (bos != null) {
                try {
                    bos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        System.out.println("Result:" + sb.toString());
        return sb.toString();
    }
    /**
     * HttpPost请求
     * @param url
     * @param headers
     * @param data
     * @return
     */
    public static String postAccess(String url, Map<String,String> headers, String data)  {
        StringBuilder sb = new StringBuilder();
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        InputStream in = null;
        try {
            final SpnegoHttpURLConnection spnego = new SpnegoHttpURLConnection("Client");
            spnego.setRequestMethod("POST");
            if(headers != null && headers.size() > 0){
                headers.forEach((K,V)->spnego.setRequestProperty(K,V));
            }
            if(data != null){
                bos.write(data.getBytes());
            }
            spnego.connect(new URL(url),bos);
            System.out.println("Kerberos data:"+data);
            System.out.println("HTTP Status Code: " + spnego.getResponseCode());
            System.out.println("HTTP Status Message: "+ spnego.getResponseMessage());
            in = spnego.getInputStream();
            byte[] b = new byte[1024];
            int len ;
            while ((len = in.read(b)) > 0) {
                sb.append(new String(b, 0, len));
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (bos != null) {
                try {
                    bos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        System.out.println("Result:" + sb.toString());
        return sb.toString();
    }
}

• Livy RESTful API调用示例代码

package com.cloudera.kerberos;
import com.cloudera.utils.KBHttpUtils;
import java.util.HashMap;
/**
 * package: com.cloudera
 * describe: Kerberos环境下Livy RESTful API接口调用
 * creat_user: Fayson
 * email: xxxx
 * creat_date: 2018/2/11
 * creat_time: 上午10:50
 * 公众号：碧茂大数据
 */
public class AppLivy {
    private static String LIVY_HOST = "http://ip-168-31-21-83.ap-southeast-1.compute.internal:8998";
    public static void main(String[] args) {
        System.setProperty("java.security.krb5.conf", "/Volumes/Transcend/keytab/krb5.conf");
        System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
//        System.setProperty("sun.security.krb5.debug", "true"); //Kerberos Debug模式
        System.setProperty("java.security.auth.login.config", "/Volumes/Transcend/keytab/login-yarn.conf");
        HashMap<String, String> headers = new HashMap<>();
        headers.put("Content-Type", "application/json");
        headers.put("Accept", "application/json");
        headers.put("X-Requested-By", "fayson");
        //创建一个交互式会话
        String kindJson = "{\"kind\": \"spark\", \"proxyUser\":\"fayson\"}";
//        KBHttpUtils.postAccess(LIVY_HOST + "/sessions", headers, kindJson);
        //执行code
        String code = "{\"code\":\"sc.parallelize(1 to 2).count()\"}";
//        KBHttpUtils.postAccess(LIVY_HOST + "/sessions/2/statements", headers, code);
        //删除会话
//        KBHttpUtils.deleteAccess(LIVY_HOST + "/sessions/3", headers);
        //封装提交Spark作业的JSON数据
        String submitJob = "{\"className\": \"org.apache.spark.examples.SparkPi\",\"executorMemory\": \"1g\",\"args\": [200],\"file\": \"/fayson-yarn/jars/spark-examples-1.6.0-cdh5.14.0-hadoop2.6.0-cdh5.14.0.jar\"}";
        //向集群提交Spark作业
        KBHttpUtils.postAccess(LIVY_HOST + "/batches", headers, submitJob);
        //通过提交作业返回的SessionID获取具体作业的执行状态及APPID
//        KBHttpUtils.getAccess(LIVY_HOST + "/batches/4", headers);
    }
}

总结

• 在Java 访问Kerberos环境的Livy API接口时需要在代码中加载krb5.conf、login-yran.conf配置文件到环境变量中，实现fayosn@FAYSON.COM用户登录
• 访问Kerberos环境下的RESTfulAPI接口需要使用HttpClient提供的spnego方式访问，这里Fayson使用第三方封装好的spnego包

大数据视频推荐：
CSDN
大数据语音推荐：
企业级大数据技术应用
大数据机器学习案例之推荐系统
自然语言处理
大数据基础
人工智能：深度学习入门到精通