Oozie API操作.Java

最新推荐文章于 2022-04-23 23:09:42 发布

江陵之月

最新推荐文章于 2022-04-23 23:09:42 发布

阅读量684

点赞数

分类专栏： Oozie

本文链接：https://blog.csdn.net/qq_27787581/article/details/110799766

版权

Oozie 专栏收录该内容

3 篇文章 0 订阅

订阅专栏

Oozie API操作

代码一

package com.spark;

import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
import org.apache.oozie.client.WorkflowJob;

import java.util.Properties;

/**
 * @program: test
 * @description:
 * @author: XiongXiao
 * @create: 2018-09-01 15:51
 **/
public class OozieTest {
    public static void main(String[] args){
        OozieClient wc = new OozieClient("http://master01:11000/oozie");
        Properties properties = wc.createConfiguration();
        properties.setProperty(OozieClient.APP_PATH,"/user/hue/oozie/workspaces/ooziespark");
        properties.setProperty("nameNode","hdfs://master01:8020");
        properties.setProperty("user.name","root");
        properties.setProperty("jobTracker","master01:8032");
        properties.setProperty("name","oozie_example");
        properties.setProperty("oozie.use.system.libpath","True");
        properties.setProperty("master","yarn");
        properties.setProperty("mode","client");
        properties.setProperty("arg","null");
        properties.setProperty("sparkOpts","--executor-memory 6G --executor-cores 6 --driver-memory 2G --num-executors 10 --conf spark.default.parallelism=50");
        properties.setProperty("jar","test.jar");
        properties.setProperty("file","/tmp/test/test.jar");
        properties.setProperty("class","com.spark.SparkTest");
        try{
            String jobId = wc.run(properties);
            System.out.println("Workflow job submitted");
            System.out.println(jobId);
            //wait until the workflow job finishes
            while(wc.getJobInfo(jobId).getStatus() == WorkflowJob.Status.RUNNING){
                System.out.println("Workflow job running...");
                try{
                    Thread.sleep(10*1000);
                }catch(InterruptedException e){
                    e.printStackTrace();
                }
            }
            System.out.println("Workflow job completed!");
            System.out.println(wc.getJobId(jobId));
        }catch(OozieClientException e){
            e.printStackTrace();
        }

    }
}

代码二

job.properties

#通用的spark程序运行的workflow位置
oozie.wf.application.path=/user/hue/oozie/workspaces/ooziespark
#nameNode的地址
nameNode=hdfs://master01:8020
#spark程序运行的名字
name=portrayal_spark2
#是否使用oozie的共享库
oozie.use.system.libpath=True
#jobTracker的端口
jobTracker=master01:8032
master=yarn-client
mode=client
#spark程序的主类
class=com.eco.datamirror.portrayal.RunMain
#程序运行的参数
arg=null
#spark虚拟机的运行参数
sparkOpts=--executor-memory 6G --executor-cores 6 --driver-memory 2G --num-executors 10 --conf spark.default.parallelism=50
#spark程序的jar包名字
jar=portrayal.jar
#spark程序的文件路径
file=/persona/jiguang/job/jar/portrayal.jar
#提交任务的用户
user.name=root
#oozie的访问地址
oozieURL=http://192.168.1.46:11000/oozie

package test;

import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;

import java.io.InputStream;
import java.util.List;
import java.util.Properties;

/**
 * @program: test
 * @description:
 * @author: XiongXiao
 * @create: 2018-09-06 14:57
 **/
public class OozieTest {
    public static void main(String[] args) throws Exception{
        Properties properties = new Properties();
        InputStream in = OozieTest.class.getResourceAsStream("/job.properties");
        properties.load(in);
        System.setProperty("user.name", properties.getProperty("user.name"));
        OozieClient oozieClient = new OozieClient(properties.getProperty("oozieURL"));
        try {
            String serverBuildVersion = oozieClient.getServerBuildVersion();
            System.out.println(serverBuildVersion);
            String job_id = oozieClient.run(properties);
            System.out.println(job_id);
            new Thread() {
                public void run() {
                    try {
                        Thread.sleep(10000l);

                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }.start();
            WorkflowJob jobInfo = oozieClient.getJobInfo(job_id);
            List<WorkflowAction> list = jobInfo.getActions();
            for (WorkflowAction action : list) {
                //输出每个Action的 Appid 即Yarn的Application ID
                System.out.println(action.getExternalId());
            }
        } catch (OozieClientException e) {
            e.printStackTrace();
        }

    }
}