Ooze 扩展自定义action节点

大家好，我是crazy_老中医，我写程序就像老中医一样，全凭感觉和经验，但是有用！

废话不多说，现在开始正文，本文将阐述如何扩展一个自定义oozie action节点。

准备阶段

oozie安装

略。

个人建议用Ambari管理工具进行安装，省心省事。

扩展action节点功能分析

本文扩展的action节点主要实现：

1、根据URL参数发送http请求（应用场景：工作流执行完毕后向通知业务平台做一些事情）；

2、根据状态参数(status)控制节点运行完毕状态（应用场景：控制工作流正常结束或者异常结束）；

编码实现

项目结构

 

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>cn.com.cloudpioneer</groupId>
    <artifactId>DDPAction</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.apache.oozie</groupId>
            <artifactId>oozie-core</artifactId>
            <version>4.2.0</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>commons-httpclient</groupId>
            <artifactId>commons-httpclient</artifactId>
            <version>3.1</version>
            <exclusions>
                <exclusion>
                    <artifactId>commons-codec</artifactId>
                    <groupId>commons-codec</groupId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.jdom</groupId>
            <artifactId>jdom</artifactId>
            <version>1.1</version>
        </dependency>
    </dependencies>

    <build>
        <finalName>DDPAction</finalName>
        <resources>
            <resource>
                <directory>src/main/resources</directory>
                <includes>
                    <include>*.*</include>
                </includes>
                <filtering>true</filtering>
            </resource>
        </resources>
        <plugins>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

 

RequestUrlAction.java

package cn.com.cloudpioneer.oozie.action;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.SchemaService;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XLog;
import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;

/**
 * Created by yangtongjie on 2017/6/9.
 */
public class RequestUrlAction extends ActionExecutor {

    private static XLog LOG = XLog.getLog(RequestUrlAction.class);

    private static String TYPE = "request-url";
    private static String URL = "url";
    private static String ARG = "arg";
    private static String STATUS = "status";

    public RequestUrlAction(){
        super(TYPE);
    }

    protected RequestUrlAction(String type) {
        super(TYPE);
    }

    @Override
    public void initActionType() {
        super.initActionType();

        try {
            String file = org.apache.commons.codec.binary.Base64.class.getProtectionDomain().getCodeSource().getLocation().getFile();
            System.err.println("initActionType:" + file);
            String s = new Base64(0).encodeToString("asdfasdfasdf".getBytes());
            System.err.println("initActionType:" + s);
        }catch (Exception ex){
            ex.printStackTrace();
        }
    }

    @Override
    public void start(Context context, WorkflowAction action) throws ActionExecutorException {
        try {
            Element actionXml = XmlUtils.parseXml(action.getConf());
            Namespace ns = actionXml.getNamespace();

            String url = actionXml.getChildTextTrim(URL, ns);
            String status = actionXml.getChildTextTrim(STATUS, ns);
            String parameters = "";
            List children = actionXml.getChildren(ARG, ns);
            for(Object obj : children){
                Element child = (Element) obj;
                String param = child.getText();
                parameters += "&" + param;
            }

            String request = url + "?status=" + status;
            request = parameters.equals("") ? request : request + parameters;

            doGet(request);

            if("ERROR".equals(status))
                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "DDPAction:" + this.getClass(), "Normal Error.");
        } catch (JDOMException e) {
            e.printStackTrace();
        } catch (Exception ex){
            ex.printStackTrace();
            if(ex instanceof ActionExecutorException) throw (ActionExecutorException)ex;
            else throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "parameters error", ex.getMessage());
        }

        context.setExecutionData("OK", null);
    }

    @Override
    public void end(Context context, WorkflowAction action) throws ActionExecutorException {
        String externalStatus = action.getExternalStatus();
        WorkflowAction.Status status = externalStatus.equals("OK") ? WorkflowAction.Status.OK :
                WorkflowAction.Status.ERROR;
        context.setEndData(status, getActionSignal(status));
    }

    @Override
    public void check(Context context, WorkflowAction action) throws ActionExecutorException {

    }

    @Override
    public void kill(Context context, WorkflowAction action) throws ActionExecutorException {

    }

    @Override
    public boolean isCompleted(String externalStatus) {
        return true;
    }

    public static void doGet(String url){
        HttpClient client = new HttpClient();
        GetMethod method = new GetMethod(url);
        try {
            client.executeMethod(method);
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //释放连接
            method.releaseConnection();
        }
    }

    private static void scanInnerSchemas() throws IOException {
        String[] schemas = ConfigurationService.getStrings(SchemaService.WF_CONF_SCHEMAS);
        Set<String> schemaNames = new HashSet<String>();
        if (schemas != null) {
            for (String schema : schemas) {
                schema = schema.trim();
                if (!schema.isEmpty()) {
                    schemaNames.add(schema);
                }
            }
        }

        for (String schemaName : schemaNames) {
            System.err.println("schema:" + schemaName);
            InputStream is = IOUtils.getResourceAsStream(schemaName, -1);
            if(is != null){
                try {
                    BufferedReader br = new BufferedReader(new InputStreamReader(is));
                    String line = br.readLine();
                    while (line != null) {
                        System.err.println(line);
                        line = br.readLine();
                    }
                }catch (Exception ex){
                    ex.printStackTrace();
                }
            }else {
                System.err.println("schema:input stream is null.");
            }
        }
    }

    public static void main(String[] args) throws JDOMException {
//        String conf = "        <request-url xmlns=\"uri:custom:ddp-action:0.1\">\n" +
//                "            <url>http://developback.ddp.dacube.com.cn/task/warning/email</url>\n" +
//                "\t\t\t<status>ERROR</status>\n" +
//                "\t\t\t<arg>subId=90dde13a-3c49-4922-82fd-ad54645bab43</arg>\n" +
//                "\t\t\t<arg>taskId=3926b022-5986-41ec-9ee7-cc22b082c4ca</arg>\n" +
//                "        </request-url>";
//        Element actionXml = XmlUtils.parseXml(conf);
//        Namespace ns = actionXml.getNamespace();
//
//        String url = actionXml.getChildTextTrim(URL, ns);
//        String status = actionXml.getChildTextTrim(STATUS, ns);
//        List<String> params = new ArrayList<String>();
//        String parameters = "";
//        List children = actionXml.getChildren(ARG, ns);
//        for(Object obj : children){
//            Element child = (Element) obj;
//            String param = child.getText();
//            params.add(param);
//            parameters += "&" + param;
//        }
//
//        String request = url + "?status=" + status;
//        request = params.size() == 0 ? request : request + parameters;
//
//        System.err.println(request);

        doGet("http://localhost:9090/task/warning/email?status=FINISHED&taskId=3926b022-5986-41ec-9ee7-cc22b082c4ca&subId=e130ff20-137b-4b86-a594-2a8a6bbbf2a3");

        System.err.println();
    }
}

 

request-url.xsd

<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
           xmlns:request-url="uri:custom:ddp-action:0.1"
           elementFormDefault="qualified"
           targetNamespace="uri:custom:ddp-action:0.1">
    <xs:complexType name="ACTION">
        <xs:sequence>

            <xs:element name="url" type="xs:string" minOccurs="1" maxOccurs="1"/>
            <xs:element name="status" type="xs:string" minOccurs="1" maxOccurs="1"/>
            <xs:element name="arg" type="xs:string" minOccurs="1" maxOccurs="unbounded"/>

        </xs:sequence>
    </xs:complexType>
    <xs:element name="request-url" type="request-url:ACTION"></xs:element>
</xs:schema>

 

打包：略

 

部署到oozie

找到oozie安装目录，并将扩展的action jar包拷贝到libext目录下，如下图：

 

添加配置到oozie-site.xml

    <property>
      <name>oozie.service.ActionService.executor.ext.classes</name>
      <value>cn.com.cloudpioneer.oozie.action.RequestUrlAction</value>
    </property>
    <property>
      <name>oozie.service.SchemaService.wf.ext.schemas</name>
      <value>request-url.xsd</value>
    </property>

 

重启oozie，如果不出意外，你的扩展节点已经可用了。

 

测试

workflow.xml

<workflow-app xmlns="uri:oozie:workflow:0.5" name="V2SubTest">
    <start to="t"/>
    <kill name="t-kill">
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <action name="t">
        <request-url xmlns="uri:custom:ddp-action:0.1">
            <url>http://developback.ddp.dacube.com.cn/task/warning/email</url>
	    <status>ERROR</status>
	    <arg>subId=90dde13a-3c49-4922-82fd-ad54645bab43</arg>
	    <arg>taskId=3926b022-5986-41ec-9ee7-cc22b082c4ca</arg>
        </request-url>
        <ok to="t-End"/>
        <error to="t-kill"/>
    </action>
    <end name="t-End"/>
</workflow-app>
	    <arg>taskId=3926b022-5986-41ec-9ee7-cc22b082c4ca</arg>
        </request-url>
        <ok to="t-End"/>
        <error to="t-kill"/>
    </action>
    <end name="t-End"/>
</workflow-app>

 

job.properties

oozie.use.system.libpath=true
oozie.wf.application.path=hdfs://master:8020/user/ddp/t
security_enabled=False
dryrun=False

 

运行结果

因为在workflow.xml中我配置了

<status>ERROR</status>

所以最终的运行结果是：ERROR。不理解的同学请读写RequestUrlAction.java就会知道，这里不赘述。

心得

如果你对oozie比较熟悉的话，你会发现，扩展oozie整个过程很简单，但是对于不熟悉的同学，肯定没有那么轻松。比如我就历时4个工作日才将oozie扩展成功，在扩展的过程中首先查阅的相关资料，资料写得很简明扼要（事实也是如此），但是对于oozie架构（这里指编码实现）不大了解的同学来说，本文中提到的“部署”阶段就会比较陌生，甚至有时不知所措，我就是这样，不知道是我搜索能力不行还是查阅文档的能力不行，我在网上没有找到我所遇到问题的答案，只能看oozie源码，其实大部分时间都是花在看源码上，通过源码了解到扩展节点是怎么被引用，加载，以及运行的整个过程，了解了这些，那扩展一个oozie节点就相当的得心应手了，祝你成功。

 

参考资料

http://oozie.apache.org/docs/4.2.0/DG_CustomActionExecutor.html