Hive自定义UDF&UDTF函数

pom.xml

    <properties>
        <project.build.sourceEncoding>UTF8</project.build.sourceEncoding>
        <hive.version>1.2.1</hive.version>
    </properties>
    <repositories>
        <repository>
            <id>spring-plugin</id>
            <url>https://repo.spring.io/plugins-release/</url>
        </repository>
    </repositories>

    <dependencies>
        <!--添加hive依赖-->
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-exec</artifactId>
            <version>${hive.version}</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13</version>
            <scope>test</scope>
        </dependency>

    </dependencies>
    <build>
        <plugins>
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

UDF

package com.chen.hive;

import org.apache.hadoop.hive.ql.exec.UDF;
import org.json.JSONException;
import org.json.JSONObject;

/**
 * 1. 集成UDF类,提供多个evaluate方法，必须有返回值，可以返回null
 * 数据格式： 时间戳 | {"ap":"","cm":{""},"et":[{},{}]}
 *
 */




public class MyUDF extends UDF {

    public String evaluate(String source,String param) throws JSONException {

        // 检查参数
        if(!source.contains(param)&&!"ts".equals(param)){
            return "";
        }

        // param合法 取出参数
        String[] words = source.split("\\|");

        // 构建json对象
        JSONObject root = new JSONObject(words[1]);

        if ("ts".equals(param)){
            return words[0].trim();
        }else if("ap".equals(param)){
            return root.getString("ap");
        }else if ("et".equals(param)){
            return root.getString("et");
        }else{
            return root.getJSONObject("cm").getString(param);
        }
    }


}

UDTF

package com.chen.hive;

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * 1.继承GenericUDTF
 * 2.父类中Initialize负责初始化操作
 * 传入数据格式 [{},{}]
 */
public class MyUDTF extends GenericUDTF {

    // 在函数运行之前会被调用一次，作用是告诉MapTask当前函数返回的结果的类型以及个数
    // 以便在MapTask运行 对函数返回值进行检查
    @Override
    public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {

        // 当前返回列的字段名  这里我们返回的是两列
        List<String> fieldNames = new ArrayList<>();
        fieldNames.add("event_name");
        fieldNames.add("event_json");

        // 返回当前两列的类型检查其
        List<ObjectInspector> fieldOIs = new ArrayList<>();

        fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

        return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);

    }

    // 执行函数的功能，处理后返回结果。该返回结果不是用过返回值返回，通过forward() 方法返回
    @Override
    public void process(Object[] args) throws HiveException {

        // 先检查是否传入
        if (args[0] == null || args.length == 0) {
            return;
        }

        try {
            // 构建JSON 数组对象
            JSONArray jsonArray = new JSONArray(args[0].toString());
            // 判断JSON数组对象是否构建成功
            if (jsonArray == null || jsonArray.length() == 0) {
                return;
            }

            // 取出json数组中每个元素中的事件名
            for (int i = 0; i < jsonArray.length(); i++) {
                try {
                    // 每次遍历写出两列数据，构建一个数组，存储一行两列数据
                    String[] result = new String[2];
                    JSONObject jsonObject = jsonArray.getJSONObject(i);

                    result[0] = jsonObject.getString("en");
                    result[1] = jsonObject.toString();

                    System.out.println(Arrays.asList(result));
                    forward(result);
                } catch (Exception e) {
                    continue;
                }
            }
        } catch (JSONException e) {
            e.printStackTrace();
        }




    }

    // 在调用函数完成之后，执行一些清理或关闭操作
    @Override
    public void close() throws HiveException {

    }
}

Test

package com.chen.hive;


import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.json.JSONException;
import org.junit.Test;

import java.util.ArrayList;

public class MyUDTFTest {

    @Test
    public void testUDF() throws JSONException, HiveException {

        MyUDF myUDF = new MyUDF();

        MyUDTF myUDTF = new MyUDTF();


        System.out.println(myUDF.evaluate(str, "ts"));

        Object [] args=new Object[2];

        args[0]=myUDF.evaluate(str, "et");

        myUDTF.process(args);
    }

}

添加到hive中

1. 打包
2. hive安装目录下创建auxlib并上传到该目录中（hive中auxlib会自动加载jar包）
3. 重新启动hive
4. hive中使用命令 创建函数 create function base_analizer as 'com.chen.hive.udf.MyUDF'