Hive UDF UDTF 实现与使用

用户自定义函数

1 UDF函数的创建 一进一出

1 添加maven依赖

    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.hive/hive-exec -->
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-exec</artifactId>
            <version>3.1.2</version>
        </dependency>
    </dependencies>

2 实现类

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

/***
 *  @description  UDF函数的定义使用
 *  @author caoyongfei
 *
 */
public class MyStringLength extends GenericUDF {
    /**
     * @param objectInspectors 输入参数的鉴别器对象
     * @return 返回值类型的鉴别器对象
     * @throws UDFArgumentException
     */
    public ObjectInspector initialize(ObjectInspector[] objectInspectors) throws UDFArgumentException {
        if (objectInspectors.length != 1)
            throw new UDFArgumentLengthException("Input Args Length Error");
        if (!objectInspectors[0].getCategory().equals(ObjectInspector.Category.PRIMITIVE)) {
            throw new UDFArgumentTypeException(0, "Input Args Type Error !");
        }
        //返回int类型的鉴别器对象
        return PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    }

    /**
     *  函数的逻辑处理
     * @param deferredObjects 输入的参数
     * @return  返回值
     * @throws HiveException
     */
    public Object evaluate(DeferredObject[] deferredObjects) throws HiveException {
        if(deferredObjects[0].get() == null)
            return 0;
        return deferredObjects[0].get().toString().length();
    }

    public String getDisplayString(String[] strings) {
        return "";
    }
}

3 打jar包上传到服务器上

rz xxx

4 添加jar包

add jar /opt/module/data/udf.jar;

5 创建函数关联jar包中的函数实现类

create temporary function my_len as "MyStringLength"

6 使用自定义函数

2 UDTF函数的创建使用 一进多出

1 实现代码

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

import java.util.ArrayList;
import java.util.List;

/***
 *  @description 自定义UDTF函数
 *  @author caoyongfei
 *
 */
public class MyUDTF extends GenericUDTF {

    private ArrayList<String> outList = new ArrayList<String>();
    @Override
    public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
        //1、定义输出数据的列明和类型
        List<String> fieldNames = new ArrayList();
        List<ObjectInspector> filedOIs = new ArrayList();

        //2、添加输出数据的列明和类型
        fieldNames.add("lineToWord");
        filedOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

        return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames,filedOIs);
    }

    public void process(Object[] objects) throws HiveException {
        //1、获取原始数据
        String arg = objects[0].toString();
        //2、获取数据传入的第二个参数，此处为分隔符
        String splitKey = objects[1].toString();
        //3、将原始数据按照分隔符进行分割
        String[] fields = arg.split(splitKey);
        //4、遍历切分后的结果，并写出内容


        for (String field : fields) {
            outList.clear();//清空集合数据
            outList.add(field);
            forward(outList);//写出
        }
    }

    public void close() throws HiveException {

    }
}

2 打包上传服务器

3 添加jar包

add jar /opt/module/data/udtf.jar

4 创建函数关联jar包中的函数实现类

create function my_udtf as "MyUDTF";

5 使用自定义UDTF函数