UDF步骤:
1.继承org.apache.hadoop.hive.ql.exec.UDF2.实现evaluate函数,evaluate函数支持重载
- package cn.sina.stat.hive.udf;
- import java.util.Arrays;
- import org.apache.hadoop.hive.ql.exec.UDF;
- public final class SortFieldContent extends UDF {
- public String evaluate( final String str, String delimiter) {
- if (str == null ) {
- return null ;
- }
- if (delimiter == null) {
- delimiter = "," ;
- }
- String[] strs = str.split(delimiter);
- Arrays. sort(strs);
- String result = "" ;
- for (int i = 0; i < strs. length; i++) {
- if (result.length() > 0) {
- result.concat(delimiter);
- }
- result.concat(strs[i]);
- }
- return result;
- }
- public String evaluate( final String str, String delimiter, String order) {
- if (str == null ) {
- return null ;
- }
- if (delimiter == null) {
- delimiter = "," ;
- }
- if (order != null && order.toUpperCase().equals( "ASC" )) {
- return evaluate(str, delimiter);
- } else {
- String[] strs = str.split(delimiter);
- Arrays. sort(strs);
- String result = "" ;
- for (int i = strs. length - 1; i >= 0; i--) {
- if (result.length() > 0) {
- result.concat(delimiter);
- }
- result.concat(strs[i]);
- }
- return result;
- }
- }
- }
UDAF步骤:
1.函数类继承org.apache.hadoop.hive.ql.exec.UDAF内部类实现接口org.apache.hadoop.hive.ql.exec.UDAFEvaluator2.Evaluator需要实现 init、iterate、terminatePartial、merge、terminate这几个函数具体执行过程如图:
- package cn.sina.stat.hive.udaf;
- import java.util.Arrays;
- import org.apache.hadoop.hive.ql.exec.UDAF;
- import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
- public class ConcatClumnGroupByKeyWithOrder extends UDAF {
- public static class ConcatUDAFEvaluator implements UDAFEvaluator {
- public static class PartialResult {
- String result;
- String delimiter;
- String order;
- }
- private PartialResult partial;
- public void init() {
- partial = null;
- }
- public boolean iterate(String value, String delimiter, String order) {
- if (value == null) {
- return true;
- }
- if (partial == null) {
- partial = new PartialResult();
- partial.result = new String("");
- if (delimiter == null || delimiter.equals("")) {
- partial.delimiter = new String(",");
- } else {
- partial.delimiter = new String(delimiter);
- }
- if (order != null
- && (order.toUpperCase().equals("ASC") || order
- .toUpperCase().equals("DESC"))) {
- partial.order = new String(order);
- } else {
- partial.order = new String("ASC");
- }
- }
- if (partial.result.length() > 0) {
- partial.result = partial.result.concat(partial.delimiter);
- }
- partial.result = partial.result.concat(value);
- return true;
- }
- public PartialResult terminatePartial() {
- return partial;
- }
- public boolean merge(PartialResult other) {
- if (other == null) {
- return true;
- }
- if (partial == null) {
- partial = new PartialResult();
- partial.result = new String(other.result);
- partial.delimiter = new String(other.delimiter);
- partial.order = new String(other.order);
- } else {
- if (partial.result.length() > 0) {
- partial.result = partial.result.concat(partial.delimiter);
- }
- partial.result = partial.result.concat(other.result);
- }
- return true;
- }
- public String terminate() {
- String[] strs = partial.result.split(partial.delimiter);
- Arrays.sort(strs);
- String result = new String("");
- if (partial.order.equals("DESC")) {
- for (int i = strs.length - 1; i >= 0; i--) {
- if (result.length() > 0) {
- result.concat(partial.delimiter);
- }
- result.concat(strs[i]);
- }
- } else {
- for (int i = 0; i < strs.length; i++) {
- if (result.length() > 0) {
- result.concat(partial.delimiter);
- }
- result.concat(strs[i]);
- }
- }
- return new String(result);
- }
- }
- }
UDTF步骤:
1.继承org.apache.hadoop.hive.ql.udf.generic.GenericUDTF
2.实现initialize, process, close三个方法
a.initialize初始化验证,返回字段名和字段类型
b.初始化完成后,调用process方法,对传入的参数进行处理,通过forword()方法把结果返回
c.最后调用close()方法进行清理工作
- package cn.sina.stat.hive.udtf;
- import java.util.ArrayList;
- import java.util.Arrays;
- import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
- import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
- import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
- import org.apache.hadoop.hive.ql.metadata.HiveException;
- import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
- import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
- import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
- import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
- public class SortFieldExplodeToPair extends GenericUDTF {
- @Override
- public void close() throws HiveException {
- // TODO Auto-generated method stub
- }
- @Override
- public StructObjectInspector initialize(ObjectInspector[] args)
- throws UDFArgumentException {
- if (args.length != 3) {
- throw new UDFArgumentLengthException(
- "SortFieldExplodeToPair takes only three argument");
- }
- if (args[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentException(
- "SortFieldExplodeToPair takes string as first parameter");
- }
- if (args[1].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentException(
- "SortFieldExplodeToPair takes string as second parameter");
- }
- if (args[2].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentException(
- "SortFieldExplodeToPair takes string as third parameter");
- }
- if (args[2] == null
- || !(args[2].toString().toUpperCase().equals("ASC") || args[2]
- .toString().toUpperCase().equals("DESC"))) {
- throw new UDFArgumentException(
- "SortFieldExplodeToPair third parameter must be \"ASC\" or \"DESC\"");
- }
- ArrayList<String> fieldNames = new ArrayList<String>();
- ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
- fieldNames.add("col1");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- return ObjectInspectorFactory.getStandardStructObjectInspector(
- fieldNames, fieldOIs);
- }
- private final String[] forwardStr = new String[1];
- @Override
- public void process(Object[] args) throws HiveException {
- String input = args[0].toString();
- String delimiter = args[1].toString();
- String order = args[2].toString();
- String[] strList = input.split(delimiter);
- Arrays.sort(strList);
- if (strList.length > 1) {
- if (order.toUpperCase().equals("DESC")) {
- for (int i = strList.length - 1; i > 0; i--) {
- forwardStr[0] = strList[i].concat(delimiter).concat(
- strList[i - 1]);
- forward(forwardStr);
- }
- } else {
- for (int i = 0; i < strList.length - 1; i++) {
- forwardStr[0] = strList[i].concat(delimiter).concat(
- strList[i + 1]);
- forward(forwardStr);
- }
- }
- } else {
- forward(strList);
- }
- }
- }