玩过hadoop,hive的你们可以回顾回顾

17 篇文章 0 订阅
2 篇文章 0 订阅

这几年,大家都是从hadoop,hive走过来的。hadoop现在都快3.0了。下面贴一下我去年写的udaf,你看每个环节你是不是都清楚了。

import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.Text;

/**
 * Created by xxx on 2014/11/23.
 */

public class ConcatUDAF extends AbstractGenericUDAFResolver {

    public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) {
        return new ConcatUDAFEvaluator();
    }

    public static class ConcatUDAFEvaluator extends GenericUDAFEvaluator {

        private transient PrimitiveObjectInspector vectorsFieldOI;
        private Text result;

        @Override
        public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
            super.init(m, parameters);
            vectorsFieldOI = (PrimitiveObjectInspector) parameters[0];
            PrimitiveTypeInfo typeInfo = TypeInfoFactory.stringTypeInfo;
            return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(typeInfo);
        }

        /**
         * class for storing string concat value.
         */
        static class ConcatHiveStringAgg extends AbstractAggregationBuffer {
            boolean empty;
            HiveVarchar str;
        }

        @Override
        public AggregationBuffer getNewAggregationBuffer() throws HiveException {
            ConcatHiveStringAgg agg = new ConcatHiveStringAgg();
            reset(agg);
            return agg;
        }

        @Override
        public void reset(AggregationBuffer agg) throws HiveException {
            ConcatHiveStringAgg bdAgg = (ConcatHiveStringAgg) agg;
            bdAgg.empty = true;
            bdAgg.str = new HiveVarchar();
            bdAgg.str.setValue("");
        }

        boolean warned = false;

        @Override
        public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException {
            try {
                ConcatHiveStringAgg myagg = (ConcatHiveStringAgg) agg;
                String str = myagg.str.toString();
                String f = "000";
                if (null != str && str.trim().length() != 0)
                    f = ",";
                myagg.str.setValue(str + f + parameters[0]);
            } catch (Exception e) {
                if (!warned) {
                    warned = true;
                }
            }
        }

        @Override
        public Object terminatePartial(AggregationBuffer agg) throws HiveException {
            ConcatHiveStringAgg myagg = (ConcatHiveStringAgg) agg;
            if (myagg.str == null) {
                result = new Text("nothing");
            }
            result = new Text("terminatePar:" + myagg.str.toString());
            return result;
        }

        @Override
        public void merge(AggregationBuffer agg, Object partial) throws HiveException {
            if (partial != null) {
                ConcatHiveStringAgg myagg = (ConcatHiveStringAgg) agg;
                String str = myagg.str.toString();
                String f = "";
                if (null != str && str.trim().length() != 0)
                    f = "|";
                myagg.str.setValue(str + f + PrimitiveObjectInspectorUtils.getHiveChar(
                        partial, vectorsFieldOI));
                myagg.empty = false;
            }
        }

        @Override
        public Object terminate(AggregationBuffer agg) throws HiveException {
            ConcatHiveStringAgg myagg = (ConcatHiveStringAgg) agg;
            if (myagg.str == null) {
                result = new Text("nothing");
            }
            result = new Text(myagg.str.toString());
            return result;
        }
    }
}
  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 1
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值