关闭

Hive+GenericUDF示例一

标签: HiveGenericUDFJava
1353人阅读 评论(3) 收藏 举报
分类:

           和UDF相比,通用GDF(GenericUDF)支持复杂类型(比如List,struct等)的输入和输出。

           下面来看一个小示例。

            Hive中whereme表中包含若干人的行程如下:  

A       2013-10-10 8:00:00      home
A       2013-10-10 10:00:00     Super Market
A       2013-10-10 12:00:00     KFC
A       2013-10-10 15:00:00     school
A       2013-10-10 20:00:00     home
A       2013-10-15 8:00:00      home
A       2013-10-15 10:00:00     park
A       2013-10-15 12:00:00     home
A       2013-10-15 15:30:00     bank
A       2013-10-15 19:00:00     home

           通过查询我们要得到如下结果: 

A	2013-10-10	08:00:00	home	10:00:00	Super Market
A	2013-10-10	10:00:00	Super Market	12:00:00	KFC
A	2013-10-10	12:00:00	KFC	15:00:00	school
A	2013-10-10	15:00:00	school	20:00:00	home
A	2013-10-15	08:00:00	home	10:00:00	park
A	2013-10-15	10:00:00	park	12:00:00	home
A	2013-10-15	12:00:00	home	15:30:00	bank
A	2013-10-15	15:30:00	bank	19:00:00	home

           1.编写GenericUDF.

package com.wz.udf;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.lazy.LazyString;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date; 
import java.util.Calendar;
import java.util.ArrayList;
 
public class helloGenericUDF extends GenericUDF {
     ////输入变量定义
     private ObjectInspector peopleObj;
     private ObjectInspector timeObj;
     private ObjectInspector placeObj;
     //之前记录保存
     String strPreTime = "";
     String strPrePlace = ""; 
     String strPrePeople = "";
 
     @Override
     //1.确认输入类型是否正确
     //2.输出类型的定义
     public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
          peopleObj = (ObjectInspector)arguments[0];
          timeObj = (ObjectInspector)arguments[1];
          placeObj = (ObjectInspector)arguments[2];
          //输出结构体定义
          ArrayList structFieldNames = new ArrayList();
          ArrayList structFieldObjectInspectors = new ArrayList();
          structFieldNames.add("people");
	  structFieldNames.add("day");
          structFieldNames.add("from_time");
          structFieldNames.add("from_place");
          structFieldNames.add("to_time");
          structFieldNames.add("to_place");
 
          structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );
          structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );
          structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );
          structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );
	  structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );
  	  structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );

          StructObjectInspector si2;
          si2 = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors); 
          return si2;
     }
 
     //遍历每条记录
     @Override
     public Object evaluate(DeferredObject[] arguments) throws HiveException{
	  LazyString LPeople = (LazyString)(arguments[0].get());
	  String strPeople = ((StringObjectInspector)peopleObj).getPrimitiveJavaObject( LPeople );

	  LazyString LTime = (LazyString)(arguments[1].get());
	  String strTime = ((StringObjectInspector)timeObj).getPrimitiveJavaObject( LTime );

	  LazyString LPlace = (LazyString)(arguments[2].get());
	  String strPlace = ((StringObjectInspector)placeObj).getPrimitiveJavaObject( LPlace );
	
	  Object[] e;	
	  e = new Object[6];

          try
	  {
                //如果是同一个人,同一天
	  	if(strPrePeople.equals(strPeople) && IsSameDay(strTime) )
	  	{
	       		e[0] = new Text(strPeople);
                        e[1] = new Text(GetYearMonthDay(strTime));
	       		e[2] = new Text(GetTime(strPreTime));
	       		e[3] = new Text(strPrePlace);
	       		e[4] = new Text(GetTime(strTime));
	       		e[5] = new Text(strPlace);
	  	}
                else
                {
	       		e[0] = new Text(strPeople);
			e[1] = new Text(GetYearMonthDay(strTime));
	       		e[2] = new Text("null");
	       		e[3] = new Text("null");
	       		e[4] = new Text(GetTime(strTime));
	       		e[5] = new Text(strPlace);
                }
          }
          catch(java.text.ParseException ex)
          {
          }
	       
	  strPrePeople = new String(strPeople);
	  strPreTime= new String(strTime);
	  strPrePlace = new String(strPlace);

          return e;
     }
 
     @Override
     public String getDisplayString(String[] children) {
          assert( children.length>0 );
 
          StringBuilder sb = new StringBuilder();
          sb.append("helloGenericUDF(");
          sb.append(children[0]);
          sb.append(")");
 
          return sb.toString();
     }

     //比较相邻两个时间段是否在同一天
     private boolean IsSameDay(String strTime) throws java.text.ParseException{   
	 if(strPreTime.isEmpty()){
	     return false;
         }
         String curDay = GetYearMonthDay(strTime);
         String preDay = GetYearMonthDay(strPreTime);
	 return curDay.equals(preDay);
     }

     //获取年月日
     private String GetYearMonthDay(String strTime)  throws java.text.ParseException{
         DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
         Date curDate = df.parse(strTime);
	 df = new SimpleDateFormat("yyyy-MM-dd");
         return df.format(curDate);
     }

     //获取时间
     private String GetTime(String strTime)  throws java.text.ParseException{
         DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
         Date curDate = df.parse(strTime);
         df = new SimpleDateFormat("HH:mm:ss");
         return df.format(curDate);
     }
}

           2.在Hive里面创建两张表,一张包含结构体的表保存执行GenericUDF查询后的结果,另外一张用于保存最终结果.

hive> create table whereresult(people string,day string,from_time string,from_place string,to_time string,to_place string);
OK
Time taken: 0.287 seconds
hive> create table tmpResult(info struct<people:string,day:string,from_time:str>ing,from_place:string,to_time:string,to_place:string>);
OK
Time taken: 0.074 seconds

           3.执行GenericUDF查询,得到最终结果。  

hive> insert overwrite table tmpResult select hellogenericudf(whereme.people,whereme.time,whereme.place) from whereme;
hive> insert overwrite table whereresult select info.people,info.day,info.from_time,info.from_place,info.to_time,info.to_place from tmpResult where info.from_time<>'null';
Total MapReduce jobs = 2
Launching Job 1 out of 2
Number of reduce tasks is set to 0 since there's no reduce operator
Starting Job = job_201312022129_0006, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201312022129_0006
Kill Command = /home/wangzhun/hadoop/hadoop-0.20.2/bin/../bin/hadoop job  -Dmapred.job.tracker=localhost:9001 -kill job_201312022129_0006
Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0
2013-12-02 22:48:40,733 Stage-1 map = 0%,  reduce = 0%
2013-12-02 22:48:49,825 Stage-1 map = 100%,  reduce = 0%
2013-12-02 22:48:52,869 Stage-1 map = 100%,  reduce = 100%
Ended Job = job_201312022129_0006
Ended Job = -383357832, job is filtered out (removed at runtime).
Moving data to: hdfs://localhost:9000/tmp/hive-root/hive_2013-12-02_22-48-24_406_2701579121398466034/-ext-10000
Loading data to table default.whereresult
Deleted hdfs://localhost:9000/user/hive/warehouse/whereresult
Table default.whereresult stats: [num_partitions: 0, num_files: 1, num_rows: 0, total_size: 346, raw_data_size: 0]
8 Rows loaded to whereresult
MapReduce Jobs Launched: 
Job 0: Map: 1   HDFS Read: 420 HDFS Write: 346 SUCESS
Total MapReduce CPU Time Spent: 0 msec
OK
Time taken: 29.098 seconds
hive> select * from whereresult;
OK
A	2013-10-10	08:00:00	home	10:00:00	Super Market
A	2013-10-10	10:00:00	Super Market	12:00:00	KFC
A	2013-10-10	12:00:00	KFC	15:00:00	school
A	2013-10-10	15:00:00	school	20:00:00	home
A	2013-10-15	08:00:00	home	10:00:00	park
A	2013-10-15	10:00:00	park	12:00:00	home
A	2013-10-15	12:00:00	home	15:30:00	bank
A	2013-10-15	15:30:00	bank	19:00:00	home
Time taken: 0.105 seconds


         
0
0

查看评论
* 以上用户言论只代表其个人观点,不代表CSDN网站的观点或立场
    个人资料
    • 访问:25429次
    • 积分:489
    • 等级:
    • 排名:千里之外
    • 原创:24篇
    • 转载:9篇
    • 译文:0篇
    • 评论:3条
    最新评论