数仓02-hdfs-ods-dwd-

最新推荐文章于 2024-08-21 15:19:17 发布

weixin_43003792

最新推荐文章于 2024-08-21 15:19:17 发布

阅读量176

点赞数

分类专栏：数仓项目

本文链接：https://blog.csdn.net/weixin_43003792/article/details/114949753

版权

数仓项目专栏收录该内容

6 篇文章 0 订阅

订阅专栏

在这里插入图片描述

package com.atguigu.udf;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.json.JSONException;
import org.json.JSONObject;
import org.stringtemplate.v4.ST;

public class BaseFieldUDF extends UDF {
public String evaluate(String line,String jsonkeysString){
StringBuilder sb = new StringBuilder();
//1.获取所有key 公共字段的mid uv
String[] jsonkeys = jsonkeysString.split(",");
//2.line切割得到服务器时间
String[] logContents = line.split("\|");
//3.校验判空操作
if(logContents.length !=2 || StringUtils.isBlank(logContents[1])){
return “”;
}
//4.对logContens[1]解析
try {
//
JSONObject jsonObject = new JSONObject(logContents[1]);
//获取公共字段的json对象
JSONObject cmJson = jsonObject.getJSONObject(“cm”);
//循环遍历
for (int i = 0; i < jsonkeys.length; i++) {
String jsonkey = jsonkeys[i].trim();
//循环赋值
if(cmJson.has(jsonkey)){
sb.append(cmJson.getString(jsonkey)).append("\t");
//空字段必须处理
}else {
sb.append("\t");
}
}
//处理事件字段和服务器时间将整个et 作为字符串
sb.append(jsonObject.getString(“et”)).append("\t");
//处理服务器时间
sb.append(logContents[0]).append("\t");
}catch (JSONException e){
e.printStackTrace();
}
return sb.toString();
}
public static void main(String[] args) {

    String line1 = "1541217850324|{\"cm\":{\"mid\":\"m7856\",\"uid\":\"u8739\",\"ln\":\"-74.8\",\"sv\":\"V2.2.2\",\"os\":\"8.1.3\",\"g\":\"P7XC9126@gmail.com\",\"nw\":\"3G\",\"l\":\"es\",\"vc\":\"6\",\"hw\":\"640*960\",\"ar\":\"MX\",\"t\":\"1541204134250\",\"la\":\"-31.7\",\"md\":\"huawei-17\",\"vn\":\"1.1.2\",\"sr\":\"O\",\"ba\":\"Huawei\"},\"ap\":\"weather\",\"et\":[{\"ett\":\"1541146624055\",\"en\":\"display\",\"kv\":{\"goodsid\":\"n4195\",\"copyright\":\"ESPN\",\"content_provider\":\"CNN\",\"extend2\":\"5\",\"action\":\"2\",\"extend1\":\"2\",\"place\":\"3\",\"showtype\":\"2\",\"category\":\"72\",\"newstype\":\"5\"}},{\"ett\":\"1541213331817\",\"en\":\"loading\",\"kv\":{\"extend2\":\"\",\"loading_time\":\"15\",\"action\":\"3\",\"extend1\":\"\",\"type1\":\"\",\"type\":\"3\",\"loading_way\":\"1\"}},{\"ett\":\"1541126195645\",\"en\":\"ad\",\"kv\":{\"entry\":\"3\",\"show_style\":\"0\",\"action\":\"2\",\"detail\":\"325\",\"source\":\"4\",\"behavior\":\"2\",\"content\":\"1\",\"newstype\":\"5\"}},{\"ett\":\"1541202678812\",\"en\":\"notification\",\"kv\":{\"ap_time\":\"1541184614380\",\"action\":\"3\",\"type\":\"4\",\"content\":\"\"}},{\"ett\":\"1541194686688\",\"en\":\"active_background\",\"kv\":{\"active_source\":\"3\"}}]}";
    String line2 ="1549902643927|{\"cm\":{\"ln\":\"-58.1\",\"sv\":\"V2.1.6\",\"os\":\"8.1.6\",\"g\":\"56CP2931@gmail.com\",\"mid\":\"982\",\"nw\":\"4G\",\"l\":\"en\",\"vc\":\"16\",\"hw\":\"640*1136\",\"ar\":\"MX\",\"uid\":\"982\",\"t\":\"1549826442022\",\"la\":\"-29.2\",\"md\":\"HTC-16\",\"vn\":\"1.0.3\",\"ba\":\"HTC\",\"sr\":\"N\"},\"ap\":\"app\",\"et\":[{\"ett\":\"1549869285780\",\"en\":\"newsdetail\",\"kv\":{\"entry\":\"2\",\"goodsid\":\"229\",\"news_staytime\":\"2\",\"loading_time\":\"2\",\"action\":\"4\",\"showtype\":\"0\",\"category\":\"63\",\"type1\":\"\"}},{\"ett\":\"1549828760263\",\"en\":\"notification\",\"kv\":{\"ap_time\":\"1549876245798\",\"action\":\"1\",\"type\":\"1\",\"content\":\"\"}},{\"ett\":\"1549880333687\",\"en\":\"active_foreground\",\"kv\":{\"access\":\"\",\"push_id\":\"1\"}},{\"ett\":\"1549836950363\",\"en\":\"error\",\"kv\":{\"errorDetail\":\"java.lang.NullPointerException\\\\n    at cn.lift.appIn.web.AbstractBaseController.validInbound(AbstractBaseController.java:72)\\\\n at cn.lift.dfdf.web.AbstractBaseController.validInbound\",\"errorBrief\":\"at cn.lift.dfdf.web.AbstractBaseController.validInbound(AbstractBaseController.java:72)\"}},{\"ett\":\"1549881120351\",\"en\":\"praise\",\"kv\":{\"target_id\":4,\"id\":1,\"type\":2,\"add_time\":\"1549807824976\",\"userid\":0}}]}";
    String x1 = new BaseFieldUDF().evaluate(line1, "mid,uid,vc,vn,l,sr,os,ar,md,ba,sv,g,hw,nw,ln,la,t");
    String x2 = new BaseFieldUDF().evaluate(line2, "mid,uid,vc,vn,l,sr,os,ar,md,ba,sv,g,hw,nw,ln,la,t");
    System.out.println(x1);
    System.out.println(x2);
}

}

=======================================================================================
package com.atguigu.udtf;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.json.JSONArray;
import org.json.JSONException;

import java.util.ArrayList;
import java.util.List;

public class EventJsonUDTF extends GenericUDTF {
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
//定义返回值的名称
List filedNames = new ArrayList();
//定义返回值类型
List filedOIs = new ArrayList();
filedNames.add(“event_name”);
filedOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
filedNames.add(“event_json”);
filedOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
//定义返回值的名称和类型
return ObjectInspectorFactory.getStandardStructObjectInspector(filedNames,filedOIs );
}

@Override
public void process(Object[] objects) throws HiveException {
    String input = objects[0].toString();
    //判空
    if(StringUtils.isBlank(input)){
        return;
    }else {
        try {
            //传进的就是json数组
            JSONArray jsonArray = new JSONArray(input);
            if(jsonArray == null){
                return;
            }
            for (int i = 0; i < jsonArray.length(); i++) {
                String[] results = new String[2];
                try {
                    //取出事件的名称 “en”
                    results[0]=jsonArray.getJSONObject(i).getString("en");//"en":"newsdetail",也就是公共字段的name   event_name,
                    results[1]=jsonArray.getString(i);//得到的是整个事件过程//也就是事件本身   event_json,
                } catch (JSONException e) {
                    e.printStackTrace();
                    continue; //获取数据异常 不要中断
                }
                forward(results);

            }
        } catch (JSONException e) {
            e.printStackTrace();
        }
    }
}

@Override
public void close() throws HiveException {
}

}