手撕项目-Flink电商实时分析六

目录

 

回顾

频道新鲜度编写

总结

---

回顾

 

之前编写了计算pvuv的mapredice，并且可以在本地进行测试，将计算结果存入hbase

 

 

频道新鲜度编写

 

新鲜度：每天的活跃用户中，新用户的数量和老用户的数量的一个比例

map代码：

public class PindaoXinXianDuMap implements FlatMapFunction<KafkaMessage,PidaoXinXianDu> {

    @Override
    public void flatMap(KafkaMessage value, Collector<PidaoXinXianDu> out) throws Exception {
        String jsonstring = value.getJsonmessage();
        long timestamp = value.getTimestamp();


        String hourtimestamp = DateUtil.getDateby(timestamp,"yyyyMMddhh");//小时
        String daytimestamp = DateUtil.getDateby(timestamp,"yyyyMMdd");//天
        String monthtimestamp = DateUtil.getDateby(timestamp,"yyyyMM");//月

        UserscanLog userscanLog = JSON.parseObject(jsonstring, UserscanLog.class);
        long pingdaoid = userscanLog.getPingdaoid();
        long userid = userscanLog.getUserid();

        UserState userState = PdvisterDao.getUserSatebyvistertime(userid+"",timestamp);
        boolean isFirsthour = userState.isFisrthour();
        boolean isFisrtday = userState.isFisrtday();
        boolean isFisrtmonth = userState.isFisrtmonth();

        PidaoXinXianDu pidaoXinXianDu = new PidaoXinXianDu();
        pidaoXinXianDu.setPingdaoid(pingdaoid);
        pidaoXinXianDu.setTimestamp(timestamp);
        /**
         * 新增用户
         */
        long newuser = 0l;
        if(userState.isIsnew()){
            newuser = 1l;
        }
        pidaoXinXianDu.setNewcount(newuser);

        /**
         * 小时
         */
        long olduser = 0l;
        if(!userState.isIsnew()&&isFirsthour){
            olduser = 1l;
        }
        pidaoXinXianDu.setOldcount(olduser);
        pidaoXinXianDu.setTimestring(hourtimestamp);
        pidaoXinXianDu.setGroupbyfield(hourtimestamp+pingdaoid);
        out.collect(pidaoXinXianDu);
        System.out.println("小时=="+pidaoXinXianDu);
        /**
         * 天
         */
        olduser = 0l;
        if(!userState.isIsnew()&&isFisrtday){
            olduser = 1l;
        }
        pidaoXinXianDu.setOldcount(olduser);
        pidaoXinXianDu.setTimestring(daytimestamp);
        pidaoXinXianDu.setGroupbyfield(daytimestamp+pingdaoid);
        out.collect(pidaoXinXianDu);
        System.out.println("小时=="+pidaoXinXianDu);
        /**
         * 月
         */
        olduser = 0l;
        if(!userState.isIsnew()&&isFisrtmonth){
            olduser = 1l;
        }
        pidaoXinXianDu.setOldcount(olduser);
        pidaoXinXianDu.setTimestring(monthtimestamp);
        pidaoXinXianDu.setGroupbyfield(monthtimestamp+pingdaoid);
        out.collect(pidaoXinXianDu);
        System.out.println("小时=="+pidaoXinXianDu);
    }
}

 

reduce代码：

public class PindaoXinxianduReduce implements ReduceFunction<PidaoXinXianDu> {

    @Override
    public PidaoXinXianDu reduce(PidaoXinXianDu value1, PidaoXinXianDu value2) throws Exception {
        System.out.println( "value1=="+value1);
        System.out.println( "value2=="+value2);
        long pingdaoid = value1.getPingdaoid();
        long timestampvalue = value1.getTimestamp();
        String timestring = value1.getTimestring();
        long newcountvalue1 = value1.getNewcount();
        long oldcountvalue1 = value1.getOldcount();

        long newcountvalue2 = value2.getNewcount();
        long oldcountvalue2 = value2.getOldcount();

        PidaoXinXianDu pidaoXinXianDu = new PidaoXinXianDu();
        pidaoXinXianDu.setPingdaoid(pingdaoid);
        pidaoXinXianDu.setTimestamp(timestampvalue);
        pidaoXinXianDu.setTimestring(timestring);
        pidaoXinXianDu.setNewcount(newcountvalue1+newcountvalue2);
        pidaoXinXianDu.setOldcount(oldcountvalue1+oldcountvalue2);
        System.out.println( "recuduce --pidaoXinXianDu=="+pidaoXinXianDu);
        return  pidaoXinXianDu;
    }
}

 

sink代码：

public class PindaoXinXiandusinkreduce implements SinkFunction<PidaoXinXianDu> {
    @Override
    public void invoke(PidaoXinXianDu value, Context context) throws Exception {
        System.out.println( "recuducesinkd --pidaoPvUv=="+value);
        long pingdaoid = value.getPingdaoid();
        long newcount = value.getNewcount();
        long oldcount = value.getOldcount();
        String timestring = value.getTimestring();
        String newcountstring = HbaseUtil.getdata("pindaoinfo",pingdaoid+timestring,"info","xinxiandunewcount");
        String oldcountstring = HbaseUtil.getdata("pindaoinfo",pingdaoid+timestring,"info","xinxianduoldcount");
        if(StringUtils.isNotBlank(newcountstring)){
            newcount += newcount + Long.valueOf(newcountstring);
        }
        if(StringUtils.isNotBlank(oldcountstring)) {
            oldcount += oldcount + Long.valueOf(oldcountstring);
        }

        Map<String,String> datamap = new HashMap<String,String>();
        datamap.put("xinxiandunewcount",newcount+"");
        datamap.put("xinxianduoldcount",oldcount+"");
        System.out.println( "xinxiandu---- HbaseUtil.put(pindaoinfo+"+","+pingdaoid+timestring+",info"+datamap+")");
        HbaseUtil.put("pindaoinfo",pingdaoid+timestring,"info",datamap);
    }
}

 

总结

 

由于flnik的addSink（）方法没有执行，还在调试，所以就不上结果图了，后续编写浏览地区分布、用户网络、浏览器使用分析等

具体代码可参照我的git项目地址,现有代码均已通过测试可以使用，后续会持续更新，直到项目结束，不懂的细节，可以关注公众号，后台留言，会细致解答。

git地址:https://github.com/jyqjyq/filnkDS.git