网站流量统计分析系统-05

最新推荐文章于 2019-09-19 16:07:08 发布

tansuoliming

最新推荐文章于 2019-09-19 16:07:08 发布

阅读量309

点赞数

分类专栏：大数据

本文链接：https://blog.csdn.net/tansuoliming/article/details/80044115

版权

大数据专栏收录该内容

52 篇文章 2 订阅

订阅专栏

定时任务计算：br、avgTime、avgDeep

br

编写两个topology

package com.liming.flux;

import java.util.UUID;

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.generated.StormTopology;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.utils.Utils;
import storm.kafka.BrokerHosts;
import storm.kafka.KafkaSpout;
import storm.kafka.SpoutConfig;
import storm.kafka.StringScheme;
import storm.kafka.ZkHosts;

public class FluxTopology {
	public static void main(String[] args) {
		//SPOUT的id 要求唯一
		String KAFKA_SPOUT_ID = "flux_spout";
		//要连接的kafka的topic
		String CONSUME_TOPIC = "flux_topic";
		//要连接的zookeeper的地址
		String ZK_HOSTS = "192.168.239.129:2181";
		//设定连接服务器的参数
		BrokerHosts hosts = new ZkHosts(ZK_HOSTS);
		SpoutConfig spoutConfig = new SpoutConfig(hosts, CONSUME_TOPIC, "/" + CONSUME_TOPIC, UUID.randomUUID().toString());
		spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
		//从kafka读取数据发射
		KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
		
		//创建TopologyBuilder类实例
		TopologyBuilder builder = new TopologyBuilder();
		builder.setSpout(KAFKA_SPOUT_ID, kafkaSpout);
		//清理数据
		builder.setBolt("ClearBolt", new ClearBolt()).shuffleGrouping(KAFKA_SPOUT_ID);
		//计算PV
		builder.setBolt("PvBolt", new PvBolt()).shuffleGrouping("ClearBolt");
		//计算Uv
		builder.setBolt("UvBolt", new UvBolt()).shuffleGrouping("PvBolt");
		//计算vv
		builder.setBolt("VvBolt", new VvBolt()).shuffleGrouping("UvBolt");
		//计算newip
		builder.setBolt("NewipBolt", new NewipBolt()).shuffleGrouping("VvBolt");
		//计算newcust
		builder.setBolt("NewcustBolt", new NewcustBolt()).shuffleGrouping("NewipBolt");
		//落地到mysql
		builder.setBolt("ToMysqlBolt", new ToMysqlBolt()).shuffleGrouping("NewcustBolt");
		builder.setBolt("PrintBolt", new PrintBolt()).shuffleGrouping("NewcustBolt");
		builder.setBolt("ToHbaseBolt", new ToHbaseBolt()).shuffleGrouping("NewcustBolt");
		StormTopology topology = builder.createTopology();
		
		/**
		 * 每隔5分钟计算br avgtime avgdeep
		 */
		TopologyBuilder builder2 = new TopologyBuilder();
		builder2.setSpout("flux_spout2", new TimeSpout());
		builder2.setBolt("br_bolt", new BrBolt()).shuffleGrouping("flux_spout2");
		//TODO:计算avgTime
		//TODO:计算avgDeep
		//TODO:写入mysql
		builder2.setBolt("PrintBolt2", new PrintBolt2()).shuffleGrouping("br_bolt");
		StormTopology topology2 = builder2.createTopology();
		
		
		/**
		 * 提交Topology给集群运行
		 */
		Config conf = new Config();
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("MyTopology", conf, topology);
		cluster.submitTopology("MyTopology2", conf, topology2);
		//--运行10秒钟后杀死Topology关闭集群
		Utils.sleep(1000 * 1000);
		cluster.killTopology("MyTopology"); 
		cluster.killTopology("MyTopology2"); 
		cluster.shutdown();
	}
}

这个spout只是定时启动这条“定时任务”流水线

package com.liming.flux;

import java.util.Map;

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

public class TimeSpout extends BaseRichSpout {

	private SpoutOutputCollector collector = null;
	private long time = 0l;
	
	@Override
	public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
		this.collector = collector;
		time = System.currentTimeMillis();
	}

	@Override
	public void nextTuple() {
		try {
			long now = System.currentTimeMillis();
			if(now - time >= 1000 * 10 /** 5*/){
				collector.emit(new Values(now));
				time = now;
			}else{
				Thread.sleep(1);
				return;
			}
		} catch (Exception e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		}
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("signal"));
	}

}

package com.liming.flux;

import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.liming.flux.dao.HBaseDao;
import com.liming.flux.domain.FluxInfo;
import com.liming.flux.utils.FluxUtils;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

public class BrBolt extends BaseRichBolt {

	private OutputCollector collector = null;
	private Map<String,Integer> map = new HashMap<>();
	@Override
	public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
		this.collector = collector;
	}

	@Override
	public void execute(Tuple input) {
		long time = input.getLongByField("signal");
		Date date = new Date(time);
		String dateStr = FluxUtils.formatDate(date);
		List<FluxInfo> list = HBaseDao.queryData("^"+dateStr+"_.*$");
		//遍历数据
		for(FluxInfo fi : list){
			String ss_id = fi.getSs_id();
			map.put(ss_id, map.containsKey(ss_id) ? map.get(ss_id)+1 : 1);
		}
		//计算跳出率
		int ssCount = map.size();
		int brCount = 0;
		for(Map.Entry<String, Integer> entry : map.entrySet()){
			if(entry.getValue() == 1)brCount++;
		}
		double br = Math.round(brCount * 100.0 / ssCount)/100.0;
		
		//输出结果
		collector.emit(new Values(br));
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("br"));
	}

}

package com.liming.flux;

import java.util.Iterator;
import java.util.Map;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;

public class PrintBolt2 extends BaseRichBolt{

	private OutputCollector collector = null;
	
	@Override
	public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
		this.collector = collector;
	}

	@Override
	public void execute(Tuple input) {
		Fields fields = input.getFields();
		StringBuffer buf = new StringBuffer();
		Iterator<String> it = fields.iterator();
		while(it.hasNext()){
			String key = it.next();
			Object value = input.getValueByField(key);
			buf.append("----"+key+":"+value+"-----");
		}
		System.out.println(buf.toString());
		
		collector.emit(input.getValues());
		
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("br"));
	}
	
}

点击一次就关闭浏览器：预订跳出率为1，验证正确