大数据Storm流处理计算wordcount

package com.neusoft.storm.wc;

import java.util.List;
import java.util.Map;
import java.util.Random;

import org.yecht.DocStage;

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import kafka.utils.threadsafe;


public class WCSpout extends BaseRichSpout{	//继承BaseRichSpout
	
	SpoutOutputCollector collector;
	Random r = new Random();	//随机函数
	String[] docs=new String[]{		//数组
			"hello hello storm",
			"hello kafka spark",
			"bye bye bye"
	};
	@Override
	public void nextTuple() {
		String line = docs[r.nextInt(docs.length)];//随机取出1行数据
		List tuple = new Values(line);
		this.collector.emit(tuple);		//执行发送数据
		System.out.println("spout line:"+line);
		try {
			Thread.sleep(1000);			//休眠1秒
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}	
	}
	
	
	//入口类
	@Override
	public void open(Map map, TopologyContext context, SpoutOutputCollector collector) {
		// TODO Auto-generated method stub
		this.collector = collector;//初始化
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("line"));	//向后发送数据
	}

}


package com.neusoft.storm.wc;

import java.util.List;
import java.util.Map;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

public class SplitBolt extends BaseRichBolt{
	//只做拆分处理
	OutputCollector collector;
	@Override
	//input接收的是一行数据 
	public void execute(Tuple input) {	//执行方法
		// TODO Auto-generated method stub
		String line = input.getString(0);	
		System.err.println("@@@split bolt line:"+line);
		String[] words = line.split(" ");
		//split拆分单词封装到tuple
		for (String word : words) {
			List tuple= new Values(word,1);	//1是key,相当于mapreduce的单词和value的1	//二元组
			this.collector.emit(tuple);
		}
	}

	//入口
	@Override
	public void prepare(Map map, TopologyContext context, OutputCollector collector) {
		// TODO Auto-generated method stub
		this.collector=collector;
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("word","n"));
	}//执行发送

	
}


package com.neusoft.storm.wc;

import java.util.HashMap;
import java.util.Map;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;

public class CountBolt extends BaseRichBolt{
	OutputCollector collector;
	Map<String, Integer> dbMap= new HashMap<>();	//单词key value1 作count聚合
	@Override
	public void execute(Tuple input) {	//splitBolt类发送的是一个单词与一个1
		// TODO Auto-generated method stub
		String word = input.getString(0);	//0位置是单词,1位置是数字1
		int ct=1;//相当于intput.getStirng(1)
		
		//单词累加,更新map结果
		if(dbMap.containsKey(word)){	//Integer.intValue()可以将Integer转化为int类型
			ct=dbMap.get(word).intValue()+1;	//word单词原来的个数加1
		}
		System.out.println("$$$countbolt word:"+word+"countbolt ct:"+ct);
		dbMap.put(word, ct);
	}

	//入口类初始化
	@Override
	public void prepare(Map map, TopologyContext context, OutputCollector collector) {
		// TODO Auto-generated method stub
		this.collector=collector;
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// TODO Auto-generated method stub
		
	}//不用将数据再次进行计算了

}


package com.neusoft.storm.wc;

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.BoltDeclarer;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;

public class WCTopology {
	
	public static void main(String[] args) {
		TopologyBuilder builder = new TopologyBuilder();	//定义拓扑对象
		builder.setSpout("wcsport", new WCSpout());			
		builder.setBolt("splitbolt", new SplitBolt()).shuffleGrouping("wcsport");
		builder.setBolt("countbolt", new CountBolt()).fieldsGrouping("splitbolt",new Fields("word") );
		LocalCluster cluster = new LocalCluster();//创建本地提交对象
		cluster.submitTopology("wc", new Config(), builder.createTopology());
	}

}