spout:
WordReader:
package spout;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
/**
Author:Benjamin
*/
public class WordReader extends BaseRichSpout{
//SpoutOutputCollector 发射器,负责向外面发射数据
//conf 配置信息
//private static final long serialVersionUID = 2756293133996402960L;
private String inputpath;
private SpoutOutputCollector collector;
//当一个task被初始化时会调用此open方法
public void open(Map conf, TopologyContext context,
SpoutOutputCollector collector) {
inputpath = (String) conf.get("INPUTPATH");
this.collector = collector;
}
//这个方法会被一直调用
public void nextTuple() {
Collection<File> listFiles = FileUtils.listFiles(new File(inputpath), new String[]{"txt"}, true);
for (File file : listFiles) {
try {
List<String> readLines = FileUtils.readLines(file);
for (String line : readLines) {
this.collector.emit(new Values(line));
}
FileUtils.moveFile(file, new File(file.getAbsolutePath()+System.currentTimeMillis()));
} catch (IOException e) {
e.printStackTrace();
}
}
}
//声明发射的字段
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
}
Bolt:
WordSpliter:
package bolt;
import java.util.Map;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
/**
Author:Benjamin
*/
public class WordSpliter extends BaseRichBolt{
//private static final long serialVersionUID = -3411512160818196264L;
OutputCollector collector;
@Override
public void prepare(Map stormConf, TopologyContext context,
OutputCollector collector) {
this.collector = collector;
}
//一直执行
@Override
public void execute(Tuple input) {
String stringByField = input.getStringByField("line");
String[] split = stringByField.split(" ");
for (String word : split) {
this.collector.emit(new Values(word));
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
}
WordCounter:
package bolt;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
/**
Author:Benjamin
*/
public class WordCounter extends BaseRichBolt{
/**
*
*/
//private static final long serialVersionUID = -8840777246416063904L;
OutputCollector collector;
HashMap<String, Integer> hashmap = new HashMap<String, Integer>();
@Override
public void prepare(Map stormConf, TopologyContext context,
OutputCollector collector) {
this.collector = collector;
//每三秒统计一次
new Thread(new Runnable() {
@Override
public void run() {
while(true){
System.out.println("=======================");
for (Entry<String, Integer> entry : hashmap.entrySet()) {
System.out.println(entry.getKey()+"====="+entry.getValue());
}
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}).start();
}
@Override
public void execute(Tuple input) {
String word = input.getStringByField("word");
Integer integer = hashmap.get(word);
if(integer == null){
hashmap.put(word, 1);
}else {
hashmap.put(word, integer+1);
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
}
Topology:
package topolopy;
import spout.WordReader;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import bolt.WordCounter;
import bolt.WordSpliter;
/**
Author:Benjamin
*/
public class WordCountTopo {
public static void main(String[] args) {
TopologyBuilder topologyBuilder = new TopologyBuilder();
topologyBuilder.setSpout("wordreader", new WordReader());
topologyBuilder.setBolt("wordspliter", new WordSpliter()).shuffleGrouping("wordreader");
topologyBuilder.setBolt("wordcounter", new WordCounter()).shuffleGrouping("wordspliter");
Config config = new Config();
config.put("INPUTPATH", "j:\\inputdir");
LocalCluster localCluster = new LocalCluster();
localCluster.submitTopology("wordcount", config, topologyBuilder.createTopology());
}
}