前段时间买了本 storm分布式实时计算模式 看了几天,今天工作之余就想着写个wordCount,扫了几处雷,把代码贴出来、让你们少走几处雷
1:pom文件
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<storm.version>0.9.4</storm.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>nexus</id>
<name>local private nexus</name>
<url>http://maven.oschina.net/content/groups/public/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>junit:junit</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
2:spout
package com.tengrong.wordCount.batch.spout;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import java.util.Map;
/**
* Created by wangmao on 16/12/6.
*/
public class SentenceSpout extends BaseRichSpout {
private SpoutOutputCollector collector;
private String[] sentences = {
"hello wm", "banyan qBao", "my Word"
};
private int index = 0;
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("sentence"));
}
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
}
public void nextTuple() {
while (true) {
this.collector.emit(new Values(sentences[index]));
index++;
if (index >= sentences.length) {
index = 0;
}
try {
Thread.sleep(100);
} catch (Exception e) {
}
}
}
}
3:splitSentence
package com.tengrong.wordCount.batch.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import java.util.Map;
/**
* Created by wangmao on 16/12/6.
*/
public class SplitSentenceBolt extends BaseRichBolt {
private OutputCollector collector;
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector=collector;
}
public void execute(Tuple input) {
String sentence=input.getStringByField("sentence");
String []words=sentence.split(" ");
for(String word:words){
this.collector.emit(new Values(word));
}
}
}
4:wordCountBolt
package com.tengrong.wordCount.batch.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;
/**
* Created by wangmao on 16/12/6.
*/
public class WordCountBolt extends BaseRichBolt {
private OutputCollector collector;
private HashMap<String,Long> counts=null;
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word","count"));
}
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector=collector;
this.counts=new HashMap<String,Long>();
}
public void execute(Tuple input) {
String word=input.getStringByField("word");
Long count=this.counts.get(word);
if(count==null){
count=0L;
}
count++;
this.counts.put(word,count);
this.collector.emit(new Values(word,count));
}
}
5:reportBolt
package com.tengrong.wordCount.batch.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
import java.util.*;
/**
* Created by wangmao on 16/12/6.
*/
public class ReportBolt extends BaseRichBolt {
private HashMap<String,Long> counts=null;
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.counts=new HashMap<String,Long>();
}
public void execute(Tuple input) {
String word=input.getStringByField("word");
Long count=input.getLongByField("count");
this.counts.put(word,count);
}
@Override
public void cleanup() {
System.out.println("----begin-----");
List<String> keys=new ArrayList<String>();
keys.addAll(this.counts.keySet());
Collections.sort(keys);
for(String key:keys){
System.out.println(key+":"+this.counts.get(key));
}
System.out.println("----end-----");
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
}
6:topology
package com.tengrong.wordCount.batch.topology;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
import com.tengrong.wordCount.batch.spout.SentenceSpout;
import com.tengrong.wordCount.batch.bolt.*;
/**
* Created by wangmao on 16/12/6.
*/
public class WordCountTopology {
private static final String SENTENCE_SPOUT_ID="sentences-spout";
private static final String SPLIT_BOLT_ID="split-bolt";
private static final String COUNT_BOLT_ID="count-bolt";
private static final String REPORT_BOLT_ID="report-bolt";
private static final String TOPOLOGY_NAME="word-count-topology";
public static void main(String args[]) {
SentenceSpout spout=new SentenceSpout();
SplitSentenceBolt splitSentenceBolt=new SplitSentenceBolt();
WordCountBolt wordCountBolt=new WordCountBolt();
ReportBolt reportBolt=new ReportBolt();
TopologyBuilder builder=new TopologyBuilder();
builder.setSpout(SENTENCE_SPOUT_ID,spout);
builder.setBolt(SPLIT_BOLT_ID,splitSentenceBolt).shuffleGrouping(SENTENCE_SPOUT_ID);
builder.setBolt(COUNT_BOLT_ID,wordCountBolt).fieldsGrouping(SPLIT_BOLT_ID,new Fields("word"));
builder.setBolt(REPORT_BOLT_ID,reportBolt).globalGrouping(COUNT_BOLT_ID);
Config config=new Config();
LocalCluster cluster=new LocalCluster();
cluster.submitTopology(TOPOLOGY_NAME,config,builder.createTopology());
try {
Thread.sleep(10000);
}catch (Exception e){
}
cluster.killTopology(TOPOLOGY_NAME);
cluster.shutdown();
}
}
7:运行topology
maven install 生成jar
java -cp storm-1.0-SNAPSHOT.jar com.tengrong.wordCount.batch.topology.WordCountTopology
----begin-----
Word:25
banyan:26
hello:26
my:25
qBao:26
wm:26
----end-----
源码我的下载里可以下载
1808568908@qq.com,交流学习