基于storm的wordCount

  前段时间买了本 storm分布式实时计算模式 看了几天，今天工作之余就想着写个wordCount,扫了几处雷，把代码贴出来、让你们少走几处雷

      1:pom文件

<properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <storm.version>0.9.4</storm.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-core</artifactId>
            <version>${storm.version}</version>
            <scope>compile</scope>
        </dependency>
    </dependencies>

    <repositories>
        <repository>
            <id>nexus</id>
            <name>local private nexus</name>
            <url>http://maven.oschina.net/content/groups/public/</url>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>
    </repositories>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>2.3</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <artifactSet>
                                <excludes>
                                    <exclude>junit:junit</exclude>
                                </excludes>
                            </artifactSet>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

2:spout

package com.tengrong.wordCount.batch.spout;

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

import java.util.Map;

/**
 * Created by wangmao on 16/12/6.
 */
public class SentenceSpout extends BaseRichSpout {

    private SpoutOutputCollector collector;
    private String[] sentences = {
            "hello wm", "banyan qBao", "my Word"
    };
    private int index = 0;

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("sentence"));
    }

    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this.collector = collector;
    }

    public void nextTuple() {
        while (true) {
            this.collector.emit(new Values(sentences[index]));
            index++;
            if (index >= sentences.length) {
                index = 0;
            }
            try {
                Thread.sleep(100);
            } catch (Exception e) {
            }
        }
    }
}

3：splitSentence

package com.tengrong.wordCount.batch.bolt;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

import java.util.Map;

/**
 * Created by wangmao on 16/12/6.
 */
public class SplitSentenceBolt extends BaseRichBolt {

    private OutputCollector collector;

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("word"));
    }

    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector=collector;
    }

    public void execute(Tuple input) {
        String sentence=input.getStringByField("sentence");
        String []words=sentence.split(" ");
        for(String word:words){
            this.collector.emit(new Values(word));
        }
    }
}

4:wordCountBolt

package com.tengrong.wordCount.batch.bolt;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

import java.util.HashMap;
import java.util.Map;

/**
 * Created by wangmao on 16/12/6.
 */
public class WordCountBolt extends BaseRichBolt {
    private OutputCollector collector;
    private HashMap<String,Long> counts=null;
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("word","count"));
    }

    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector=collector;
        this.counts=new HashMap<String,Long>();
    }

    public void execute(Tuple input) {
        String word=input.getStringByField("word");
        Long count=this.counts.get(word);
        if(count==null){
            count=0L;
        }
        count++;
        this.counts.put(word,count);
        this.collector.emit(new Values(word,count));
    }
}

5:reportBolt

package com.tengrong.wordCount.batch.bolt;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;

import java.util.*;

/**
 * Created by wangmao on 16/12/6.
 */
public class ReportBolt extends BaseRichBolt {
    private HashMap<String,Long> counts=null;
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.counts=new HashMap<String,Long>();
    }

    public void execute(Tuple input) {
        String word=input.getStringByField("word");
        Long count=input.getLongByField("count");
        this.counts.put(word,count);
    }

    @Override
    public void cleanup() {
        System.out.println("----begin-----");
        List<String> keys=new ArrayList<String>();
        keys.addAll(this.counts.keySet());
        Collections.sort(keys);
        for(String key:keys){
            System.out.println(key+":"+this.counts.get(key));
        }
        System.out.println("----end-----");
    }
    public void declareOutputFields(OutputFieldsDeclarer declarer) {

    }
}

6:topology

package com.tengrong.wordCount.batch.topology;

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
import com.tengrong.wordCount.batch.spout.SentenceSpout;

import com.tengrong.wordCount.batch.bolt.*;
/**
 * Created by wangmao on 16/12/6.
 */
public class WordCountTopology {
    private static final String SENTENCE_SPOUT_ID="sentences-spout";
    private static final String SPLIT_BOLT_ID="split-bolt";
    private static final String COUNT_BOLT_ID="count-bolt";
    private static final String REPORT_BOLT_ID="report-bolt";
    private static final String TOPOLOGY_NAME="word-count-topology";

    public static void main(String args[]) {
        SentenceSpout spout=new SentenceSpout();
        SplitSentenceBolt splitSentenceBolt=new SplitSentenceBolt();
        WordCountBolt wordCountBolt=new WordCountBolt();
        ReportBolt reportBolt=new ReportBolt();
        TopologyBuilder builder=new TopologyBuilder();
        builder.setSpout(SENTENCE_SPOUT_ID,spout);
        builder.setBolt(SPLIT_BOLT_ID,splitSentenceBolt).shuffleGrouping(SENTENCE_SPOUT_ID);
        builder.setBolt(COUNT_BOLT_ID,wordCountBolt).fieldsGrouping(SPLIT_BOLT_ID,new Fields("word"));
        builder.setBolt(REPORT_BOLT_ID,reportBolt).globalGrouping(COUNT_BOLT_ID);
        Config config=new Config();
        LocalCluster cluster=new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME,config,builder.createTopology());
        try {
            Thread.sleep(10000);
        }catch (Exception e){

        }
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
    }
}

7:运行topology

maven install 生成jar

java -cp storm-1.0-SNAPSHOT.jar com.tengrong.wordCount.batch.topology.WordCountTopology

----begin-----

Word:25

banyan:26

hello:26

my:25

qBao:26

wm:26

----end-----

源码我的下载里可以下载

1808568908@qq.com，交流学习