Storm安装和使用Demo

一,Storm集群的安装

1.下载安装包:

wget  http://archive.apache.org/dist/storm/apache-storm-0.9.7/apache-storm-0.9.7.tar.gz

2.解压移动到软件安装目录

tar -zxvf   xxxx

3.修改配置文件, conf目录下:

vim storm.yaml

storm.zookeeper.servers:

      - "hadoop6"

      - "hadoop7"

      - "hadoop8"

nimbus.host: "hadoop5"

# 

nimbus.childopts: "-Xmx1024m"

#

supervisor.childopts: "-Xmx1024m"

#

worker.childopts: "-Xmx1024m"

#

ui.childopts: "-Xmx1024m"

#

supervisor.slots.ports:

      - 6700

      - 6701

      - 6702

      - 6703

4.分发到运行的节点上:

scp -r storm  hadoop@hadoop8:/home/hadoop/app/

......

5.在各个节点启动服务:

• 在nimbus.host所属的机器上启动 nimbus服务

cd /export/servers/storm/bin/

nohup ./storm nimbus &

• 在nimbus.host所属的机器上启动ui服务

cd /export/servers/storm/bin/

nohup ./storm ui &

• 在其它个点击上启动supervisor服务

cd /export/servers/storm/bin/

nohup ./storm supervisor &

6.查看

访问nimbus.host:/8080，即可看到storm的ui界面。

二,Storm得使用Demo

1.官方统计字数例子

bin/storm jar examples/storm-starter/storm-starter-topologies-0.9.7.jar storm.starter.WordCountTopology wordcount

2.运行结果

3.自己书写一个字符统计的Demo

package com.demo.storm.wordcount;

import java.util.HashMap;
import java.util.Map;

import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

/**
 * @Description: Storm 单词统计主类
 * @author: songqinghu
 * @date: 2017年11月2日 下午2:18:19
 * Version:1.0
 */
public class WordCountStart {


    public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
        //建造者
        TopologyBuilder builder = new TopologyBuilder();
        //构建spout模块
        builder.setSpout("mySpout", new MyWordSpout(), Integer.valueOf(4));
        //构建bolt模块 --切割  定义 mySpout为随机发送规则
        builder.setBolt("mySplitBolt", new MySplitWordBolt(), Integer.valueOf(5)).shuffleGrouping("mySpout");
        //构建bolt模块 --统计  定义 mySplitBolt为Hash分发
        builder.setBolt("myCountBolt", new MyCountWordBolt(), Integer.valueOf(5)).fieldsGrouping("mySplitBolt", new Fields("word"));

        Config config = new Config();

        config.setNumWorkers(Integer.valueOf(3));
        //向集群提交任务
        if(args.length==1){
            StormSubmitter.submitTopology(args[0],config, builder.createTopology());
        }else{
            StormSubmitter.submitTopology("mywordcount",config, builder.createTopology());
        }
    }
}
/**
 * @Description: 对接受到的数据进行处理 --统计
 * @author: songqinghu
 * @date: 2017年11月2日 下午2:42:33
 * Version:1.0
 */
class MyCountWordBolt extends BaseRichBolt {


    private Map<String, Integer> wordCount = new HashMap<String,Integer>();

    @Override
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {

    }

    @Override
    public void execute(Tuple input) {
        String word = input.getString(0);

        if(wordCount.containsKey(word)){
            wordCount.put(word, wordCount.get(word)+1);
        }else{
            wordCount.put(word, 1);
        }

        System.out.println(wordCount);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        //不在向下发送
    }

}


/**
 * @Description: 对接受到的数据进行处理 --切割  分组发送
 * @author: songqinghu
 * @date: 2017年11月2日 下午2:42:33
 * Version:1.0
 */
class MySplitWordBolt extends BaseRichBolt {

    private OutputCollector collector;

    @Override
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector =collector;
    }

    @Override
    public void execute(Tuple input) {
        String words = input.getString(0);
        String[] split = words.split(" ");
        for (String word : split) {
            collector.emit(new Values(word.trim()));
        }
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("word"));
    }

}



/**
 * @Description: 数据源接入类,完成数据源的接入和传递工作
 * @author: songqinghu
 * @date: 2017年11月2日 下午2:41:23
 * Version:1.0
 */
class MyWordSpout extends BaseRichSpout{

    private SpoutOutputCollector collector;


    @Override
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        //初始执行 
        this.collector = collector;
    }

    @Override
    public void nextTuple() {
        //发送数据
        collector.emit(new Values("My life is in these books Read these and know my heart"));
    }


    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        //定义发送的tuple对应字段
        declarer.declare(new Fields("words"));
    }

}

4.打包运行即可:

bin/storm jar wordcount.jar  com.demo.storm.wordcount.WordCountStart mywordcount