我的大数据之路 -- storm简单demo和理解

Storm

Maven依赖

<dependencies>
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-core</artifactId>
            <version>1.2.2</version>
        </dependency>
</dependencies>

1. Storm基本概念
   Topologies ：拓扑，也称一个任务
   Spouts ： 拓扑的数据源 – （获取数据）
   Bolts：拓扑的处理逻辑单位 –
   tuple：消息元组（传递数据的一种格式）
   Streams：流
   Stream groupings ：流的分组策略
   Tasks：任务的处理单位
   Executor：工作线程
   Workers：工作进程
   Configuration：topology的配置

高并发的处理

现在来写第一个程序

1. 启动zookeeper

2. 启动storm，在nimbus主机上(Master)

   nohup bin/storm nimbus 1>/dev/null 2>&1 &
   nohup bin/storm ui 1>/dev/null 2>&1 &
   

3. 在supervisor主机上(Slave1,Slave2)

   nohup storm supervisor 1>/dev/null 2>&1 &
   

写代码

RandomWordSpout.java
--------------------------------------------------------
package cn.storm.demo;

import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

import javax.rmi.CORBA.Util;
import java.util.Map;
import java.util.Random;

public class RandomWordSpout extends BaseRichSpout {

    private SpoutOutputCollector collector;

    //模拟数据
    private String[] words={"xiaomi","huawie","meizu","sanxing","xiaolajiao"};

    public void nextTuple() {
        Random random=new Random();
        int index=random.nextInt(words.length);

        String goName= words[index];

        collector.emit(new Values(goName));
        Utils.sleep(500);


    }

    //初始化方法，在spout组件实例化调用一次
    public void open(Map map, TopologyContext context, SpoutOutputCollector collector) {
                this.collector=collector;

    }

    //声明输出的字段
    public void declareOutputFields(OutputFieldsDeclarer declarer)
    {
        //只有一个商品名
        declarer.declare(new Fields("orignname"));
    }
}

SuffixBolt.java
---------------------------------------------
package cn.storm.demo;

import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;

import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;

public class SuffixBolt extends BaseBasicBolt {
    private FileWriter fileWrite=null;

    public void prepare(Map stormConf, TopologyContext context){
        try {
            fileWrite=new FileWriter("/user/input"+ UUID.randomUUID());

        }catch (IOException e){
            throw new RuntimeException(e);
        }
    }

    public void execute(Tuple tuple, BasicOutputCollector collector) {
                    String upper_name=tuple.getString(0);
                    String suffix_name=upper_name + "_ysh";

                    try {
                        fileWrite.write(suffix_name);
                        fileWrite.write('\n');
                        fileWrite.flush();
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
    }

    //不声名字段来
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {

    }
}

TopoMain.java
-------------------------------------------------
package cn.storm.demo;


import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.topology.TopologyBuilder;

public class TopoMain {
        public static void main(String[] args) throws InvalidTopologyException, AuthorizationException, AlreadyAliveException {

            TopologyBuilder builder = new TopologyBuilder();

            //将我们的spout组件设置到topology中去
            builder.setSpout("randomspout",new RandomWordSpout(),4);

            //将大写转换成bolt组件设置到topology，并且指定randomspout组件的消息
            builder.setBolt("upperbolt",new UpperBolt(),4).shuffleGrouping("randomspout");

            //添加后缀，并且指定upperbolt组件的消息
            builder.setBolt("suffixbolt",new SuffixBolt(),4).shuffleGrouping("upperbolt");

            StormTopology demotopo4 =builder.createTopology();

            Config conf=new Config();
            conf.setNumWorkers(4);
            conf.setDebug(true);
            conf.setNumAckers(0);

            StormSubmitter.submitTopology("demotopo4",conf,demotopo4);
        }
}

UpperBolt.java
-----------------------------------
package cn.storm.demo;


import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.topology.TopologyBuilder;

public class TopoMain {
        public static void main(String[] args) throws InvalidTopologyException, AuthorizationException, AlreadyAliveException {

            TopologyBuilder builder = new TopologyBuilder();

            //将我们的spout组件设置到topology中去
            builder.setSpout("randomspout",new RandomWordSpout(),4);

            //将大写转换成bolt组件设置到topology，并且指定randomspout组件的消息
            builder.setBolt("upperbolt",new UpperBolt(),4).shuffleGrouping("randomspout");

            //添加后缀，并且指定upperbolt组件的消息
            builder.setBolt("suffixbolt",new SuffixBolt(),4).shuffleGrouping("upperbolt");

            StormTopology demotopo4 =builder.createTopology();

            Config conf=new Config();
            conf.setNumWorkers(4);
            conf.setDebug(true);
            conf.setNumAckers(0);

            StormSubmitter.submitTopology("demotopo4",conf,demotopo4);
        }
}

打成jar包上传到集群上运行

运行命令,在storm目录下

bin/storm jar order_jar/storm_study.jar cn.storm.demo.TopoMain

IDEA打jar包



然后xftp将jar包上传即可。

BUG

Exception in thread “main” java.lang.ExceptionInInitializerError

解决方案，在打jar包的时候，将jstorm-core这个jar包去掉

BUG2

[hadoop@Master bin]$ ./storm jar ../order_jar/stoem_study.jar cn.storm.demo.TopoMain
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/user/storm-1.2.2/lib/log4j-slf4j-impl-2.8.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/user/storm-1.2.2/order_jar/stoem_study.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]

jar包冲突，则把
file:/user/storm-1.2.2/lib/log4j-slf4j-impl-2.8.2.jar
file:/user/storm-1.2.2/order_jar/stoem_study.jar
删掉一个即可