storm 入门学习实例

前段时间，项目中使用到了流式处理方面的技术，学习了一下storm，编写了一个小实例。

1.引入jar包

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.3.12.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.example</groupId>
    <artifactId>control</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>control</name>
    <description>Demo project for Spring Boot</description>
    <properties>
        <java.version>1.8</java.version>
    </properties>
    <dependencies>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
            <exclusions>
                <exclusion>
                    <artifactId>log4j-to-slf4j</artifactId>
                    <groupId>org.apache.logging.log4j</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>jul-to-slf4j</artifactId>
                    <groupId>org.slf4j</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>logback-classic</artifactId>
                    <groupId>ch.qos.logback</groupId>
                </exclusion>
            </exclusions>
        </dependency>


        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>


        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-core</artifactId>
            <version>2.2.1</version>
        </dependency>

        <dependency>
            <groupId>com.codahale.metrics</groupId>
            <artifactId>metrics-core</artifactId>
            <version>3.0.2</version>
            <exclusions>
                <exclusion>
                    <artifactId>slf4j-api</artifactId>
                    <groupId>org.slf4j</groupId>
                </exclusion>
            </exclusions>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <createDependencyReducedPom>false</createDependencyReducedPom>
                    <artifactSet>
                        <excludes>
                            <exclude>commons-logging:commons-logging</exclude>
                            <exclude>javax.servlet:servlet-api</exclude>
                            <exclude>javax.mail:javax.mail-api</exclude>
                        </excludes>
                    </artifactSet>
                </configuration>
            </plugin>

        </plugins>
    </build>

</project>

2.编写程序：

（1）编写输入流类
package test.storm;

/**
 * @author gjp
 * @className WordSpout class
 * @date 2021/11/23
 * @description: 测试信息
 */
import java.util.Map;
import java.util.Random;


import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;

/**
 * 随机产生星期几，然后交给spot 进行输出
 * @author Administrator
 *
 */
public class WordSpout extends BaseRichSpout {

    private SpoutOutputCollector collector;
    private static String[] words = {"星期一","星期二","星期三","星期四","星期五","星期六","星期日"};

    public WordSpout() {
        System.out.println("--====================WordSpout===---------------");
    }

    public void nextTuple() {
        //随机取 words 字符串中一个词。
        String word = words[new Random().nextInt(words.length)];
        //发射元组到输出收集器
        collector.emit(new Values(word));

    }

    public void open(Map arg0, TopologyContext arg1, SpoutOutputCollector arg2) {
        this.collector=arg2;
        //定义数据源输出收集器
    }

    public void declareOutputFields(OutputFieldsDeclarer arg0) {
        // TODO Auto-generated method stub
        //声明输出字段的名称为为 word
        arg0.declare(new Fields("word"));
    }

}

（2）整理数据流并输出

package test.storm;

/**
 * @author gjp
 * @className BaseRichBolt class
 * @date 2021/11/24
 * @description: 处理字符流
 */
import java.util.Map;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;

public class ProcessBolt extends BaseRichBolt {

    public ProcessBolt() {
        // TODO Auto-generated constructor stub
        System.out.println("-----------ProcessBolt==============");
    }

    @Override
    public void execute(Tuple arg0) {
        // TODO Auto-generated method stub
        //此处直接对接受到的元组进行处理，然后输出到控制台，这里没有将处理后的数据再送到输出收集器中。
        //取得元组的数据
        String word = (String) arg0.getValue(0);
        String out = "Hello ：" + word + "!";
        //输出到控制台，使用 err.println 会显示红色，所以这里使用 err
        System.err.println(out);
    }
    @Override
    public void prepare(Map arg0, TopologyContext arg1, OutputCollector arg2) {
        // TODO Auto-generated method stub
        System.out.println("-----------prepare==============");

    }
    @Override
    public void declareOutputFields(OutputFieldsDeclarer arg0) {
        // TODO Auto-generated method stub
        System.out.println("-----------declareOutputFields==============");

    }

}
 

（3）编写启动类：

package test.storm;

/**
 * @author gjp
 * @className WordStorm class
 * @date 2021/11/24
 * @description: 主类
 */

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;



public class WordStorm {

    public WordStorm() {
        // TODO Auto-generated constructor stub
    }

    public static void main(String[] args)  throws Exception {
        // TODO Auto-generated method stub
        //定义 TopologyBuilder
        TopologyBuilder builder=new TopologyBuilder();
        //定义 Spout
        builder.setSpout("Spout_ID", new WordSpout());
        //定义 Bolt
        builder.setBolt("Bolt_ID", new ProcessBolt()).localOrShuffleGrouping("Spout_ID");

        //下面开始定义运行模式
        final Config config=new Config();
        config.setDebug(true);
        //设置workers
        config.setNumWorkers(1);
        config.setMaxSpoutPending(1);

        if (args != null && args.length > 0) {
            //集群运行模式
            config.setNumWorkers(3);

            StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.createTopology());
        }else {
            //使用本地模式运行
            final LocalCluster localCluster=new LocalCluster();

            localCluster.submitTopology(WordStorm.class.getSimpleName(), config, builder.createTopology());
            org.apache.storm.utils.Utils.sleep(20000);

            localCluster.killTopology(WordStorm.class.getSimpleName());
            localCluster.shutdown();
        }


    }

}

运行结果：