Storm的流式计算
-
storm和sparkstreaming的对比
- storm非常严格的一条数据一次计算,流式处理
- sparkstreaming 一批数据算一次(每批次的时间非常短,用户可以根据自己的想法指定)把数据流按照时间端切分成许多离散的RDD
然后SparkCore执行计算 现阶段用的比较多 - 都属于流式处理
- 在flink的世界里,所有的计算都是流式处理,把离线处理看成是流式处理的一种特例。离散的批处理基于流式处理。
- mapreduce是每个一段时间执行一次任务,对于两次任务之间的数据进行累积。
-
storm的安装
下载安装包并解压 apache-storm-1.2.2.tar.gz tar -zxvf apache-storm-1.2.2.tar.gz 进行配置 conf下的 storm.yaml storm.zookeeper.servers: - "hadoop" nimbus.seeds: ["hadoop"] nohup storm nimbus 1>~/logs/storms-nibus.log 2>&1& nohup storm ui 1>~/logs/storms-ui.log 2>&1& nohup storm supervisor 1>~/logs/storms-supervisor.log 2>&1& 查看进程jps nimbus supervisor则启动成功 页面的端口是8080
-
idea中的程序包含四个类以词频统计为例
-
数据源
package com.wc; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichSpout; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; import java.util.Map; import static jdk.nashorn.internal.parser.TokenType.values; /** * 数据源 要继承 BaseRichSpout 实现下面的三个方法 * 用于产生数据发送到WCBolt1 */ public class WCSpout extends BaseRichSpout { private static String[] lines = new String[]{ "hello huanbo", "hello xuzheng", "hello wangbaoqiang", "hello liudehua" }; SpoutOutputCollector spoutOutputCollector; int index = 0; /** * 跟mapper中的setup类似 * 在WCSpout初始化之后会首先执行一次open方法 * 对于一个spout来说只会执行一次 * * @param map * @param topologyContext * @param spoutOutputCollector */ @Override public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) { this.spoutOutputCollector = spoutOutputCollector; } /** * nextTuple的作用就是用来向下发送句子或消息的 * 另外这个方法由topology这个组件进行死循环的调用 */ @Override public void nextTuple() { if (index == 4) { return; } try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } String line = lines[index]; index++; //发wcbolt1发送句子模拟每个一秒发送一个line中的一句话 spoutOutputCollector.emit(new Values(line)); } /** * 给 nextTuple 往后发送的信息 取个名称 * * @param outputFieldsDeclarer */ @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { outputFieldsDeclarer.declare(new Fields("line")); } }
-
数据处理的第一个组件
package com.wc; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; import java.util.Map; /** * 把数据源WCSpout发送过来的数据切割成单词发送给WCBolt2 */ public class WCBolt1 extends BaseRichBolt { OutputCollector outputCollector; /** * 工作机制和spout的机制一样 * * @param map * @param topologyContext * @param outputCollector */ @Override public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { this.outputCollector = outputCollector; } /** * 表示执行 * 会在storm执行的过程中不停的调用 * * @param tuple */ @Override public void execute(Tuple tuple) { String line = tuple.getStringByField("line"); String[] words = line.trim().split(" "); for (String word : words) { outputCollector.emit(new Values(word)); } } @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { outputFieldsDeclarer.declare(new Fields("word")); } }
-
数据处理的第二个组件
package com.wc; import clojure.lang.Var; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Tuple; import java.util.HashMap; import java.util.Map; /** * 数据处理组件 * 用于接收WCBolt1发送过来的数据,然后做词频统计 */ public class WCBolt2 extends BaseRichBolt { private Map<String,Integer> wordCount; @Override public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { wordCount=new HashMap<String,Integer>(); } public void pringMap(){ for (String key:wordCount.keySet()) { System.out.print(key+":"+wordCount.get(key)+"\t"); } System.out.println(); } /** * 接收一个单把单词进行统计汇总 * * @param tuple */ @Override public void execute(Tuple tuple) { String word = tuple.getStringByField("word"); if(wordCount.containsKey(word)){ Integer newcount=wordCount.get(word)+1; wordCount.replace(word,newcount); }else{ wordCount.put(word,1); } } @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { //因为execute方法中并没有向后发送消息了所以这里没有必要再编写代码了 } @Override public void cleanup() { pringMap(); } }
-
storm的应用程序的驱动程序
package com.wc; import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.StormSubmitter; import org.apache.storm.generated.AlreadyAliveException; import org.apache.storm.generated.AuthorizationException; import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.generated.StormTopology; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.tuple.Fields; /** * storm的应用程序的驱动程序 */ public class WCTopology { public static void main(String[] args) { //获取一个Topology的构建器 TopologyBuilder topologyBuilder = new TopologyBuilder(); //给Topology设置数据源 topologyBuilder.setSpout("wcspout",new WCSpout()); topologyBuilder.setBolt("wcbolt1",new WCBolt1()).shuffleGrouping("wcspout"); topologyBuilder.setBolt("wcbolt2",new WCBolt2()) .fieldsGrouping("wcbolt1",new Fields("word")); StormTopology topology = topologyBuilder.createTopology(); Config config = new Config(); if(args[0].equals("local")){ LocalCluster cluster = new LocalCluster(); cluster.submitTopology("wctopo",config,topology); try { Thread.sleep(12000); } catch (InterruptedException e) { e.printStackTrace(); } cluster.shutdown(); }else if(args[0].equals("cluster")){ StormSubmitter submitter = new StormSubmitter(); try { submitter.submitTopology("wctopo",config,topology); } catch (AlreadyAliveException e) { e.printStackTrace(); } catch (InvalidTopologyException e) { e.printStackTrace(); } catch (AuthorizationException e) { e.printStackTrace(); } } } }
-
-
提交到本地运行需要在program arguments 传入参数local
-
打jar提交到集群运行
storm jar /home/hadoop/original-storm-1.0-SNAPSHOT.jar com.wc.WCTopology cluster 查看集群的中的应用程序storm list 杀掉应用程序的命令 storm kill 应用程序的名字 查看结果文件 在安装目录下日志里面