python 流式计算框架_Storm 流式计算框架

简单的WorldCount实例

WordCountSpout

package com.ronnie.storm.wordCount;

import backtype.storm.spout.SpoutOutputCollector;

import backtype.storm.task.TopologyContext;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.topology.base.BaseRichSpout;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Values;

import java.util.Map;

import java.util.Random;

public class WordCountSpout extends BaseRichSpout {

private Random random = new Random();

SpoutOutputCollector collector;

String[] lines = {

"Well done Gaben well fucking done",

"What is going wrong with you Bro",

"You are so fucking retard",

"What the hell is it",

"hadoop spark storm flink",

"mysql oracle memcache redis mongodb"

};

@Override

public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector collector) {

this.collector = collector;

}

/**

* 1. storm 会一直(死循环)调用此方法

* 2. 每次调用此方法, 往下游发输出

*

* while(flag){

* nextTuple();

* }

*/

@Override

public void nextTuple() {

int index = random.nextInt(lines.length);

String line = lines[index];

System.out.println("line: " + line);

collector.emit(new Values(line));

try {

Thread.sleep(1000);

} catch (InterruptedException e) {

e.printStackTrace();

}

}

@Override

public void declareOutputFields(OutputFieldsDeclarer declarer) {

declarer.declare(new Fields("liner"));

}

}

WordCountSplit

package com.ronnie.storm.wordCount;

import backtype.storm.task.OutputCollector;

import backtype.storm.task.TopologyContext;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.topology.base.BaseRichBolt;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Tuple;

import backtype.storm.tuple.Values;

import java.util.Map;

public class WordCountSplit extends BaseRichBolt {

// 提升作用域

OutputCollector collector;

@Override

public void prepare(Map map, TopologyContext topologyContext, OutputCollector collector) {

System.err.println(this + "=============================");

this.collector = collector;

}

@Override

public void execute(Tuple input) {

// 从域中获取数据, 要与之前Spout中 declareOutputFields 的域名称一致

String line = input.getStringByField("liner");

// 根据什么分离

String[] words = line.split(" ");

for (String word: words){

// Value是一个ArrayList, 其中存的对象要与后面声明的域中属性相对应

collector.emit(new Values(word,"ronnie"));

}

}

/**

* Fields中 的名称 与 前面 value 中的属性相应

* @param declarer

*/

@Override

public void declareOutputFields(OutputFieldsDeclarer declarer) {

declarer.declare(new Fields("word", "name"));

}

}

WordCount

package com.ronnie.storm.wordCount;

import backtype.storm.task.OutputCollector;

import backtype.storm.task.TopologyContext;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.topology.base.BaseRichBolt;

import backtype.storm.tuple.Tuple;

import java.util.HashMap;

import java.util.Map;

public class WordCount extends BaseRichBolt {

Map result = new HashMap<>();

/**

* 初始化任务

* @param map

* @param topologyContext

* @param outputCollector

*/

@Override

public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {

}

/**

* 最核心方法

* 上游传tuple数据给它, 并调用此方法

* @param input

*/

@Override

public void execute(Tuple input) {

String word = input.getString(0);

Integer integer = result.get(word);

if (null == integer){

integer = 1;

} else {

integer += 1;

}

result.put(word, integer);

System.err.println(word + " : " + integer);

}

/**

* 声明输出的域类型

* @param outputFieldsDeclarer

*/

@Override

public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {

}

}

WordCountTopology

package com.ronnie.storm.wordCount;

import backtype.storm.Config;

import backtype.storm.LocalCluster;

import backtype.storm.StormSubmitter;

import backtype.storm.generated.AlreadyAliveException;

import backtype.storm.generated.AuthorizationException;

import backtype.storm.generated.InvalidTopologyException;

import backtype.storm.generated.StormTopology;

import backtype.storm.topology.TopologyBuilder;

import backtype.storm.tuple.Fields;

public class WordCountTopology {

public static void main(String[] args) {

TopologyBuilder topologyBuilder = new TopologyBuilder();

topologyBuilder.setSpout("wcSpout", new WordCountSpout());

// setBolt 的第三个参数为并行量 setNumTasks 修改 任务数量为 4

topologyBuilder.setBolt("wcSplit", new WordCountSplit(), 2).setNumTasks(4).shuffleGrouping("wcSpout");

topologyBuilder.setBolt("wcCount", new WordCount(), 5).fieldsGrouping("wcSplit", new Fields("word"));

StormTopology topology = topologyBuilder.createTopology();

Config config = new Config();

// 修改配置文件中的worker数量为3

config.setNumWorkers(3);

// 只要参数存在

if (args.length > 0){

try {

StormSubmitter.submitTopology(args[0],config,topology);

} catch (AlreadyAliveException e) {

e.printStackTrace();

} catch (InvalidTopologyException e) {

e.printStackTrace();

} catch (AuthorizationException e) {

e.printStackTrace();

}

} else {

// 不存在就执行本地任务

LocalCluster localCluster = new LocalCluster();

localCluster.submitTopology("wordCount", config, topology);

}

}

}