首先创建一个一个storm的应用
public class TopoMain {
//创建日志
/**
* @param args
*/
public static void main(String[] args) throws Exception {
创建拓扑对象
TopologyBuilder builder = new TopologyBuilder();
设置spout和bolt之间的关联关系
builder.setSpout("random", new RandomWordSpout(), 2); 启动二个线程执行spout
builder.setBolt("writer", new WriterBolt(), 4).fieldsGrouping("transfer", new Fields("word")); 启动四个线程执行bolt
Config conf = new Config();
conf.setNumWorkers(2); 启动2个worker进程
conf.setDebug(true);
log.warn("$$$$$$$$$$$ submitting topology...");
StormSubmitter.submitTopology("life-cycle", conf, builder.createTopology());
log.warn("$$$$$$$4$$$ topology submitted !");
}
}
这个主应用程序为启动2个worker,占用2个slot,2个端口。10个线程,10个task。默认情况下一个线程处理一个task。一个task即spout或者bolt。
RandomWordSpout。java
/**
* 随机从String数组当中读取一个单词发送给下一个bolt
* @author Administrator
*
*/
public class RandomWordSpout extends BaseRichSpout {
private static final long serialVersionUID = -4287209449750623371L;
private static final Log log = LogFactory.getLog(RandomWordSpout.class);
private SpoutOutputCollector collector;
private String[] words = new String[]{"storm", "hadoop", "hive", "flume"};
private Random random = new Random();
public RandomWordSpout() {
log.warn("&&&&&&&&&&&&&&&&& RandomWordSpout constructor method invoked");
}
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
log.warn("################# RandomWordSpout open() method invoked");
this.collector = collector;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
log.warn("################# RandomWordSpout declareOutputFields() method invoked");
declarer.declare(new Fields("str"));
}
@Override
public void nextTuple() {
log.warn("################# RandomWordSpout nextTuple() method invoked");
Utils.sleep(500);
String str = words[random.nextInt(words.length)];
collector.emit(new Values(str));
}
@Override
public void activate() {
log.warn("################# RandomWordSpout activate() method invoked");
}
@Override
public void deactivate() {
log.warn("################# RandomWordSpout deactivate() method invoked");
}
}
TransferBolt。java
/**
* 将数据做简单的 传递的Bolt
* @author Administrator
*
*/
public class TransferBolt extends BaseBasicBolt {
private static final long serialVersionUID = 4223708336037089125L;
private static final Log log = LogFactory.getLog(TransferBolt.class);
public TransferBolt() {
log.warn("&&&&&&&&&&&&&&&&& TransferBolt constructor method invoked");
}
@Override
public void prepare(Map stormConf, TopologyContext context) {
log.warn("################# TransferBolt prepare() method invoked");
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
log.warn("################# TransferBolt declareOutputFields() method invoked");
declarer.declare(new Fields("word"));
}
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
log.warn("################# TransferBolt execute() method invoked");
String word = input.getStringByField("str");
collector.emit(new Values(word));
}
}
WriterBolt。java
/**
* 将接收到的单词写入到一个文件当中
* @author Administrator
*
*/
public class WriterBolt extends BaseBasicBolt {
private static final long serialVersionUID = -6586283337287975719L;
private static final Log log = LogFactory.getLog(WriterBolt.class);
private FileWriter writer = null;
public WriterBolt() {
log.warn("&&&&&&&&&&&&&&&&& WriterBolt constructor method invoked");
}
@Override
public void prepare(Map stormConf, TopologyContext context) {
log.warn("################# WriterBolt prepare() method invoked");
try {
writer = new FileWriter("/home/" + this);
} catch (IOException e) {
log.error(e);
throw new RuntimeException(e);
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
log.warn("################# WriterBolt declareOutputFields() method invoked");
}
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
log.warn("################# WriterBolt execute() method invoked");
String s = input.getString(0);
try {
writer.write(s);
writer.write("\n");
writer.flush();
} catch (IOException e) {
log.error(e);
throw new RuntimeException(e);
}
}
}
将上面的项目打成一个randomStorm。jar文件上传到集群
./storm jar randomStorm.jar 包名.类名
./storm list
./storm kill 进程名字
storm的工作过程
(1)nimbus讲数据任务写到zookeeper上,然后supervisor监控zookeeper
(2)supervisor根据client中配置项启动worker进程
(3)在不同的worker进程内启动设置的线程
(4)使用线程处理spout和bolt任务task。