storm-kafka 学习

最近几天学习用storm-kafka，被一个问题困扰了一阵，发现是版本的问题。storm的官方文档中的storm-contrib中推荐的storm-kafka和kafka 0.7.0匹配，而我用的是kafka 0.8.2-beta，当时发现不能用，想着这是一个常见需求，偶然间发现storm的release的external下storm-kafka-0.9.3包，可以和kafka 0.8匹配，用之问题解决。

在找问题的过程中发现使用local模式调试问题确实很方便

上代码,  github地址：https://github.com/xueyumusic/kafka-storm

package stormtry;
import java.util.List;
import java.util.ArrayList;
import backtype.storm.Config;
import backtype.storm.topology.TopologyBuilder;
import storm.kafka.SpoutConfig;
import storm.kafka.KafkaSpout;
import storm.kafka.ZkHosts;
import backtype.storm.StormSubmitter;
import org.apache.storm.guava.collect.ImmutableList;
import backtype.storm.LocalCluster;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Kstest {
private static final Logger LOG = LoggerFactory.getLogger(Kstest.class);
private final TopologyBuilder builder;
private final Config topologyConfig;
private final String topologyName;
public Kstest(String topologyName) throws InterruptedException {
builder = new TopologyBuilder();
this.topologyName = topologyName;
topologyConfig = createTopologyConfiguration();
WireTopology();
}
private static Config createTopologyConfiguration() {
Config conf = new Config();
conf.setDebug(true);
return conf;
}
private void WireTopology() throws InterruptedException {
String spoutId = "kafkaGenerator";
String boltId = "kafkaBolt";
List<String> hosts = new ArrayList<String>();
hosts.add("172.18.14.101");
//hosts.add("172.18.14.101:9093");
//hosts.add("172.18.14.101:9094");
List<String> zkServers = new ArrayList<String>();
zkServers.add("172.18.14.101");
SpoutConfig spoutConf = new SpoutConfig(new ZkHosts("172.18.14.101:2181", "/brokers"), "kstesttopic", "/mykstest", "ksid");
/*SpoutConfig spoutConfig = new SpoutConfig(
ImmutableList.of("172.18.14.101"), // list of Kafka brokers
1, // number of partitions per host
"xueyutest", // topic to read from
"/xueyukafkastorm", // the root path in Zookeeper for the spout to store the consumer offsets
"kdid");*/ // an id for this consumer for storing the consumer offsets in Zookeeper
KafkaSpout kafkaSpout = new KafkaSpout(spoutConf);
builder.setSpout(spoutId, kafkaSpout);
builder.setBolt(boltId, new StubBolt()).globalGrouping(spoutId);
}
public void runRemotely() throws Exception {
//StormRunner.runTopologyRemotely(builder.createTopology(), topologyName, topologyConfig);
StormSubmitter.submitTopology(topologyName, topologyConfig, builder.createTopology());
}
public void runLocal() throws Exception {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(topologyName, topologyConfig, builder.createTopology());
Thread.sleep(100000000);
cluster.shutdown();
}
public static void main(String[] args) throws Exception {
String topologyName = "KstestTopo";
if (args.length >= 2) {
topologyName = args[1];
}
Kstest kst = new Kstest(topologyName);
if (args[0].equals("local")) {
kst.runLocal();
} else {
kst.runRemotely();
}
}
}