前言
在使用storm框架消费kafka数据时,之前一直是使用storm-kafka.jar中的KafkaSpout实现的,其局限性在于一次只能传入一个topic(public SpoutConfig(BrokerHosts hosts,String topic,String zkRoot,String id)。当消费多个topic时,需要实例化多个KafkaSpout。那么是否有其他方法可以一次性传入多个topic?下面给出了两种解决思路。
思路
(1)使用storm-kafka-client.jar实现
(2)自己实现spout,spout中集成kafka的消费者实现,从而完成多个kafka的消费
实现
(1)思路一的实现可以参考源码包中apache-storm-版本-src——external——storm-kafka-client——src——test部分。
pom.xml(引入适合版本的storm-core和storm-kafka-client)
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>1.0.2</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-kafka-client</artifactId>
<version>1.0.2</version>
<type>jar</type>
</dependency>
Topology1.java
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.spout.KafkaSpout;
import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.topology.TopologyBuilder;
import java.util.HashMap;
import java.util.Map;
/*
调用storm-kafka-client.jar实现spout的多topic消费
*/
public class Topology1 {
public static void main(String[] args) {
Topology1 topology1 = new Topology1();
StormTopology stormTopology = topology1.buildTopology();
Config config = new Config();
try {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("topoName", config, stormTopology);
} catch (Exception e) {
e.printStackTrace();
}
}
//构建拓扑
public StormTopology buildTopology() {
TopologyBuilder builder = new TopologyBuilder();
Map<String, Object> props = new HashMap<String, Object>();
props.put(KafkaSpoutConfig.Consumer.ENABLE_AUTO_COMMIT, "true");
props.put(KafkaSpoutConfig.Consumer.BOOTSTRAP_SERVERS, "localhost:9092");
props.put(KafkaSpoutConfig.Consumer.GROUP_ID, "group_1");
props.put(KafkaSpoutConfig.Consumer.KEY_DESERIALIZER, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(KafkaSpoutConfig.Consumer.VALUE_DESERIALIZER, "