storm+kafka实现多topic消费

前言

在使用storm框架消费kafka数据时，之前一直是使用storm-kafka.jar中的KafkaSpout实现的，其局限性在于一次只能传入一个topic(public SpoutConfig(BrokerHosts hosts,String topic,String zkRoot,String id)。当消费多个topic时，需要实例化多个KafkaSpout。那么是否有其他方法可以一次性传入多个topic？下面给出了两种解决思路。

思路

(1)使用storm-kafka-client.jar实现

(2)自己实现spout，spout中集成kafka的消费者实现，从而完成多个kafka的消费

实现

(1)思路一的实现可以参考源码包中apache-storm-版本-src——external——storm-kafka-client——src——test部分。

pom.xml（引入适合版本的storm-core和storm-kafka-client）

<dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-core</artifactId>
    <version>1.0.2</version>
</dependency>
<dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-kafka-client</artifactId>
    <version>1.0.2</version>
    <type>jar</type>
</dependency>

Topology1.java  

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.spout.KafkaSpout;
import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.topology.TopologyBuilder;

import java.util.HashMap;
import java.util.Map;

/*
调用storm-kafka-client.jar实现spout的多topic消费
 */
public class Topology1 {
    public static void main(String[] args) {
        Topology1 topology1 = new Topology1();
        StormTopology stormTopology = topology1.buildTopology();

        Config config = new Config();
        try {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("topoName", config, stormTopology);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //构建拓扑
    public StormTopology buildTopology() {
        TopologyBuilder builder = new TopologyBuilder();

        Map<String, Object> props = new HashMap<String, Object>();
        props.put(KafkaSpoutConfig.Consumer.ENABLE_AUTO_COMMIT, "true");
        props.put(KafkaSpoutConfig.Consumer.BOOTSTRAP_SERVERS, "localhost:9092");
        props.put(KafkaSpoutConfig.Consumer.GROUP_ID, "group_1");
        props.put(KafkaSpoutConfig.Consumer.KEY_DESERIALIZER, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(KafkaSpoutConfig.Consumer.VALUE_DESERIALIZER, "