storm集成kafka简单使用示例

KafkaStormSample.java

package kafkaUse.kafkaUse.withStorm;

import java.util.Properties;
import java.util.UUID;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.kafka.BrokerHosts;
import org.apache.storm.kafka.KafkaSpout;
import org.apache.storm.kafka.SpoutConfig;
import org.apache.storm.kafka.StringScheme;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.kafka.bolt.KafkaBolt;
import org.apache.storm.kafka.bolt.mapper.FieldNameBasedTupleToKafkaMapper;
import org.apache.storm.kafka.bolt.selector.DefaultTopicSelector;
import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;

public class KafkaStormSample 
{
    public static void main(String[] args) throws Exception
 {
        Config config = new Config();
        config.setDebug(true);
        config.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

        String zkConnString = "192.168.153.233:2181";
        String topic = "test";
        BrokerHosts hosts = new ZkHosts(zkConnString);

        SpoutConfig kafkaSpoutConfig = new SpoutConfig(hosts, topic, "/"
                + topic, UUID.randomUUID().toString());
        kafkaSpoutConfig.bufferSizeBytes = 1024 * 1024 * 4;
        kafkaSpoutConfig.fetchSizeBytes = 1024 * 1024 * 4;
        // kafkaSpoutConfig.forceFromStart = true;
        kafkaSpoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());

        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.153.233:9092");
        props.put("acks", "1");
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        KafkaBolt<String, String> bolt = new KafkaBolt<String, String>()
                .withProducerProperties(props)
                .withTopicSelector(new DefaultTopicSelector("topic3"))
                .withTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper<String, String>("word", "count"));

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("kafkaspout", new KafkaSpout(kafkaSpoutConfig));
        builder.setBolt("wordspitter", new SplitBolt()).shuffleGrouping(
                "kafkaspout");
        builder.setBolt("wordcounter", new CountBolt()).shuffleGrouping(
                "wordspitter");
        builder.setBolt("kafkabolt", bolt).shuffleGrouping("wordcounter");

        if (args != null && args.length > 0) {
            config.setNumWorkers(3);
            StormSubmitter.submitTopology(args[0], config,
                    builder.createTopology());
        } else {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Topo", config, builder.createTopology());
            Utils.sleep(100000);
            cluster.killTopology("Topo");
            cluster.shutdown();
        }

    }
}

CountBolt.java

package kafkaUse.kafkaUse.withStorm;

import java.util.HashMap;
import java.util.Map;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

public class CountBolt implements IRichBolt
{
    Map<String, Integer> counters;
    private OutputCollector collector;

    @Override
    public void cleanup() 
    {
         for(Map.Entry<String, Integer> entry:counters.entrySet())
         {
             System.out.println(entry.getKey()+" : " + entry.getValue());
          }
    }
    @Override
    public void execute(Tuple input) 
    {
        String str = input.getString(0);
        Integer c = 0;
          if(!counters.containsKey(str)){
             c = 1; 
             counters.put(str, c);
          }else {
             c = counters.get(str) +1;
             counters.put(str, c);
          }

         // collector.ack(input);
        String value = str + " : " + String.valueOf(c);  
        collector.emit(new Values(str, value));

    }
    @Override
    public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) 
    {
        this.counters = new HashMap<String, Integer>();
        this.collector = collector;

    }
    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer)
    {
        declarer.declare(new Fields("word", "count"));
    }
    @Override
    public Map<String, Object> getComponentConfiguration()
    {
        return null;
    }



}

SplitBolt.java

package kafkaUse.kafkaUse.withStorm;

import java.util.Map;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

public class SplitBolt implements IRichBolt 
{
    private OutputCollector collector;

    @Override
    public void cleanup() {

    }

    @Override
    public void execute(Tuple input) 
    {
        String sentence = input.getString(0);
        String[] words = sentence.split(" ");

        for (String word : words) {
            word = word.trim();

            if (!word.isEmpty()) {
                word = word.toLowerCase();
                collector.emit(new Values(word));
            }

        }

        collector.ack(input);

    }

    @Override
    public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) {
        this.collector = collector;

    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) 
    {
        declarer.declare(new Fields("word"));

    }

    @Override
    public Map<String, Object> getComponentConfiguration() {
        return null;
    }

}