kafka consumer 0.8.2.1示例代码

package test_kafka;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.ConsumerTimeoutException;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

public class KafkaConsumerApi {

    public ConsumerConnector consumer;
    private AtomicInteger sendCount;

    public Properties initProperties(String broker, String groupId) {
        Properties properties = new Properties();
        properties.put("zookeeper.connect", broker);
        properties.put("group.id", groupId);
        properties.put("auto.offset.reset", "smallest");
        properties.put("auto.commit.enable", "false");
        properties.put("zookeeper.session.timeout.ms", "4000");
        properties.put("zookeeper.sync.time.ms", "200");
        properties.put("consumer.timeout.ms", "10000");
        return properties;
    }

    public static void main(String[] args) throws Exception {
        new KafkaConsumerApi().start();
    }

    public void start() {
        String topic = "test_010_kafka";
        String groupId = "test3";
        Properties properties = initProperties("192.168.137.131:2181", groupId);
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, new Integer(1));
        ConsumerConfig config = new ConsumerConfig(properties);
        sendCount = new AtomicInteger(0);
        while (true) {
            System.out.println("start to consumer Messages.");
            List<byte[]> bytes = new ArrayList<byte[]>();
            try {
                consumer = Consumer.createJavaConsumerConnector(config);
                Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
                        .createMessageStreams(topicCountMap);
                KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);
                ConsumerIterator<byte[], byte[]> it = stream.iterator();
                while (true) {
                    try {
                        while (it.hasNext()) {
                            bytes.add(it.next().message());
                            if (bytes.size() >= 8) {
                                submitMessage(bytes);
                            }
                        }
                    } catch (ConsumerTimeoutException e) {
                        System.out.println("-------------------------ConsumerTimeoutException---------------------------");
                        if (bytes.size() > 0) {
                            submitMessage(bytes);
                        }
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
                sleepSeconds(60);
            }
        }
    }

    public void submitMessage(List<byte[]> bytes) {
        int size = bytes.size();
        if (exceute(bytes)) {
            consumer.commitOffsets();
            System.out.println("consumer message size:" + size + ", and consumer total count:" + sendCount.addAndGet(size));
            bytes.clear();
        }
    }

    public boolean exceute(List<byte[]> bytes) {
        for (byte[] str : bytes) {
            System.out.println(new String(str));
        }
        return true;
    }

    public void sleepSeconds(int seconds) {
        try {
            Thread.sleep(seconds * 1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

}

 

转载于:https://www.cnblogs.com/yixihanfengwu/p/8569013.html