public class KafkaConsumer {
public static void main(String[] args) {
Properties props = new Properties();
props.put("zookeeper.connect", "m6:2181,m7:2181,m8:2181");
props.put("group.id", "1111");
props.put("auto.offset.reset", "smallest");
ConsumerConfig conf = new ConsumerConfig(props);
ConsumerConnector consumer = Consumer.createJavaConsumerConnector(conf);
Map<String, Integer> topicStrams = new HashMap<String, Integer>();
topicStrams.put("test", 1);
Map<String, List<KafkaStream<byte[], byte[]>>> messageStreamsMap = consumer.createMessageStreams(topicStrams);
List<KafkaStream<byte[], byte[]>> messageStreams = messageStreamsMap.get("test");
for(final KafkaStream<byte[], byte[]> kafkaStream : messageStreams){
new Thread(new Runnable() {
@Override
public void run() {
for(MessageAndMetadata<byte[], byte[]> mm : kafkaStream){
String msg = new String(mm.message());
System.out.println(msg);
}
}
}).start();
}
}
}
package cn.bigdata.kafka;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.javaapi.consumer.ZookeeperConsumerConnector;
import kafka.message.MessageAndMetadata;
public class KafkaConsumerDemo {
public static void main(String[] args) {
Properties props = new Properties();
// zookeeper地址
props.put("zookeeper.connect", "m1:2181,m2:2181,m3:2181");
// 消费者组id
props.put("group.id", "22w2");
// smallest : 从头消费
// largest : 从最后消费
props.put("auto.offset.reset", "smallest");
ConsumerConfig conf = new ConsumerConfig(props);
ConsumerConnector consumer = new ZookeeperConsumerConnector(conf);
Map<String, Integer> topicStrams = new HashMap<String, Integer>();
// 第二个数字是返回几个流,topic几个分区就陪几个流比较合理
topicStrams.put("test2", 2);
Map<String, List<KafkaStream<byte[], byte[]>>> messageStreamsMap = consumer.createMessageStreams(topicStrams);
List<KafkaStream<byte[], byte[]>> messageStreams = messageStreamsMap.get("test2");
for(final KafkaStream<byte[], byte[]> kafkaStream : messageStreams){
new Thread(new Runnable() {
public void run() {
for(MessageAndMetadata<byte[], byte[]> mm : kafkaStream){
String msg = new String(mm.message());
System.out.println(msg);
}
}
}).start();
}
}
}