kafka java示例_Kafka Java示例

由于项目原因需要升级kafka，需要重新申请kafka服务，用新的Java API来编写producer和consumer，顺便接触一下Kafka。准备工作就是安装zookeeper和kafka，由于关于安装方面的文章很多，在此就不细说了。

一、一些问题

1.1 kafka错误

错误：

WARN Error while fetching metadata with correlation id 0 : {test-topic=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)

解决方案：https://github.com/wurstmeister/kafka-docker/issues/85https://github.com/wurstmeister/kafka-docker/issues/85

config/server.properties：

>set advertised.host.name = ip。

1.2 kafka命令行

>创建topic：sh kafka-topics.sh --create --zookeeper 10.95.177.100:2181,10.95.97.175:2181,10.95.176.44:2181  --replication-factor 1 --partitions 1 --topic  topic-test

>查看topic：sh kafka-topics.sh --list --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181

>删除topic：

sh kafka-topics.sh --delete --topic topic-test --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181

>生产消息：sh kafka-console-producer.sh --broker-list 10.95.177.192:9092,10.95.97.175:9092 --topic topic-test

>消费消息：sh kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic-test --from-beginning  --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181

>启动kafka服务：

./kafka-server-start.sh -daemon ../config/server.properties

>停止kafka服务：

./kafka-server-stop

1.3 Java API版本

一种：kafka.javaapi.*：

import kafka.javaapi.producer.Producer;

import kafka.producer.KeyedMessage;

import kafka.producer.ProducerConfig;另一种：org.apache.kafka.*:

import org.apache.kafka.clients.producer.KafkaProducer在Kafka 0.8.2之前，kafka.javaapi.producer.Producer是为唯一官方用Scala实现的Java Client。

在Kafka 0.8.2之后，有新的Java Producer API，org.apache.kafka.clients.producer.KafkaProducer,完全用Java实现的。

二、Java

Kafka示例

pom.xml:

org.apache.kafka

kafka-clients

0.10.2.0

org.apache.kafka

kafka_2.11

0.10.2.0

2.1 老版本Producer：

KafkaProducerOld.java:

import kafka.javaapi.producer.Producer;

import kafka.producer.KeyedMessage;

import kafka.producer.ProducerConfig;

import java.util.Properties;

/**

* kafka.javaapi.producer.Producer

* */

public class KafkaProducerOld {

private Producer producer;

public final static String TOPIC = "didi-topic-test";

private KafkaProducerOld() {

Properties props = new Properties();

props.put("metadata.broker.list", "10.95.177.192:9092,10.95.97.175:9092");

props.put("serializer.class", "kafka.serializer.StringEncoder");

ProducerConfig config = new ProducerConfig(props);

producer = new Producer(config);

}

public void produce() {

int messageNo = 0;

final int COUNT = 10;

while(messageNo < COUNT) {

String data = String.format("hello kafka.javaapi.producer message %d", messageNo);

KeyedMessage msg = new KeyedMessage(TOPIC, data);

try {

producer.send(msg);

} catch (Exception e) {

e.printStackTrace();

}

messageNo++;

}

producer.close();

}

public static void main(String[] args) {

new KafkaProducerOld().produce();

}

}2.2 老版本Consumer

KafkaConsumerOld.java:

import kafka.consumer.ConsumerConfig;

import kafka.consumer.ConsumerIterator;

import kafka.consumer.KafkaStream;

import kafka.javaapi.consumer.ConsumerConnector;

import kafka.serializer.StringDecoder;

import kafka.utils.VerifiableProperties;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Properties;

/**

* Consumer: kafka.javaapi.consumer.Consumer

* */

public class KafkaConsumerOld {

private final ConsumerConnector consumerConnector;

public final static String TOPIC = "didi-topic-test";

private KafkaConsumerOld() {

Properties props = new Properties();

props.put("zookeeper.connect", "10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181");

props.put("group.id", "group-1");

//zk连接超时

props.put("zookeeper.session.timeout.ms", "400");

props.put("zookeeper.sync.time.ms", "200");

props.put("auto.commit.interval.ms", "1000");

props.put("auto.offset.reset", "smallest");

//序列化类

props.put("serializer.class", "kafka.serializer.StringDecoder");

ConsumerConfig config = new ConsumerConfig(props);

consumerConnector = kafka.consumer.Consumer.createJavaConsumerConnector(config);

}

public void consume() {

Map topicCountMap = new HashMap();

topicCountMap.put(KafkaProducerNew.TOPIC, new Integer(1));

StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());

StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

Map>> consumerMap = consumerConnector.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);

List> streams = consumerMap.get(TOPIC);

for (final KafkaStream stream : streams) {

ConsumerIterator it = stream.iterator();

while (it.hasNext()) {

System.out.println("this is kafka consumer : " + new String( it.next().message().toString()) );

}

}

}

public static void main(String[] args) {

new KafkaConsumerOld().consume();

}

}2.3 新版本Producer：

KafkaProducerNew.java:

import org.apache.kafka.clients.producer.KafkaProducer;

import org.apache.kafka.clients.producer.ProducerConfig;

import org.apache.kafka.clients.producer.ProducerRecord;

import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;

/**

* producer: org.apache.kafka.clients.producer.KafkaProducer;

* */

public class KafkaProducerNew {

private final KafkaProducer producer;

public final static String TOPIC = "didi-topic-test";

private KafkaProducerNew() {

Properties props = new Properties();

props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.95.177.192:9092,10.95.97.175:9092");

props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

producer = new KafkaProducer(props);

}

public void produce() {

int messageNo = 1;

final int COUNT = 10;

while(messageNo < COUNT) {

String key = String.valueOf(messageNo);

String data = String.format("hello KafkaProducer message %s", key);

try {

producer.send(new ProducerRecord(TOPIC, data));

} catch (Exception e) {

e.printStackTrace();

}

messageNo++;

}

producer.close();

}

public static void main(String[] args) {

new KafkaProducerNew().produce();

}

}2.4 新版本Consumer：

KafkaConsumerNew.java:

import org.apache.kafka.clients.consumer.*;

import java.util.Arrays;

import java.util.Properties;

/**

* consumer: org.apache.kafka.clients.consumer.Consumer

* */

public class KafkaConsumerNew {

private Consumer consumer;

private static String group = "group-1";

private static String TOPIC = "didi-topic-test";

private KafkaConsumerNew() {

Properties props = new Properties();

props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.95.177.192:9092,10.95.97.175:9092");

props.put(ConsumerConfig.GROUP_ID_CONFIG, group);

props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");

props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动commit

props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动commit的间隔

props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");

props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");

props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");

consumer = new KafkaConsumer(props);

}

private void consume() {

consumer.subscribe(Arrays.asList(TOPIC)); // 可消费多个topic,组成一个list

while (true) {

ConsumerRecords records = consumer.poll(1000);

for (ConsumerRecord record : records) {

System.out.printf("offset = %d, key = %s, value = %s \n", record.offset(), record.key(), record.value());

try {

Thread.sleep(1000);

} catch (InterruptedException e) {

e.printStackTrace();

}

}

}

}

public static void main(String[] args) {

new KafkaConsumerNew().consume();

}

}

Author：忆之独秀

Email：leaguenew@qq.com

注明出处：http://blog..net/lavorange/article/details/78970977