由于项目原因需要升级kafka,需要重新申请kafka服务,用新的Java API来编写producer和consumer,顺便接触一下Kafka。准备工作就是安装zookeeper和kafka,由于关于安装方面的文章很多,在此就不细说了。
一、一些问题
1.1 kafka错误
错误:
WARN Error while fetching metadata with correlation id 0 : {test-topic=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
config/server.properties:
>set advertised.host.name = ip。
1.2 kafka命令行
>创建topic:sh kafka-topics.sh --create --zookeeper 10.95.177.100:2181,10.95.97.175:2181,10.95.176.44:2181 --replication-factor 1 --partitions 1 --topic topic-test
>查看topic:sh kafka-topics.sh --list --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181
>删除topic:
sh kafka-topics.sh --delete --topic topic-test --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181
>生产消息:sh kafka-console-producer.sh --broker-list 10.95.177.192:9092,10.95.97.175:9092 --topic topic-test
>消费消息:sh kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic-test --from-beginning --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181
>启动kafka服务:
./kafka-server-start.sh -daemon ../config/server.properties
>停止kafka服务:
./kafka-server-stop
>删除topic:
sh kafka-topics.sh --delete --topic topic-test --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181
>生产消息:sh kafka-console-producer.sh --broker-list 10.95.177.192:9092,10.95.97.175:9092 --topic topic-test
>消费消息:sh kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic-test --from-beginning --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181
>启动kafka服务:
./kafka-server-start.sh -daemon ../config/server.properties
>停止kafka服务:
./kafka-server-stop
1.3 Java API版本
一种:kafka.javaapi.*:
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
另一种:
org.apache.kafka.*:
import org.apache.kafka.clients.producer.KafkaProducer<K,V>
在Kafka 0.8.2之前,kafka.javaapi.producer.Producer是为唯一官方用Scala实现的Java Client。
在Kafka 0.8.2之后,有新的Java Producer API,org.apache.kafka.clients.producer.KafkaProducer,完全用Java实现的。
二、Java Kafka示例
pom.xml:
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.10.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>0.10.2.0</version>
</dependency>
</dependencies>
2.1 老版本Producer:
KafkaProducerOld.java:
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import java.util.Properties;
/**
* kafka.javaapi.producer.Producer
* */
public class KafkaProducerOld {
private Producer<String, String> producer;
public final static String TOPIC = "didi-topic-test";
private KafkaProducerOld() {
Properties props = new Properties();
props.put("metadata.broker.list", "10.95.177.192:9092,10.95.97.175:9092");
props.put("serializer.class", "kafka.serializer.StringEncoder");
ProducerConfig config = new ProducerConfig(props);
producer = new Producer<String, String>(config);
}
public void produce() {
int messageNo = 0;
final int COUNT = 10;
while(messageNo < COUNT) {
String data = String.format("hello kafka.javaapi.producer message %d", messageNo);
KeyedMessage<String, String> msg = new KeyedMessage<String, String>(TOPIC, data);
try {
producer.send(msg);
} catch (Exception e) {
e.printStackTrace();
}
messageNo++;
}
producer.close();
}
public static void main(String[] args) {
new KafkaProducerOld().produce();
}
}
2.2 老版本Consumer
KafkaConsumerOld.java:
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.serializer.StringDecoder;
import kafka.utils.VerifiableProperties;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Consumer: kafka.javaapi.consumer.Consumer
* */
public class KafkaConsumerOld {
private final ConsumerConnector consumerConnector;
public final static String TOPIC = "didi-topic-test";
private KafkaConsumerOld() {
Properties props = new Properties();
props.put("zookeeper.connect", "10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181");
props.put("group.id", "group-1");
//zk连接超时
props.put("zookeeper.session.timeout.ms", "400");
props.put("zookeeper.sync.time.ms", "200");
props.put("auto.commit.interval.ms", "1000");
props.put("auto.offset.reset", "smallest");
//序列化类
props.put("serializer.class", "kafka.serializer.StringDecoder");
ConsumerConfig config = new ConsumerConfig(props);
consumerConnector = kafka.consumer.Consumer.createJavaConsumerConnector(config);
}
public void consume() {
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(KafkaProducerNew.TOPIC, new Integer(1));
StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
Map<String, List<KafkaStream<String, String>>> consumerMap = consumerConnector.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
List<KafkaStream<String, String>> streams = consumerMap.get(TOPIC);
for (final KafkaStream stream : streams) {
ConsumerIterator<String, String> it = stream.iterator();
while (it.hasNext()) {
System.out.println("this is kafka consumer : " + new String( it.next().message().toString()) );
}
}
}
public static void main(String[] args) {
new KafkaConsumerOld().consume();
}
}
2.3 新版本Producer:
KafkaProducerNew.java:
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;
/**
* producer: org.apache.kafka.clients.producer.KafkaProducer;
* */
public class KafkaProducerNew {
private final KafkaProducer<String, String> producer;
public final static String TOPIC = "didi-topic-test";
private KafkaProducerNew() {
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.95.177.192:9092,10.95.97.175:9092");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
producer = new KafkaProducer<String, String>(props);
}
public void produce() {
int messageNo = 1;
final int COUNT = 10;
while(messageNo < COUNT) {
String key = String.valueOf(messageNo);
String data = String.format("hello KafkaProducer message %s", key);
try {
producer.send(new ProducerRecord<String, String>(TOPIC, data));
} catch (Exception e) {
e.printStackTrace();
}
messageNo++;
}
producer.close();
}
public static void main(String[] args) {
new KafkaProducerNew().produce();
}
}
2.4 新版本Consumer:
KafkaConsumerNew.java:
import org.apache.kafka.clients.consumer.*;
import java.util.Arrays;
import java.util.Properties;
/**
* consumer: org.apache.kafka.clients.consumer.Consumer
* */
public class KafkaConsumerNew {
private Consumer<String, String> consumer;
private static String group = "group-1";
private static String TOPIC = "didi-topic-test";
private KafkaConsumerNew() {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.95.177.192:9092,10.95.97.175:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, group);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动commit
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动commit的间隔
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
consumer = new KafkaConsumer<String, String>(props);
}
private void consume() {
consumer.subscribe(Arrays.asList(TOPIC)); // 可消费多个topic,组成一个list
while (true) {
ConsumerRecords<String, String> records = consumer.poll(1000);
for (ConsumerRecord<String, String> record : records) {
System.out.printf("offset = %d, key = %s, value = %s \n", record.offset(), record.key(), record.value());
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
public static void main(String[] args) {
new KafkaConsumerNew().consume();
}
}
Author:忆之独秀
Email:leaguenew@qq.com
注明出处:http://blog.csdn.net/lavorange/article/details/78970977