Author:赵志乾
Date:2018-10-20
Declaration:All Right Reserved!!!
1、源码位置:
kafka源码包中的案例代码在/kafka-2.0.0-src/examples/src/main目录下。导入IDEA后,对应的项目为main4。其包含一个package包:kafka.examples;在该包下共有4个类文件,分别为Consumer.java、Producer.java、KafkaProperties.java和KafkaConsumerProducerDemo.java。
2、KafkaProperties.java
该类用于提供配置信息,其内容如下。实际应用中,配置信息通常放入xml或者property配置文件中,方便配置参数的在线修改。
package kafka.examples;
public class KafkaProperties {
//主题名称
public static final String TOPIC = "topic1";
public static final String TOPIC2 = "topic2";
public static final String TOPIC3 = "topic3";
//域名和端口号,客户端用其来定位kafka服务器
public static final String KAFKA_SERVER_URL = "localhost";
public static final int KAFKA_SERVER_PORT = 9092;
//生产者客户端缓存大小
public static final int KAFKA_PRODUCER_BUFFER_SIZE = 64 * 1024;
//连接超时时间
public static final int CONNECTION_TIMEOUT = 100000;
//用于标识客户端
public static final String CLIENT_ID = "SimpleConsumerDemoClient";
private KafkaProperties() {}
}
3、Producer.java
该类用于构建生产者线程,其内容如下。在生产者线程中,无限循环向kafka服务器发送消息。
package kafka.examples;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
public class Producer extends Thread {
//kafka生产者实例引用,该生产者发送消息的value值为String类型,Key值为Integer类型
private final KafkaProducer<Integer, String> producer;
//生产者发送消息所属的主题
private final String topic;
//同步发送和异步发送开关值
private final Boolean isAsync;
public Producer(String topic, Boolean isAsync) {
//生产者属性配置
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
props.put(ProducerConfig.CLIENT_ID_CONFIG, "DemoProducer");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
IntegerSerializer.class.getName());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
StringSerializer.class.getName());
producer = new KafkaProducer<>(props);
this.topic = topic;
this.isAsync = isAsync;
}
public void run() {
int messageNo = 1;
while (true) {
//生产者所发消息的value值构建
String messageStr = "Message_" + messageNo;
//消息发送起始时间,用于在回调函数中计算消息发送的耗时
long startTime = System.currentTimeMillis();
if (isAsync) { // 异步发送
producer.send(new ProducerRecord<>(topic,
messageNo,
messageStr), new DemoCallBack(startTime, messageNo, messageStr));
} else { // 同步发送
try {
producer.send(new ProducerRecord<>(topic,
messageNo,
messageStr)).get();
System.out.println("Sent message: (" + messageNo + ", " +
messageStr + ")");
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
++messageNo;
}
}
}
class DemoCallBack implements Callback {
private final long startTime;
private final int key;
private final String message;
public DemoCallBack(long startTime, int key, String message) {
this.startTime = startTime;
this.key = key;
this.message = message;
}
/* 回调方法:当异步处理完成时,该回调函数会被调用。其中,参数metadata为消息记录的元数
据(包括所属分区和偏移量),当处理过程出错时,该参数为null;参数exception为消息记录处理出
错时抛出的异常,如果处理过程没有出错,该参数为null;即回调函数中的两个参数有且仅有一个不是null;*/
public void onCompletion(RecordMetadata metadata, Exception exception) {
/*以下处理逻辑:消息记录处理正常时,打印消息的key、value、所属分区、偏移量以及耗时;
消息处理异常时,打印堆栈信息;开发人员可以根据应用需要,在该函数中实现自己的回调逻辑*/
long elapsedTime = System.currentTimeMillis() - startTime;
if (metadata != null) {
System.out.println(
"message(" + key + ", " + message + ") sent to partition(" +
metadata.partition() +
"), " +
"offset(" + metadata.offset() + ") in " + elapsedTime + " ms");
} else {
exception.printStackTrace();
}
}
}
4、Consumer.java
该类用于构建消费者线程,其内容如下。该类继承了ShutdownableThread,而ShutdownableThread是由scala语言写的,位于目录kafka-2.0.0-src/core/src/main/scala/kafka/utils下。在IDEA导入的工程中,位于工程main8。
package kafka.examples;
import kafka.utils.ShutdownableThread;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.util.Collections;
import java.util.Properties;
public class Consumer extends ShutdownableThread {
//kafka消费者实例引用
private final KafkaConsumer<Integer, String> consumer;
//主题名称
private final String topic;
public Consumer(String topic) {
super("KafkaConsumerExample", false);
//kafka消费者属性配置
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
props.put(ConsumerConfig.GROUP_ID_CONFIG, "DemoConsumer");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.IntegerDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.StringDeserializer");
//创建消费者实例
consumer = new KafkaConsumer<>(props);
this.topic = topic;
}
//该方法会持续不断的被执行,直到线程关闭或者该方法抛出异常。
@Override
public void doWork() {
//消费者订阅主题
consumer.subscribe(Collections.singletonList(this.topic));
/*从kafka服务器获取消息记录,最长等待时间1秒,该函数在2.0版本开始,已被标记为
deprecated方法*/
ConsumerRecords<Integer, String> records = consumer.poll(1000);
/*遍历获取到的消息进行消费处理,具体处理逻辑取决于实际应用要求,此处只是简单的
将收到消息记录的key、value和偏移量打印出来*/
for (ConsumerRecord<Integer, String> record : records) {
System.out.println("Received message: (" + record.key() + ", " +
record.value() + ") at offset " + record.offset());
}
}
@Override
public String name() {
return null;
}
@Override
public boolean isInterruptible() {
return false;
}
}
5、KafkaConsumerProducerDemo.java
该类用于测试生产者和消费者功能入口。其通过启动一个生产者线程和消费者线程,演示kafka作为消息系统的基本功能,内容如下。
package kafka.examples;
public class KafkaConsumerProducerDemo {
public static void main(String[] args) {
boolean isAsync = args.length == 0 || !args[0].trim().equalsIgnoreCase("sync");
//启动生产者线程
Producer producerThread = new Producer(KafkaProperties.TOPIC, isAsync);
producerThread.start();
//启动消费者线程
Consumer consumerThread = new Consumer(KafkaProperties.TOPIC);
consumerThread.start();
}
}
6、总结
通过kafka源码中所带的案例代码可以看出,kafka的应用模式是:在一个服务器上拉起一个消息代理broker,之后需要在负责消息生产的应用中起一个线程,循环不断的向消息代理发送消息记录;而需要消费消息的应用也会对应的起一个线程,循环不断的从消息代理拉取消息进行处理消费。