kafka java API模拟生产者消费者
1.Linux启动服务
zkServer.sh start
kafka-server-start.sh -daemon /opt/server.properties
[root@hadoop001 ~]# kafka-topics.sh --create --zookeeper 192.168.116.60:2181 --topic kb09two --partitions 3 --replication-factor 1
[root@hadoop001 ~]# kafka-topics.sh --zookeeper 192.168.116.60:2181 --topic kb09two --describe
//返回的队列基本信息
Topic:kb09two PartitionCount:3 ReplicationFactor:1 Configs:
Topic: kb09two Partition: 0 Leader: 0 Replicas: 0 Isr: 0
Topic: kb09two Partition: 1 Leader: 0 Replicas: 0 Isr: 0
Topic: kb09two Partition: 2 Leader: 0 Replicas: 0 Isr: 0
2.生产者代码
Producer API
消息发送流程
Kafka 的 Producer 发送消息采用的是异步发送的方式。
在消息发送的过程中涉及到了两个线程——main 线程和 Sender 线程
以及一个线程共享变量——RecordAccumulator
main 线程将消息发送给 RecordAccumulator
Sender 线程不断从 RecordAccumulator 中拉取消息发送到 Kafka broker
代码如下:
package cn.bright;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;
/**
* @Author Bright
* @Date 2020/12/7
* @Description
*/
public class MyProducer {
public static void main(String[] args) {
Properties prop = new Properties();
//获取连接 prop.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.116.60:9092");
prop.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
prop.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,StringSerializer.class);
prop.put(ProducerConfig.ACKS_CONFIG,"-1");
//创建KafkaProducer对象
KafkaProducer<String,String> producer=new KafkaProducer<String, String>(prop);
for (int i = 0; i < 200; i++) {
ProducerRecord<String, String> producerRecord
= new ProducerRecord<>("kb09two", "hello world" + i);
producer.send(producerRecord);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
System.out.println("game over!!!");
}
}
Linux上查看生产者队列内容
//查看队列内容:
[root@hadoop001 ~]# kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list 192.168.116.60:9092 --topic kb09two -time -1 --offsets 1
//返回的分区以及对应的偏移量开始位置
kb09two:0:66
kb09two:1:67
kb09two:2:67
3.消费者代码
Consumer API
Consumer 消费数据时的可靠性是很容易保证的,因为数据在 Kafka 中是持久化的,故不用担心数据丢失问题。
由于 consumer 在消费过程中可能会出现断电宕机等故障, consumer 恢复后,需要从故障前的位置的继续消费,所以 consumer 需要实时记录自己消费到了哪个 offset,以便故障恢复后继续消费。
所以 offset 的维护是 Consumer 消费数据是必须考虑的问题。
代码如下:
package cn.bright;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Collections;
import java.util.Properties;
/**
* @Author Bright
* @Date 2020/12/7
* @Description
*/
public class MyConsumer {
public static void main(String[] args) {
Properties prop = new Properties();
prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.116.60:9092");
prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
prop.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
//earliest latest none
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
//模拟一个消费者组G1只有一个消费者
// prop.put(ConsumerConfig.GROUP_ID_CONFIG,"G1");
// KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
// consumer.subscribe(Collections.singleton("kb09two"));
// while (true){
// ConsumerRecords<String, String> poll = consumer.poll(100);
// for (ConsumerRecord<String,String> record:poll) {
// System.out.println(record.offset()+"\t"+record.key()+"\t"+record.value());
// }
// }
//模拟多消费者在在同一个分组G2
prop.put(ConsumerConfig.GROUP_ID_CONFIG, "G2");
new Thread(new Runnable() {
@Override
public void run() {
//创建KafkaConsumer对象
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
consumer.subscribe(Collections.singleton("kb09two"));
while (true) {
ConsumerRecords<String, String> poll = consumer.poll(100);
for (ConsumerRecord<String, String> record : poll) {
System.out.println(Thread.currentThread().getName() + "\t" +
record.offset() + "\t" + record.key() + "\t" + record.value());
}
}
}
}).start();
}
}
Linux上查看消费者队列内容
[root@hadoop001 ~]# kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9092 --topic kb09two --from-beginning
//消费的消息
hello world2
hello world5
hello world8
hello world11
hello world14
hello world17
hello world20
hello world23
4.maven依赖
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.0.0</version>
</dependency>
</dependencies>