kafka之java API模拟生产者消费者

1.Linux启动服务

zkServer.sh start
kafka-server-start.sh -daemon /opt/server.properties
[root@hadoop001 ~]# kafka-topics.sh --create --zookeeper 192.168.116.60:2181 --topic kb09two --partitions 3 --replication-factor 1
[root@hadoop001 ~]# kafka-topics.sh --zookeeper 192.168.116.60:2181 --topic kb09two --describe
//返回的队列基本信息
Topic:kb09two   PartitionCount:3        ReplicationFactor:1     Configs:
        Topic: kb09two  Partition: 0    Leader: 0       Replicas: 0     Isr: 0
        Topic: kb09two  Partition: 1    Leader: 0       Replicas: 0     Isr: 0
        Topic: kb09two  Partition: 2    Leader: 0       Replicas: 0     Isr: 0

2.生产者代码

Producer API

消息发送流程

Kafka 的 Producer 发送消息采用的是异步发送的方式。
在消息发送的过程中涉及到了两个线程——main 线程和 Sender 线程
以及一个线程共享变量——RecordAccumulator
main 线程将消息发送给 RecordAccumulator
Sender 线程不断从 RecordAccumulator 中拉取消息发送到 Kafka broker

代码如下：

package cn.bright;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;

/**
 * @Author Bright
 * @Date 2020/12/7
 * @Description
 */
public class MyProducer {
    public static void main(String[] args) {
        Properties prop = new Properties();
//获取连接        prop.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.116.60:9092");
        prop.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        prop.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,StringSerializer.class);
        prop.put(ProducerConfig.ACKS_CONFIG,"-1");
//创建KafkaProducer对象
        KafkaProducer<String,String> producer=new KafkaProducer<String, String>(prop);

        for (int i = 0; i < 200; i++) {
            ProducerRecord<String, String> producerRecord
                    = new ProducerRecord<>("kb09two", "hello world" + i);
            producer.send(producerRecord);
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        System.out.println("game over!!!");
    }
}

Linux上查看生产者队列内容

//查看队列内容：
[root@hadoop001 ~]# kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list 192.168.116.60:9092 --topic kb09two -time -1 --offsets 1
//返回的分区以及对应的偏移量开始位置
kb09two:0:66
kb09two:1:67
kb09two:2:67

3.消费者代码

Consumer API

Consumer 消费数据时的可靠性是很容易保证的，因为数据在 Kafka 中是持久化的，故不用担心数据丢失问题。
由于 consumer 在消费过程中可能会出现断电宕机等故障， consumer 恢复后，需要从故障前的位置的继续消费，所以 consumer 需要实时记录自己消费到了哪个 offset，以便故障恢复后继续消费。
所以 offset 的维护是 Consumer 消费数据是必须考虑的问题。

代码如下：

package cn.bright;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Collections;
import java.util.Properties;

/**
 * @Author Bright
 * @Date 2020/12/7
 * @Description
 */
public class MyConsumer {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.116.60:9092");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        prop.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        //earliest   latest   none
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");

        //模拟一个消费者组G1只有一个消费者
//        prop.put(ConsumerConfig.GROUP_ID_CONFIG,"G1");
//        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
//        consumer.subscribe(Collections.singleton("kb09two"));
//        while (true){
//            ConsumerRecords<String, String> poll = consumer.poll(100);
//            for (ConsumerRecord<String,String> record:poll) {
//                System.out.println(record.offset()+"\t"+record.key()+"\t"+record.value());
//            }
//        }

        //模拟多消费者在在同一个分组G2
        prop.put(ConsumerConfig.GROUP_ID_CONFIG, "G2");
        new Thread(new Runnable() {
            @Override
            public void run() {
            //创建KafkaConsumer对象
                KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
                consumer.subscribe(Collections.singleton("kb09two"));
                while (true) {
                    ConsumerRecords<String, String> poll = consumer.poll(100);
                    for (ConsumerRecord<String, String> record : poll) {
                        System.out.println(Thread.currentThread().getName() + "\t" +
                                record.offset() + "\t" + record.key() + "\t" + record.value());
                    }
                }
            }
        }).start();
    }
}

Linux上查看消费者队列内容

[root@hadoop001 ~]# kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9092 --topic kb09two --from-beginning
//消费的消息
hello world2
hello world5
hello world8
hello world11
hello world14
hello world17
hello world20
hello world23

4.maven依赖

<dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.flume</groupId>
      <artifactId>flume-ng-core</artifactId>
      <version>1.6.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>2.0.0</version>
    </dependency>
  </dependencies>