Kafka 史上最全面的大数据学习（十一）javaAPI topic操作

五、Java API

创建Topic

@Test
    public void create() {

        Properties properties = new Properties();

        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        AdminClient adminClient = KafkaAdminClient.create(properties);
        List<NewTopic> topic01 = Arrays.asList(new NewTopic("topic01", 3, (short) 3));

        adminClient.createTopics(topic01);

        adminClient.close();


    }

删除Topic

 @Test
    public void del() {

        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        AdminClient adminClient = KafkaAdminClient.create(properties);

        List<String> topic01 = Arrays.asList("topic01");

        adminClient.deleteTopics(topic01);

        adminClient.close();

    }

列出Topic

@Test
    public void list() throws Exception {
        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        AdminClient adminClient = KafkaAdminClient.create(properties);

        adminClient.listTopics().names().get().forEach((name) -> {
            System.out.println(name);
        });
    }

描述Topic

@Test
    public void describe() throws Exception {


        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");


        AdminClient adminClient = KafkaAdminClient.create(properties);
        List<String> topic01 = Arrays.asList("t01");

        DescribeTopicsResult result = adminClient.describeTopics(topic01);

        /*KafkaFuture<Map<String, TopicDescription>> all = result.all();


        Map<String, TopicDescription> stringTopicDescriptionMap = all.get();

        Set<String> strings = stringTopicDescriptionMap.keySet();

        for (String string : strings) {


            TopicDescription topicDescription = stringTopicDescriptionMap.get(string);

            System.out.println(topicDescription.toString());
        }*/

        result.all().get().forEach((key, value) -> {
            System.out.println(key + "---" + value);
        });

    }

发布Topic

 @Test
    public void publish() {

        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        /*
         * 对key记性序列化的操作
         * */
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        /*
         * 对value记性序列化的操作
         * */
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

        /*
         * 默认的分区操作
         * */
        properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);


        KafkaProducer<String, String> kafkaProducer = new KafkaProducer<>(properties);

        for (int i = 0; i < 10; i++) {
            ProducerRecord<String, String> producerRecord = new ProducerRecord<>("topic01", UUID.randomUUID().toString(), "this is zs"+i);

            kafkaProducer.send(producerRecord);
        }


        kafkaProducer.flush();

        kafkaProducer.close();


    }

订阅Topic

在使用.subscribe订阅某个topic的时候，必须制定消费者组的ID，这样kafka才能实现负载均衡，才能实现组内均分和组件广播的策略。

 @Test
    public void subscribe() {

        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        /*
         * key的反序列化处理
         * */
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        /*
         * value的反序列化处理
         * */
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

        /*
         * 指定组名
         * */

        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");

        /*
         * 创建KafkaConsume 对象  订阅某个topic
         * */
        KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(properties);


        /*
         * 订阅topic01
         * */
        kafkaConsumer.subscribe(Arrays.asList("topic01"));

        /*
         * 通过kafkaConsumer.poll 获取记录
         * */
        while (true) {
            ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);

            for (ConsumerRecord<String, String> record : consumerRecords) {

                String key = record.key();
                String value = record.value();
                long offset = record.offset();
                int partition = record.partition();
                long timestamp = record.timestamp();


                System.out.println("key:" + key + " value:" + value + " 偏移量:" + offset + " 分区：" + partition + " 时间戳：" + timestamp);

            }


        }


    }

对象传输

 @Test
    public void subscribe() {

        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        /*
         * key的反序列化处理
         * */
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        /*
         * value的反序列化处理
         * */
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

        /*
         * 指定组名
         * */

        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");

        /*
         * 创建KafkaConsume 对象  订阅某个topic
         * */
        KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(properties);


        /*
         * 订阅topic01
         * */
        kafkaConsumer.subscribe(Arrays.asList("topic01"));

        /*
         * 通过kafkaConsumer.poll 获取记录
         * */
        while (true) {
            ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);

            for (ConsumerRecord<String, String> record : consumerRecords) {

                String key = record.key();
                String value = record.value();
                long offset = record.offset();
                int partition = record.partition();
                long timestamp = record.timestamp();


                System.out.println("key:" + key + " value:" + value + " 偏移量:" + offset + " 分区：" + partition + " 时间戳：" + timestamp);

            }


        }


    }

package com.baizhi.test01;

import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;
import java.util.UUID;

public class KafkaUserProducer {
    public static void main(String[] args) {


        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        /*
         * 对key记性序列化的操作
         * */
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        /*
         * 对value记性序列化的操作
         * */
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ObjectSerializer.class);

        /*
         * 默认的分区操作
         * */
        properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);


        KafkaProducer<String, User> kafkaProducer = new KafkaProducer<>(properties);

        for (int i = 0; i < 10; i++) {
            ProducerRecord<String, User> producerRecord = new ProducerRecord<>("topic03", UUID.randomUUID().toString(), new User(i + "", "zs" + i));

            kafkaProducer.send(producerRecord);
        }


        kafkaProducer.flush();

        kafkaProducer.close();
    }
}

package com.baizhi.test01;

import org.apache.commons.lang3.SerializationUtils;
import org.apache.kafka.common.serialization.Deserializer;

import java.util.Map;

public class ObjectDeserializer implements Deserializer<Object> {
    @Override
    public void configure(Map<String, ?> map, boolean b) {

    }

    /*
     * 反序列化  解码
     * */
    @Override
    public Object deserialize(String s, byte[] bytes) {

        return SerializationUtils.deserialize(bytes);
    }

    @Override
    public void close() {

    }
}

package com.baizhi.test01;

import org.apache.commons.lang3.SerializationUtils;
import org.apache.kafka.common.serialization.Serializer;

import java.io.Serializable;
import java.util.Map;

public class ObjectSerializer implements Serializer<Object> {
    @Override
    public void configure(Map<String, ?> map, boolean b) {

    }

    /*
     * 序列化 编码
     * */
    @Override
    public byte[] serialize(String s, Object o) {

        return SerializationUtils.serialize((Serializable) o);
    }

    @Override
    public void close() {

    }
}

package com.baizhi.test01;

import java.io.Serializable;

public class User implements Serializable {
    private String id;
    private String name;

    public User() {
    }

    public User(String id, String name) {
        this.id = id;
        this.name = name;
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    @Override
    public String toString() {
        return "User{" +
                "id='" + id + '\'' +
                ", name='" + name + '\'' +
                '}';
    }
}

偏移量控制

默认用户使用Subscribe订阅消息的时候，默认首次offset是latest 其实就是读取左后一个偏移量。

用户可以auto.offset.reset控制消费行为

properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 默认为latest

使用AUTO_OFFSET_RESET_CONFIG进行消费行为的时候，首次本组一次是读取0偏移量以后的数据，本组在进行第二次消费后，因为偏移量在第一次已经提交，第二次进行消费的时候就会读取上一次提交偏移量之后的数据。如果换个组，就会重新的从0之后开始读。

因为在kafkaConsumer.poll的时候，kafka的客户端会定时的向kafka服务端提交偏移量，如果用户不想自动提交偏移量，则需要进行手动控制

enable.auto.commit=true
auto.commit.interval.ms=5000

手动偏移量提交

package com.rechen.test02;

import com.rechen.test01.ObjectDeserializer;
import com.rechen.test01.User;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;

public class TestConsumer {
    public static void main(String[] args) {


        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        /*
         * key的反序列化处理
         * */
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        /*
         * value的反序列化处理
         * */
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ObjectDeserializer.class);

        /*
         * 指定组名
         * */

        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");


        properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 默认为latest


        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
        /*
         * 创建KafkaConsume 对象  订阅某个topic
         * */
        KafkaConsumer<String, User> kafkaConsumer = new KafkaConsumer<>(properties);


        /*
         * 订阅topic01
         * */
        kafkaConsumer.subscribe(Arrays.asList("topic03"));

        /*
         * 通过kafkaConsumer.poll 获取记录
         * */
        while (true) {
            ConsumerRecords<String, User> consumerRecords = kafkaConsumer.poll(1000);

            if (consumerRecords != null && !consumerRecords.isEmpty()) {
                HashMap<TopicPartition, OffsetAndMetadata> map = new HashMap<>();


                for (ConsumerRecord<String, User> record : consumerRecords) {


                    String key = record.key();

                    User user = record.value();
                    long offset = record.offset();
                    int partition = record.partition();
                    long timestamp = record.timestamp();

                    System.out.println("key:" + key + " value:" + user + " 偏移量:" + offset + " 分区：" + partition + " 时间戳：" + timestamp);

                    TopicPartition topicPartition = new TopicPartition("topic03", partition);

                    OffsetAndMetadata metadata = new OffsetAndMetadata(offset + 1);


                    map.put(topicPartition, metadata);

                }

                kafkaConsumer.commitSync(map);
            }


        }


    }
}

指定消费者分区

通过assign的方式进行指定分区的消费，那么组的管理策略就失效了。

package com.rechen.test03;

import com.rechen.test01.ObjectDeserializer;
import com.rechen.test01.User;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;

public class TestConsumer {
    public static void main(String[] args) {


        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");

        /*
         * key的反序列化处理
         * */
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        /*
         * value的反序列化处理
         * */
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ObjectDeserializer.class);

        /*
         * 指定组名
         * */

        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");


        properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 默认为latest


        /*
         * 创建KafkaConsume 对象  订阅某个topic
         * */
        KafkaConsumer<String, User> kafkaConsumer = new KafkaConsumer<>(properties);


        /*
         * 订阅topic01
         * */
        //kafkaConsumer.subscribe(Arrays.asList("topic04"));

        /*
         * 指定分区
         * */
        kafkaConsumer.assign(Arrays.asList(new TopicPartition("topic04", 0)));

        kafkaConsumer.seek(new TopicPartition("topic04", 0), 0);
        /*
         * 通过kafkaConsumer.poll 获取记录
         * */
        while (true) {
            ConsumerRecords<String, User> consumerRecords = kafkaConsumer.poll(1000);


            for (ConsumerRecord<String, User> record : consumerRecords) {


                String key = record.key();

                User user = record.value();
                long offset = record.offset();
                int partition = record.partition();
                long timestamp = record.timestamp();

                System.out.println("key:" + key + " value:" + user + " 偏移量:" + offset + " 分区：" + partition + " 时间戳：" + timestamp);


            }


        }


    }


}

通过assign 方式进行消费，手动提交偏移量将不再有意义，因为在seek中指定了可以从哪里开始消费，此时就不用纠结偏移量的概念了，因为偏移量此时有用户决定。

生产者幂等性

https://www.cnblogs.com/javalyy/p/8882144.html 幂等性概念参考

在真正的生产环境中，往往会因为网络等种种原因，导致生产者会重复生产消息。生产者在重试的时候，往往可能前一个消息已经生产过了，但是仍然生了重复的消息。这样就会导致数据紊乱，比如在使用MongoDB的时候，如果不加控制的话有可能会出现了两个重复的用户。

所以为了应对此等场景，kafka也有其解决办法，那就是kafka的幂等性

幂等:多次操作最终的影响等价与一次操作称为幂等性操作,所有的读操作一定是幂等的.所有的写操作一定不是幂等的.当 生产者和broker默认有acks应答机制,如果当生产者发送完数据给broker之后如果没有在规定的时间内收到应答,生产者可以考虑重发数据.可以通过一下配置参数提升生产者的可靠性.

acks = all // 0 无需应答  n 应答个数 -1所有都需要
retries = 3 // 表示重试次数
request.timeout.ms = 3000 //等待应答超时时间
enable.idempotence = true //开启幂等性

public class KafkaProducerDemo {
    public static void main(String[] args) {
        //0.配置生产者了连接属性
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");

        props.put(ProducerConfig.ACKS_CONFIG,"all");//等待所有从机应答
        props.put(ProducerConfig.RETRIES_CONFIG,3);//重试3次
        props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);//等待3s应答
        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);//开启幂等性

        //1.创建Kafka生产者
        KafkaProducer<String, String> producer = new KafkaProducer<String, String>(props);

        //2.构建ProducerRecord
        for (int i=15;i<20;i++){
            DecimalFormat decimalFormat = new DecimalFormat("000");
            User user = new User(i, "name" + i, i % 2 == 0);
            ProducerRecord<String, String> record = new ProducerRecord<String, String>("topic06", decimalFormat.format(i), "user"+i);
            //3.发送消息
            producer.send(record);
        }
        //4.清空缓冲区
        producer.flush();
        //5.关闭生产者
        producer.close();
    }

生产者批量发送

生产者会尝试缓冲record，实现批量发送，通过一下配置控制发送时机，记住如果开启可batch，一定在关闭producer之前需要flush。

batch.size = 16384 //16KB 缓冲16kb数据本地
linger.ms = 2000 //默认逗留时间

public static void main(String[] args) {
    //0.配置生产者了连接属性
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");

    props.put(ProducerConfig.ACKS_CONFIG,"all");
    props.put(ProducerConfig.RETRIES_CONFIG,3);
    props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);
    props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);

    props.put(ProducerConfig.BATCH_SIZE_CONFIG,1024);//1kb缓冲区
    props.put(ProducerConfig.LINGER_MS_CONFIG,1000);//设置逗留时常


    //1.创建Kafka生产者
    KafkaProducer<String, String> producer = new KafkaProducer<String, String>(props);

    //2.构建ProducerRecord
    for (int i=15;i<20;i++){
        DecimalFormat decimalFormat = new DecimalFormat("000");
        User user = new User(i, "name" + i, i % 2 == 0);
        ProducerRecord<String, String> record = new ProducerRecord<String, String>("topic06", decimalFormat.format(i), "user"+i);
        //3.发送消息
        producer.send(record);
    }
    //4.清空缓冲区
    producer.flush();
    //5.关闭生产者
    producer.close();
}

生产者事务

kafka生产者事务指的是在发送多个数据的时候，保证多个Record记录发送的原子性。如果有一条发送失败就回退，但是需要注意在使用kafka事务的时候需要调整消费者的事务隔离级别设置为read_committed因为kafka默认的事务隔离策略是read_uncommitted

开启事务

transactional.id=transaction-1 //必须保证唯一
enable.idempotence=true //开启kafka的幂等性

生产者 Only

public class KafkaProducerDemo {
    public static void main(String[] args) {

        //1.创建Kafka生产者
        KafkaProducer<String, String> producer = buildKafkaProducer();

        //2.初始化事务和开启事务
        producer.initTransactions();
        producer.beginTransaction();
        try {
            for (int i=5;i<10;i++){
                DecimalFormat decimalFormat = new DecimalFormat("000");
                User user = new User(i, "name" + i, i % 2 == 0);
                ProducerRecord<String, String> record = new ProducerRecord<String, String>("topic07", decimalFormat.format(i), "user"+i);
                producer.send(record);
            }
            producer.flush();
            //3.提交事务]
            producer.commitTransaction();
        } catch (Exception e) {
            System.err.println(e.getMessage());
            //终止事务
            producer.abortTransaction();
        }
        //5.关闭生产者
        producer.close();
    }

    private static KafkaProducer<String, String> buildKafkaProducer() {
        //0.配置生产者了连接属性
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");

        props.put(ProducerConfig.ACKS_CONFIG,"all");
        props.put(ProducerConfig.RETRIES_CONFIG,3);
        props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);
        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);

        props.put(ProducerConfig.BATCH_SIZE_CONFIG,1024);//1kb缓冲区
        props.put(ProducerConfig.LINGER_MS_CONFIG,1000);//设置逗留时常

        //开启事务
        props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG,"transaction-"+UUID.randomUUID().toString());
        return new KafkaProducer<String, String>(props);
    }
}

消费者那方需要将事务隔离级别设置为read_committed

public class KafkaConsumerDemo {
    public static void main(String[] args) {

        //1.创建Kafka消费者
        KafkaConsumer<String, String> consumer = buildKafkaConsumer();

        //2.订阅topics
        consumer.subscribe(Arrays.asList("topic07"));
        //3.死循环读取消息
        while(true){
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));
            if(records!=null && !records.isEmpty()){
                for (ConsumerRecord<String, String> record : records) {
                    int partition = record.partition();
                    long offset = record.offset();
                    long timestamp = record.timestamp();
                    String key = record.key();
                    String value = record.value();
                    System.out.println(partition+"\t"+offset+"\t"+timestamp+"\t"+key+"\t"+value);
                }
            }
        }
    }

    private static KafkaConsumer<String, String> buildKafkaConsumer() {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.GROUP_ID_CONFIG,"group1");
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG,"read_committed");
        return new KafkaConsumer<String, String>(props);
    }
}

生产者&消费者

package com.baizhi.demo08;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.Serializer;

import java.util.Properties;
import java.util.UUID;

public class KafkaUtils {
    public static KafkaConsumer<String, String> buildKafkaConsumer(String servers, Class<? extends Deserializer> keyDeserializer,
                                                                   Class<? extends Deserializer> valueDeserializer,String group) {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,servers);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,keyDeserializer);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,valueDeserializer);
        props.put(ConsumerConfig.GROUP_ID_CONFIG,group);
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG,"read_committed");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,false);//设置为手动提交
        return new KafkaConsumer<String, String>(props);
    }
    public static KafkaProducer<String, String> buildKafkaProducer(String servers, Class<? extends Serializer> keySerializer,
                                                                   Class<? extends Serializer> valueSerializer) {
        //0.配置生产者了连接属性
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,servers);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,keySerializer);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,valueSerializer);

        props.put(ProducerConfig.ACKS_CONFIG,"all");
        props.put(ProducerConfig.RETRIES_CONFIG,3);
        props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);
        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);

        props.put(ProducerConfig.BATCH_SIZE_CONFIG,1024);//1kb缓冲区
        props.put(ProducerConfig.LINGER_MS_CONFIG,1000);//设置逗留时常

        //开启事务
        props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG,"transaction-"+ UUID.randomUUID().toString());
        return new KafkaProducer<String, String>(props);
    }
}

KafkaProducerAndConsumer

package com.rechen.demo08;

import com.rechen.demo05.User;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;

import java.text.DecimalFormat;
import java.time.Duration;
import java.util.*;

public class KafkaProducerAndConsumer {
    public static void main(String[] args) {

        String servers = "CentOSA:9092,CentOSB:9092,CentOSC:9092";
        String group="g1";
        //1.创建Kafka生产者
        KafkaProducer<String, String> producer = KafkaUtils.buildKafkaProducer(servers, StringSerializer.class, StringSerializer.class);
        KafkaConsumer<String, String> consumer = KafkaUtils.buildKafkaConsumer(servers, StringDeserializer.class, StringDeserializer.class,group);

        consumer.subscribe(Arrays.asList("topic08"));
        //初始化事务
        producer.initTransactions();

        while (true) {
            producer.beginTransaction();
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));
            try {
                Map<TopicPartition, OffsetAndMetadata> commits = new HashMap<TopicPartition, OffsetAndMetadata>();
                for (ConsumerRecord<String, String> record : records) {
                    TopicPartition partition = new TopicPartition(record.topic(), record.partition());
                    OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(record.offset() + 1);
                    commits.put(partition, offsetAndMetadata);

                    System.out.println(record);

                    ProducerRecord<String, String> srecord = new ProducerRecord<String, String>("topic09", record.key(), record.value());
                    producer.send(srecord);
                }
                producer.flush();

                //并没使用 consumer提交，而是使用producer帮助消费者提交偏移量
                producer.sendOffsetsToTransaction(commits,group);
                //提交生产者的偏移量
                producer.commitTransaction();
            } catch (Exception e) {
                //System.err.println(e.getMessage());
                producer.abortTransaction();
            }
        }
    }
}

献给每一个正在努力的我们，就算在忙，也要注意休息和饮食哦！我就是我，一个在互联网跌跌撞撞，摸爬滚打的热忱，给个三连吧~
还有就是不要只看，多动手才行！努力呀！