Kafka原生API使用Java代码-生产者-分区策略-自定义分区器

1、分区策略

kafka的生产者分区策略

1. 默认分区策略：减少重新建立分区连接的性能损耗 开发使用最多的分区方式，采用黏性分区，默认向第一次连接上的主题分区发送消息，直到消息累积到 batch.size大小（16kb）
2. 轮询分区策略：每个分区接收一次消息（linger.ms决定生产者一次批量发送多少条消息 到一个分区中），开发中一定不会用轮询分区策略，顶多自定义，因为轮询性能太差，频繁跟不同的分区建立连接，大数据会用轮询策略
3. 自定义分区策略：通过自定义分区器来实现，如果发送消息中包含 某某 发送到分区2，包含 某某 发送到分区1，其它发送到分区0

1.1、pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>3.0.5</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>

    <!-- Generated by https://start.springboot.io -->
    <!-- 优质的 spring/boot/data/security/cloud 框架中文文档尽在 => https://springdoc.cn -->
    <groupId>com.atguigu.kafka</groupId>
    <artifactId>kafka-producer</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>kafka-producer</name>
    <description>kafka-producer</description>
    <properties>
        <java.version>17</java.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>3.0.0</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

</project>

1.2、自定义分区 MyPartitioner

package com.atguigu.kafka.producer;
import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;
import java.util.Map;
public class MyPartitioner implements Partitioner {

    //s 表示主题
    //o 和 bytes 表示发送消息的 key
    //o1 表示发送的消息
    @Override
    public int partition(String s, Object o, byte[] bytes, Object o1, byte[] bytes1, Cluster cluster) {
        if (o1.toString().contains("小明")) {
            return 1;
        }else if (o1.toString().contains("小兰")) {
            return 2;
        }
        return 0;
    }

    @Override
    public void close() {

    }


    @Override
    public void configure(Map<String, ?> map) {

    }
}

1.3、KafkaProducerPartitioningStrategy.java

package com.atguigu.kafka.producer;
import org.apache.kafka.clients.producer.*;
import java.util.Properties;
import java.util.UUID;
public class KafkaProducerPartitioningStrategy {

    public static void main(String[] args) {
        // 初始化Kafka生产者配置
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.74.148:9092"); // 指定Kafka broker的地址和端口
        // 确认消息写入策略，"acks"设置为all表示所有副本都确认消息接收后才会响应
        props.put("acks", "all");
        // 消息发送失败时的重试次数，设置为0表示不重试
        props.put("retries", 0);
        // 发送缓冲区等待时间，等待1秒后，发送
        //props.put(ProducerConfig.LINGER_MS_CONFIG, 1000);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); // 指定键的序列化器
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); // 指定值的序列化器
        // 分区器选择RoundRobinPartitioner，实现消息在主题分区间的轮询分配
        //props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, "org.apache.kafka.clients.producer.RoundRobinPartitioner"  );

        //配置自定义分区策略
        props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, "com.atguigu.kafka.producer.MyPartitioner");
        // 创建Kafka生产者实例
        Producer<String, String> producer = new KafkaProducer<>(props);

        for (int i = 0; i < 1000; i++) {
            producer.send(new ProducerRecord<String, String>("my_topic3", "小兰-" + UUID.randomUUID().toString()+"-"+100), new Callback() {
                @Override
                public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                    if (e == null) {
                        System.out.println("recordMetadata.partition() = " + recordMetadata.partition());
                    }
                }
            });

        }
        for (int i = 0; i < 2000; i++) {
            producer.send(new ProducerRecord<String, String>("my_topic3", "小明-" + UUID.randomUUID().toString()+"-"+100), new Callback() {
                @Override
                public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                    if (e == null) {
                        System.out.println("recordMetadata.partition() = " + recordMetadata.partition());
                    }
                }
            });

        }
        for (int i = 0; i < 3000; i++) {
            producer.send(new ProducerRecord<String, String>("my_topic3", "我是外星人-" + UUID.randomUUID().toString()+"-"+100), new Callback() {
                @Override
                public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                    if (e == null) {
                        System.out.println("recordMetadata.partition() = " + recordMetadata.partition());
                    }
                }
            });

        }
        producer.close();
    }
}