<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.12</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>28.2-jre</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.68</version>
</dependency>
</dependencies>
- ACKS_CONFIG acks 消息保障(
0:至少一次,只发不管你收没收到,最快;
1:至多一次,broker没响应就会再发;
all:正好一次,broker没响应会再发,producer会带上id+消息,broker会去重,并响应已收到)- RETRIES_CONFIG:message.send.max.retries 发送失败后,重试次数
- BATCH_SIZE_CONFIG:batch.num.messages 批量发送最大条数
- LINGER_MS_CONFIG:linger.ms 单次批量发送,消息追加所用时间,到达该时间则发送
- BUFFER_MEMORY_CONFIG:buffer.memory 单次批量发送,消息追加内存大小
- KEY_SERIALIZER_CLASS_CONFIG:key序列化
- VALUE_SERIALIZER_CLASS_CONFIG:value序列化
- PARTITIONER_CLASS_CONFIG:自定义Partition负载均衡
异步直接发送
/*
Producer异步发送演示
ACKS_CONFIG acks 消息保障 (0:至少一次,只发不管你收没收到,最快;1:至多一次,broker没响应就会再发;all:正好一次,broker没响应会再发,producer会带上id+消息,broker会去重,并响应已收到)4
RETRIES_CONFIG:message.send.max.retries 发送失败后,重试次数
BATCH_SIZE_CONFIG:batch.num.messages 批量发送最大条数
LINGER_MS_CONFIG:linger.ms 单次批量发送,消息追加所用时间,到达该时间则发送
BUFFER_MEMORY_CONFIG:buffer.memory 单次批量发送,消息追加内存大小
KEY_SERIALIZER_CLASS_CONFIG:key序列化
VALUE_SERIALIZER_CLASS_CONFIG:value序列化
*/
public static void producerSend(){
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.174.128:9092");
properties.put(ProducerConfig.ACKS_CONFIG,"all");
properties.put(ProducerConfig.RETRIES_CONFIG,"0");
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,"16384");
properties.put(ProducerConfig.LINGER_MS_CONFIG,"1");
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"33554432");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
// Producer的主对象
Producer<String,String> producer = new KafkaProducer<>(properties);
// 消息对象 - ProducerRecoder
for(int i=0;i<10;i++){
ProducerRecord<String,String> record =
new ProducerRecord<>(TOPIC_NAME,"key-"+i,"value-"+i);
producer.send(record);
}
// 所有的通道打开都需要关闭
producer.close();
}
异步阻塞发送
/*
Producer异步阻塞发送
ACKS_CONFIG acks 消息保障 (0:至少一次,只发不管你收没收到,最快;1:至多一次,broker没响应就会再发;all:正好一次,broker没响应会再发,producer会带上id+消息,broker会去重,并响应已收到)4
RETRIES_CONFIG:message.send.max.retries 发送失败后,重试次数
BATCH_SIZE_CONFIG:batch.num.messages 批量发送最大条数
LINGER_MS_CONFIG:linger.ms 单次批量发送,消息追加所用时间,到达该时间则发送
BUFFER_MEMORY_CONFIG:buffer.memory 单次批量发送,消息追加内存大小
KEY_SERIALIZER_CLASS_CONFIG:key序列化
VALUE_SERIALIZER_CLASS_CONFIG:value序列化
*/
public static void producerSyncSend() throws ExecutionException, InterruptedException {
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.174.128:9092");
properties.put(ProducerConfig.ACKS_CONFIG,"all");
properties.put(ProducerConfig.RETRIES_CONFIG,"0");
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,"16384");
properties.put(ProducerConfig.LINGER_MS_CONFIG,"1");
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"33554432");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
// Producer的主对象
Producer<String,String> producer = new KafkaProducer<>(properties);
// 消息对象 - ProducerRecoder
for(int i=0;i<10;i++){
String key = "key-"+i;
ProducerRecord<String,String> record =
new ProducerRecord<>(TOPIC_NAME,key,"value-"+i);
Future<RecordMetadata> send = producer.send(record);
RecordMetadata recordMetadata = send.get();
System.out.println(key + "partition : "+recordMetadata.partition()+" , offset : "+recordMetadata.offset());
}
// 所有的通道打开都需要关闭
producer.close();
}
异步发送带回调函数
/*
Producer异步发送带回调函数
ACKS_CONFIG acks 消息保障 (0:至少一次,只发不管你收没收到,最快;1:至多一次,broker没响应就会再发;all:正好一次,broker没响应会再发,producer会带上id+消息,broker会去重,并响应已收到)4
RETRIES_CONFIG:message.send.max.retries 发送失败后,重试次数
BATCH_SIZE_CONFIG:batch.num.messages 批量发送最大条数
LINGER_MS_CONFIG:linger.ms 单次批量发送,消息追加所用时间,到达该时间则发送
BUFFER_MEMORY_CONFIG:buffer.memory 单次批量发送,消息追加内存大小
KEY_SERIALIZER_CLASS_CONFIG:key序列化
VALUE_SERIALIZER_CLASS_CONFIG:value序列化
*/
public static void producerSendWithCallback(){
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.174.128:9092");
properties.put(ProducerConfig.ACKS_CONFIG,"all");
properties.put(ProducerConfig.RETRIES_CONFIG,"0");
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,"16384");
properties.put(ProducerConfig.LINGER_MS_CONFIG,"1");
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"33554432");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
// Producer的主对象
Producer<String,String> producer = new KafkaProducer<>(properties);
// 消息对象 - ProducerRecoder
for(int i=0;i<10;i++){
ProducerRecord<String,String> record =
new ProducerRecord<>(TOPIC_NAME,"key-"+i,"value-"+i);
producer.send(record, new Callback() {
@Override
public void onCompletion(RecordMetadata recordMetadata, Exception e) {
System.out.println(
"partition : "+recordMetadata.partition()+" , offset : "+recordMetadata.offset());
}
});
}
// 所有的通道打开都需要关闭
producer.close();
}
异步发送带回调函数和Partition负载均衡
/*
Producer异步发送带回调函数和Partition负载均衡
ACKS_CONFIG acks 消息保障 (0:至少一次,只发不管你收没收到,最快;1:至多一次,broker没响应就会再发;all:正好一次,broker没响应会再发,producer会带上id+消息,broker会去重,并响应已收到)4
RETRIES_CONFIG:message.send.max.retries 发送失败后,重试次数
BATCH_SIZE_CONFIG:batch.num.messages 批量发送最大条数
LINGER_MS_CONFIG:linger.ms 单次批量发送,消息追加所用时间,到达该时间则发送
BUFFER_MEMORY_CONFIG:buffer.memory 单次批量发送,消息追加内存大小
KEY_SERIALIZER_CLASS_CONFIG:key序列化
VALUE_SERIALIZER_CLASS_CONFIG:value序列化
PARTITIONER_CLASS_CONFIG:自定义Partition负载均衡
*/
public static void producerSendWithCallbackAndPartition(){
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.174.128:9092");
properties.put(ProducerConfig.ACKS_CONFIG,"all");
properties.put(ProducerConfig.RETRIES_CONFIG,"0");
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,"16384");
properties.put(ProducerConfig.LINGER_MS_CONFIG,"1");
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"33554432");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG,"com.imooc.jiangzh.kafka.producer.SamplePartition");
// Producer的主对象
Producer<String,String> producer = new KafkaProducer<>(properties);
// 消息对象 - ProducerRecoder
for(int i=0;i<10;i++){
ProducerRecord<String,String> record =
new ProducerRecord<>(TOPIC_NAME,"key-"+i,"value-"+i);
producer.send(record, new Callback() {
@Override
public void onCompletion(RecordMetadata recordMetadata, Exception e) {
System.out.println(
"partition : "+recordMetadata.partition()+" , offset : "+recordMetadata.offset());
}
});
}
// 所有的通道打开都需要关闭
producer.close();
}
事务支持
/*
Producer 事务支持
*/
public static void producerWithTransactional() throws ExecutionException, InterruptedException {
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.174.128:9092");
properties.put(ProducerConfig.ACKS_CONFIG,"all");
properties.put(ProducerConfig.RETRIES_CONFIG,"0");
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,"16384");
properties.put(ProducerConfig.LINGER_MS_CONFIG,"1");
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"33554432");
// 事务支持配置
properties.put(ProducerConfig.RETRIES_CONFIG, "2");
properties.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "zsl-trans-id");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
// Producer的主对象
Producer<String,String> producer = new KafkaProducer<>(properties);
producer.initTransactions();
producer.beginTransaction();
try{
// 消息对象 - ProducerRecoder
for(int i=0;i<10;i++){
String key = "key-"+i;
ProducerRecord<String,String> record =
new ProducerRecord<>(TOPIC_NAME,key,"value-"+i);
//
// if(i == 8){
// throw new Exception();
// }
producer.send(record);
// RecordMetadata recordMetadata = send.get();
// System.out.println(key + " , partition : "+recordMetadata.partition()+" , offset : "+recordMetadata.offset());
}
// 提交事务
producer.commitTransaction();
}catch (Exception e){
e.printStackTrace();
// 回滚事务
producer.abortTransaction();
}finally {
producer.close();
}
}
带SSL异步发送
/*
Producer异步发送演示 并携带SSL
*/
public static void producerSendWithSSL(){
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.174.128:8989");
properties.put(ProducerConfig.ACKS_CONFIG,"all");
properties.put(ProducerConfig.RETRIES_CONFIG,"0");
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,"16384");
properties.put(ProducerConfig.LINGER_MS_CONFIG,"1");
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"33554432");
properties.put("security.protocol","SSL");
properties.put("ssl.endpoint.identification.algorithm","");
properties.put("ssl.truststore.location","client.truststore.jks");
properties.put("ssl.truststore.password","zsl");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
// Producer的主对象
Producer<String,String> producer = new KafkaProducer<>(properties);
// 消息对象 - ProducerRecoder
for(int i=0;i<10;i++){
ProducerRecord<String,String> record =
new ProducerRecord<>(TOPIC_NAME,"key-"+i,"value-"+i);
producer.send(record);
}
// 所有的通道打开都需要关闭
producer.close();
}