五、Java API
创建Topic
@Test
public void create() {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
AdminClient adminClient = KafkaAdminClient.create(properties);
List<NewTopic> topic01 = Arrays.asList(new NewTopic("topic01", 3, (short) 3));
adminClient.createTopics(topic01);
adminClient.close();
}
删除Topic
@Test
public void del() {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
AdminClient adminClient = KafkaAdminClient.create(properties);
List<String> topic01 = Arrays.asList("topic01");
adminClient.deleteTopics(topic01);
adminClient.close();
}
列出Topic
@Test
public void list() throws Exception {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
AdminClient adminClient = KafkaAdminClient.create(properties);
adminClient.listTopics().names().get().forEach((name) -> {
System.out.println(name);
});
}
描述Topic
@Test
public void describe() throws Exception {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
AdminClient adminClient = KafkaAdminClient.create(properties);
List<String> topic01 = Arrays.asList("t01");
DescribeTopicsResult result = adminClient.describeTopics(topic01);
/*KafkaFuture<Map<String, TopicDescription>> all = result.all();
Map<String, TopicDescription> stringTopicDescriptionMap = all.get();
Set<String> strings = stringTopicDescriptionMap.keySet();
for (String string : strings) {
TopicDescription topicDescription = stringTopicDescriptionMap.get(string);
System.out.println(topicDescription.toString());
}*/
result.all().get().forEach((key, value) -> {
System.out.println(key + "---" + value);
});
}
发布Topic
@Test
public void publish() {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
/*
* 对key记性序列化的操作
* */
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
/*
* 对value记性序列化的操作
* */
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
/*
* 默认的分区操作
* */
properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
KafkaProducer<String, String> kafkaProducer = new KafkaProducer<>(properties);
for (int i = 0; i < 10; i++) {
ProducerRecord<String, String> producerRecord = new ProducerRecord<>("topic01", UUID.randomUUID().toString(), "this is zs"+i);
kafkaProducer.send(producerRecord);
}
kafkaProducer.flush();
kafkaProducer.close();
}
订阅Topic
在使用.subscribe订阅某个topic的时候,必须制定消费者组的ID,这样kafka才能实现负载均衡,才能实现组内均分和组件广播的策略。
@Test
public void subscribe() {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
/*
* key的反序列化处理
* */
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
/*
* value的反序列化处理
* */
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
/*
* 指定组名
* */
properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");
/*
* 创建KafkaConsume 对象 订阅某个topic
* */
KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(properties);
/*
* 订阅topic01
* */
kafkaConsumer.subscribe(Arrays.asList("topic01"));
/*
* 通过kafkaConsumer.poll 获取记录
* */
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);
for (ConsumerRecord<String, String> record : consumerRecords) {
String key = record.key();
String value = record.value();
long offset = record.offset();
int partition = record.partition();
long timestamp = record.timestamp();
System.out.println("key:" + key + " value:" + value + " 偏移量:" + offset + " 分区:" + partition + " 时间戳:" + timestamp);
}
}
}
对象传输
@Test
public void subscribe() {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
/*
* key的反序列化处理
* */
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
/*
* value的反序列化处理
* */
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
/*
* 指定组名
* */
properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");
/*
* 创建KafkaConsume 对象 订阅某个topic
* */
KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(properties);
/*
* 订阅topic01
* */
kafkaConsumer.subscribe(Arrays.asList("topic01"));
/*
* 通过kafkaConsumer.poll 获取记录
* */
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);
for (ConsumerRecord<String, String> record : consumerRecords) {
String key = record.key();
String value = record.value();
long offset = record.offset();
int partition = record.partition();
long timestamp = record.timestamp();
System.out.println("key:" + key + " value:" + value + " 偏移量:" + offset + " 分区:" + partition + " 时间戳:" + timestamp);
}
}
}
package com.baizhi.test01;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;
import java.util.UUID;
public class KafkaUserProducer {
public static void main(String[] args) {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
/*
* 对key记性序列化的操作
* */
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
/*
* 对value记性序列化的操作
* */
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ObjectSerializer.class);
/*
* 默认的分区操作
* */
properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
KafkaProducer<String, User> kafkaProducer = new KafkaProducer<>(properties);
for (int i = 0; i < 10; i++) {
ProducerRecord<String, User> producerRecord = new ProducerRecord<>("topic03", UUID.randomUUID().toString(), new User(i + "", "zs" + i));
kafkaProducer.send(producerRecord);
}
kafkaProducer.flush();
kafkaProducer.close();
}
}
package com.baizhi.test01;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.kafka.common.serialization.Deserializer;
import java.util.Map;
public class ObjectDeserializer implements Deserializer<Object> {
@Override
public void configure(Map<String, ?> map, boolean b) {
}
/*
* 反序列化 解码
* */
@Override
public Object deserialize(String s, byte[] bytes) {
return SerializationUtils.deserialize(bytes);
}
@Override
public void close() {
}
}
package com.baizhi.test01;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.kafka.common.serialization.Serializer;
import java.io.Serializable;
import java.util.Map;
public class ObjectSerializer implements Serializer<Object> {
@Override
public void configure(Map<String, ?> map, boolean b) {
}
/*
* 序列化 编码
* */
@Override
public byte[] serialize(String s, Object o) {
return SerializationUtils.serialize((Serializable) o);
}
@Override
public void close() {
}
}
package com.baizhi.test01;
import java.io.Serializable;
public class User implements Serializable {
private String id;
private String name;
public User() {
}
public User(String id, String name) {
this.id = id;
this.name = name;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Override
public String toString() {
return "User{" +
"id='" + id + '\'' +
", name='" + name + '\'' +
'}';
}
}
偏移量控制
默认用户使用Subscribe订阅消息的时候,默认首次offset是latest 其实就是读取左后一个偏移量。
用户可以
auto.offset.reset
控制消费行为
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 默认为latest
使用AUTO_OFFSET_RESET_CONFIG进行消费行为的时候,首次本组一次是读取0偏移量以后的数据,本组在进行第二次消费后,因为偏移量在第一次已经提交,第二次进行消费的时候就会读取上一次提交偏移量之后的数据。如果换个组,就会重新的从0之后开始读。
因为在kafkaConsumer.poll的时候,kafka的客户端会定时的向kafka服务端提交偏移量,如果用户不想自动提交偏移量,则需要进行手动控制
enable.auto.commit=true
auto.commit.interval.ms=5000
手动偏移量提交
package com.rechen.test02;
import com.rechen.test01.ObjectDeserializer;
import com.rechen.test01.User;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;
public class TestConsumer {
public static void main(String[] args) {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
/*
* key的反序列化处理
* */
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
/*
* value的反序列化处理
* */
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ObjectDeserializer.class);
/*
* 指定组名
* */
properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 默认为latest
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
/*
* 创建KafkaConsume 对象 订阅某个topic
* */
KafkaConsumer<String, User> kafkaConsumer = new KafkaConsumer<>(properties);
/*
* 订阅topic01
* */
kafkaConsumer.subscribe(Arrays.asList("topic03"));
/*
* 通过kafkaConsumer.poll 获取记录
* */
while (true) {
ConsumerRecords<String, User> consumerRecords = kafkaConsumer.poll(1000);
if (consumerRecords != null && !consumerRecords.isEmpty()) {
HashMap<TopicPartition, OffsetAndMetadata> map = new HashMap<>();
for (ConsumerRecord<String, User> record : consumerRecords) {
String key = record.key();
User user = record.value();
long offset = record.offset();
int partition = record.partition();
long timestamp = record.timestamp();
System.out.println("key:" + key + " value:" + user + " 偏移量:" + offset + " 分区:" + partition + " 时间戳:" + timestamp);
TopicPartition topicPartition = new TopicPartition("topic03", partition);
OffsetAndMetadata metadata = new OffsetAndMetadata(offset + 1);
map.put(topicPartition, metadata);
}
kafkaConsumer.commitSync(map);
}
}
}
}
指定消费者分区
通过assign的方式进行指定分区的消费,那么组的管理策略就失效了。
package com.rechen.test03;
import com.rechen.test01.ObjectDeserializer;
import com.rechen.test01.User;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;
public class TestConsumer {
public static void main(String[] args) {
Properties properties = new Properties();
properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "HadoopNode01:9092,HadoopNode02:9092,HadoopNode03:9092");
/*
* key的反序列化处理
* */
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
/*
* value的反序列化处理
* */
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ObjectDeserializer.class);
/*
* 指定组名
* */
properties.put(ConsumerConfig.GROUP_ID_CONFIG, "G1");
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 默认为latest
/*
* 创建KafkaConsume 对象 订阅某个topic
* */
KafkaConsumer<String, User> kafkaConsumer = new KafkaConsumer<>(properties);
/*
* 订阅topic01
* */
//kafkaConsumer.subscribe(Arrays.asList("topic04"));
/*
* 指定分区
* */
kafkaConsumer.assign(Arrays.asList(new TopicPartition("topic04", 0)));
kafkaConsumer.seek(new TopicPartition("topic04", 0), 0);
/*
* 通过kafkaConsumer.poll 获取记录
* */
while (true) {
ConsumerRecords<String, User> consumerRecords = kafkaConsumer.poll(1000);
for (ConsumerRecord<String, User> record : consumerRecords) {
String key = record.key();
User user = record.value();
long offset = record.offset();
int partition = record.partition();
long timestamp = record.timestamp();
System.out.println("key:" + key + " value:" + user + " 偏移量:" + offset + " 分区:" + partition + " 时间戳:" + timestamp);
}
}
}
}
通过assign 方式进行消费,手动提交偏移量将不再有意义,因为在seek中指定了可以从哪里开始消费,此时就不用纠结偏移量的概念了,因为偏移量此时有用户决定。
生产者幂等性
https://www.cnblogs.com/javalyy/p/8882144.html 幂等性概念参考
在真正的生产环境中,往往会因为网络等种种原因,导致生产者会重复生产消息。生产者在重试的时候,往往可能前一个消息已经生产过了,但是仍然生了重复的消息。这样就会导致数据紊乱,比如在使用MongoDB的时候,如果不加控制的话有可能会出现了两个重复的用户。
所以为了应对此等场景,kafka也有其解决办法,那就是kafka的幂等性
幂等:多次操作最终的影响等价与一次操作称为幂等性操作,所有的读操作一定是幂等的.所有的写操作一定不是幂等的.当 生产者和broker默认有acks应答机制,如果当生产者发送完数据给broker之后如果没有在规定的时间内收到应答,生产者可以考虑重发数据.可以通过一下配置参数提升生产者的可靠性.
acks = all // 0 无需应答 n 应答个数 -1所有都需要
retries = 3 // 表示重试次数
request.timeout.ms = 3000 //等待应答超时时间
enable.idempotence = true //开启幂等性
public class KafkaProducerDemo {
public static void main(String[] args) {
//0.配置生产者了连接属性
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.ACKS_CONFIG,"all");//等待所有从机应答
props.put(ProducerConfig.RETRIES_CONFIG,3);//重试3次
props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);//等待3s应答
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);//开启幂等性
//1.创建Kafka生产者
KafkaProducer<String, String> producer = new KafkaProducer<String, String>(props);
//2.构建ProducerRecord
for (int i=15;i<20;i++){
DecimalFormat decimalFormat = new DecimalFormat("000");
User user = new User(i, "name" + i, i % 2 == 0);
ProducerRecord<String, String> record = new ProducerRecord<String, String>("topic06", decimalFormat.format(i), "user"+i);
//3.发送消息
producer.send(record);
}
//4.清空缓冲区
producer.flush();
//5.关闭生产者
producer.close();
}
生产者批量发送
生产者会尝试缓冲record,实现批量发送,通过一下配置控制发送时机,记住如果开启可batch,一定在关闭producer之前需要flush。
batch.size = 16384 //16KB 缓冲16kb数据本地
linger.ms = 2000 //默认逗留时间
public static void main(String[] args) {
//0.配置生产者了连接属性
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.ACKS_CONFIG,"all");
props.put(ProducerConfig.RETRIES_CONFIG,3);
props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);
props.put(ProducerConfig.BATCH_SIZE_CONFIG,1024);//1kb缓冲区
props.put(ProducerConfig.LINGER_MS_CONFIG,1000);//设置逗留时常
//1.创建Kafka生产者
KafkaProducer<String, String> producer = new KafkaProducer<String, String>(props);
//2.构建ProducerRecord
for (int i=15;i<20;i++){
DecimalFormat decimalFormat = new DecimalFormat("000");
User user = new User(i, "name" + i, i % 2 == 0);
ProducerRecord<String, String> record = new ProducerRecord<String, String>("topic06", decimalFormat.format(i), "user"+i);
//3.发送消息
producer.send(record);
}
//4.清空缓冲区
producer.flush();
//5.关闭生产者
producer.close();
}
生产者事务
kafka生产者事务指的是在发送多个数据的时候,保证多个Record记录发送的原子性。如果有一条发送失败就回退,但是需要注意在使用kafka事务的时候需要调整消费者的事务隔离级别设置为read_committed
因为kafka默认的事务隔离策略是read_uncommitted
开启事务
transactional.id=transaction-1 //必须保证唯一
enable.idempotence=true //开启kafka的幂等性
生产者 Only
public class KafkaProducerDemo {
public static void main(String[] args) {
//1.创建Kafka生产者
KafkaProducer<String, String> producer = buildKafkaProducer();
//2.初始化事务和开启事务
producer.initTransactions();
producer.beginTransaction();
try {
for (int i=5;i<10;i++){
DecimalFormat decimalFormat = new DecimalFormat("000");
User user = new User(i, "name" + i, i % 2 == 0);
ProducerRecord<String, String> record = new ProducerRecord<String, String>("topic07", decimalFormat.format(i), "user"+i);
producer.send(record);
}
producer.flush();
//3.提交事务]
producer.commitTransaction();
} catch (Exception e) {
System.err.println(e.getMessage());
//终止事务
producer.abortTransaction();
}
//5.关闭生产者
producer.close();
}
private static KafkaProducer<String, String> buildKafkaProducer() {
//0.配置生产者了连接属性
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.ACKS_CONFIG,"all");
props.put(ProducerConfig.RETRIES_CONFIG,3);
props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);
props.put(ProducerConfig.BATCH_SIZE_CONFIG,1024);//1kb缓冲区
props.put(ProducerConfig.LINGER_MS_CONFIG,1000);//设置逗留时常
//开启事务
props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG,"transaction-"+UUID.randomUUID().toString());
return new KafkaProducer<String, String>(props);
}
}
消费者那方需要将事务隔离级别设置为read_committed
public class KafkaConsumerDemo {
public static void main(String[] args) {
//1.创建Kafka消费者
KafkaConsumer<String, String> consumer = buildKafkaConsumer();
//2.订阅topics
consumer.subscribe(Arrays.asList("topic07"));
//3.死循环读取消息
while(true){
ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));
if(records!=null && !records.isEmpty()){
for (ConsumerRecord<String, String> record : records) {
int partition = record.partition();
long offset = record.offset();
long timestamp = record.timestamp();
String key = record.key();
String value = record.value();
System.out.println(partition+"\t"+offset+"\t"+timestamp+"\t"+key+"\t"+value);
}
}
}
}
private static KafkaConsumer<String, String> buildKafkaConsumer() {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"CentOSA:9092,CentOSB:9092,CentOSC:9092");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.GROUP_ID_CONFIG,"group1");
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG,"read_committed");
return new KafkaConsumer<String, String>(props);
}
}
生产者&消费者
package com.baizhi.demo08;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.Serializer;
import java.util.Properties;
import java.util.UUID;
public class KafkaUtils {
public static KafkaConsumer<String, String> buildKafkaConsumer(String servers, Class<? extends Deserializer> keyDeserializer,
Class<? extends Deserializer> valueDeserializer,String group) {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,servers);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,keyDeserializer);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,valueDeserializer);
props.put(ConsumerConfig.GROUP_ID_CONFIG,group);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG,"read_committed");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,false);//设置为手动提交
return new KafkaConsumer<String, String>(props);
}
public static KafkaProducer<String, String> buildKafkaProducer(String servers, Class<? extends Serializer> keySerializer,
Class<? extends Serializer> valueSerializer) {
//0.配置生产者了连接属性
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,servers);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,keySerializer);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,valueSerializer);
props.put(ProducerConfig.ACKS_CONFIG,"all");
props.put(ProducerConfig.RETRIES_CONFIG,3);
props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,3000);
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,true);
props.put(ProducerConfig.BATCH_SIZE_CONFIG,1024);//1kb缓冲区
props.put(ProducerConfig.LINGER_MS_CONFIG,1000);//设置逗留时常
//开启事务
props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG,"transaction-"+ UUID.randomUUID().toString());
return new KafkaProducer<String, String>(props);
}
}
KafkaProducerAndConsumer
package com.rechen.demo08;
import com.rechen.demo05.User;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import java.text.DecimalFormat;
import java.time.Duration;
import java.util.*;
public class KafkaProducerAndConsumer {
public static void main(String[] args) {
String servers = "CentOSA:9092,CentOSB:9092,CentOSC:9092";
String group="g1";
//1.创建Kafka生产者
KafkaProducer<String, String> producer = KafkaUtils.buildKafkaProducer(servers, StringSerializer.class, StringSerializer.class);
KafkaConsumer<String, String> consumer = KafkaUtils.buildKafkaConsumer(servers, StringDeserializer.class, StringDeserializer.class,group);
consumer.subscribe(Arrays.asList("topic08"));
//初始化事务
producer.initTransactions();
while (true) {
producer.beginTransaction();
ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));
try {
Map<TopicPartition, OffsetAndMetadata> commits = new HashMap<TopicPartition, OffsetAndMetadata>();
for (ConsumerRecord<String, String> record : records) {
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(record.offset() + 1);
commits.put(partition, offsetAndMetadata);
System.out.println(record);
ProducerRecord<String, String> srecord = new ProducerRecord<String, String>("topic09", record.key(), record.value());
producer.send(srecord);
}
producer.flush();
//并没使用 consumer提交,而是使用producer帮助消费者提交偏移量
producer.sendOffsetsToTransaction(commits,group);
//提交生产者的偏移量
producer.commitTransaction();
} catch (Exception e) {
//System.err.println(e.getMessage());
producer.abortTransaction();
}
}
}
}
献给每一个正在努力的我们,就算在忙,也要注意休息和饮食哦!我就是我,一个在互联网跌跌撞撞,摸爬滚打的热忱,给个三连吧~
还有就是不要只看,多动手才行!努力呀!