kafka简单应用:
生产者:
package com.learn.kafka.autocommit;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.producer.*;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
@Slf4j
public class ProducerKafka extends Thread {
Producer<Integer, String> producer;
String topic;
public ProducerKafka(String topic) {
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.40.137:9092,192.168.40.138:9092,192.168.40.139:9092");
properties.put(ProducerConfig.CLIENT_ID_CONFIG, "learn-producer");
//异步发送涉及到的两个配置
/**
*producer(生产者)缓存每个分区未发送的消息。缓存的大小是通过 batch.size 配置指定的。
*值较大的话将会产生更大的批。并需要更多的内存(因为每个“活跃”的分区都有1个缓冲区)。
* 默认缓冲可立即发送,即便缓冲空间还没有满,但是,如果你想减少请求的数量,可以设置linger.ms大于0。
* 这将指示生产者发送请求之前等待一段时间,希望更多的消息填补到未满的批中。
* 这类似于TCP的算法,
* 例如上面的代码段,可能100条消息在一个请求发送,
* 因为我们设置了linger(逗留)时间为1毫秒,然后,如果我们没有填满缓冲区,
* 这个设置将增加1毫秒的延迟请求以等待更多的消息。
* 需要注意的是,在高负载下,相近的时间一般也会组成批,即使是linger.ms=0。
* 在不处于高负载的情况下,如果设置比0大,以少量的延迟代价换取更少的,更有效的请求。
*/
properties.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
properties.put(ProducerConfig.LINGER_MS_CONFIG, 1);
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
producer = new KafkaProducer<>(properties);
this.topic = topic;
}
@Override
public void run() {
int num = 0;
log.info("start");
while (num < 20) {
try {
String msg = "learn kafka msg" + num;
//get会拿到发送的结果,同步发送
// RecordMetadata recordMetadata = producer.send(new ProducerRecord<>(topic, msg)).get();
//log.info("recordMetadata offset:{}, partition:{}, topic:{}", recordMetadata.offset(), recordMetadata.partition(), recordMetadata.topic());
producer.send(new ProducerRecord<>(topic, msg), (recordMetadata, e) -> {
log.info("recordMetadata offset:{}, partition:{}, topic:{}", recordMetadata.offset(), recordMetadata.partition(), recordMetadata.topic());
});
TimeUnit.SECONDS.sleep(2);
++num;
} catch (InterruptedException e) {
log.error("InterruptedException:{}", e.getMessage());
}
}
}
public static void main(String[] args) {
new ProducerKafka("learn").start();
}
}
消费者:
package com.learn.kafka.autocommit;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.IntegerDeserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;
@Slf4j
public class ConsumerKafka extends Thread {
KafkaConsumer<Integer, String> kafkaConsumer;
String topic;
public ConsumerKafka(String topic) {
Properties properties = new Properties();
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.40.137:9092,192.168.40.138:9092,192.168.40.139:9092");
properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "learn-consumer");
properties.put(ConsumerConfig.GROUP_ID_CONFIG, "group1");
properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "6000");
//自动提交(批量确认)
properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000 ");
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
//这个属性,它能够消费最早的数据(对于一个新的group来说)
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
kafkaConsumer = new KafkaConsumer<>(properties);
this.topic = topic;
}
@Override
public void run() {
kafkaConsumer.subscribe(Collections.singleton(topic));
while (true) {
ConsumerRecords<Integer, String> consumerRecords = kafkaConsumer.poll(Duration.ofSeconds(1));
consumerRecords.forEach(record -> {
log.info("key:{}, value:{}, partition:{}, offset:{}", record.key(), record.value(), record.partition(), record.offset());
});
}
}
public static void main(String[] args) {
new ConsumerKafka("learn").start();
}
}