1. 引依赖包
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
2. 配置类
/**
* @author :yepk
* @version :1.0
* @apiNote :kafka配置
* @date :2020-03-13-10:34
*/
@Configuration
@PropertySource("classpath:application.properties")
public class KafkaConfiguration {
// 删除手动配置项 使用自动配置
}
3. 生产者自定义类
/**
* @author :yepk
* @version :1.0
* @apiNote :生产者
* @date :2020-03-13-10:38
*/
@Component
public class KafkaProducer {
@Resource
private KafkaTemplate<String, String> kafkaTemplate;
/**
* 向topic中发送消息
*/
public void send(String topic, String msg) {
try {
kafkaTemplate.send(topic, msg);
} catch (Exception e) {
log.error("kafka send error:{}",e);
}
}
/**
* 向topic中发送消息
*/
public void send(String topic, List<String> msgs) {
msgs.forEach(msg -> kafkaTemplate.send(topic, msg));
}
}
4. 消费者基类
/**
* @author :yepk
* @version :1.0
* @apiNote :消费者
* @date :2020-03-13-11:02
*/
public abstract class BaseKafkaConsumer {
/***
* @apiNote 消费类
* topics 在配置文件配置 例:spring.kafka.consumer.topic=topic1,topic2
* @author yepk
* @date 2020/3/13 11:06
* @param consumerRecord 消费的消息
*/
@KafkaListener(topics = "#{'${spring.kafka.consumer.topic}'.split(',')}")
public void consumer(ConsumerRecord<String,Object> consumerRecord){
String topic = consumerRecord.topic();
Optional<Object> kafkaMassage = Optional.ofNullable(consumerRecord.value());
if(kafkaMassage.isPresent()){
String msg = (String) kafkaMassage.get();
consume(topic,msg);
}
}
/***
* 继承基础消费类则继承此方法
* @apiNote 消费方法
* @author yepk
* @date 2020/3/13 11:08
* @param topic 主题
* @param msg 消费消息
*/
protected abstract void consume(String topic, String msg);
}
5. 消费子类(具体消费逻辑)
/**
* @author :yepk
* @version :1.0
* @apiNote :消费了
* @date :2020-03-13-11:11
*/
@Service
@Slf4j
public class LogKafkaConsumer extends BaseKafkaConsumer {
@Override
protected void consume(String topic, String msg) {
// 消费逻辑
}
6. 配置文件
### kafka配置
#============== kafka ===================
# 指定kafka 代理地址,可以多个
spring.kafka.bootstrap-servers=47.104.205.107:9092
#=============== provider =======================
spring.kafka.producer.retries=1
# 每次批量发送消息的数量
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432
# 指定消息key和消息体的编解码方式
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
#=============== consumer =======================
# 指定默认消费者group id
spring.kafka.consumer.group-id=etl
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=1000
spring.kafka.consumer.max-poll-records=1000
# 指定消息key和消息体的编解码方式
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
## 多个配置,分割
spring.kafka.consumer.topic=