SpringBoot使用集成kafka实现生产与消费

集成kafka之前需要安装kafka。详见上一篇博客安装zookeeper+kafka。本文不再赘述
无网络环境的Centos7中安装Docker

集成kafka

pom.xml引入包

<dependency>
        <groupId>org.springframework.kafka</groupId>
        <artifactId>spring-kafka</artifactId>
</dependency>

spring:
  kafka:
  	#自定义的，不需要可以不要
    enable: false
    #生产者配置
    producer:
      acks: all #all为传到所有副本和-1一样
      retries: 3 #重试3次
      bootstrap-servers: 192.168.1.206:9092 #kafka地址
      key-serializer: org.apache.kafka.common.serialization.StringSerializer #key序列化
      value-serializer: org.apache.kafka.common.serialization.StringSerializer #value序列化
    #消费者配置
    consumer:
      enable-auto-commit: false #是否自动提交。此处我手动提交
      bootstrap-servers: ${spring.kafka.producer.bootstrap-servers} #kafka消费地址
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      group-id: lgDefaultGroup #消费组
      #其他配置
      properties: {
      	#配置拉取消息的最大间隔时间，超过这个时间可能会出发rebalance
        max.poll.interval.ms: 500000,
        #一次拉取的消息量
        max.poll.records: 50
      }
    template:
      #默认主题
      default-topic: my-topic

其他参数可以参照KafkaProperties.class
springboot中kafka配置类为KafkaAutoConfiguration.class

消费消息

配置消费监听

/**
 * @Description
 * @Author Singh
 * @Date 2020-06-27 11:01
 * @Version
 **/

@Configuration
@EnableKafka
@ConditionalOnProperty(name = "spring.kafka.enable",havingValue = "true")
public class KafkaConfig {

    /**
     * 配置@KafkaListener监听
     * @param kafkaConsumerFactory
     * @return
     */
    @Bean
    @Primary
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<Integer, String>>
    kafkaListenerContainerFactory(ConsumerFactory<Integer, String> kafkaConsumerFactory) {
        ConcurrentKafkaListenerContainerFactory<Integer, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(kafkaConsumerFactory);
        factory.setConcurrency(3);//分区数量
        factory.getContainerProperties().setPollTimeout(3000);//拉消息超时时间
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);//设置手动提交ackMode
        return factory;
    }

    @Bean
    @ConditionalOnMissingBean(StringKafkaTemplate.class)
    public StringKafkaTemplate stringKafkaTemplate(
            ProducerFactory<String, String> kafkaProducerFactory,KafkaProperties properties) {
        StringKafkaTemplate kafkaTemplate = new StringKafkaTemplate(kafkaProducerFactory);
        kafkaTemplate.setDefaultTopic(properties.getTemplate().getDefaultTopic());
        return kafkaTemplate;
    }

}

消费者监听DefaultKafkaListener

/**
 * @Description  消费监听，默认主题 **此处需要手动提交**
 * @Author Singh
 * @Date 2020-06-27 11:11
 * @Version
 **/

@Slf4j
@Component
@ConditionalOnProperty(name = "spring.kafka.enable",havingValue = "true")
public class DefaultKafkaListener implements BeanPostProcessor {

    @Autowired
    private StringRedisTemplate redisTemplate;

	//缓存自定义消费者
    private Map<Integer, LgKafkaConsumer> cacheHandlers = new HashMap<>();

    public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
        Class<?> targetCls = bean.getClass();
        KafkaConsumer[] kafkaHandlers = targetCls.getAnnotationsByType(KafkaConsumer.class);
        if(kafkaHandlers.length > 0){
            int type = kafkaHandlers[0].type();
            if(bean instanceof LgKafkaConsumer){
                cacheHandlers.put(type, (LgKafkaConsumer) bean);
            }
        }
        return bean;
    }

    @KafkaListener(id = "singhCustomer" ,topics = {"my-topic"})
    public void listen(Consumer consumer, ConsumerRecord<String, String> record, Acknowledgment ack){
        log.info("kafka consume -------------> partition={},offset={} ",record.partition(),record.offset());
        String id = null;
        try{
            KafkaMessage kafkaMessage = JSON.parseObject(record.value(),KafkaMessage.class);
            int type = kafkaMessage.getType();
            id = id(kafkaMessage);
            if(redisTemplate.hasKey(id)){
                ack.acknowledge();
                return ;
            }
            LgKafkaConsumer messageHandler = cacheHandlers.get(type);
            if(messageHandler == null){
                log.error("type = {} kafka message handler not exist！" , type);
                ack.acknowledge();
                return;
            }
            Type type0 = messageHandler.getClass().getGenericInterfaces()[0];
            Type type1 = ((ParameterizedType)type0).getActualTypeArguments()[0];
            Class<?> clzz = (Class<?>) type1;
            KafkaMessage msg = (KafkaMessage) JSON.parseObject(record.value(),clzz);
            messageHandler.handle(msg);//handle kafka message
            ack.acknowledge();
            redisTemplate.opsForValue().set(id,"0",1, TimeUnit.HOURS);
        }catch (Exception e){
            if(id == null){
                return ;
            }
            String retryTimesStr = redisTemplate.opsForValue().get(id);
            int retryTimes = retryTimesStr == null ? 0 : Integer.valueOf(retryTimesStr);
            if(retryTimes < 1){//重试1次
                consumer.seek(new TopicPartition(record.topic(),record.partition()),record.offset());
                redisTemplate.opsForValue().increment(id,1);
            }else{
                ack.acknowledge();
                redisTemplate.delete(id);
            }
        }
    }

    private String id(KafkaMessage message){
        return CommonConstant.PREFIX_KAFKA_CACHE_ID  + message.getId();
    }
}

注解消费者

/**
 * @Description 消费者处理注解
 * @Author Singh
 * @Date 2020-06-28 15:03
 * @Version
 **/
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Component
public @interface KafkaConsumer {

    int type() ;

}

/**
 * @Description
 * @Author Singh
 * @Date 2020-06-28 15:00
 * @Version
 **/
public interface LgKafkaConsumer<T extends KafkaMessage> {

    void handle(T object);

}

生产消息

模仿redis弄一个StringKafkaTemplate.java

/**
 * @Description 构建一个字符串生产端
 * @Author Singh
 * @Date 2020-06-28 11:15
 * @Version
 **/
public class StringKafkaTemplate extends KafkaTemplate<String,String>{

    /**
     * Create an instance using the supplied producer factory and autoFlush false.
     *
     * @param producerFactory the producer factory.
     */
    public StringKafkaTemplate(ProducerFactory<String, String> producerFactory) {
        super(producerFactory);
    }

}

消息实体类KafkaMessage.java

/**
 * @Description 消息实体类
 * @Author Singh
 * @Date 2020-06-28 13:37
 * @Version
 **/
@Data
public abstract class KafkaMessage {

    private String id;//消息唯一标示

    private int retryTimes;//重试次数

    private int type;//类型

    public KafkaMessage(int type,int retryTimes,String id){
        this.id = id;
        this.type = type;
        this.retryTimes = retryTimes;
    }
}

封装一个简单的生产者LgKafkaProducer.java

/**
 * @Description 发送消息
 * @Author Singh
 * @Date 2020-06-28 18:05
 * @Version
 **/
@Service
@ConditionalOnProperty(name = "spring.kafka.enable",havingValue = "true")
public class LgKafkaProducer {

    @Autowired
    private StringKafkaTemplate kafkaTemplate;

    /**
     *  异步发送消息，默认主题
     * @param message
     * @param callback
     * @param <T>
     */
    public <T extends KafkaMessage> void send(T message,ListenableFutureCallback<SendResult<String, String>> callback){
        ListenableFuture<SendResult<String, String>> future = kafkaTemplate.sendDefault(JSON.toJSONString(message));
        future.addCallback(callback);
    }

    /**
     * 同步发送消息，默认主题
     * @param message
     * @param <T>
     * @return
     */
    public <T extends KafkaMessage> SendResult<String, String> send(T message){
        try{
            ListenableFuture<SendResult<String, String>> future = kafkaTemplate.sendDefault(JSON.toJSONString(message));
            return future.get();
        }catch (Exception e){
            throw new RRException("发送消息失败！");
        }
    }

    /**
     * 异步发送消息
     * @param topic
     * @param message
     * @param callback
     * @param <T>
     */
    public <T extends KafkaMessage> void send(String topic,T message,ListenableFutureCallback<SendResult<String, String>> callback){
        ListenableFuture<SendResult<String, String>> future = kafkaTemplate.sendDefault(JSON.toJSONString(message));
        future.addCallback(callback);
    }

    /**
     * 同步发送消息
     * @param topic
     * @param message
     * @param <T>
     * @return
     */
    public <T extends KafkaMessage> SendResult<String, String> send(String topic,T message){
        try{
            ListenableFuture<SendResult<String, String>> future = kafkaTemplate.sendDefault(JSON.toJSONString(message));
            return future.get();
        }catch (Exception e){
            throw new RRException("发送消息失败！");
        }
    }
}

测试

/**
 * @Description junit
 * @Author Singh
 * @Date 2020-06-24 15:11
 * @Version
 **/
@Slf4j
@RunWith(SpringRunner.class)
@SpringBootTest(classes = {LigoApplication.class})
public class T {

    @Autowired
    private LgKafkaProducer kafkaProducer;

    @Test
    public void testAsync(){
        KafkaMessage message = new KafkaMessage(1,1,UUID.randomUUID().toString()){};
        kafkaProducer.send(message);
    }

}

Kafka在新增一个broker或者掉线一个broker的时候会rebalance
rebalance
欢迎大家留言多多指正！！！