kafka+spring定时任务拉取消费

通过实践获取一下结论：

1.topic可以被多个group消费
group之间消费位移互不干扰
2.topic被group消费时，若有多个消费者实例，同一条消息只会被一个消费者处理
3.一个group可以消费多个topic
4.一个消费者group可以拉取多个topic消息

pom依赖包：

<dependency>
   <groupId>org.springframework.cloud</groupId>
   <artifactId>spring-cloud-starter-stream-kafka</artifactId>
</dependency>

KafkaConsumerFactory.java(消费者工厂)

package cn.shipeng.friend.consumer;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.util.Properties;
/*
消费者工厂方法
*/
public class KafkaConsumerFactory {
    static KafkaConsumer<String, String> consumer;
    public static KafkaConsumer getKafkaConsumer(String group) {
        Properties props = new Properties();
        props.put("bootstrap.servers", "localhost:9092");
        //每个消费者分配独立的组号
        props.put("group.id", group);
        //如果value合法，则自动提交偏移量
        props.put("enable.auto.commit", "false");
        // 每次拉取10条
        props.put("max.poll.records", 10);
        //设置多久一次更新被消费消息的偏移量
        props.put("auto.commit.interval.ms", "1000");
        //设置会话响应的时间，超过这个时间kafka可以选择放弃消费或者消费下一条消息
        props.put("session.timeout.ms", "30000");
        //自动重置offset
        props.put("auto.offset.reset", "earliest");
        props.put("key.deserializer",
            "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer",
            "org.apache.kafka.common.serialization.StringDeserializer");

        return consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<String, String>(props);
    }
}

GroupQueue.java (消费者)定时任务拉取消息队列

package cn.shipeng.friend.consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.UUID;
/**
消费者
*/
@EnableAsync
@Component
public class GroupQueue {
    static boolean RUN = false;

    @Async
    @Scheduled(cron = "0 * * * * ?")
    public void task() {
        if (RUN) {
            return;
        }
        RUN = true;
        KafkaConsumer consumer = KafkaConsumerFactory.getKafkaConsumer("group2");
        consumer.subscribe(Arrays.asList("GroupQueue", "FriendQueue"));

        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            System.out.println("批次：" + UUID.randomUUID().toString());

            for (ConsumerRecord<String, String> record : records) {
                System.out.printf(record.topic() +
                    "一条新消息 offset = %d, key = %s, value = %s", record.offset(),
                    record.key(), record.value());
                System.out.println(record.topic() + "partition:" +
                    record.partition());

                // 业务处理 TODO
            }

            // 同步提交
            if (records.count() > 0) {
                consumer.commitSync();
                System.out.println("批次提交");
            }
        }
    }
}

SimpleKafkaProducer.java 生产者

package test;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.util.Properties;
/**
 * 生产者
 */
public class SimpleKafkaProducer {
    private static KafkaProducer<String, String> producer;
    private final static String TOPIC = "GroupQueue";

    public SimpleKafkaProducer() {
        Properties props = new Properties();
        props.put("bootstrap.servers", "localhost:9092");
        props.put("acks", "all");
        props.put("retries", 0);
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);
        props.put("key.serializer",
            "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer",
            "org.apache.kafka.common.serialization.StringSerializer");
        //设置分区类,根据key进行数据分区
        producer = new KafkaProducer<String, String>(props);
    }

    public void produce() {
        for (int i = 0; i < 100; i++) {
            String key = String.valueOf(i);
            String data = "hello kafka message：" + key;
            producer.send(new ProducerRecord<String, String>(TOPIC, key, data));
            System.out.println(data);
        }

        producer.close();
    }

    public static void main(String[] args) {
        new SimpleKafkaProducer().produce();
    }
}