Kafka 多维度系统精讲（10）- kafka consumer（4）

1 consumer 控制 offset 起始位置

 /**
     * 手动指定 offset 的起始位置，及手动提交 offset
     */
    private static void controlOffset() {
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.168.10.103:9092");
        props.setProperty("group.id", "test");
        props.setProperty("enable.auto.commit", "true");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);

        // tzb-new-topic, 0,1 两个 partition
        TopicPartition p0 = new TopicPartition(TOPIC_NAME, 0);

        // 订阅一个或者多个 topic
//        consumer.subscribe(Arrays.asList(TOPIC_NAME));

        // 消费订阅某个 topic 的某个分区
        consumer.assign(Arrays.asList(p0));

        while (true) {

            // 手动指定 offset 起始位置
            /**
             * 1、人为控制 offset 起始位置
             * 2、如果出现程序错误，重复消费
             */
            /**
             * 1.第一次从 0 消费【一般情况】
             * 2.比如一次消费了100条， offset 置为101，并且存入 redis
             * 3.每次 poll 之前，从redis中获取最新的 offset 位置
             * 4.每次从这个位置开始消费
             */
            consumer.seek(p0, 400);

            // 定时间隔去拉取
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));

            // 每个 partition 单独处理
            for (TopicPartition partition : records.partitions()) {
                List<ConsumerRecord<String, String>> pRecord = records.records(partition);
                for (ConsumerRecord<String, String> record : pRecord) {
                    System.out.printf("partition = %d, offset = %d, key = %s, value = %s%n",
                            record.partition(), record.offset(), record.key(), record.value());
                }

                long lastOffset = pRecord.get(pRecord.size() - 1).offset();

                // 单个 partition 中的 offset，并且进行提交
                Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
                offset.put(partition, new OffsetAndMetadata(lastOffset + 1));
                // 提交 offset
                consumer.commitSync(offset);
                System.out.println("===========  partition - " + partition +"====================");
            }

        }
    }

2 Consumer 限流

 /**
     * 流量控制：限流
     */
    private static void controlPause() {
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.168.10.103:9092");
        props.setProperty("group.id", "test");
        props.setProperty("enable.auto.commit", "true");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);

        // tzb-new-topic, 0,1 两个 partition
        TopicPartition p0 = new TopicPartition(TOPIC_NAME, 0);
        TopicPartition p1 = new TopicPartition(TOPIC_NAME, 1);

        // 消费订阅某个 topic 的某个分区
        consumer.assign(Arrays.asList(p0, p1));

        long totalNum = 40;


        while (true) {
            // 定时间隔去拉取
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));

            // 每个 partition 单独处理
            for (TopicPartition partition : records.partitions()) {
                List<ConsumerRecord<String, String>> pRecord = records.records(partition);

                long num = 0;

                for (ConsumerRecord<String, String> record : pRecord) {
                    System.out.printf("partition = %d, offset = %d, key = %s, value = %s%n",
                            record.partition(), record.offset(), record.key(), record.value());

                    /*
                     * 1.接收到record 信息后，去令牌桶中拿取令牌
                     * 2.如果获取到令牌，则继续业务处理
                     * 3.如果获取不到令牌，则 pause 等待令牌
                     * 4.当令牌桶中的令牌足够，则将 consumer 置为 resume 状态
                     */
                    num++;
                    if (record.partition() == 0) {
                        if (num >= totalNum) {
                            consumer.pause(Arrays.asList(p0));
                        }
                    }

                    if(record.partition() == 1){
                        if(num == 40){
                            consumer.resume(Arrays.asList(p0));
                        }
                    }

                }

                long lastOffset = pRecord.get(pRecord.size() - 1).offset();

                // 单个 partition 中的 offset，并且进行提交
                Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
                offset.put(partition, new OffsetAndMetadata(lastOffset + 1));
                // 提交 offset
                consumer.commitSync(offset);
                System.out.println("===========  partition - " + partition + "====================");
            }

        }
    }

3 Consumer Rebalance