package com.wenbronk.kafka;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.util.*;public classKafkaConsumerMain {/**
* 自动提交offset*/
public void commitAuto(Listtopics) throws FileNotFoundException {
Properties props= KafkaUtils.getProperties("cosumer");
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
KafkaConsumer consumer = new KafkaConsumer<>(props);
consumer.subscribe(topics);while (true) {
ConsumerRecords records = consumer.poll(1000);for (ConsumerRecordrecord : records)
System.err.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
}
}/**
* 手动提交offset
*
* @throws FileNotFoundException*/
public void commitControl(Listtopics) throws FileNotFoundException {
Properties props= KafkaUtils.getProperties("cosumer");
props.put("enable.auto.commit", "false");
KafkaConsumer consumer = new KafkaConsumer<>(props);
consumer.subscribe(topics);
finalint minBatchSize = 2;
List> buffer = new ArrayList<>();while (true) {
ConsumerRecords records = consumer.poll(1000);for (ConsumerRecordrecord : records) {
buffer.add(record);
}if (buffer.size() >=minBatchSize) {
insertIntoDb(buffer);//阻塞同步提交
consumer.commitSync();
buffer.clear();
}
}
}/**
* 手动设置分区*/
public void setOffSet(Listtopics) throws FileNotFoundException {
Properties props= KafkaUtils.getProperties("cosumer");
props.put("enable.auto.commit", "false");
KafkaConsumer consumer = new KafkaConsumer<>(props);
consumer.subscribe(topics);while (true) {
ConsumerRecords records =consumer.poll(Long.MAX_VALUE);//处理每个分区消息后, 提交偏移量
for(TopicPartition partition : records.partitions()) {
List> partitionRecords =records.records(partition);for (ConsumerRecordrecord : partitionRecords) {
System.out.println(record.offset() + ":" +record.value());
}long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset();
consumer.commitSync(Collections.singletonMap(partition,new OffsetAndMetadata(lastOffset + 1)));
}
}
}/**
* 手动设置消息offset*/
public void setSeek(Listtopics) throws FileNotFoundException {
Properties props= KafkaUtils.getProperties("cosumer");
props.put("enable.auto.commit", "false");
KafkaConsumer consumer = new KafkaConsumer<>(props);
consumer.subscribe(topics);
consumer.seek(new TopicPartition("http_zhixin", 0), 797670770);
ConsumerRecords records = consumer.poll(100);for (ConsumerRecordrecord : records) {
System.err.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
consumer.commitSync();
}
}
@Testpublic voidtest() throws FileNotFoundException {
ArrayList topics = new ArrayList<>();
topics.add("http_zhixin");//commitAuto(topics);//commitControl(topics);//setOffSet(topics);
setSeek(topics);
}/**
* doSomethings*/
private void insertIntoDb(List>buffer) {
buffer.stream().map(x->x.value()).forEach(System.err::println);
}
}