java连接kafka_kafka-java客户端连接

package com.wenbronk.kafka;

import org.apache.kafka.clients.consumer.ConsumerRecord;

import org.apache.kafka.clients.consumer.ConsumerRecords;

import org.apache.kafka.clients.consumer.KafkaConsumer;

import org.apache.kafka.clients.consumer.OffsetAndMetadata;

import org.apache.kafka.common.TopicPartition;

import org.junit.Test;

import java.io.FileNotFoundException;

import java.util.*;public classKafkaConsumerMain {/**

* 自动提交offset*/

public void commitAuto(Listtopics) throws FileNotFoundException {

Properties props= KafkaUtils.getProperties("cosumer");

props.put("enable.auto.commit", "true");

props.put("auto.commit.interval.ms", "1000");

KafkaConsumer consumer = new KafkaConsumer<>(props);

consumer.subscribe(topics);while (true) {

ConsumerRecords records = consumer.poll(1000);for (ConsumerRecordrecord : records)

System.err.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());

}

}/**

* 手动提交offset

*

* @throws FileNotFoundException*/

public void commitControl(Listtopics) throws FileNotFoundException {

Properties props= KafkaUtils.getProperties("cosumer");

props.put("enable.auto.commit", "false");

KafkaConsumer consumer = new KafkaConsumer<>(props);

consumer.subscribe(topics);

finalint minBatchSize = 2;

List> buffer = new ArrayList<>();while (true) {

ConsumerRecords records = consumer.poll(1000);for (ConsumerRecordrecord : records) {

buffer.add(record);

}if (buffer.size() >=minBatchSize) {

insertIntoDb(buffer);//阻塞同步提交

consumer.commitSync();

buffer.clear();

}

}

}/**

* 手动设置分区*/

public void setOffSet(Listtopics) throws FileNotFoundException {

Properties props= KafkaUtils.getProperties("cosumer");

props.put("enable.auto.commit", "false");

KafkaConsumer consumer = new KafkaConsumer<>(props);

consumer.subscribe(topics);while (true) {

ConsumerRecords records =consumer.poll(Long.MAX_VALUE);//处理每个分区消息后, 提交偏移量

for(TopicPartition partition : records.partitions()) {

List> partitionRecords =records.records(partition);for (ConsumerRecordrecord : partitionRecords) {

System.out.println(record.offset() + ":" +record.value());

}long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset();

consumer.commitSync(Collections.singletonMap(partition,new OffsetAndMetadata(lastOffset + 1)));

}

}

}/**

* 手动设置消息offset*/

public void setSeek(Listtopics) throws FileNotFoundException {

Properties props= KafkaUtils.getProperties("cosumer");

props.put("enable.auto.commit", "false");

KafkaConsumer consumer = new KafkaConsumer<>(props);

consumer.subscribe(topics);

consumer.seek(new TopicPartition("http_zhixin", 0), 797670770);

ConsumerRecords records = consumer.poll(100);for (ConsumerRecordrecord : records) {

System.err.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());

consumer.commitSync();

}

}

@Testpublic voidtest() throws FileNotFoundException {

ArrayList topics = new ArrayList<>();

topics.add("http_zhixin");//commitAuto(topics);//commitControl(topics);//setOffSet(topics);

setSeek(topics);

}/**

* doSomethings*/

private void insertIntoDb(List>buffer) {

buffer.stream().map(x->x.value()).forEach(System.err::println);

}

}