我们的项目是springboot集成kafka的,配置文件在application.yml里如下:
#kafka配置
kafka:
consumer:
bootstrap:
maxPollRecords: 1000
enableAutoCommit: true
autoCommitIntervalMs: 100
sessionTimeoutMs: 15000
keyDeserializer: org.apache.kafka.common.serialization.StringDeserializer
valueDeserializer: org.apache.kafka.common.serialization.StringDeserializer
maxPollIntervalMs: 500000
log:
group: test
topic: test_log
method:
group: test1
topic: test_method
listen:
pullTimeOut: 5000
concurrency: 3
消费者类如下:
package consumer.consumer;
import common.utils.JsonUtils;
import consumer.method.MethodMoinitorIndexOperator;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
@Service
@Slf4j
public class MethodMonitorConsumer {
@Autowired
private MethodMoinitorIndexOperator methodMoinitorIndexOperator;
@Autowired
private TransportClient transportClient;
@KafkaListener(id ="${kafka.consumer.method.topic}",topics = "${kafka.consumer.method.topic}", groupId = "${kafka.consumer.method.group}")
public void methodBatchMessage(List<ConsumerRecord<?, ?>> records){
log.info("共消费数据"+records.size()+"条。");
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
try {
for (ConsumerRecord record : records) {
Map<String, Object> recordMap = JsonUtils.fromJson(record.value().toString());
methodMoinitorIndexOperator.handleMessages(bulkRequestBuilder, recordMap);
log.debug("消费的数据:"+recordMap);
}
} catch (Exception exception) {
log.error("consumerBatchMessage异常: {}", exception);
}
//批量写入es
methodMoinitorIndexOperator.executeBuilder(bulkRequestBuilder);
}
}
其他类如下:
import common.constans.ESConstans;
import common.utils.DateFormatUtil;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Date;
import java.util.Map;
@Slf4j
@Component
public class EsIndexHandler {
private final String indexPrefixName = "method_monitor";
private final String defaultInexType = "monitor_method";
private final String pattern = "yyyy_MM_dd";
@Autowired
private TransportClient transportClient;
public void handle(BulkRequestBuilder bulkRequestBuilder, Map<String, Object> methodIndexMap) {
String indexId = String.valueOf(methodIndexMap.get(ESConstans.ID));
StringBuilder sb = new StringBuilder();
sb.append(indexPrefixName);
String str = String.valueOf(methodIndexMap.get(ESConstans.INVOKE_TIME));
long l = Long.parseLong(str);
Date date = new Date(l);
String time = DateFormatUtil.format(date, pattern);
sb.append(time);
String indexName = sb.toString();
methodIndexMap.remove(ESConstans.ID);
IndexRequestBuilder indexRequestBuilder = transportClient.prepareIndex(indexName, defaultInexType, indexId).setSource(methodIndexMap);
bulkRequestBuilder.add(indexRequestBuilder);
log.debug("--------"+indexRequestBuilder);
}
}
以及另一个类:
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
@Slf4j
@Component
public class MethodIndexOperator {
@Autowired
private EsIndexHandler esIndexHandler;
public void handleMessages(BulkRequestBuilder bulkRequestBuilder, Map<String, Object> methodIndexMap){
esIndexHandler.handle(bulkRequestBuilder, methodIndexMap);
}
public void executeBuilder(BulkRequestBuilder bulkRequestBuilder) {
if (bulkRequestBuilder.numberOfActions() == 0) {
return;
}
BulkResponse response = bulkRequestBuilder.execute().actionGet();
log.debug("========"+response);
}
}