线上有一个消息消费服务xxx-consumer,使用spring-kafka框架,主线程批量从消费队列(kafka)拉取交易系统生产的消息,然后提交到子线程池中挨个处理消费。
public abstract class AbstractMessageDispatchListener implements
BatchAcknowledgingMessageListener<String, Msg>, ApplicationListener<ApplicationReadyEvent>
{
private ThreadPoolExecutor executor;
public abstract MessageWorker chooseWorker(ConsumerRecord<String, Msg> data);
@Override
public void onMessage(List<ConsumerRecord<String, Msg>> datas, Acknowledgment acknowledgment) {
List<Future<?>> futureList = new ArrayList<>(datas.size());
try {
CountDownLatch countDownLatch = new CountDownLatch(datas.size());
for (ConsumerRecord<String, Msg> data : datas) {
Future<?> future = executor.submit(new Worker(data, countDownLatch));
futureList.add(future);
}
countDownLatch.await(20000L - 2000, TimeUnit.MILLISECONDS);
long countDownLatchCount = countDownLatch.getCount();
if (countDownLatchCount > 0) {
return;
}
acknowledgment.acknowledge();
} catch (Exception e) {
logger.error("onMessage error ", e);
} finally {
for (Future<?> future : futureList) {
if (future.isDone() || future.isCancelled()) {
continue;
}
future.cancel(true);
}
}
}
@Override
public void onApplicationEvent(ApplicationReadyEvent event) {
ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
builder.setNameFormat(this.getClass().getSimpleName() + "-pool-%d");
builder.setDaemon(false);
executor = new ThreadPoolExecutor(12,
12 * 2,
60L,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(100),
builder.build());
}
private class Worker implements Runnable {
private ConsumerRecord<String, Msg> data;
private CountDownLatch countDownLatch;
Worker(ConsumerRecord<String, Msg> data, CountDownLatch countDownLatch) {
this.data = data;
this.countDownLatch = countDownLatch;
}
@Override
public void run() {
try {
MessageWorker worker = chooseWorker(data);
worker.work(data.value());
} finally {
countDownLatch.countDown();
}
}
}