前言
在现代分布式系统中，消息队列处理是核心组件之一。今天我们将深入解析一个高性能、高可用的消息队列线程池实现——FindMessageQueue，并探讨如何将其优化应用于实际项目中。
一、核心架构设计
1.1 整体架构图
┌─────────────────────────────────────────────────┐ │ FindMessageQueue │ │ │ │ ┌─────────────────────────────────────────┐ │ │ │ ThreadPoolExecutor │ │ │ │ │ │ │ │ ┌─────────────┐ ┌─────────────────┐ │ │ │ │ │ 核心线程池 │ │ 有界任务队列 │ │ │ │ │ │ (Core Pool) │ │ (Bounded Queue) │ │ │ │ │ └─────────────┘ └─────────────────┘ │ │ │ └─────────────────────────────────────────┘ │ │ │ │ ┌─────────────────────────────────────────┐ │ │ │ 熔断器机制 │ │ │ │ (Circuit Breaker) │ │ │ └─────────────────────────────────────────┘ │ │ │ │ ┌─────────────────────────────────────────┐ │ │ │ 监控系统 │ │ │ │ (Monitoring) │ │ │ └─────────────────────────────────────────┘ │ │ │ │ ┌─────────────────────────────────────────┐ │ │ │ 指标统计系统 │ │ │ │ (Metrics System) │ │ │ └─────────────────────────────────────────┘ │ └─────────────────────────────────────────────────┘
1.2 核心组件介绍
// 核心线程池配置
private final ThreadPoolExecutor executorService;
private final int queueCapacity;
// 熔断器机制
private final AtomicBoolean circuitBreakerOpen = new AtomicBoolean(false);
private final AtomicLong circuitBreakerOpenedTime = new AtomicLong(0);
// 监控指标
private final AtomicLong totalTasksSubmitted = new AtomicLong(0);
private final AtomicLong totalTasksRejected = new AtomicLong(0);
private final AtomicLong totalTasksCompleted = new AtomicLong(0);
private final AtomicLong totalTasksFailed = new AtomicLong(0);
二、详细源码解析
2.1 线程池初始化
public FindMessageQueue(int threadPoolSize) {
this.queueCapacity = 1000;
this.executorService = new ThreadPoolExecutor(
threadPoolSize, // 核心线程数
threadPoolSize, // 最大线程数
60L, TimeUnit.SECONDS, // 线程空闲存活时间
new LinkedBlockingQueue<>(queueCapacity), // 有界队列
new ThreadPoolExecutor.DiscardPolicy() // 拒绝策略
);
startMonitorThread(); // 启动监控线程
}
关键参数说明：
-
corePoolSize=maximumPoolSize：创建固定大小线程池 -
keepAliveTime= 60秒：空闲线程回收时间 -
LinkedBlockingQueue：有界队列防止内存溢出 -
DiscardPolicy：队列满时由调用线程执行任务
2.2 熔断器机制实现
// 熔断检查逻辑
if (rejectionRate > rejectionRateThreshold && !circuitBreakerOpen.get()) {
logger.warn("拒绝率过高({}%)，触发熔断机制", rejectionRate * 100);
circuitBreakerOpen.set(true);
circuitBreakerOpenedTime.set(System.currentTimeMillis());
}
// 熔断恢复逻辑
if (circuitBreakerOpen.get() &&
System.currentTimeMillis() - circuitBreakerOpenedTime.get() > circuitBreakerResetTimeout) {
if (rejectionRate < rejectionRateThreshold / 2) {
circuitBreakerOpen.set(false); // 恢复服务
}
}
2.3 任务提交机制
public boolean addTask(Runnable task, long timeout, TimeUnit unit) {
totalTasksSubmitted.incrementAndGet();
// 熔断器检查
if (circuitBreakerOpen.get()) {
totalTasksRejected.incrementAndGet();
return false;
}
try {
if (timeout <= 0) {
executorService.execute(wrapTask(task)); // 异步执行
return true;
} else {
Future<?> future = executorService.submit(wrapTask(task));
future.get(timeout, unit); // 同步等待结果
return true;
}
} catch (RejectedExecutionException e) {
totalTasksRejected.incrementAndGet();
return false;
}
}
2.4 监控系统实现
private void monitorQueueHealth() {
int queueSize = executorService.getQueue().size();
int activeCount = executorService.getActiveCount();
double queueUsage = (double) queueSize / queueCapacity;
double rejectionRate = (double) totalTasksRejected.get() / totalTasksSubmitted.get();
logger.info("线程池监控 - 活跃线程: {}, 队列大小: {}/{}, 使用率: {}%, 拒绝率: {}%",
activeCount, queueSize, queueCapacity,
queueUsage * 100, rejectionRate * 100);
}
三、优化改进方案
3.1 使用Spring Boot集成
@Configuration
public class ThreadPoolConfig {
@Bean
public FindMessageQueue findMessageQueue(
@Value("${thread.pool.size:10}") int poolSize,
@Value("${thread.queue.capacity:1000}") int queueCapacity) {
return new FindMessageQueue(poolSize) {
@Override
protected void init(int threadPoolSize) {
// 可重写初始化逻辑
super.queueCapacity = queueCapacity;
}
};
}
}
3.2 添加Prometheus监控指标
@Component
public class ThreadPoolMetrics {
private final FindMessageQueue messageQueue;
// 注册监控指标
public void registerMetrics() {
Gauge.builder("thread_pool_queue_size", messageQueue, FindMessageQueue::getQueueSize)
.description("当前任务队列大小")
.register(MeterRegistry);
Gauge.builder("thread_pool_rejection_rate", messageQueue,
q -> (double) q.getRejectedCount() / q.getSubmittedCount())
.description("任务拒绝率")
.register(MeterRegistry);
}
}
3.3 增强的熔断策略
// 多维度熔断条件
private boolean shouldTriggerCircuitBreaker() {
double rejectionRate = getRejectionRate();
double queueUsage = getQueueUsage();
long avgTaskTime = getAverageTaskTime();
return rejectionRate > rejectionRateThreshold ||
queueUsage > 0.9 ||
avgTaskTime > maxAllowedTaskTime;
}
3.4 动态配置调整
@RefreshScope
@Component
public class DynamicThreadPoolConfig {
@Autowired
private FindMessageQueue messageQueue;
@EventListener
public void onConfigUpdate(EnvironmentChangeEvent event) {
// 动态调整线程池参数
if (event.getKeys().contains("thread.pool.size")) {
adjustThreadPoolSize();
}
}
}
总结
核心优势：
-
高可用性：熔断器机制防止系统雪崩
-
可观测性：完善的监控和指标统计
-
弹性伸缩：动态调整线程池参数
-
错误隔离：任务失败不影响主线程
适用场景：
-
消息队列处理
-
批量数据处理
-
异步任务执行
-
高并发请求处理
注意事项：
-
合理设置线程池大小和队列容量
-
监控关键指标并及时调整参数
-
实现恰当的错误处理和重试机制
-
定期进行压力测试和性能调优
这个FindMessageQueue实现提供了一个生产级别的线程池解决方案，通过熔断器、监控系统和弹性设计，确保了系统的高可用性和稳定性。
附赠：完整代码：
package com.baotademo.controller;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
public class FindMessageQueue {
private static final Logger logger = LoggerFactory.getLogger(FindMessageQueue.class);
private final ThreadPoolExecutor executorService;
private final int queueCapacity;
// 熔断器状态
private final AtomicBoolean circuitBreakerOpen = new AtomicBoolean(false);
private final AtomicLong circuitBreakerOpenedTime = new AtomicLong(0);
private final long circuitBreakerResetTimeout = 30000; // 30秒后尝试恢复
// 监控指标
private final AtomicLong totalTasksSubmitted = new AtomicLong(0);
private final AtomicLong totalTasksRejected = new AtomicLong(0);
private final AtomicLong totalTasksCompleted = new AtomicLong(0);
private final AtomicLong totalTasksFailed = new AtomicLong(0);
// 监控阈值
private final double queueUsageThreshold = 0.8; // 队列使用率超过80%警告
private final double rejectionRateThreshold = 0.1; // 拒绝率超过10%触发熔断
public FindMessageQueue(int threadPoolSize) {
this.queueCapacity = 1000;
// 使用有界队列+合适的拒绝策略
this.executorService = new ThreadPoolExecutor(
threadPoolSize, // 核心线程数
threadPoolSize, // 最大线程数
60L, TimeUnit.SECONDS, // 空闲线程存活时间
new LinkedBlockingQueue<>(queueCapacity), // 有界任务队列
new ThreadPoolExecutor.DiscardPolicy() // 拒绝策略：由调用线程执行
);
// 启动监控线程
startMonitorThread();
}
// 启动监控线程
private void startMonitorThread() {
ScheduledExecutorService monitorExecutor = Executors.newSingleThreadScheduledExecutor();
monitorExecutor.scheduleAtFixedRate(() -> {
try {
monitorQueueHealth();
} catch (Exception e) {
logger.error("监控线程执行异常", e);
}
}, 1, 5, TimeUnit.SECONDS); // 5秒监控一次
}
// 监控队列健康状态
private void monitorQueueHealth() {
int queueSize = executorService.getQueue().size();
int activeCount = executorService.getActiveCount();
long completedTaskCount = executorService.getCompletedTaskCount();
long submittedTasks = totalTasksSubmitted.get();
long rejectedTasks = totalTasksRejected.get();
// 计算队列使用率
double queueUsage = (double) queueSize / queueCapacity;
// 计算拒绝率
double rejectionRate = submittedTasks > 0 ? (double) rejectedTasks / submittedTasks : 0;
// 记录监控指标
logger.info("线程池监控 - 活跃线程: {}, 队列大小: {}/{}, 队列使用率: {}%, 拒绝率: {}%, 已完成任务: {}",
activeCount, queueSize, queueCapacity,
String.format("%.2f", queueUsage * 100),
String.format("%.2f", rejectionRate * 100),
completedTaskCount);
// 检查是否需要触发熔断
if (rejectionRate > rejectionRateThreshold && !circuitBreakerOpen.get()) {
logger.warn("拒绝率过高({}%)，触发熔断机制", String.format("%.2f", rejectionRate * 100));
circuitBreakerOpen.set(true);
circuitBreakerOpenedTime.set(System.currentTimeMillis());
}
// 检查是否可以恢复熔断
if (circuitBreakerOpen.get() &&
System.currentTimeMillis() - circuitBreakerOpenedTime.get() > circuitBreakerResetTimeout) {
logger.info("尝试恢复熔断器，当前拒绝率: {}%", String.format("%.2f", rejectionRate * 100));
// 如果拒绝率下降到阈值以下，恢复服务
if (rejectionRate < rejectionRateThreshold / 2) {
logger.info("拒绝率已恢复正常({}%)，关闭熔断器", String.format("%.2f", rejectionRate * 100));
circuitBreakerOpen.set(false);
} else {
// 否则重置熔断时间，继续熔断
circuitBreakerOpenedTime.set(System.currentTimeMillis());
}
}
// 队列使用率过高警告
if (queueUsage > queueUsageThreshold) {
logger.warn("任务队列使用率过高: {}%", String.format("%.2f", queueUsage * 100));
}
}
// 向队列添加任务
public boolean addTask(Runnable task) {
return addTask(task, 0, TimeUnit.MILLISECONDS);
}
// 带超时的任务添加
public boolean addTask(Runnable task, long timeout, TimeUnit unit) {
totalTasksSubmitted.incrementAndGet();
// 检查熔断器状态
if (circuitBreakerOpen.get()) {
logger.warn("熔断器已打开，拒绝新任务");
totalTasksRejected.incrementAndGet();
return false;
}
try {
// 尝试提交任务
if (timeout <= 0) {
executorService.execute(task);
return true;
} else {
// 带超时的提交
Future<?> future = executorService.submit(task);
try {
future.get(timeout, unit);
return true;
} catch (TimeoutException e) {
logger.warn("任务执行超时，已取消");
future.cancel(true);
totalTasksFailed.incrementAndGet();
return false;
}
}
} catch (RejectedExecutionException e) {
logger.warn("任务被线程池拒绝，当前队列大小: {}", executorService.getQueue().size());
totalTasksRejected.incrementAndGet();
return false;
} catch (Exception e) {
logger.error("添加任务时发生异常", e);
totalTasksFailed.incrementAndGet();
return false;
}
}
// 获取当前队列大小
public int getQueueSize() {
return executorService.getQueue().size();
}
// 获取活跃线程数
public int getActiveCount() {
return executorService.getActiveCount();
}
// 获取熔断器状态
public boolean isCircuitBreakerOpen() {
return circuitBreakerOpen.get();
}
// 手动重置熔断器
public void resetCircuitBreaker() {
circuitBreakerOpen.set(false);
circuitBreakerOpenedTime.set(0);
logger.info("熔断器已手动重置");
}
// 获取监控指标
public String getMetrics() {
return String.format(
"任务统计 - 已提交: %d, 已拒绝: %d, 已完成: %d, 失败: %d, 拒绝率: %.2f%%",
totalTasksSubmitted.get(),
totalTasksRejected.get(),
totalTasksCompleted.get(),
totalTasksFailed.get(),
totalTasksSubmitted.get() > 0 ?
(double) totalTasksRejected.get() / totalTasksSubmitted.get() * 100 : 0
);
}
// 优雅关闭
public void shutdown() {
logger.info("开始关闭线程池...");
executorService.shutdown();
try {
if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
logger.warn("线程池未正常关闭，尝试强制关闭");
executorService.shutdownNow();
}
} catch (InterruptedException e) {
executorService.shutdownNow();
Thread.currentThread().interrupt();
}
logger.info("线程池已关闭");
}
// 立即关闭
public void shutdownNow() {
logger.info("立即关闭线程池");
executorService.shutdownNow();
}
// 包装任务以跟踪完成情况
private Runnable wrapTask(Runnable task) {
return () -> {
try {
task.run();
totalTasksCompleted.incrementAndGet();
} catch (Exception e) {
totalTasksFailed.incrementAndGet();
logger.error("任务执行失败", e);
throw e;
}
};
}
}
使用方法：
1.实例化：
private static final FindMessageQueue findMessageQueue = new FindMessageQueue(50);
2.调用：
public CompletableFuture<R> sendQueneSms(@RequestBody Map<String, Object> request,
HttpServletRequest requesthead) {
CompletableFuture<R> future = new CompletableFuture<>();
// 设置超时
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
ScheduledFuture<?> timeoutFuture = scheduler.schedule(() -> {
if (!future.isDone()) {
logger.warn("请求处理超时");
future.complete(R.error("处理超时，请稍后重试"));
}
}, 10, TimeUnit.SECONDS); // 10秒超时
// 创建任务
Runnable task = () -> {
try {
R result = loadHistoryMessage(request, requesthead);
future.complete(result);
} catch (Exception e) {
logger.error("处理历史消息失败", e);
future.complete(R.error("处理失败: " + e.getMessage()));
} finally {
// 取消超时检查
timeoutFuture.cancel(true);
scheduler.shutdown();
}
};
// 添加任务到队列
boolean success = findMessageQueue.addTask(task, 5, TimeUnit.SECONDS); // 5秒提交超时
if (!success) {
// 任务提交失败，直接返回降级响应
timeoutFuture.cancel(true);
scheduler.shutdown();
if (findMessageQueue.isCircuitBreakerOpen()) {
future.complete(R.error("系统繁忙，熔断器已打开，请稍后重试"));
} else {
future.complete(R.error("系统繁忙，请稍后重试"));
}
}
return future;
}
扫一扫
123
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
