针对不同的业务场景需要不同的线程处理方式,分享下自己项目使用的线程:
第一种 注解形式使用
1.创建线程池
@Configuration
@EnableAsync
public class TaskExecutorConfig {
@Bean("taskExecutor")
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 设置核心线程数
executor.setCorePoolSize(5);
// 设置最大线程数
executor.setMaxPoolSize(10);
// 设置队列容量
executor.setQueueCapacity(40);
// 设置线程空闲活跃时间(秒),超时强制关闭
executor.setKeepAliveSeconds(60);
// 设置默认线程名称
executor.setThreadNamePrefix("subcribe-thread-");
// 设置拒绝策略,队列满,线程被拒绝执行策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 等待所有任务结束后再关闭线程池,异步任务的销毁就会先于依赖的外部资源(如Redis)线程池的销毁。
executor.setWaitForTasksToCompleteOnShutdown(true);
//等待终止秒,设置线程池中任务的等待时间,如果超过这个时候还没有销毁就强制销毁,以确保应用最后能够被关闭,而不是阻塞住
executor.setAwaitTerminationSeconds(60);
// //初始化
// executor.initialize();
return executor;
}
}
2.在需要使用多线程的方法上使用@Async("taskExecutor")
@Override
@Async("taskExecutor")
public void taskKfk(TaskRecordInfo taskRecordInfo) {
long startTime = System.currentTimeMillis();
}
这种线程池使用起来简单方便,没有什么特别的业务需求。
第二种 需要等待所有线程执行完毕 才执行后续逻辑操作
@Slf4j
@Service("initEcacheService")
public class InitEcacheService {
@Autowired
RedisUtil redisUtil;
@Autowired
SubscribeInfoDao subscribeInfoDao;
@Autowired
SubscribeInfoService subscribeInfoService;
@Autowired
TaskSubscribeService taskSubscribeService;
@Autowired
SubscribeRecordService subscribeRecordService;
@Autowired
ResultMappingInfoService resultMappingInfoService;
/**
* 是否使用Redis
*/
@Value("${discernRedis.employRedis:true}")
private boolean employRedis;
/**
* 每页行数
*/
@Value("${discernRedis.rowsPerPage:10000}")
private int rowsPerPage;
@Value("${subscribe.subscribeTaskRepeatTime:10}")
private int subscribeTaskRepeatTime;
@Value("${subscribe.taskIsEnable:false}")
private boolean taskIsEnable;
/**
* 项目启动后执行
*/
@EventListener(ApplicationReadyEvent.class)
public void doSomethingAfterStartup() {
try {
if (taskIsEnable && employRedis) {
long startTime = System.currentTimeMillis();
Date maxUpdaDate = subscribeInfoService.getMaxLastUpdateTime();
log.info("当前订阅最大更新时间:" + DateUtil.dateToStr17(maxUpdaDate));
Date validDate = new Date();
//1,加分页。2,加线程。3,有效期
int count = subscribeInfoService.sumSubscribe(validDate);
int page = count % rowsPerPage == 0 ? (count / rowsPerPage) : (count / rowsPerPage) + 1;
log.info("总数量:" + count + ",总页数:" + page);
//参数说明:5条核心线程 10条最大线程60秒回收空闲线程ThreadPoolExecutor.CallerRunsPolicy:由调用线程处理该任务
ExecutorService executorService = new ThreadPoolExecutor(5, 10, 60,
TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
new MyThread(),
new ThreadPoolExecutor.CallerRunsPolicy());
for (int i = 1; i <= page; i++) {
InitEcacheService.PutInfoRunnable putInfoRunnable = new InitEcacheService.PutInfoRunnable();
putInfoRunnable.setPageIndex(i);
putInfoRunnable.setVDate(validDate);
putInfoRunnable.setUpdaDate(maxUpdaDate);
executorService.execute(putInfoRunnable);
}
executorService.shutdown();
boolean loop = true;
do {
loop = !executorService.awaitTermination(120, TimeUnit.MINUTES);
} while (loop); //等待
subscribeInfoService.loadTemporaryEcache(maxUpdaDate);
redisUtil.set("_FLAG#isHaveRedisData", "true");
Long spendTime = System.currentTimeMillis() - startTime;
log.info("加载缓存结束,用时:" + spendTime);
}
} catch (Exception e) {
log.error("redis缓存加载异常:" + e.getMessage(), e);
}
}
public class PutInfoRunnable implements Runnable {
private int pageIndex;
private Date vDate;
private Date updaDate;
@Override
public void run() {
try {
subscribeInfoService.pagingSubscribeEcache(pageIndex, rowsPerPage, vDate, updaDate);
} catch (Exception e) {
log.info("分页加载订阅信息异常:" + e.getMessage(), e);
e.printStackTrace();
}
}
public int getPageIndex() {
return pageIndex;
}
public void setPageIndex(int pageIndex) {
this.pageIndex = pageIndex;
}
public Date getVDate() {
return vDate;
}
public void setVDate(Date vDate) {
this.vDate = vDate;
}
public Date getUpdaDate() {
return updaDate;
}
public void setUpdaDate(Date updaDate) {
this.updaDate = updaDate;
}
}
//实现ThreadFactory 接口,重写newThread方法。
public class MyThread implements ThreadFactory {
@Override
public Thread newThread(@NonNull Runnable r) {
Thread thread = new Thread(r);
thread.setName("page_subscribe_" + r.hashCode());
return thread;
}
}
}
第三种 多线程下获取返回值
@Service("taskSubscribeService")
@Slf4j
public class TaskSubscribeServiceImpl implements TaskSubscribeService {
@Autowired private AllocateTaskService allocateTaskService;
@PersistenceContext(unitName = "masterDbUnit")
private EntityManager masterEntityManager;
@Value("${kafka.topic:test}")
private String topic;
@Value("${threadPool.taskAllocation.corePoolSize:5}")
private int corePoolSize;
@Value("${threadPool.taskAllocation.maximumPoolSize:10}")
private int maximumPoolSize;
@Value("${threadPool.taskAllocation.keepAliveTime:60}")
private int keepAliveTime;
@Override
public void queryTaskRlation() {
log.info("###### 开始任务轮询 时间:" + DateUtil.getTimeString() + " ######");
//.......
//.......业务
//.....
int taskSum = rlationList.size();
int taskRlationSum = 0;
int noTaskRlationSum = 0;
int taskRlationErrorNumber = 0;
int waitTime = 1;
// 参数说明:5条核心线程 10条最大线程60秒回收空闲线程ThreadPoolExecutor.CallerRunsPolicy:由调用线程处理该任务
ExecutorService taskPool =
new ThreadPoolExecutor(
corePoolSize,
maximumPoolSize,
keepAliveTime,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new MyThread(),
new ThreadPoolExecutor.CallerRunsPolicy());
Map<String, Object> resultMap = new HashMap(); // 存放线程结果的对象,为了让main线程循环询问子线程结果,进行等待
try {
for (SubscribeRlation rlation : rlationList) {
Callable<Map<String, Object>> c1 = new MyCallable(rlation, taskRecordInfoMap, parentUuid);
Future<Map<String, Object>> f1 = taskPool.submit(c1);
resultMap.put(rlation.getTaskKey(), f1);
}
// latch.await();
// 进行判断线程是否结束,目的是让main主线程等待子线程
boolean isDone = false;
for (Map.Entry<String, Object> m : resultMap.entrySet()) {
System.out.println("key:" + m.getKey() + " value:" + m.getValue());
Future<Map<String, Object>> f1 = (Future<Map<String, Object>>) m.getValue();
Map<String, String> taskMap = (Map<String, String>) f1.get().get(m.getKey());
do {
isDone = f1.isDone();
if (isDone) {
if (taskMap != null && taskMap.size() > 0) {
if (taskMap.containsKey("noTaskRlation")) {
noTaskRlationSum++;
} else if (taskMap.containsKey("uuid")) {
taskRlationSum++;
} else if (taskMap.containsKey("error")) {
taskRlationErrorNumber++;
}
} else {
log.info(" ### 此分配线程的任务 没有返回值 taskKey:" + m.getKey());
}
} else {
// // 等待两秒,因为它一直询问,执行者很烦,效率会耽误执行者
// log.info("主线程进行等待...%s毫秒(ms)\n", waitTime);
// Thread.sleep(waitTime);
}
} while (!isDone);
isDone = false;
taskMap = null;
}
} catch (InterruptedException | ExecutionException e) {
isSuccess = "0";
e.printStackTrace();
} catch (Exception e) {
isSuccess = "0";
log.error("分配任务异常, " + e.getMessage(), e);
}
// 关闭线程池
taskPool.shutdown();
rlationList = null;
taskRecordlist = null;
taskRecordInfoMap = null;
resultMap = null;
BizLogger.info(
bizloginfo
.setBizloginfo("taskSum", taskSum)
.setBizloginfo("taskRlationSum", taskRlationSum)
.setBizloginfo("noTaskRlationSum", noTaskRlationSum)
.setBizloginfo("taskRlationErrorNumber", taskRlationErrorNumber)
.setBizloginfo("taskRlationEndTime", DateUtil.getCurrentTimeISO8061())
.setBizloginfo("spendTime", System.currentTimeMillis() - startTime)
.setBizloginfo("isSuccess", isSuccess)
.getMap());
} catch (Exception e) {
String errMsg = e.getMessage();
log.error("分配任务失败,parentUuid:" + parentUuid + ",异常:" + errMsg, e);
isSuccess = "0";
BizLogger.info(
bizloginfo
.setBizloginfo("taskRlationEndTime", DateUtil.getCurrentTimeISO8061())
.setBizloginfo("spendTime", System.currentTimeMillis() - startTime)
.setBizloginfo("isSuccess", isSuccess)
.setBizloginfo(
LOGKEYS.ERR_INFO,
"分配任务异常:" + errMsg.substring(0, errMsg.length() > 200 ? 200 : errMsg.length()))
.getMap());
}
log.info("###### 任务轮询 结束 ,耗时:" + (System.currentTimeMillis() - startTime));
}
public class MyCallable implements Callable<Map<String, Object>> {
private SubscribeRlation rlation;
private Map<String, TaskRecordInfo> taskRecordInfoMap;
private String parentUuid;
private Map<String, Object> resultMap = new HashMap();
public MyCallable(
SubscribeRlation rlation,
Map<String, TaskRecordInfo> taskRecordInfoMap,
String parentUuid) {
this.rlation = rlation;
this.taskRecordInfoMap = taskRecordInfoMap;
this.parentUuid = parentUuid;
}
@Override
public Map<String, Object> call() throws Exception {
Map<String, String> stringMap = new HashMap<>();
try {
stringMap = allocateTaskService.allocateTask(rlation, taskRecordInfoMap, parentUuid);
if (stringMap != null) {
resultMap.put(rlation.getTaskKey(), stringMap);
}
stringMap = null;
} catch (Exception e) {
log.info("分配任务异常,TaskKey:" + rlation.getTaskKey() + ", " + e.getMessage(), e);
e.printStackTrace();
}
return resultMap;
}
public SubscribeRlation getRlation() {
return this.rlation;
}
public void setRlation(SubscribeRlation rlation) {
this.rlation = rlation;
}
public Map<String, TaskRecordInfo> getTaskRecordInfoMap() {
return this.taskRecordInfoMap;
}
public void setTaskRecordInfoMap(Map<String, TaskRecordInfo> taskRecordInfoMap) {
this.taskRecordInfoMap = taskRecordInfoMap;
}
}
public class MyThread implements ThreadFactory {
@Override
public Thread newThread(@NonNull Runnable r) {
Thread thread = new Thread(r);
thread.setName("subscribe_task_" + r.hashCode());
return thread;
}
}
}
注:① 计数类的变量也可以用AtomicInteger,多线程下方便计数
AtomicInteger httpSuccessSum = new AtomicInteger(0);
httpSuccessSum.addAndGet(1);
Integer sum = httpSuccessSum.get();
②开始计时,也可以在方法外使用ThreadLocal<Long> 为每个线程建一个计时
ThreadLocal<Long> startTime = new ThreadLocal<>();
@Override
@Async("taskExecutor")
public void dealTask(TaskInfo task) {
// 执行任务开始时间
startTime.set(System.currentTimeMillis());
//..业务逻辑...
Long spendTime = (System.currentTimeMillis() - startTime.get());
}
第四种 线程设置超时
生产有些任务需要占用很长时间的CPU,从而对其他任务造成不良影响,此时我们需要壮士断臂的精神,把这个耗内存的任务干掉。
@EnableAsync
@Api(value = "UserSubscibe", description = "接收任务处理", produces = MediaType.APPLICATION_JSON_VALUE)
@Slf4j
@ResponseBody
public class Task extends BaseController {
@Autowired private TaskKfkService taskKfkService;
@Autowired private TaskRecordInfoDao taskRecordInfoDao;
/**
* @Description:kafka任务 topic监听方法 主要用来监听定时任 务放入kafka的任务获取后执行比对操作。见Task.class 任务方法
*
* @param record
*/
@KafkaListener(topics = "${kafka.topic:subscribeTask}")
public void receiptTopic(ConsumerRecord<String, String> record) {
log.info("### 监听任务 对象信息:" + record.toString());
long startTime = System.currentTimeMillis();
TaskRecordInfo recordInfo = new TaskRecordInfo();
SubscribeRlation rlation = new SubscribeRlation();
ExecutorService taskResultPool =
new ThreadPoolExecutor(
1,
1,
60,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new MyThread(),
new ThreadPoolExecutor.CallerRunsPolicy());
try {
String param = record.value();
Map paramMap = JSON.parseObject(param, Map.class);
SubscribeRlation subscribeRlation =
JSON.parseObject(paramMap.get("rlation").toString(), SubscribeRlation.class);
rlation = subscribeRlation;
TaskRecordInfo taskRecordInfo =
JSON.parseObject(paramMap.get("taskRecordInfo").toString(), TaskRecordInfo.class);
recordInfo = taskRecordInfo;
final List<Future> threadList = new ArrayList<Future>();
Future future1 =
taskResultPool.submit(
new Runnable() {
@Override
public void run() {
try {
taskKfkService.taskKfk(subscribeRlation, taskRecordInfo);
} catch (Exception e) {
String errMsg = e.getMessage();
log.error("##### 任务处理模块异常:" + errMsg, e);
BizLogger.info(
new Bizloginfo()
.setBizloginfo(
"classMethod",
"com.easipass.subscribe.controller.TaskController.receiptTopic")
.setBizloginfo(LOGKEYS.EVENT_TYPE, "taskResult")
.setBizloginfo("uuid", taskRecordInfo.getUuid())
.setBizloginfo("taskKey", taskRecordInfo.getTaskKey())
.setBizloginfo("beginTime", taskRecordInfo.getBeginTime().toString())
.setBizloginfo("endTime", taskRecordInfo.getEndTime().toString())
.setBizloginfo(
LOGKEYS.ERR_INFO,
"任务处理模块异常:"
+ errMsg.substring(
0, errMsg.length() > 200 ? 200 : errMsg.length()))
.setBizloginfo("chargeType", "0")
.setBizloginfo("uuid", taskRecordInfo.getUuid())
.setBizloginfo("isSuccess", "0")
.setInfoNotEmpty(LOGKEYS.EVENT_TIME, DateUtil.getCurrentTimeISO8061())
.setBizloginfo("spendTime", System.currentTimeMillis() - startTime)
.getMap());
}
}
});
threadList.add(future1);
for (Future future : threadList) {
final Future futureTemp = future;
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
String errInfo = "";
try {
futureTemp.get(1000 * 60 * 30, TimeUnit.MILLISECONDS);
//futureTemp.get(1000 * 60 * 30, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
futureTemp.cancel(true);
log.error("InterruptedException发生线程强行终止:" + e.getMessage(), e);
errInfo = "InterruptedException发生线程强行终止";
// pool.shutdown();
// pool.shutdownNow();
} catch (ExecutionException e) {
futureTemp.cancel(true);
log.error("InterruptedException发生:" + e.getMessage(), e);
errInfo = "InterruptedException发生线程中异常";
// pool.shutdown();
// pool.shutdownNow();
} catch (TimeoutException e) {
futureTemp.cancel(true);
log.error("TimeoutException发生,意味着线程超时报错:" + e.getMessage(), e);
errInfo = "TimeoutException发生,意味着线程超时报错";
// pool.shutdown();
// pool.shutdownNow();
}
if (StringUtils.isNotBlank(errInfo)) {
BizLogger.info(
new Bizloginfo()
.setBizloginfo(
"classMethod",
"com.easipass.subscribe.controller.TaskController.receiptTopic")
.setBizloginfo(LOGKEYS.EVENT_TYPE, "taskResult")
.setBizloginfo("uuid", taskRecordInfo.getUuid())
.setBizloginfo("taskKey", taskRecordInfo.getTaskKey())
.setBizloginfo("beginTime", taskRecordInfo.getBeginTime().toString())
.setBizloginfo("endTime", taskRecordInfo.getEndTime().toString())
.setBizloginfo(
LOGKEYS.ERR_INFO,
"任务处理异常:"
+ errInfo.substring(
0, errInfo.length() > 200 ? 200 : errInfo.length()))
.setBizloginfo("chargeType", "0")
.setBizloginfo("uuid", taskRecordInfo.getUuid())
.setBizloginfo("isSuccess", "0")
.setInfoNotEmpty(LOGKEYS.EVENT_TIME, DateUtil.getCurrentTimeISO8061())
.setBizloginfo("spendTime", System.currentTimeMillis() - startTime)
.getMap());
}
}
});
t.start();
}
} catch (Exception e) {
String errInfo = e.getMessage();
log.error("任务处理未知异常:" + errInfo, e);
BizLogger.info(
new Bizloginfo()
.setBizloginfo(
"classMethod", "com.easipass.subscribe.controller.TaskController.receiptTopic")
.setBizloginfo(LOGKEYS.EVENT_TYPE, "taskResult")
.setBizloginfo("uuid", recordInfo.getUuid())
.setBizloginfo("taskKey", recordInfo.getTaskKey())
.setBizloginfo("beginTime", recordInfo.getBeginTime().toString())
.setBizloginfo("endTime", recordInfo.getEndTime().toString())
.setBizloginfo(
LOGKEYS.ERR_INFO,
"任务处理未知异常:"
+ errInfo.substring(0, errInfo.length() > 200 ? 200 : errInfo.length()))
.setBizloginfo("chargeType", "0")
.setBizloginfo("uuid", recordInfo.getUuid())
.setBizloginfo("isSuccess", "0")
.setInfoNotEmpty(LOGKEYS.EVENT_TIME, DateUtil.getCurrentTimeISO8061())
.setBizloginfo("spendTime", System.currentTimeMillis() - startTime)
.getMap());
} finally {
taskResultPool.shutdown();
}
log.info(
"######## 监听任务处理 结束 taskKey:"
+ rlation.getTaskKey()
+ " ,uuid:"
+ recordInfo.getUuid()
+ " #######");
}
public class MyThread implements ThreadFactory {
@Override
public Thread newThread(@NonNull Runnable r) {
Thread thread = new Thread(r);
thread.setName("task_result_" + r.hashCode());
return thread;
}
}
}
水平有限,欢迎各位同行指导。