java ExecutorService 多线程更新表数据
ExecutorServiceImportDataTask 类定时任务配置调度方法
package com.realize.framework.task;
import com.realize.project.system.service.IBusiPublicNoticeSanshiService;
import com.realize.project.system.service.IBusiPublicNoticeService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.*;
/**
* Java 多线程:有返回值的线程
*
*/
@SuppressWarnings("unchecked")
@Slf4j
@Component("executorServiceImportDataTask")
public class ExecutorServiceImportDataTask {
@Autowired
private IBusiPublicNoticeSanshiService busiPublicNoticeSanshiService;
@Autowired
private IBusiPublicNoticeService busiPublicNoticeService;
/**
* 1-2 三市公告500万-只同步OSS表信息
*/
public void synchJianweiToRzSanshiOnlyOss(){
log.info("----三市公告500万 OnlyOss 多线程ExecutorService 程序开始运行----");
Date dateStart = new Date();
int taskSize = 5;
// 创建一个线程池
ExecutorService pool = Executors.newFixedThreadPool(taskSize);
// 创建多个有返回值的任务
List<Future> list = new ArrayList<Future>();
for (int i = 0; i < taskSize; i++) {
Callable c = new MyOssMetaShszUpdateCallable(i + " ",busiPublicNoticeSanshiService);
// 执行任务并获取Future对象
Future f = pool.submit(c);
// System.out.println(">>>" + f.get().toString());
list.add(f);
}
// 关闭线程池
pool.shutdown();
// 获取所有并发任务的运行结果
for (Future f : list) {
// 从Future对象上获取任务的返回值,并输出到控制台
try {
log.info("----ExecutorServiceImportDataTask" + f.get().toString());
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
Date dateEnd = new Date();
log.info("----三市公告500万 OnlyOss 多线程ExecutorService 程序结束运行----,程序运行时间【" + (dateEnd.getTime() - dateStart.getTime()) + "毫秒】");
//busiPublicNoticeSanshiService.synchOldDataJianweiToRzSanshiOnlyOss();
}
/**
* 2-2 股权激励公告80万-只同步OSS表
*/
public void synchJianweiToRzOnlyOss(){
log.info("----股权激励公告80万 OnlyOss 多线程ExecutorService 程序开始运行----");
Date dateStart = new Date();
int taskSize = 5;
// 创建一个线程池
ExecutorService pool = Executors.newFixedThreadPool(taskSize);
// 创建多个有返回值的任务
List<Future> list = new ArrayList<Future>();
for (int i = 0; i < taskSize; i++) {
Callable c = new MyOssMetaGqUpdateCallable(i + " ",busiPublicNoticeService);
// 执行任务并获取Future对象
Future f = pool.submit(c);
// System.out.println(">>>" + f.get().toString());
list.add(f);
}
// 关闭线程池
pool.shutdown();
// 获取所有并发任务的运行结果
for (Future f : list) {
// 从Future对象上获取任务的返回值,并输出到控制台
try {
log.info("----ExecutorServiceImportDataTask" + f.get().toString());
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
Date dateEnd = new Date();
log.info("----股权激励公告80万 OnlyOss 多线程ExecutorService 程序结束运行----,程序运行时间【" + (dateEnd.getTime() - dateStart.getTime()) + "毫秒】");
}
}
/**
* 内部任务类-500万三市公告
*/
class MyOssMetaShszUpdateCallable implements Callable<Object> {
private String taskNum;
private IBusiPublicNoticeSanshiService busiPublicNoticeSanshiService;
MyOssMetaShszUpdateCallable(String taskNum,IBusiPublicNoticeSanshiService busiPublicNoticeSanshiService) {
this.taskNum = taskNum;
this.busiPublicNoticeSanshiService = busiPublicNoticeSanshiService;
}
public Object call() throws Exception {
System.out.println(">>>" + taskNum + "任务启动");
Date dateTmp1 = new Date();
//循环更新200条表记录状态值
busiPublicNoticeSanshiService.synchOldDataJianweiToRzSanshiOnlyOss();
Date dateTmp2 = new Date();
long time = dateTmp2.getTime() - dateTmp1.getTime();
System.out.println(">>>" + taskNum + "任务终止");
return taskNum + "任务返回运行结果,当前任务时间【" + time + "毫秒】";
}
}
/**
* 内部任务类-80万股权激励公告
*/
class MyOssMetaGqUpdateCallable implements Callable<Object> {
private String taskNum;
private IBusiPublicNoticeService busiPublicNoticeService;
MyOssMetaGqUpdateCallable(String taskNum,IBusiPublicNoticeService busiPublicNoticeService) {
this.taskNum = taskNum;
this.busiPublicNoticeService = busiPublicNoticeService;
}
public Object call() throws Exception {
System.out.println(">>>" + taskNum + "任务启动");
Date dateTmp1 = new Date();
//循环更新200条表记录状态值
busiPublicNoticeService.synchOldDataJianweiToRzOnlyOss();
Date dateTmp2 = new Date();
long time = dateTmp2.getTime() - dateTmp1.getTime();
System.out.println(">>>" + taskNum + "任务终止");
return taskNum + "任务返回运行结果,当前任务时间【" + time + "毫秒】";
}
}
IBusiPublicNoticeSanshiService 实现类 BusiPublicNoticeSanshiServiceImpl,
循环更新200条记录状态提交到es
@DataSource(value = DataSourceType.SLAVE)
public Boolean synchOldDataJianweiToRzSanshiOnlyOss() {
String uuid=UUID.randomUUID().toString();
log.info(uuid+"只是同步OSS表【560万条三市公告】开始同步到ES synchJianweiToRzSanshi()"+DateUtils.dateTimeNow());
// 记录起始时间
long start = System.currentTimeMillis();
//2查询出MySQL 需要同步的公告数据 | 1增量更新pdf路径 /2增量同步到ES
OssMetaGq ossMetaGq=new OssMetaGq();
// ossMetaGq.setLimitIndex(p*10L);
ossMetaGq.setStatus(3);
//ossMetaGq.setPublishdate(DateUtils.parseDate("2019-01-01"));
List<OssMetaGq> searchList = ossMetaGqService.selectJwOssMetaSanshiListSqlOnlyOss(ossMetaGq);
List<String> ossMetaGqList = searchList.stream().filter(ossMeta -> StringUtils.isNotBlank(ossMeta.getStockcode())).map(OssMetaGq::getStockcode).distinct().collect(Collectors.toList());
Map<String, AshareOwnership> ashareOwnershipMap=null;
if(ossMetaGqList.size()>0){
List<AshareOwnership> ashareOwnershipList = ashareOwnershipService.selectAshareOwnershipByCompanyCode(ossMetaGqList);
ashareOwnershipMap = ashareOwnershipList.stream().collect(Collectors.toMap(AshareOwnership::getCompanyCode, Function.identity(),(key1,key2)->key2));
}
if(CollectionUtils.isNotEmpty(searchList)) {
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("60m");
for (int i = 0; i < searchList.size(); i++) {
OssMetaGq busiPublicNoticeEs = searchList.get(i);
AshareOwnership ashareOwnership = ashareOwnershipMap.get(busiPublicNoticeEs.getStockcode());
if (Objects.nonNull(ashareOwnership)){
busiPublicNoticeEs.setOwnership(ashareOwnership.getOwnership());
busiPublicNoticeEs.setOwnershipcode(ashareOwnership.getOwnCode());
}
Long publicId = busiPublicNoticeEs.getId();
//5同步结果无异常记录Mysql;
OssMetaGq ossMetaGqOne = new OssMetaGq();
ossMetaGqOne.setId(busiPublicNoticeEs.getId());
ossMetaGqOne.setStatus(4);//0初始化,1完成,2失败;
ossMetaGqMapper.updateOssMetaGqSanshi(ossMetaGqOne);
boolean idHitsExist = filterReappear(publicId);
try {
if(idHitsExist){//已存在则更新
//log.info("三市公告 Notice UpdateRequest publicId ="+publicId);
bulkRequest.add(new UpdateRequest(clusterName,publicId.toString()).doc(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON));
}else{//插入
//log.info("三市公告 Notice IndexRequest publicId ="+publicId);
bulkRequest.add(new IndexRequest(clusterName).source(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON).id(publicId.toString()));
}
//bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
//restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
//ossMetaGqOne.setParseContent(convertTxtContent.toString());
//ossMetaGqMapper.updateOssMetaGq(ossMetaGqOne);
//}
//}
} catch (Exception e) {
//6同步结果有异常记录Mysql;
//repeatIdsMap.remove(publicId);//删除缓存
OssMetaGq ossMetaGqOneTry = new OssMetaGq();
ossMetaGqOneTry.setId(busiPublicNoticeEs.getId());
ossMetaGqOneTry.setStatus(5);//手动重置0状态,下次继续尝试;
//ossMetaGqOne.setProcessStatus(4);
ossMetaGqMapper.updateOssMetaGqSanshi(ossMetaGqOneTry);
if(idHitsExist){//已存在则更新
//log.info("三市公告 catch UpdateRequest JSON="+publicId);
bulkRequest.add(new UpdateRequest(clusterName,publicId.toString()).doc(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON));
}else{//插入
//log.info("三市公告 catch IndexRequest JSON="+publicId);
bulkRequest.add(new IndexRequest(clusterName).source(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON).id(publicId.toString()));
}
log.error("三市公告 6按行读取见微TXT文本内容 同步结果有异常记录Mysql" + e.getMessage());
e.printStackTrace();
}
}
if(bulkRequest.requests()!=null && bulkRequest.requests().size()>0){
try {//批量提交
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}catch(IOException e){
log.info("三市公告 批量提交失败:{}", e);
e.printStackTrace();
}
}
Long mitue = (System.currentTimeMillis() - start)/1000;
//log.info("三市公告 批量处理总计耗时分钟结果:{}", mitue);
//return !bulk.hasFailures();
}
log.info(uuid+"只是同步OSS表【560万条三市公告】结束同步到ES synchJianweiToRzSanshi()"+DateUtils.dateTimeNow());
return true;
}