java ExecutorService 多线程更新表数据

java ExecutorService 多线程更新表数据

ExecutorServiceImportDataTask 类定时任务配置调度方法

package com.realize.framework.task;

import com.realize.project.system.service.IBusiPublicNoticeSanshiService;
import com.realize.project.system.service.IBusiPublicNoticeService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.*;

/**
 * Java 多线程：有返回值的线程
 *
 */
@SuppressWarnings("unchecked")
@Slf4j
@Component("executorServiceImportDataTask")
public class ExecutorServiceImportDataTask {


    @Autowired
    private IBusiPublicNoticeSanshiService busiPublicNoticeSanshiService;

    @Autowired
    private IBusiPublicNoticeService busiPublicNoticeService;

    /**
     * 1-2 三市公告500万-只同步OSS表信息
     */
    public void  synchJianweiToRzSanshiOnlyOss(){
        log.info("----三市公告500万 OnlyOss 多线程ExecutorService 程序开始运行----");
        Date dateStart = new Date();
        int taskSize = 5;
        // 创建一个线程池
        ExecutorService pool = Executors.newFixedThreadPool(taskSize);
        // 创建多个有返回值的任务
        List<Future> list = new ArrayList<Future>();
        for (int i = 0; i < taskSize; i++) {
            Callable c = new MyOssMetaShszUpdateCallable(i + " ",busiPublicNoticeSanshiService);
            // 执行任务并获取Future对象
            Future f = pool.submit(c);
            // System.out.println(">>>" + f.get().toString());
            list.add(f);
        }
        // 关闭线程池
        pool.shutdown();
        // 获取所有并发任务的运行结果
        for (Future f : list) {
            // 从Future对象上获取任务的返回值，并输出到控制台
            try {
                log.info("----ExecutorServiceImportDataTask" + f.get().toString());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }
        Date dateEnd = new Date();
        log.info("----三市公告500万 OnlyOss 多线程ExecutorService 程序结束运行----，程序运行时间【" + (dateEnd.getTime() - dateStart.getTime()) + "毫秒】");
        //busiPublicNoticeSanshiService.synchOldDataJianweiToRzSanshiOnlyOss();
    }

    /**
     * 2-2 股权激励公告80万-只同步OSS表
     */
    public void  synchJianweiToRzOnlyOss(){
        log.info("----股权激励公告80万 OnlyOss 多线程ExecutorService 程序开始运行----");
        Date dateStart = new Date();
        int taskSize = 5;
        // 创建一个线程池
        ExecutorService pool = Executors.newFixedThreadPool(taskSize);
        // 创建多个有返回值的任务
        List<Future> list = new ArrayList<Future>();
        for (int i = 0; i < taskSize; i++) {
            Callable c = new MyOssMetaGqUpdateCallable(i + " ",busiPublicNoticeService);
            // 执行任务并获取Future对象
            Future f = pool.submit(c);
            // System.out.println(">>>" + f.get().toString());
            list.add(f);
        }
        // 关闭线程池
        pool.shutdown();
        // 获取所有并发任务的运行结果
        for (Future f : list) {
            // 从Future对象上获取任务的返回值，并输出到控制台
            try {
                log.info("----ExecutorServiceImportDataTask" + f.get().toString());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }
        Date dateEnd = new Date();
        log.info("----股权激励公告80万 OnlyOss 多线程ExecutorService 程序结束运行----，程序运行时间【" + (dateEnd.getTime() - dateStart.getTime()) + "毫秒】");
    }

}


/**
 * 内部任务类-500万三市公告
 */
class MyOssMetaShszUpdateCallable implements Callable<Object> {
    private String taskNum;
    private IBusiPublicNoticeSanshiService busiPublicNoticeSanshiService;
    MyOssMetaShszUpdateCallable(String taskNum,IBusiPublicNoticeSanshiService busiPublicNoticeSanshiService) {
        this.taskNum = taskNum;
        this.busiPublicNoticeSanshiService = busiPublicNoticeSanshiService;
    }
    public Object call() throws Exception {
        System.out.println(">>>" + taskNum + "任务启动");
        Date dateTmp1 = new Date();
        //循环更新200条表记录状态值
        busiPublicNoticeSanshiService.synchOldDataJianweiToRzSanshiOnlyOss();
        Date dateTmp2 = new Date();
        long time = dateTmp2.getTime() - dateTmp1.getTime();
        System.out.println(">>>" + taskNum + "任务终止");
        return taskNum + "任务返回运行结果,当前任务时间【" + time + "毫秒】";
    }
}

/**
 * 内部任务类-80万股权激励公告
 */
class MyOssMetaGqUpdateCallable implements Callable<Object> {
    private String taskNum;
    private IBusiPublicNoticeService busiPublicNoticeService;
    MyOssMetaGqUpdateCallable(String taskNum,IBusiPublicNoticeService busiPublicNoticeService) {
        this.taskNum = taskNum;
        this.busiPublicNoticeService = busiPublicNoticeService;
    }
    public Object call() throws Exception {
        System.out.println(">>>" + taskNum + "任务启动");
        Date dateTmp1 = new Date();
       //循环更新200条表记录状态值
        busiPublicNoticeService.synchOldDataJianweiToRzOnlyOss();
        Date dateTmp2 = new Date();
        long time = dateTmp2.getTime() - dateTmp1.getTime();
        System.out.println(">>>" + taskNum + "任务终止");
        return taskNum + "任务返回运行结果,当前任务时间【" + time + "毫秒】";
    }
}

IBusiPublicNoticeSanshiService 实现类 BusiPublicNoticeSanshiServiceImpl，

循环更新200条记录状态提交到es 

@DataSource(value = DataSourceType.SLAVE)
    public Boolean synchOldDataJianweiToRzSanshiOnlyOss() {
        String uuid=UUID.randomUUID().toString();
        log.info(uuid+"只是同步OSS表【560万条三市公告】开始同步到ES  synchJianweiToRzSanshi()"+DateUtils.dateTimeNow());
        // 记录起始时间
        long start = System.currentTimeMillis();
        //2查询出MySQL 需要同步的公告数据 | 1增量更新pdf路径 /2增量同步到ES
        OssMetaGq ossMetaGq=new OssMetaGq();
        // ossMetaGq.setLimitIndex(p*10L);
        ossMetaGq.setStatus(3);
        //ossMetaGq.setPublishdate(DateUtils.parseDate("2019-01-01"));
        List<OssMetaGq> searchList = ossMetaGqService.selectJwOssMetaSanshiListSqlOnlyOss(ossMetaGq);
        List<String> ossMetaGqList = searchList.stream().filter(ossMeta -> StringUtils.isNotBlank(ossMeta.getStockcode())).map(OssMetaGq::getStockcode).distinct().collect(Collectors.toList());
        Map<String, AshareOwnership> ashareOwnershipMap=null;
        if(ossMetaGqList.size()>0){
            List<AshareOwnership> ashareOwnershipList = ashareOwnershipService.selectAshareOwnershipByCompanyCode(ossMetaGqList);
            ashareOwnershipMap = ashareOwnershipList.stream().collect(Collectors.toMap(AshareOwnership::getCompanyCode, Function.identity(),(key1,key2)->key2));
        }
        if(CollectionUtils.isNotEmpty(searchList)) {
            BulkRequest bulkRequest = new BulkRequest();
            bulkRequest.timeout("60m");
            for (int i = 0; i < searchList.size(); i++) {
                OssMetaGq busiPublicNoticeEs = searchList.get(i);
                AshareOwnership ashareOwnership = ashareOwnershipMap.get(busiPublicNoticeEs.getStockcode());
                if (Objects.nonNull(ashareOwnership)){
                    busiPublicNoticeEs.setOwnership(ashareOwnership.getOwnership());
                    busiPublicNoticeEs.setOwnershipcode(ashareOwnership.getOwnCode());
                }
                Long publicId = busiPublicNoticeEs.getId();
                //5同步结果无异常记录Mysql;
                OssMetaGq ossMetaGqOne = new OssMetaGq();
                ossMetaGqOne.setId(busiPublicNoticeEs.getId());
                ossMetaGqOne.setStatus(4);//0初始化，1完成，2失败；
                ossMetaGqMapper.updateOssMetaGqSanshi(ossMetaGqOne);
                boolean idHitsExist = filterReappear(publicId);
                try {
                    if(idHitsExist){//已存在则更新
                        //log.info("三市公告 Notice UpdateRequest publicId ="+publicId);
                        bulkRequest.add(new UpdateRequest(clusterName,publicId.toString()).doc(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON));
                    }else{//插入
                        //log.info("三市公告 Notice IndexRequest publicId ="+publicId);
                        bulkRequest.add(new IndexRequest(clusterName).source(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON).id(publicId.toString()));
                    }
                    //bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
                    //restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
                    //ossMetaGqOne.setParseContent(convertTxtContent.toString());
                    //ossMetaGqMapper.updateOssMetaGq(ossMetaGqOne);
                    //}
                    //}
                } catch (Exception e) {
                    //6同步结果有异常记录Mysql;
                    //repeatIdsMap.remove(publicId);//删除缓存
                    OssMetaGq ossMetaGqOneTry = new OssMetaGq();
                    ossMetaGqOneTry.setId(busiPublicNoticeEs.getId());
                    ossMetaGqOneTry.setStatus(5);//手动重置0状态，下次继续尝试；
                    //ossMetaGqOne.setProcessStatus(4);
                    ossMetaGqMapper.updateOssMetaGqSanshi(ossMetaGqOneTry);
                    if(idHitsExist){//已存在则更新
                        //log.info("三市公告 catch UpdateRequest JSON="+publicId);
                        bulkRequest.add(new UpdateRequest(clusterName,publicId.toString()).doc(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON));
                    }else{//插入
                        //log.info("三市公告 catch IndexRequest JSON="+publicId);
                        bulkRequest.add(new IndexRequest(clusterName).source(JSON.toJSONString(busiPublicNoticeEs), XContentType.JSON).id(publicId.toString()));
                    }
                    log.error("三市公告 6按行读取见微TXT文本内容  同步结果有异常记录Mysql" + e.getMessage());
                    e.printStackTrace();
                }
            }
            if(bulkRequest.requests()!=null && bulkRequest.requests().size()>0){
                try {//批量提交
                    bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
                    restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
                }catch(IOException e){
                    log.info("三市公告 批量提交失败：{}", e);
                    e.printStackTrace();
                }
            }
            Long mitue = (System.currentTimeMillis() - start)/1000;
            //log.info("三市公告 批量处理总计耗时分钟结果：{}", mitue);
            //return !bulk.hasFailures();
        }
        log.info(uuid+"只是同步OSS表【560万条三市公告】结束同步到ES synchJianweiToRzSanshi()"+DateUtils.dateTimeNow());
        return true;
    }