多线程处理大量数据

一、背景

最近在在建设银行代发功能，需要批量处理代发数据，线性处理效率比较低，大概1秒钟一条数据；然后考虑用多线程处理。刚开始使用的是Executors.newFixedThreadPool(10)这种方式创建固定数量线程池；后面考虑到数据量大的时候，可能队列装不下，导致异常，然后换成了ExecutorService executorSer = new ThreadPoolExecutor(10,30,10,TimeUnit.SECONDS, new LinkedBlockingDeque<>(2000),new ThreadPoolExecutor.CallerRunsPolicy());

二、代码

CountDownLatch latch = new CountDownLatch(10);
ExecutorService executorSer = new ThreadPoolExecutor(10,30,10,TimeUnit.SECONDS, new LinkedBlockingDeque<>(2000),new ThreadPoolExecutor.CallerRunsPolicy());
for (ZfPayrollBatchRecDO batchRecDO:zfPayrollBatchRecDOS){
	executorSer.execute(new PayRollBatchRunnable(latch, batchRecDO, tyPubSysParamCacheClient, dsk075Service, zfPayrollBatchRecService, zfPayrollBatchIntfRecService, dsk074Service));
}
latch.wait();
executorSer.shutdown();

public class PayRollBatchRunnable implements Runnable {

    private static final Logger logger = LoggerFactory.getLogger(PayRollBatchRunnable.class);

    private ZfPayrollBatchService zfPayrollBatchService;

    private ZfPayrollBatchRecService zfPayrollBatchRecService;

    private DSK075Service dsk075Service;

    private DSK074Service dsk074Service;

    private ZfPayrollBatchIntfRecService zfPayrollBatchIntfRecService;

    private TyPubSysParamCacheClient tyPubSysParamCacheClient;

    private final CountDownLatch latch;

    private final ZfPayrollBatchRecDO batchRecDO;

    public PayRollBatchRunnable(CountDownLatch latch,ZfPayrollBatchRecDO batchRecDO,TyPubSysParamCacheClient client,DSK075Service dsk075Service,ZfPayrollBatchRecService zfPayrollBatchRecService,ZfPayrollBatchIntfRecService zfPayrollBatchIntfRecService,DSK074Service dsk074Service){
        this.latch = latch;
        this.batchRecDO = batchRecDO;
        this.tyPubSysParamCacheClient = client;
        this.dsk075Service = dsk075Service;
        this.dsk074Service = dsk074Service;
        this.zfPayrollBatchRecService = zfPayrollBatchRecService;
        this.zfPayrollBatchIntfRecService = zfPayrollBatchIntfRecService;
    }

    @Override
    public void run() {
        try {
            doWork(this.batchRecDO);
            logger.info("当前代发处理信息:{}",JSON.toJSONString(batchRecDO));
            latch.countDown();
        } catch (Exception ex) {
            logger.info("异常信息：{}",ex.getMessage());
        }
    }

    public void doWork(ZfPayrollBatchRecDO batchRecDO) {
        logger.info("work开始处理");
        //2.代发完成后修改-代发业务文件批次表处理状态
        DSK074Vo dsk074Vo = new DSK074Vo();
        dsk074Vo.setAmount(batchRecDO.getTranAmt().toString());
        dsk074Vo.setCorpSerno(batchRecDO.getCorpSerNo());//前置请求流水号
        dsk074Vo.setRecSvcID(batchRecDO.getSvcId());
        dsk074Vo.setRecCstAccNo(batchRecDO.getCstAccNo());
        dsk074Vo.setTrdPCstAccNo(tyPubSysParamCacheClient.getParamValue("CCB_THRD_PLAT_CST_ACC_NO","44201501100056411087"));//代发第三方平台账号
        long startTime = System.currentTimeMillis();
        DSK074Res dsk074Res = dsk074Service.dsk074(dsk074Vo);
        long endTime =  System.currentTimeMillis();
        long  intfExeTime = endTime - startTime;
        boolean incRetFlag = false;//接口调用成功标识
        if ("0000".equals(dsk074Res.getReturnCode())){
            //代发业务批次流水表记录状态
            ZfPayrollBatchRecDO zfPayrollBatchRecDO = new ZfPayrollBatchRecDO();
            zfPayrollBatchRecDO.setBtcId(batchRecDO.getBtcId());
            zfPayrollBatchRecDO.setSeqNo(batchRecDO.getSeqNo());
            zfPayrollBatchRecDO.setDealStatus("1");//处理状态：0：待处理，1：处理成功，2：处理失败
            zfPayrollBatchRecDO.setEvtTraceId(dsk074Res.getSysEvtTraceId());
            try {
                zfPayrollBatchRecService.updateSelective(zfPayrollBatchRecDO);
            }catch (Exception  e){
                logger.error("代发业务批次流水表记录更新失败:{}",e.getMessage());
            }
            incRetFlag = true;
        }else {
            //如果失败，反查询下代发结果
            DSK075Vo dsk075Vo = new DSK075Vo();
            dsk075Vo.setOrigCorpSerno(batchRecDO.getCorpSerNo());
            DSK075Res dsk075Res = dsk075Service.dsk075(dsk075Vo);
            if ("0000".equals(dsk075Res.getReturnCode())){
                ZfPayrollBatchRecDO zfPayrollBatchRecDO = new ZfPayrollBatchRecDO();
                zfPayrollBatchRecDO.setBtcId(batchRecDO.getBtcId());
                zfPayrollBatchRecDO.setSeqNo(batchRecDO.getSeqNo());
                zfPayrollBatchRecDO.setDealStatus("1");//处理状态：0：待处理，1：处理成功，2：处理失败
                zfPayrollBatchRecDO.setEvtTraceId(dsk075Res.getSysEvtTraceId());
                try {
                    zfPayrollBatchRecService.updateSelective(zfPayrollBatchRecDO);
                }catch (Exception  e){
                    logger.error("代发业务批次流水表记录更新失败:{}",e.getMessage());
                }
                incRetFlag = true;
            }else {
                ZfPayrollBatchRecDO zfPayrollBatchRecDO = new ZfPayrollBatchRecDO();
                zfPayrollBatchRecDO.setBtcId(batchRecDO.getBtcId());
                zfPayrollBatchRecDO.setSeqNo(batchRecDO.getSeqNo());
                zfPayrollBatchRecDO.setDealStatus("2");//处理状态：0：待处理，1：处理成功，2：处理失败
                try {
                    zfPayrollBatchRecService.updateSelective(zfPayrollBatchRecDO);
                }catch (Exception  e){
                    logger.error("代发业务批次流水表记录更新失败:{}",e.getMessage());
                }
                incRetFlag = false;
            }

        }
        //新增代发接口调用记录
        ZfPayrollBatchIntfRecDO intfRecDO = new ZfPayrollBatchIntfRecDO();
        intfRecDO.setBtcId(batchRecDO.getBtcId());
        intfRecDO.setSeqNo(batchRecDO.getSeqNo());
        intfRecDO.setAcctDay(DateUtils.getCurrentDateNumber());
        intfRecDO.setInParam(JSON.toJSONString(dsk074Vo));
        intfRecDO.setOutParam(JSON.toJSONString(dsk074Res));
        intfRecDO.setAddTime(new Date());
        if (incRetFlag){
            intfRecDO.setIncRetFlag("1");//接口调用结果 0：未返回，1：返回成功，2：返回失败
        }else {
            intfRecDO.setIncRetFlag("2");//接口调用结果 0：未返回，1：返回成功，2：返回失败
        }
        intfRecDO.setIncRetCode(dsk074Res.getReturnCode());
        intfRecDO.setIncRetRemark(dsk074Res.getReturnMsg());
        intfRecDO.setEvtTraceId(dsk074Res.getSysEvtTraceId());
        intfRecDO.setIncCallNum(1);
        intfRecDO.setIncLastTime(new Date());
        intfRecDO.setIncExeTime(CallTimeUnit.getTimeUnit(intfExeTime));
        intfRecDO.setPartDate(DateUtils.getCurrentDateNumber());
        try {
            zfPayrollBatchIntfRecService.insertSelective(intfRecDO);
        }catch (Exception e){
            logger.error("代发调用接口记录失败:{}",e.getMessage());
        }

        logger.info("work结束处理");
    }
}

三、总结

        原本120条的数据，线性执行，需要将近120秒。差不多一秒一条数据；调整后，500条数据只需要12-14秒；多线程时真的强大。以后要多研究多线程，这个功能还是挺牛叉的？