java使用多线程进行分页数据采集

最新推荐文章于 2024-07-26 11:11:16 发布

冰雪Love齐迹

最新推荐文章于 2024-07-26 11:11:16 发布

阅读量5.2k

点赞数

分类专栏： web后端开发文章标签： java java.util.concurrent ThreadPoolExecutor

本文链接：https://blog.csdn.net/liqi_q/article/details/65935285

版权

web后端开发专栏收录该内容

46 篇文章 0 订阅

订阅专栏

public  void executeList(Date startDate, Date endDate,String subgamekind) {
   System.out.println("定时任务：抓取任务--->开始------->"+ sd1.format(new Date()));
   long start1 = System.currentTimeMillis();
   int totalPage = 0;

   try {
      String start = sd1.format(startDate);
      String end = sd1.format(endDate);
      BBINBetResultApi result = GameUtil.getBBINBetRecordResult(start, end, "5", "1","",subgamekind);
      if (result != null) {
         if(result.getResult().equals("true")){
            if(result.getPagination()!=null){
               totalPage = Integer.valueOf(result.getPagination().getTotalPage());
            }else {
               totalPage = 0;
            }
         }
      }
      if(totalPage!=0) {
         // 创建一个线程池
         ThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(totalPage);
         CountDownLatch countDownLatch = new CountDownLatch(totalPage);
         for (int i = 0; i < totalPage; i++) {
            if(i==0){
               BBINBetsRunnable bbinBetsRunnable = new BBINBetsRunnable(start,end,"5",String.valueOf(i + 1),"",countDownLatch,bbinBetsLstMapper,bbinBetsSlotMapper,subgamekind,"single",result!=null?result.getData():null);
               executor.execute(bbinBetsRunnable);
               continue;
            }
            Thread.sleep(2000);
            BBINBetsRunnable bbinBetsRunnable = new BBINBetsRunnable(start,end,"5",String.valueOf(i + 1),"",countDownLatch,bbinBetsLstMapper,bbinBetsSlotMapper,subgamekind,"multi",null);
            executor.execute(bbinBetsRunnable);

         }
         countDownLatch.await();
         executor.shutdown();
      }
   } catch (Exception e) {
      e.printStackTrace();
   }
   long end1 = System.currentTimeMillis();
   logger.info("定时任务：抓取任务--->结束" + sd.format(new Date())+" 定时插入耗时："+(end1-start1));
}

package com.ig.bbin.service.impl;

import com.ig.bbin.dao.BBINBetsLstMapper;
import com.ig.bbin.dao.BBINBetsSlotMapper;
import com.ig.biservice.config.GetUserCodeUtil;
import com.ig.biservice.config.SetCustMap;
import com.ig.common.util.UtilTool;
import com.ig.gapi.result.bbin.BBINBetResultApi;
import com.ig.gapi.result.bbin.BBINbet;
import com.ig.oths.util.GameUtil;
import com.ig.sid.syssetting.domain.SidCustMap;
import org.apache.log4j.Logger;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;

/**
 *
 */
public class BBINBetsRunnable implements Runnable {
    private static final Logger logger = Logger.getLogger(BBINBetsRunnable.class);

    private String startDate ;

    private String endDate ;

    private String pageNum;

    private String gameKind;

    private String gameType;

    private String subgamekind;

    private BBINBetsLstMapper bbinBetLstMapper;

    private BBINBetsSlotMapper bbinBetSlotsMapper;

    private CountDownLatch countDownLatch ;

    private String type;

    private BBINbet[] bbinBet_Data;

    public BBINBetsRunnable(String startDate, String endDate,String gameKind, String pageNum,String gameType, CountDownLatch countDownLatch, BBINBetsLstMapper bbinBetLstMapper,BBINBetsSlotMapper bbinBetSlotsMapper,String subgamekind,String type,BBINbet[] bbinBet_Data){

        this.startDate  = startDate ;

        this.endDate = endDate ;

        this.gameKind = gameKind;

        this.pageNum = pageNum;

        this.gameType = gameType;

        this.bbinBetLstMapper = bbinBetLstMapper ;

        this.bbinBetSlotsMapper = bbinBetSlotsMapper;

        this.countDownLatch = countDownLatch ;

        this.subgamekind= subgamekind;

        this.type = type;

        this.bbinBet_Data=bbinBet_Data;

    }

    @Override
    public void run() {
        try {
            BBINbet[] bbinBetData = null;
            System.out.println("--------------------- BetRecord   注单抓取  --------------------------");
            if(type.equals("multi")){
                BBINBetResultApi bbinBetResultApi = GameUtil.getBBINBetRecordResult(startDate, endDate,gameKind, pageNum,gameType,subgamekind);
                if(null!=bbinBetResultApi && bbinBetResultApi.getResult().equals("true")){
                    if(null!=bbinBetResultApi.getData()){
                        bbinBetData = bbinBetResultApi.getData();
                    }
                }
            }else{
                bbinBetData = bbinBet_Data;
            }


            BBINbet bbiNbet = null;
            List<BBINbet> bbiNbetList = new ArrayList<BBINbet>();
            if(bbinBetData!=null && bbinBetData.length>0) {
                if (GetUserCodeUtil.custMap.isEmpty()) {
                    SetCustMap setCustMap = new SetCustMap();
                    setCustMap.addCustMap();
                }
            }
            if(bbinBetData!=null) {
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                //2.入库
                for(int i=0;i<bbinBetData.length;i++){
                    bbiNbet = bbinBetData[i];
                    String userName = bbiNbet.getUserName();
                    bbiNbet.setGameKind(gameKind);
                    bbiNbet.setWagersDate(bbiNbet.getWagersDate() == null ? null : UtilTool.headDate(sdf.format(bbiNbet.getWagersDate()), 12));

                    long hallId = 0l;
                    String gamePwd = "0";
                    SidCustMap custmapobj = GetUserCodeUtil.custCodeMap.get(userName);
                    if(null==custmapobj){
                        String newUserName = userName.toUpperCase();
                        SidCustMap custobj = GetUserCodeUtil.custCodeMap.get(newUserName);
                        if(null!=custobj){
                            gamePwd = custobj.getGamePwd();
                            hallId = custobj.getHallId();
                        }
                    }else {
                        gamePwd = custmapobj.getGamePwd();
                        hallId = custmapobj.getHallId();
                    }

                    bbiNbet.setHallId(hallId);//GetUserCodeUtil.custCodeMap.get(userName) == null ? 0l : GetUserCodeUtil.custCodeMap.get(userName).getHallId()
                    bbiNbet.setUserId(Long.valueOf(gamePwd));//GetUserCodeUtil.custCodeMap.get(userName) == null ? 0l : Long.valueOf(GetUserCodeUtil.custCodeMap.get(userName).getGamePwd())
                    if(bbiNbet.getHallId()==0l){
                        continue;
                    }
                    bbiNbetList.add(bbiNbet);
                }

                if (bbiNbetList.size() > 0) {
                 
                    if(gameKind.equals("1") || gameKind.equals("3") || gameKind.equals("12") || gameKind.equals("30")){
                        bbinBetLstMapper.replaceBBinBet(bbiNbetList);
                    }else {
                        bbinBetSlotsMapper.replaceBBinBet(bbiNbetList);
                    }

                }
            }
        } catch (Exception e) {
            logger.error("插入数据异常，异常消息："+e);
            e.printStackTrace();
        } finally {
            countDownLatch.countDown();
        }
    }
}