分页多线程执行ThreadPoolTaskExecutor

public class ScheduleConfig implements SchedulingConfigurer {

  @Bean(name = "initTaskExecutor")
  public ThreadPoolTaskExecutor initTaskExecutor() {
    ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
    taskExecutor.setCorePoolSize(4);
    taskExecutor.setMaxPoolSize(20);
    taskExecutor.setQueueCapacity(100);
    taskExecutor.setKeepAliveSeconds(3600);
    taskExecutor.setThreadNamePrefix("initTaskExecutor-");
    taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
    taskExecutor.initialize();
    return taskExecutor;
  }

  @Bean(name = "mainTaskExecutor")
  public ThreadPoolTaskExecutor mainTaskExecutor() {
    ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
    taskExecutor.setCorePoolSize(20);
    taskExecutor.setMaxPoolSize(100);
    taskExecutor.setQueueCapacity(200);
    taskExecutor.setKeepAliveSeconds(3600);
    taskExecutor.setThreadNamePrefix("mainTaskExecutor-");
    taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
    taskExecutor.initialize();
    return taskExecutor;
  }

  @Bean(name = "methodTaskExecutor")
  public ThreadPoolTaskExecutor methodTaskExecutor() {
    ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
    taskExecutor.setCorePoolSize(20);
    taskExecutor.setMaxPoolSize(100);
    taskExecutor.setQueueCapacity(200);
    taskExecutor.setKeepAliveSeconds(3600);
    taskExecutor.setThreadNamePrefix("methodTaskExecutor-");
    taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
    taskExecutor.initialize();
    return taskExecutor;
  }

  /**
   * 重置spring task默认的Executors.newSingleThreadScheduledExecutor()
   */
  @Override
  public void configureTasks(ScheduledTaskRegistrar taskRegistrar) {
    taskRegistrar.setTaskScheduler(taskScheduler());
  }

  @Bean(name = ScheduledAnnotationBeanPostProcessor.DEFAULT_TASK_SCHEDULER_BEAN_NAME)
  public ThreadPoolTaskScheduler taskScheduler() {
    ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler();
    taskScheduler.setPoolSize(20);
    /** @see java.util.concurrent.ExecutorService#awaitTermination(long, TimeUnit) */
    taskScheduler.setAwaitTerminationSeconds(60 * 10);
    taskScheduler.setWaitForTasksToCompleteOnShutdown(true);
    taskScheduler.setThreadNamePrefix("taskScheduler-");
    taskScheduler.initialize();
    return taskScheduler;
  }
public abstract class BaseInitDateTask implements InitDateTask {

  /**
   * 处理数据条数
   */
  volatile Integer processCount = 0;

  /**
   * 默认四个线程工资
   */
  volatile Integer defaultWorkThreadCount = 4;
  /**
   * 锁超时时间
   **/
  final static Integer LOCK_TIME_OUT = 10;
  /**
   * 任务数
   */
  volatile Integer taskCount = 0;
  /**
   * 拆分任务的最小行数
   */
  volatile Integer subTaskMinNum = 40;
  /**
   * 分布式锁对象
   */
  DistributedLock distributedLock = null;
  /**
   * 是否首次(有的数据做增量同步的话可以用这个字段来区分)
   */
  volatile Boolean isFirst = false;
  /**
   * 每页的处理数量，默认1000，会根据任务重算
   */
  volatile Integer defaultPageSize = 200;
  /**
   * 任务集合
   */
  List<Runnable> taskList;

  /**
   * 待处理数据集合
   */
  List<Integer> changeIdList;

  @Resource(name = "initTaskExecutor")
  private ThreadPoolTaskExecutor initTaskExecutor;

  @Override
  public boolean tryLock() {
    return false;
  }

  @Override
  public void initFlow() {
    try {
      //step 1:获取锁
      if (!tryLock()) {
        return;
      }
      //step 2:设置执行参数
      setParameter();
      //如果需要处理的数据行小于等于0 则退出
      if (processCount <= 0) {
        return;
      }
      //step 3:执行任务
      process();
    } catch (Exception e) {
      log.error("initFlow error,message:{}", e.getMessage());
      e.printStackTrace();
    } finally {
      if (distributedLock != null) {
        distributedLock.release();
      }
    }
  }

  @Override
  public void setParameter() {
    setIsFirstFlag();
    //根据需要需要重写默认的初始化配置，比如defaultPageSize，subTaskMinNum等等
    setDefaultParameter();
    //设置需要处理数据条数
    setProcessCount();
    //如果需要处理的数据行小于等于0 则退出
    if (processCount <= 0) {
      return;
    }
    //设置任务数
    setTaskCount();
    log.info("className:{},isFirst:{},taskCount:{}", this.getClass(), isFirst, taskCount);
    //设置任务
    taskList = new CopyOnWriteArrayList<>();
    setTaskList();
  }

  void setDefaultParameter() {
    //默认不需要修改参数，子类如有需要自己实现
  }

  /**
   * 设置是否是第一次初始化
   * 子类实现(每个实例的逻辑不一样)
   */
  abstract void setIsFirstFlag();

  private void process() {
    resetData();
    taskList.forEach(initTaskExecutor::execute);
  }

  /**
   * 重置数据集，(hospital是全量的，因为DIS那边的数据修改，本系统是不可知的)
   */
  abstract void resetData();

  /**
   * 设置需要处理的数据条数
   * 子类实现(每个实例的逻辑不一样)
   */
  abstract void setProcessCount();

  /**
   * 设置执行任务
   * 子类可以重写实现(每个实例的逻辑可能不一样)
   */
  void setTaskList() {
    Runnable task;
    if (isFirst) {
      for (int i = 0; i < taskCount; i++) {
        task = getThread(i, defaultPageSize);
        taskList.add(task);
      }
    } else {
      List<List<Integer>> subLists = ListUtil.groupList(changeIdList, taskCount);
      for (List<Integer> subList : subLists) {
        task = getThread(subList);
        taskList.add(task);
      }
    }
  }

  /**
   * 获取线程任务
   *
   * @return
   */
  abstract Runnable getThread(Integer pageNum, Integer pageSize);

  /**
   * 获取线程任务
   *
   * @return
   */
  abstract Runnable getThread(List<Integer> idList);

  /**
   * 设置任务的数量，默认每次处理1000条数据，如果是增量同步的话，会根据需要处理的数据除以最大的工作线程得出任务数
   * 子类可以通过重新该方法来自定义
   */
  private void setTaskCount() {
    if (!isFirst) {
      //少于40条则不需要拆分线程去执行
      if (processCount >= subTaskMinNum) {
        int newPageSize = processCount / defaultWorkThreadCount;
        //如果大于1000则每页最多处理1000条
        if (newPageSize < defaultPageSize) {
          defaultPageSize = newPageSize;
        }
      } else {
        //如果修改数量少于线程数，则直接一次执行完
        defaultPageSize = processCount;
      }
    }
    //需要处理的次数
    int count = processCount / defaultPageSize;
    if (count % defaultPageSize != 0) {
      count = count + 1;
    }
    taskCount = count;
  }

@Component
public class DoctorCooperateInitDateTask extends BaseInitDateTask{

  /**
   * 分布式锁
   **/
  private final static String LOCK_KEY = "init:doctor:cooperateNum";

  @Resource
  private DcDoctorCooperateInfoService dcDoctorCooperateInfoService;
  @Resource
  private DcTrialDoctorService dcTrialDoctorService;

  @Override
  public boolean tryLock() {
    distributedLock = DistributedLockUtil.getDistributedLock(LOCK_KEY, LOCK_TIME_OUT);
    return distributedLock.acquire();
  }

  @Override
  void setIsFirstFlag() {
    Integer totalCount = dcDoctorCooperateInfoService.getTotalCount();
    isFirst = totalCount == null || totalCount == 0;
    log.info("doctorCooperate totalCount:{}",totalCount);
  }

  @Override
  void resetData() {
  }

  @Override
  void setProcessCount() {
    //首次全量初始化，非首次增量更新
    if (isFirst) {
      processCount = dcTrialDoctorService.getMultipleTrialsDoctorCount();
    } else {
      changeIdList = dcDoctorCooperateInfoService.findLastDayChangeDoctorIdList();
      processCount = changeIdList.size();
    }
    log.info("doctorCooperate processCount:{}",processCount);
  }

  @Override
  Runnable getThread(Integer pageNum,Integer pageSize) {
    return new InitThead(isFirst, pageNum, pageSize);
  }

  @Override
  Runnable getThread(List<Integer> idList) {
    return new InitThead(isFirst, idList);
  }


  public class InitThead implements Runnable {
    //分页开始index
    private Integer pageNum;
    //分页每页条数
    private Integer pageSize;

    private Boolean isFirst;

    private List<Integer> doctorIdList;

    public InitThead(Boolean isFirst,Integer page, Integer pageSize) {
      this.pageNum = page * pageSize;
      this.pageSize = pageSize;
      this.isFirst = isFirst;
    }

    public InitThead(Boolean isFirst, List<Integer> doctorIdList) {
      this.isFirst = isFirst;
      this.doctorIdList = doctorIdList;
    }

    @Override
    public void run() {
      if (isFirst) {
        dcDoctorCooperateInfoService.initDate(pageNum,pageSize);
      } else {
        dcDoctorCooperateInfoService.initDate(doctorIdList);
      }
    }
  }


public class DoctorCooperateJobHandler extends IJobHandler {
  @Resource
  BaseInitDateTask doctorCooperateInitDateTask;
  @Resource(name = "mainTaskExecutor")
  private ThreadPoolTaskExecutor mainTaskExecutor;

  @Override
  public ReturnT<String> execute(String s) {
    mainTaskExecutor.submit(new CommonInitThead(doctorCooperateInitDateTask));
    return SUCCESS;
  }

分页执行任务，提供参考。