使用springbatch实现批处理（自带分页）

       我个人理解springbatch虽然是一门古老的艺术，在传统金融行业的日切/日终结算中发挥着余光，但是在目前的微服务架构或者中台架构设计中还是存有一席之地的。

       在微服务和中台架构设计中经常碰到的场景，如数据批处理（群发短信/邮件，批量数据同步分发）。

       个人整理一个数据分发场景，为了不侵入业务，分为待分发数据初始化 / 增量待分发数据初始化 / 数据分发。

       首先，maven加入依赖：

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-batch</artifactId>
            <version>2.3.2.RELEASE</version>
        </dependency>

      数据初始化Reader(支持分页):

public class InitDataReader extends JdbcPagingItemReader<Map<String,Object>> {


    public SkuInitDataReader(InitStepContext context){
        log.info("begin to init InitDataReader.");
        setDataSource(context.getDataSource());
        /**
         * 设置一次最大读取条数
         */
        setFetchSize(100);
        setRowMapper(new ResultMapper());

        setParameterValues(MapTools.parseObj2Map(context));

        MySqlPagingQueryProvider queryProvider = ProviderUtils.buildProviderByJsql(context.getQuerySql());
        /**
         * 设置查询配置
         */
        setQueryProvider(queryProvider);

        log.info("InitDataReader created.");
        try {
            afterPropertiesSet();
        } catch (Exception e) {
            log.error("InitDataReader created failed.",e);
        }
    }


}

数据处理Processor:

public class InitDataProcessor implements ItemProcessor<Map<String,Object>,Map<String,Object>> {

    @Setter
    @Getter
    private InitStepContext context;

    @Override
    public Map<String,Object> process(Map<String,Object> item) throws Exception {

        //具体处理逻辑    
        processHandler(item);

        return processMap ;
    }

数据结果保存Writer:

public class InitDataWriter implements ItemWriter<Map<String,Object>> {


    @Setter
    @Getter
    private InitStepContext context;

    @Override
    public void write(List<? extends Map<String,Object>> items){
        try {
            writeSQL((List<Map<String,Object>>)items);
        }catch (Exception e){
            log.error("InitDataWriter writer error : {}", e);
            throw new BatchWriteException("InitDataWriter writer error,context is " + context);
        }
    }

    private void writeSQL(List<Map<String,Object>> items){
        if (!CollectionUtils.isEmpty(items)) {
            NamedParameterJdbcTemplate namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(context.getDataSource());
            namedParameterJdbcTemplate.batchUpdate(context.getInsertSQL(), SqlParameterSourceUtils.createBatch(items));
        }
    }
}

数据Step构建：

@Component
public class InitDataStep extends AbstractBatchStep<Map<String,Object>, Map<String,Object>> {

    @Override
    protected ItemProcessor<Map<String,Object>, Map<String,Object>> createBatchItemProcessor(SkuInitStepContext context) {
        InitDataProcessor itemProcessor = new InitDataProcessor();
        itemProcessor.setContext(context);
        return itemProcessor;
    }


    @Override
    protected JdbcPagingItemReader<Map<String,Object>> createBatchReader(SkuInitStepContext context) {
        InitDataReader reader = new InitDataReader(context);
        return reader;
    }

    @Override
    protected ItemWriter<Map<String,Object>> createBatchWriter(InitStepContext context) {
        InitDataWriter writer =new InitDataWriter();
        writer.setContext(context);
        return writer;
    }
}

public abstract class AbstractBatchStep<I, O> {

    @Setter
    @Getter
    private int chuntSize = 2000;

    @Setter
    @Getter
    private boolean hasProcessorValidator = false;

    @Setter
    @Getter
    private ItemProcessor<I, O> processor;

    @Setter
    @Getter
    private JdbcPagingItemReader<I> reader;

    @Setter
    @Getter
    private Step step;

    @Setter
    @Getter
    private StepBuilderFactory stepBuilderFactory;

    @Setter
    @Getter
    private ItemWriter<O> writer;

    @Setter
    @Getter
    private int skipLimit =10;

    private boolean isNeedskip = false;

    public AbstractBatchStep<I, O> buildBatchStep(String stepName, StepBuilderFactory stepBuilderFactory
            , ThreadPoolTaskExecutor executor, InitStepContext context) {
        this.stepBuilderFactory = stepBuilderFactory;
        // 创建reader
        this.reader = createBatchReader(context);
        // 创建writer
        this.writer = createBatchWriter(context);
        // 创建processor
        this.processor = createBatchItemProcessor(context);

        // 创建step
        this.step = createBatchStep(stepName, executor);

        return this;
    }

    protected abstract ItemProcessor<I, O> createBatchItemProcessor(InitStepContext task);

    protected abstract JdbcPagingItemReader<I> createBatchReader(InitStepContext task);

    protected Step createBatchStep(String stepName, ThreadPoolTaskExecutor executor) {
        log.info("begin to create batchStep,name is {}.",stepName);
        TaskletStep step = null;
        if (getBatchExceptionHandler() != null ) {
            if(isNeedskip) {
                step = stepBuilderFactory.get(stepName).<I, O>chunk(getChuntSize())
                        .reader(this.reader)
                        .processor(this.processor).writer(this.writer)
                        .faultTolerant()
                        .skipLimit(10)
                        .skip(BatchReaderException.class)
                        .taskExecutor(executor)
                        .startLimit(10)
                        .allowStartIfComplete(true)
                        .exceptionHandler(getBatchExceptionHandler())
                        /**
                         * 发任务数为 10,默认为4
                         */
                        .throttleLimit(10)
                        .build();
            }else {
                step = stepBuilderFactory.get(stepName).<I, O>chunk(getChuntSize()).reader(this.reader)
                        .processor(this.processor).writer(this.writer).exceptionHandler(getBatchExceptionHandler()).build();
            }
        } else {
            if(isNeedskip) {
                step = stepBuilderFactory.get(stepName).<I, O>chunk(getChuntSize()).reader(this.reader)
                        .processor(this.processor).writer(this.writer)
                        .faultTolerant()
                        .skipLimit(10)
                        .skip(BatchReaderException.class)
                        .build();
            }else {
                step = stepBuilderFactory.get(stepName).<I, O>chunk(getChuntSize()).reader(this.reader)
                        .processor(this.processor).writer(this.writer).build();
            }


        }
        log.info("batchStep created,name is {}.",stepName);

        return step;
    }

    protected abstract ItemWriter<O> createBatchWriter(SkuInitStepContext task);

    /**
     * @return the batchExceptionHandler
     */
    public BatchStepExceptionHandler getBatchExceptionHandler() {
        return new BatchStepExceptionHandler();
    }


    public void isNeedskip(boolean isNeedskip) {
        this.isNeedskip = isNeedskip;
    }

}

数据处理Job构建

@Component
public class InitDataJob {

    private Job job;

    @Autowired
    private JobLauncher jobLauncher;

    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    @Autowired
    private DefaultBatchJobListener listener;

    /**
     * 描述信息：数据分发Job
     * @param step,name
     */
    public Job buildBatchJob(Step step,String name) {
        assertNotNull(jobLauncher);
        assertNotNull(jobBuilderFactory);
        // 创建job
        this.job = jobBuilderFactory.get(name)
                .listener(listener)
                .incrementer(new RunIdIncrementer())
                .start(step)
                .build();

        return job;
    }


    /**
     * 运行Job
     *
     * @param jobParameters job的参数
     * @return
     * @throws JobExecutionAlreadyRunningException
     * @throws JobRestartException
     * @throws JobInstanceAlreadyCompleteException
     * @throws JobParametersInvalidException
     */
    public JobExecution run(JobParameters jobParameters) throws JobExecutionAlreadyRunningException,
            JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException {
        return this.jobLauncher.run(this.job, jobParameters);
    }
}

测试job，是用xxl-job集成调用：

@JobHandler(value="initDataJobHandler")
@Service
@Slf4j
public class InitDataJobHandler extends IJobHandler {

    @Autowired
    InitDataJob job;

    @Autowired
    private InitDataStep step;

    @Autowired
    private StepBuilderFactory stepBuilderFactory;
    @Autowired
    @Qualifier("batchTaskExecutor")
    private ThreadPoolTaskExecutor executor;

    @Value("${init.query.sql}")
    String querySql;

    @Value("${init.insert.sql}")
    String insertSql;

    @Autowired
    @Qualifier("dataSource")
    DataSource dataSource;

    @Override
    public ReturnT<String> execute(String param){
        // 业务逻辑
        InputPara para =
                JSONObject.parseObject(param, InputPara.class);
        assertNotNull(para);

        JobLogger.log("参数 = {}", param);
        JobLogger.log("querySql = {}", querySql);
        JobLogger.log("insertSql = {}", insertSql);

        InitStepContext context = new InitStepContext();
        context.setDataSource(dataSource);
        context.setQuerySql(querySql);
        context.setInsertSQL(insertSql);

        //设置是否跳过异常
        step.isNeedskip(true);
        //设置跳过次数
        step.setSkipLimit(10);
        step.buildBatchStep(String.join(Constants.TASK_SYMBOL,Constants.INIT_PREFIX, "task_name"), stepBuilderFactory,executor,context);

        //创建Job
        job.buildBatchJob(step.getStep(),String.join(Constants.TASK_SYMBOL,SkuConstants.SKU_INIT_PREFIX, "task_name"));
        try{
            // 任务执行
            log.info("InitDataJobHandler begin to run job {}.",String.join(Constants.SKU_TASK_SYMBOL,SkuConstants.SKU_INIT_PREFIX, "task_name"));
            // 设置Job的参数,开始执行
            JobExecution jobExecution = job.run(new JobParametersBuilder().addLong(Constants.TASK_TIME, System.currentTimeMillis())
                    .toJobParameters());
            log.info("InitDataJobHandler end to run job {}.",String.join(Constants.TASK_SYMBOL,SkuConstants.INIT_PREFIX, "task_name"));
            if(jobExecution.getStatus() == BatchStatus.COMPLETED){
                return SUCCESS;
            }else{
                return FAIL;
            }
        }catch (Exception e){
            JobLogger.log(e);
            log.error("InitDataJobHandler execute failed ,Exception is：{}",e);
        }
        return SUCCESS;
    }

}

测试的SQL如：

## init.query.sql
select id,code,update_time from info where status='online' order by id

##sku.init.insert.sql
insert ignore into data_distribution_record(biz_code,sys_code,sku_id,sku_code,release_time) value(:bizCode,:sysCode,:skuId,:skuCode,:releaseTime)

如果querySQL想支持动态传参

select id,code,update_time from info where status='online' and id in (:ids) order by id