springBatch批处理框架(二)实战

一·、概览

之前我们了解了如何创建一个springBatch的项目，这篇文章我们一起来学习一下，如何应用框架进行数据。

二、springBatch架构图

首先我看看到最底层的Job Respository伴随着我们整个任务的始终，那他究竟是做什么的呢？

1.Job Respository : 记录任务状态信息,SpringBatch会在数据库中默认创建一些记录整个工作状态的表.

2.JobLauncher: 启动job的运行器。

3.Job: 代表一个任务

4.Step(关键):如我们的批处理示例所示，批处理过程通常由一个包含多个步骤的Job封装。 每个步骤通常有一个ItemReader(用来读取数据的接口)、ItemProcessor(用来处理数据的接口)和ItemWriter(用来写入数据的接口)。

三、代码结构

1.@EnableBatchProcessing：该注释支持Spring Batch特性，并为设置批处理作业提供基本配置。

package com.example.demo;

import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
@EnableBatchProcessing
public class DemoApplication {

	public static void main(String[] args) {
		SpringApplication.run(DemoApplication.class, args);
	}

}

2.创建job的config

package com.example.demo.config;
import com.example.demo.step.DemoProcesser;
import com.example.demo.step.DemoReader;
import com.example.demo.step.DemoWriter;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;

/**
 * x DemoJobConfig
 * <p>
 */
@Configuration
public class DemoJobConfig {

  @Autowired
  private JobBuilderFactory jobs;

  @Autowired
  private StepBuilderFactory steps;

  private Integer skipLimit = 100;

  @Bean
  public Job initialLoadTncMuniDataJob(DemoReader DemoDataReader) {
    return (Job) jobs.get("demoJob")
        .start(loadMuniDataStep(DemoDataReader))
        .build();
  }

  @Bean
  public Step loadMuniDataStep(DemoReader demoReader) {
    return steps.get("demoStep")
        .<String, Integer>chunk(1)
        .reader(demoReader)
        .processor(demoProcesser())
        .writer(demoWriter())
        .faultTolerant()
        .skipLimit(skipLimit) //异常跳过次数最大值，超过这个值 job会停止
        .skip(Exception.class)//跳过异常的种类
        .taskExecutor(stepMuniTaskExecutor())//配置多线程
        .throttleLimit(1)//多线程的个数
        .build();
  }

  @Bean
  @StepScope
  public DemoReader demoReader() {
    return new DemoReader();
  }

  @Bean
  public DemoProcesser demoProcesser() {
    return new DemoProcesser();
  }

  @Bean
  public DemoWriter demoWriter() {
    return new DemoWriter();
  }

  @Bean
  public SimpleAsyncTaskExecutor stepMuniTaskExecutor() {
    return new SimpleAsyncTaskExecutor();
  }

}

 3.最关键的，定义我们业务实现的step相关item. 

reader:

package com.example.demo.step;

import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.NonTransientResourceException;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.UnexpectedInputException;

/**
 * TODO
 * <p>
 **/
public class DemoReader implements ItemReader {

  Integer loopSize = 2; //标志位，需求：只需要跑两遍就可以,    如果不return null job会一直loop

  Integer end = 0;
  @Override
  public String read(){
    String age = "今年18岁";
    end++;
    if(end>=loopSize){
      return null;// return null     ==   shut down job
    }
    return age;
  }
}

processer:

package com.example.demo.step;

import org.springframework.batch.item.ItemProcessor;

/**
 * TODO
 * <p>
 **/
public class DemoProcesser implements ItemProcessor<String,Integer> {

  @Override
  public Integer process(String s) {
    if(s.equals("今年18岁")){
      return 18;
    }else {
      return 888;
    }
  }
}

weiter:

package com.example.demo.step;


import java.util.List;
import org.springframework.batch.item.ItemWriter;

/**
 * TODO
 * <p>
 **/
public class DemoWriter implements ItemWriter<Integer> {

  @Override
  public void write(List<? extends Integer> list) throws Exception {
    System.out.print(list.get(0));
  }
}

上面demo实现的需求为，通过reader读取数据，再processer对数据进行处理，最后再writer中将数据写入目标位置。

四、程序主动触发job的启动

1.我们上面设置的是springboot启动自动执行我们的job， 那么如果我们想通过程序去trigger这个job的启动(比如定时任务)，如何设置呢？

首先：spring.batch.job.enabled = false 在我们的yaml里面加上这个参数，那么springboot启动的时候就不会默认执行job了     

  其次启动类上面去掉注解@EnableBatchProcessing

我们来看一下新的applicatioin的样子

package com.example.demo;

import com.example.demo.config.DemoJobConfig;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.configuration.JobRegistry;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;

@SpringBootApplication(scanBasePackages = {"com.example.demo"})
@EnableScheduling
public class DemoApplication{

	@Autowired
	private ApplicationContext context;

	public static void main(String[] args) {
		System.exit(SpringApplication.exit(SpringApplication.run(DemoApplication.class, args), () -> 0));
	}

	@Scheduled(fixedRate =1000)
	public void scheduleJob()
			throws Exception {
		JobLauncher jobLauncher = context.getBean(JobLauncher.class);
		JobRegistry jobRegistry = context.getBean(JobRegistry.class);
		Job job = jobRegistry.getJob("demoJob");
		JobExecution run = jobLauncher.run(job, new JobParameters());
	}




}

我们新增一个类，专门去管理所有的job

package com.example.demo.config;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.support.ApplicationContextFactory;
import org.springframework.batch.core.configuration.support.GenericApplicationContextFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;


/**
 * BatchConfig
 * <p>
 */
@Configuration
@EnableBatchProcessing(modular = true)
public class BatchConfig {

  @Bean
  public ApplicationContextFactory executeJobContext() {
    return new GenericApplicationContextFactory(DemoJobConfig.class);
  }
}

如有错误欢迎指出，