springboot之springbatch入门

官网地址:

轻量级，全面的批处理框架，旨在开发对企业系统日常运营至关重要的强大批处理应用程序。

Spring Batch提供了可重复使用的功能，这些功能对于处理大量记录至关重要，包括记录/跟踪，事务管理，作业处理统计，作业重启，跳过和资源管理。它还提供更高级的技术服务和功能，通过优化和分区技术实现极高容量和高性能的批处理作业。简单和复杂的大批量批处理作业可以高度可扩展的方式利用框架来处理大量信息。

特征

• 交易管理

• 基于块的处理

• 声明性I / O.

• 启动/停止/重新启动

• 重试/跳过

• 基于Web的管理界面

 

数据库表:

CREATE TABLE person(
id int PRIMARY KEY AUTO_INCREMENT,
name VARCHAR(20),
age int,
nation VARCHAR(20),
address VARCHAR(20)
);

pom.xml文件

  <!--我们首先配置Spring Batch 在Spring Boot 中的使用,数据库用的是mysql，
            pom文件如下，因为Spring Boot 中的Spring Batch 包含 hsqsldb 所以我们将其去除-->
            <exclusions> <!-- 注意这里-->
                <exclusion>
                    <groupId>org.hsqldb</groupId>
                    <artifactId>hsqldb</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jdbc</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>


        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.41</version>
        </dependency>
        <dependency>
            <groupId>org.hibernate</groupId>
            <artifactId>hibernate-validator</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.batch</groupId>
            <artifactId>spring-batch-test</artifactId>
            <scope>test</scope>
        </dependency>

application.properties

spring.datasource.url=jdbc:mysql://127.0.0.1:3306/数据库名?useUnicode=true&characterEncoding=utf-8&useSSL=false
spring.datasource.username=用户名
spring.datasource.password=密码
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
server.port=9001
#启动时要执行的Job，默认执行全部Job
spring.batch.job.names = importJob
#是否自动执行定义的Job，默认是
spring.batch.job.enabled=true
#是否初始化Spring Batch的数据库，默认为是
spring.batch.initializer.enabled=true
spring.batch.initialize-schema=ALWAYS
#spring.batch.schema=
#spring.batch.table-prefix= #设置SpringBatch的数据库表的前缀

spring.main.allow-bean-definition-overriding=true

resource下创建person.csv

Person

package com.hlvy.entity;


import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author heng
 * @since 2019-06-03
 */
public class Person implements Serializable {

    private static final long serialVersionUID = 1L;


    private Integer id;

    private String name;

    private Integer age;

    private String nation;

    private String address;

    public Integer getId() {
        return id;
    }

    public void setId(Integer id) {
        this.id = id;
    }
    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }
    public Integer getAge() {
        return age;
    }

    public void setAge(Integer age) {
        this.age = age;
    }
    public String getNation() {
        return nation;
    }

    public void setNation(String nation) {
        this.nation = nation;
    }
    public String getAddress() {
        return address;
    }

    public void setAddress(String address) {
        this.address = address;
    }

    @Override
    public String toString() {
        return "Person{" +
        "id=" + id +
        ", name=" + name +
        ", age=" + age +
        ", nation=" + nation +
        ", address=" + address +
        "}";
    }
}

 

 

CsvBatchConfig

package com.hlvy.config;

import com.hlvy.entity.Person;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.core.launch.support.SimpleJobLauncher;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.validator.Validator;
import org.springframework.batch.support.DatabaseType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.transaction.PlatformTransactionManager;

import javax.sql.DataSource;

/**
 * BatchConfiguration
 *
 * @author heng
 * @date 2019/6/3
 **/
@Configuration
@EnableBatchProcessing
public class CsvBatchConfig {
    /**1 首先我们通过 FlatFileItemReader 读取我们需要的文件 通过setResource来实现
     * 2 设置map  在这里通过先设置解析器 setLineTokenizer  来解析我们csv文件中的数     据
     * 3  setFieldSetMapper  将我们需要的数据转化为我们的实体对象 存储
     * 4 如果想 跳过前面的几行 需要使用setLinesToSkip就可以实现
     */
    @Bean
    public ItemReader<Person> reader() throws Exception {
        FlatFileItemReader<Person> reader = new FlatFileItemReader<Person>(); //1
        reader.setResource(new ClassPathResource("person.csv")); //2
        reader.setLineMapper(new DefaultLineMapper<Person>() {{ //3
            setLineTokenizer(new DelimitedLineTokenizer() {{
                setNames(new String[] { "name","age", "nation" ,"address"});
            }});
            setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
                setTargetType(Person.class);
            }});
        }});
        reader.setLinesToSkip(3);
        return reader;
    }

    @Bean
    public ItemProcessor<Person, Person> processor() {
        CsvItemProcessor processor = new CsvItemProcessor(); //1
        processor.setValidator(csvBeanValidator()); //2
        return processor;
    }


    /**
     *写入数据到数据库中
     * 1执行的sql 语句  2 设置数据源
     */
    @Bean
    public ItemWriter<Person> writer(DataSource dataSource) {//1
        JdbcBatchItemWriter<Person> writer = new JdbcBatchItemWriter<Person>(); //2
        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Person>());
        String sql = "insert into person " + "(name,age,nation,address) "
                + "values(:name, :age, :nation,:address)";
        writer.setSql(sql); //3
        writer.setDataSource(dataSource);
        return writer;
    }

    //  作业的仓库 就是设置数据源
    @Bean
    public JobRepository jobRepository(DataSource dataSource, PlatformTransactionManager transactionManager)
            throws Exception {
        JobRepositoryFactoryBean jobRepositoryFactoryBean = new JobRepositoryFactoryBean();
        jobRepositoryFactoryBean.setDataSource(dataSource);
        jobRepositoryFactoryBean.setTransactionManager(transactionManager);
        jobRepositoryFactoryBean.setDatabaseType("mysql");
        return jobRepositoryFactoryBean.getObject();
    }

    //调度器  使用它来执行 我们的批处理
    @Bean
    public SimpleJobLauncher jobLauncher(DataSource dataSource, PlatformTransactionManager transactionManager)
            throws Exception {
        SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
        jobLauncher.setJobRepository(jobRepository(dataSource, transactionManager));
        return jobLauncher;
    }

    //将监听器加入到job中
    @Bean
    public Job importJob(JobBuilderFactory jobs, Step s1) {
        return jobs.get("importJob")
                .incrementer(new RunIdIncrementer())
                .flow(s1) //1
                .end()
                .listener(csvJobListener()) //2
                .build();
    }
    //步骤绑定 reader 与writer  一次性处理65000条记录
    @Bean
    public Step step1(StepBuilderFactory stepBuilderFactory, ItemReader<Person> reader, ItemWriter<Person> writer,
                      ItemProcessor<Person,Person> processor) {
        return stepBuilderFactory
                .get("step1")
                .<Person, Person>chunk(65000) //1
                .reader(reader) //2
                .processor(processor) //3
                .writer(writer) //4
                .build();
    }



    @Bean
    public CsvJobListener csvJobListener() {
        return new CsvJobListener();
    }

    @Bean
    public Validator<Person> csvBeanValidator() {
        return new CsvBeanValidator<Person>();
    }

}

CsvBeanValidator

package com.hlvy.config;

import org.springframework.batch.item.validator.ValidationException;
import org.springframework.batch.item.validator.Validator;
import org.springframework.beans.factory.InitializingBean;

import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.ValidatorFactory;
import java.util.Set;

/**
 * CsvBeanValidator
 *
 * @author heng
 * @date 2019/6/3
 **/
public class CsvBeanValidator<T> implements Validator<T>,InitializingBean {

    private javax.validation.Validator validator;

    @Override
    public void validate(T value) throws ValidationException {
        Set<ConstraintViolation<T >> constraintViolations=validator.validate(value);
        if(constraintViolations.size()>0){

            StringBuilder message=new StringBuilder();
            for(ConstraintViolation<T> constraintViolation:constraintViolations){
                message.append(constraintViolation.getMessage() +"\n");
            }
            throw  new ValidationException(message.toString());
        }


    }

    //在这里我们使用的是JSR-303校验数据，在此进行初始化
    @Override
    public void afterPropertiesSet() throws Exception {
        ValidatorFactory  validatorFactory= Validation.buildDefaultValidatorFactory();
        validator=validatorFactory.usingContext().getValidator();
    }


}

CsvItemProcessor

package com.hlvy.config;

import com.hlvy.entity.Person;
import org.springframework.batch.item.validator.ValidatingItemProcessor;
import org.springframework.batch.item.validator.ValidationException;

/**
 * CsvItemProcessor
 *
 * @author heng
 * @date 2019/6/3
 **/
public class CsvItemProcessor extends ValidatingItemProcessor<Person> {


    @Override
    public Person process(Person item) throws ValidationException {
        // 在这里启动 然后才会调用我们自定义的校验器，否则不能通过 。
        super.process(item);
//        if (item.getNation().equals("汉族")){
//            item.setNation("01");
//        }else{
//            item.setNation("02");
//        }
        return item;

    }
}

CsvJobListener

package com.hlvy.config;

import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionListener;


/**
 * CsvJobListener
 *
 * @author heng
 * @date 2019/6/3
 **/
public class CsvJobListener implements JobExecutionListener {
        long startTime;
        long endTime;

@Override
public void beforeJob(JobExecution jobExecution) {
        startTime = System.currentTimeMillis();
        System.out.println("任务处理开始...");
        }

@Override
public void afterJob(JobExecution jobExecution) {
        endTime = System.currentTimeMillis();
        System.out.println("任务处理结束...");
        System.out.println("耗时:" + (endTime - startTime) + " ms");
        }


}