EE颠覆者第九章2 spring Batch

spring Batch

用来处理大量数据操作的一个框架，

主要用来读取大量数据，然后进行一定处理后输出成指定的形式。

名称	用途
Job Repository	用来注册Job容器
Job Launcher	用来启动Job接口
Job	实际要执行的任务，包含一个或多个Step
Step	包含Item Reader Processor Writer
Item Reader	读取数据的接口
Item Processor	处理数据的接口
Item Writer	输出数据的接口

主要组成部分注册成 Spring Bean

//@Configuration
@EnableBatchProcessing

数据读取：Spring Batch 为我们提供了大量的 ItemReader的实现

数据处理和校验都是通过 ItemProcessor接口完成

实现ItemProcessor 重写 process 方法， 从 ItemReader，读取到数据，返回给 Item Writer

1. pom

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-batch</artifactId>
			<exclusions>
				<exclusion>
					<groupId>org.hsqldb</groupId>
					<artifactId>hsqldb</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-jdbc</artifactId>
		</dependency>


		<dependency>
			<groupId>com.oracle</groupId>
			<artifactId>ojdbc6</artifactId>
			<version>11.2.0.2.0</version>
		</dependency>

		<dependency>
			<groupId>org.hibernate</groupId>
			<artifactId>hibernate-validator</artifactId>
		</dependency> //作为校验使用
	
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>

2. 测试 people.csv

汪某某,11,汉族,合肥
张某某,12,汉族,上海
李某某,13,非汉族,武汉
刘某,14,非汉族,南京
欧阳某某,115,汉族,北京

3. 数据表定义

src/main/resources/schema.sql
create table PERSON
(
  id          NUMBER not null primary key,
  name        VARCHAR2(20),
  age         NUMBER,
  nation      VARCHAR2(20),
  address     VARCHAR2(20)
);

4. 领域模型类

public class Person {
	
	@Size(max=4,min=2) //1
	private String name;
	
	private int age;
	
	private String nation;
	
	private String address;
	}

5. 数据处理及校验

public class CsvItemProcessor  extends ValidatingItemProcessor<Person>{

	@Override
	public Person process(Person item) throws ValidationException {
		super.process(item); //1 才会调用自定义校验器
		
		if(item.getNation().equals("汉族")){ //2 如果汉族就转成01
			item.setNation("01");
		}else{
			item.setNation("02");
		}
		return item;
	}


}

public class CsvBeanValidator<T> implements Validator<T>,InitializingBean {
    private javax.validation.Validator validator; 
    @Override
    public void afterPropertiesSet() throws Exception { //1 JSR-303 的Validator的初始化
        ValidatorFactory validatorFactory = Validation.buildDefaultValidatorFactory();
        validator = validatorFactory.usingContext().getValidator();
    }

    @Override
    public void validate(T value) throws ValidationException {
        Set<ConstraintViolation<T>> constraintViolations = validator.validate(value); //2
        if(constraintViolations.size()>0){
            
            StringBuilder message = new StringBuilder();
            for (ConstraintViolation<T> constraintViolation : constraintViolations) {
                message.append(constraintViolation.getMessage() + "\n");
            }
            throw new ValidationException(message.toString());

        }

    }

}

6. job监听

public class CsvJobListener implements JobExecutionListener{  //实现这个类

    long startTime;
    long endTime;
    @Override
    public void beforeJob(JobExecution jobExecution) { //重写
        startTime = System.currentTimeMillis();
        System.out.println("任务处理开始");
    }

    @Override
    public void afterJob(JobExecution jobExecution) { //重写
        endTime = System.currentTimeMillis();
        System.out.println("任务处理结束");
        System.out.println("耗时:" + (endTime - startTime) + "ms");
    }

}

7. 配置

//@Configuration
@EnableBatchProcessing  //开户被批处理的支持
public class CsvBatchConfig {

	@Bean
	public ItemReader<Person> reader() throws Exception {
		FlatFileItemReader<Person> reader = new FlatFileItemReader<Person>(); //1这个类读取文件
		reader.setResource(new ClassPathResource("people.csv")); //2 csv文件的位置
	        reader.setLineMapper(new DefaultLineMapper<Person>() {{ //3 数据和领域模型映射
	            setLineTokenizer(new DelimitedLineTokenizer() {{
	                setNames(new String[] { "name","age", "nation" ,"address"});
	            }});
	            setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
	                setTargetType(Person.class);
	            }});
	        }});
	        return reader;
	}
	
	@Bean
	public ItemProcessor<Person, Person> processor() { 
		CsvItemProcessor processor = new CsvItemProcessor(); //1 自定义的csv
		processor.setValidator(csvBeanValidator()); //2 指定校验器
		return processor;
	}
	
	

	@Bean
	public ItemWriter<Person> writer(DataSource dataSource) {//1 注入boot 定义的dataSource
		JdbcBatchItemWriter<Person> writer = new JdbcBatchItemWriter<Person>(); //2 jdbc的批处理 JdbcBatchItemWriter，写数据到数据库
		writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Person>());
		String sql = "insert into person " + "(id,name,age,nation,address) "
				+ "values(hibernate_sequence.nextval, :name, :age, :nation,:address)";
		writer.setSql(sql); //3 执行要处理的批处理 SQL语句
		writer.setDataSource(dataSource);
		return writer;
	}

	@Bean //工作仓库
	public JobRepository jobRepository(DataSource dataSource, PlatformTransactionManager transactionManager)
			throws Exception {
		JobRepositoryFactoryBean jobRepositoryFactoryBean = new JobRepositoryFactoryBean();
		jobRepositoryFactoryBean.setDataSource(dataSource);
		jobRepositoryFactoryBean.setTransactionManager(transactionManager);
		jobRepositoryFactoryBean.setDatabaseType("oracle");
		return jobRepositoryFactoryBean.getObject();
	}

	@Bean //启动job的接口
	public SimpleJobLauncher jobLauncher(DataSource dataSource, PlatformTransactionManager transactionManager) //需要dataSource，transactionManager
			throws Exception {
		SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
		jobLauncher.setJobRepository(jobRepository(dataSource, transactionManager));
		return jobLauncher;
	}

	@Bean //导入job
	public Job importJob(JobBuilderFactory jobs, Step s1) {
		return jobs.get("importJob")
				.incrementer(new RunIdIncrementer())
				.flow(s1) //1 为job指定 step
				.end()
				.listener(csvJobListener()) //2 注册并绑定 监听器到job
				.build();
	}

	@Bean //步骤
	public Step step1(StepBuilderFactory stepBuilderFactory, ItemReader<Person> reader, ItemWriter<Person> writer,
			ItemProcessor<Person,Person> processor) {
		return stepBuilderFactory
				.get("step1")
				.<Person, Person>chunk(65000) //1 批处理 每次提交 65000条数据
				.reader(reader) //2 给step 绑定 reader
				.processor(processor) //3 绑定 processor
				.writer(writer) //4 绑定 writer
				.build();
	}



	@Bean
	public CsvJobListener csvJobListener() {
		return new CsvJobListener();
	}

	@Bean
	public Validator<Person> csvBeanValidator() {
		return new CsvBeanValidator<Person>();
	}
	

}

8.手动触发任务

//@Configuration 注释这个配置
@EnableBatchProcessing
public class CsvBatchConfig {
}

新建TriggerBatchConfig。内容与 CsvBachConfig保持一致

除了 ItemReader 这个bean

	@Bean
	@StepScope
	public FlatFileItemReader<Person> reader(@Value("#{jobParameters['input.file.name']}") String pathToFile) throws Exception {
		FlatFileItemReader<Person> reader = new FlatFileItemReader<Person>(); //1 用 FlatFile。而不是Item Reader
		 reader.setResource(new ClassPathResource(pathToFile)); //2
	        reader.setLineMapper(new DefaultLineMapper<Person>() {{ //3
	            setLineTokenizer(new DelimitedLineTokenizer() {{
	                setNames(new String[] { "name","age", "nation" ,"address"});
	            }});
	            setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
	                setTargetType(Person.class);
	            }});
	        }});
	       
	        return reader;
	}

9. 定义控制器

@RestController
public class DemoController {
	
	    @Autowired
	    JobLauncher jobLauncher;

	    @Autowired
	    Job importJob;
	    public JobParameters   jobParameters;
	    
	    @RequestMapping("/read")
	    public String imp(String fileName) throws Exception{
	    	
	        String path = fileName+".csv";
	        jobParameters = new JobParametersBuilder()
	                .addLong("time", System.currentTimeMillis())
	                .addString("input.file.name", path)
	                .toJobParameters();
	        jobLauncher.run(importJob,jobParameters);
	        return "ok";
	    }

}

10. 配置文件

spring.datasource.driverClassName=oracle.jdbc.OracleDriver
spring.datasource.url=jdbc\:oracle\:thin\:@192.168.31.183\:49161\:xe
spring.datasource.username=system
spring.datasource.password=oracle

spring.batch.job.enabled=false #关闭自动执行job的配置

logging.level.org.springframework.web = DEBUG

http://localhost:8080/read?fileName=people

表里就会生成处理好的，对应的数据，就像 hadoop一样。