书接上回
从数据库读取数据
关键类:JdbcPagingItemReader属于ItemReader的子类
例子:
package com.example.demo.config2;
import com.example.demo.po.Teacher;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.JdbcPagingItemReader;
import org.springframework.batch.item.database.Order;
import org.springframework.batch.item.database.support.MySqlPagingQueryProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.RowMapper;
import javax.annotation.Resource;
import javax.sql.DataSource;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
@Configuration
public class ItemReaderDemo {
@Resource
private JobBuilderFactory jobBuilderFactory;
@Resource
private StepBuilderFactory stepBuilderFactory;
@Resource
private DataSource dataSource;
@Resource
@Qualifier("dbJdbcWriter") // 注入指定名字的bean
private ItemWriter<Teacher> dbJdbcWriter;
@Bean
public Job itemReaderJob() {
return jobBuilderFactory.get("itemReaderJob").start(itemReaderStem()).build();
}
@Bean
public Step itemReaderStem() {
return stepBuilderFactory.get("itemReaderStem")
.<Teacher,Teacher>chunk(2)
.reader(itemReaderDemoRead())
.writer(dbJdbcWriter)
.build();
}
@Bean
@StepScope // 只使用在step范围之内
public JdbcPagingItemReader<Teacher> itemReaderDemoRead() {
JdbcPagingItemReader<Teacher> reader = new JdbcPagingItemReader<>();
// p配置数据源(使用spring数据源)
reader.setDataSource(dataSource);
// 一次读取多少(分页效果)
reader.setFetchSize(2);
// 把读取到的数据转换为实体类
reader.setRowMapper(new RowMapper<Teacher>() {
@Override
// 设置结果映射
public Teacher mapRow(ResultSet rs, int rowNum) throws SQLException {
Teacher teacher = new Teacher();
teacher.setClasss(rs.getString(1));
teacher.setTname(rs.getString(2));
teacher.setTsex(rs.getString(3));
teacher.setTbirthday(rs.getDate(4));
teacher.setProf(rs.getString(5));
teacher.setDepart(rs.getString(6));
return teacher;
}
});
// 设置Sql语句
MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider();
// 指定所查找的字段
provider.setSelectClause("classs, Tname, Tsex, Tbirthday, Prof, Depart");
// 指定表
provider.setFromClause("from teacher");
// 指定根据哪个地段排序
Map<String, Order> map = new HashMap<>();
map.put("classs",Order.ASCENDING);
provider.setSortKeys(map);
reader.setQueryProvider(provider);
return reader;
}
}
package com.example.demo.config2;
import com.example.demo.po.Teacher;
import org.springframework.batch.item.ItemWriter;
import org.springframework.stereotype.Component;
import java.util.List;
@Component("dbJdbcWriter")
public class DbJdbcWriter implements ItemWriter<Teacher> {
@Override
public void write(List<? extends Teacher> items) throws Exception {
for (Teacher item : items) {
System.out.println(item);
}
}
}
从普通文件读取数据
关键类:FlatFileItemReader
804,李诚,男,1958-12-02 00:00:00,副教授,计算机系
825,王萍,女,1972-05-05 00:00:00,助教,计算机系
831,刘冰,女,1977-08-14 00:00:00,助教,电子工程系
856,张旭,男,1969-03-12 00:00:00,讲师,电子工程系
@Bean
@StepScope
public FlatFileItemReader<Teacher> fileReader() {
FlatFileItemReader<Teacher> reader = new FlatFileItemReader<>();
reader.setResource(new ClassPathResource("test.txt"));
// 指定跳过第一行
// reader.setLinesToSkip(1);
// 解析数据
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
// 指明表头
tokenizer.setNames("classs","tname","tsex","tbirthday","prof","depart");
// 把解析出来的一行数据映射成指定对象
DefaultLineMapper<Teacher> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
mapper.setFieldSetMapper(new FieldSetMapper<Teacher>() {
@Override
public Teacher mapFieldSet(FieldSet fieldSet) throws BindException {
Teacher teacher = new Teacher();
teacher.setClasss(fieldSet.readString("classs"));
teacher.setTname(fieldSet.readString("tname"));
teacher.setTsex(fieldSet.readString("tsex"));
teacher.setTbirthday(fieldSet.readDate("tbirthday"));
teacher.setProf(fieldSet.readString("prof"));
teacher.setDepart(fieldSet.readString("depart"));
return teacher;
}
});
mapper.afterPropertiesSet();
reader.setLineMapper(mapper);
return reader;
}
从xml读取数据
关键类:StaxEventItemReater
1、导入依赖:
<!-- https://mvnrepository.com/artifact/org.springframework/spring-oxm -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-oxm</artifactId>
<version>5.3.23</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.thoughtworks.xstream/xstream -->
<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.19</version>
</dependency>
2、创建xml文件
<teachers>
<teacher>
<classs>804</classs>
<tname>李诚</tname>
<tsex>男</tsex>
<tbirthday>1958-12-02 00:00:00</tbirthday>
<prof>副教授</prof>
<depart>计算机系</depart>
</teacher>
<teacher>
<classs>825</classs>
<tname>王萍</tname>
<tsex>女</tsex>
<tbirthday>1958-12-02 00:00:00</tbirthday>
<prof>助教</prof>
<depart>计算机系</depart>
</teacher>
</teachers>
3、使用:
@Bean
@StepScope
public StaxEventItemReader<? extends Teacher> xmlReader() {
StaxEventItemReader<Teacher> reader = new StaxEventItemReader<>();
reader.setResource(new ClassPathResource("test.xml"));
// 指定需要处理的跟标签
reader.setFragmentRootElementName("teacher");
// 把xml转换为指定对象
XStreamMarshaller marshaller = new XStreamMarshaller();
Map<String, Class> map = new HashMap<>();
// 当对象中存在时间类型时设置格式
marshaller.setConverters(new DateConverter("yyyy-MM-dd HH:mm:ss",
new String[]{"yyyy-MM-dd","HH:mm:ss","yyyyMMdd","HHmmss","yyyyMMdd HHmmss","yyyyMMddHHmmss"},
TimeZone.getTimeZone("Asia/Shanghai")));
// 设置类型权限,当属性为私有属性时设置
marshaller.setTypePermissions(AnyTypePermission.ANY);
// teacher指的是<teacher>标签
map.put("teacher",Teacher.class);
marshaller.setAliases(map);
reader.setUnmarshaller(marshaller);
return reader;
}
从多个文件中读取数据
关键类:MultiResourceItemReader
package com.example.demo.config2;
import com.example.demo.po.Teacher;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.FieldSet;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.bind.BindException;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import javax.annotation.Resource;
@Configuration
@EnableBatchProcessing
public class MultiFileReaderDemo {
@Resource
private JobBuilderFactory jobBuilderFactory;
@Resource
private StepBuilderFactory stepBuilderFactory;
@Resource
@Qualifier("file1") // 注入指定名字的bean
private ItemWriter<Teacher> file1;
@Value("classpath:/*.txt")
private org.springframework.core.io.Resource[] multiResources;
@Bean
public Job multiFileReaderDemoJob() {
return jobBuilderFactory.get("multiFileReaderDemoJob1").start(multiFileReaderDemoStep()).build();
}
public Step multiFileReaderDemoStep() {
return stepBuilderFactory.get("multiFileReaderDemoStep1")
.<Teacher,Teacher>chunk(2).reader(multiFileReaderDemoReader()).writer(file1).build();
}
@Bean
public MultiResourceItemReader<? extends Teacher> multiFileReaderDemoReader() {
MultiResourceItemReader<Teacher> reader = new MultiResourceItemReader<>();
reader.setDelegate(fileReader1());
reader.setResources(multiResources);
return reader;
}
@Bean
@StepScope
public FlatFileItemReader<Teacher> fileReader1() {
FlatFileItemReader<Teacher> reader = new FlatFileItemReader<>();
reader.setResource(new ClassPathResource("test.txt"));
// 指定跳过第一行
// reader.setLinesToSkip(1);
// 解析数据
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
// 指明表头
tokenizer.setNames("classs","tname","tsex","tbirthday","prof","depart");
// 把解析出来的一行数据映射成指定对象
DefaultLineMapper<Teacher> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
mapper.setFieldSetMapper(new FieldSetMapper<Teacher>() {
@Override
public Teacher mapFieldSet(FieldSet fieldSet) throws BindException {
Teacher teacher = new Teacher();
teacher.setClasss(fieldSet.readString("classs"));
teacher.setTname(fieldSet.readString("tname"));
teacher.setTsex(fieldSet.readString("tsex"));
teacher.setTbirthday(fieldSet.readDate("tbirthday"));
teacher.setProf(fieldSet.readString("prof"));
teacher.setDepart(fieldSet.readString("depart"));
return teacher;
}
});
mapper.afterPropertiesSet();
reader.setLineMapper(mapper);
return reader;
}
}
ItemReader处理异常
实现ItemStreamReader
package com.example.demo.config2;
import com.example.demo.po.Teacher;
import org.springframework.batch.item.*;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.FieldSet;
import org.springframework.boot.context.properties.bind.BindException;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
@Service
public class ItemReaderError implements ItemStreamReader<Teacher> {
// 读取文件的对象
private FlatFileItemReader<Teacher> threadFlatFileItemReader = new FlatFileItemReader<>();
// 记录当前是第几行
private Long curLine = 0L;
private boolean restart = false;
// 向数据库持久化一些信息
/**
* 之所以要持久化当前行数,是为了当读取出现异常是能断点续传,当不记录条数时,出现异常重启会重新读取
*/
private ExecutionContext executionContext;
public ItemReaderError(){
threadFlatFileItemReader.setResource(new ClassPathResource("test.txt"));
// 指定跳过第一行
// reader.setLinesToSkip(1);
// 解析数据
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
// 指明表头
tokenizer.setNames("classs","tname","tsex","tbirthday","prof","depart");
// 把解析出来的一行数据映射成指定对象
DefaultLineMapper<Teacher> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
mapper.setFieldSetMapper(new FieldSetMapper<Teacher>() {
@Override
public Teacher mapFieldSet(FieldSet fieldSet) throws BindException {
Teacher teacher = new Teacher();
teacher.setClasss(fieldSet.readString("classs"));
teacher.setTname(fieldSet.readString("tname"));
teacher.setTsex(fieldSet.readString("tsex"));
teacher.setTbirthday(fieldSet.readDate("tbirthday"));
teacher.setProf(fieldSet.readString("prof"));
teacher.setDepart(fieldSet.readString("depart"));
return teacher;
}
});
mapper.afterPropertiesSet();
threadFlatFileItemReader.setLineMapper(mapper);
}
@Override
public Teacher read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
Teacher teacher = null;
this.curLine++;
if (restart){
// 设置读取时要跳过的行数
threadFlatFileItemReader.setLinesToSkip(this.curLine.intValue()-1);
restart = false;
System.out.println("start reading from line:"+this.curLine);
}
threadFlatFileItemReader.open(this.executionContext);
teacher = threadFlatFileItemReader.read();
if (teacher != null && teacher.getClasss().equals("857")){
throw new RuntimeException("somehing wrong customer id: "+teacher.getTname());
}
return teacher;
}
/**
* 在每个step读取之前执行
* @param executionContext
* @throws ItemStreamException
*/
@Override
public void open(ExecutionContext executionContext) throws ItemStreamException {
this.executionContext = executionContext;
if (executionContext.containsKey("curLine")){
this.curLine = executionContext.getLong("curLine");
this.restart = true;
}{
this.curLine = 0L;
executionContext.put("curLine",this.curLine);
System.out.println("start reading from line: "+this.curLine +1);
}
}
/**
* 每个chunk读完一批执行
* @param executionContext
* @throws ItemStreamException
*/
@Override
public void update(ExecutionContext executionContext) throws ItemStreamException {
executionContext.put("curLine",this.curLine);
System.out.println("currentLine:"+this.curLine);
}
/**
* 读完之后执行
* @throws ItemStreamException
*/
@Override
public void close() throws ItemStreamException {
}
}
ItemWriter输出到数据库
ItemReader是一个一个读取数据,而ItemWriter是一批一批输出
- Neo4jItemWriter
- MongoItemWriter
- RepositoryItemWriter
- HibernateItemWriter
- JdbcBatchItemWriter
- JpaItemWriter
- GemfireItemWriter
@Resource
private DataSource dataSource;
@Bean
public JdbcBatchItemWriter<Teacher> itemWriterDbDemoWriter(){
JdbcBatchItemWriter<Teacher> writer = new JdbcBatchItemWriter<>();
writer.setDataSource(dataSource);
writer.setSql("insert into thread(classs, Tname, Tsex, Tbirthday, Prof, Depart) values"+
"(:classs, :Tname, :Tsex, :Tbirthday, :Prof, :Depart)");
writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>());
return writer;
}
输出数据到普通文件中
关键类:FlatFileItemWriter
@Configuration
public class FileItemWriterConfig {
@Bean
public FlatFileItemWriter<Teacher> fileItemWriter() throws Exception {
// 吧实体对象Teacher转换成某种格式的字符串输出到文件中
FlatFileItemWriter<Teacher> writer = new FlatFileItemWriter<>();
// 指定文件路径
String path = "F:\\test.txt";
writer.setResource(new FileSystemResource(path));
writer.setLineAggregator(new LineAggregator<Teacher>() {
@NotNull
@Override
public String aggregate(@NotNull Teacher item) {
return JSONUtils.toJSONString(item);
}
});
writer.afterPropertiesSet();
return writer;
}
}
输出数据到Xml中
关键类: StaxEventItemWriter
@Configuration
public class XmlItemWriterConfig {
@Bean
public StaxEventItemWriter<Teacher> staxEventItemWriter() throws Exception {
StaxEventItemWriter<Teacher> writer = new StaxEventItemWriter<>();
XStreamMarshaller marshaller = new XStreamMarshaller();
// 指定父标签,对象中的属性对应子标签
HashMap<String, Class> map = new HashMap<>();
map.put("teacher",Teacher.class);
marshaller.setAliases(map);
// 设置根标签
writer.setRootTagName("teachers");
writer.setMarshaller(marshaller);
// 指定输入的xml文件路径
String path = "F\\cus.xml";
writer.setResource(new FileSystemResource(path));
writer.afterPropertiesSet();
return writer;
}
}
数据输出到多个文件
关键类:CompositeItemWriter
ClassiflerCompositeItemWriter—> 可以将不同的分类读取到不同的文件中
结合上面两种读取方式
@Bean
public CompositeItemWriter<Teacher> multiFileItemWriter() throws Exception {
CompositeItemWriter<Teacher> writer = new CompositeItemWriter<>();
// 把输出到普通文件中和xml文件中的ItemWriter
writer.setDelegates(Arrays.asList(fileItemWriter(),staxEventItemWriter()));
writer.afterPropertiesSet();
return writer;
}
@Bean
public ClassifierCompositeItemWriter<Teacher> multiFileItemWriter2(){
ClassifierCompositeItemWriter<Teacher> writer = new ClassifierCompositeItemWriter<>();
// 指定分类方式
writer.setClassifier(new Classifier<Teacher, ItemWriter<? super Teacher>>() {
@Override
public ItemWriter<? super Teacher> classify(Teacher teacher) {
ItemWriter<Teacher> itemWriter = null;
try {
itemWriter = teacher.getTname().hashCode() % 2 == 0?fileItemWriter():staxEventItemWriter();
} catch (Exception e) {
e.printStackTrace();
}
return itemWriter;
}
});
return writer;
}
@Resource
@Qualifier("fileItemWriter")
private ItemStreamWriter<? super Teacher> fileItemWriter;
@Resource
@Qualifier("staxEventItemWriter")
private ItemStreamWriter<? super Teacher> staxEventItemWriter;
@Bean
public Step itemWriterDbDemoStep() {
return stepBuilderFactory.get("itemWriterDbDemoStep")
.<Teacher,Teacher>chunk(2)
.reader(itemWriterDbDemoReader())
.writer(itemWriterDbDemoWriter)
.stream(fileItemWriter)
.stream(staxEventItemWriter)
.build();
}