SpringBatch介绍与使用（中）

最新推荐文章于 2023-07-19 15:13:59 发布

识时务者J

最新推荐文章于 2023-07-19 15:13:59 发布

阅读量477

点赞数

分类专栏： java学习文章标签： java 数据库 spring spring boot

本文链接：https://blog.csdn.net/qq_65377318/article/details/128438308

版权

java学习专栏收录该内容

8 篇文章 0 订阅

订阅专栏

文章目录

书接上回

从数据库读取数据

关键类：JdbcPagingItemReader属于ItemReader的子类

例子：

package com.example.demo.config2;

import com.example.demo.po.Teacher;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.JdbcPagingItemReader;
import org.springframework.batch.item.database.Order;
import org.springframework.batch.item.database.support.MySqlPagingQueryProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.RowMapper;

import javax.annotation.Resource;
import javax.sql.DataSource;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;

@Configuration
public class ItemReaderDemo {

    @Resource
    private JobBuilderFactory jobBuilderFactory;

    @Resource
    private StepBuilderFactory stepBuilderFactory;

    @Resource
    private DataSource dataSource;

    @Resource
    @Qualifier("dbJdbcWriter") // 注入指定名字的bean
    private ItemWriter<Teacher> dbJdbcWriter;

    @Bean
    public Job itemReaderJob() {
        return jobBuilderFactory.get("itemReaderJob").start(itemReaderStem()).build();
    }

    @Bean
    public Step itemReaderStem() {
        return stepBuilderFactory.get("itemReaderStem")
                .<Teacher,Teacher>chunk(2)
                .reader(itemReaderDemoRead())
                .writer(dbJdbcWriter)
                .build();
    }


    @Bean
    @StepScope // 只使用在step范围之内
    public JdbcPagingItemReader<Teacher> itemReaderDemoRead() {
        JdbcPagingItemReader<Teacher> reader = new JdbcPagingItemReader<>();
        // p配置数据源（使用spring数据源）
        reader.setDataSource(dataSource);
        // 一次读取多少（分页效果）
        reader.setFetchSize(2);
        // 把读取到的数据转换为实体类
        reader.setRowMapper(new RowMapper<Teacher>() {

            @Override
            // 设置结果映射
            public Teacher mapRow(ResultSet rs, int rowNum) throws SQLException {
                Teacher teacher = new Teacher();
                teacher.setClasss(rs.getString(1));
                teacher.setTname(rs.getString(2));
                teacher.setTsex(rs.getString(3));
                teacher.setTbirthday(rs.getDate(4));
                teacher.setProf(rs.getString(5));
                teacher.setDepart(rs.getString(6));
                return teacher;
            }
        });

        // 设置Sql语句
        MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider();
        // 指定所查找的字段
        provider.setSelectClause("classs, Tname, Tsex, Tbirthday, Prof, Depart");
        // 指定表
        provider.setFromClause("from teacher");
        // 指定根据哪个地段排序
        Map<String, Order> map = new HashMap<>();
        map.put("classs",Order.ASCENDING);
        provider.setSortKeys(map);
        reader.setQueryProvider(provider);
        return reader;
    }

}

package com.example.demo.config2;

import com.example.demo.po.Teacher;
import org.springframework.batch.item.ItemWriter;
import org.springframework.stereotype.Component;

import java.util.List;

@Component("dbJdbcWriter")
public class DbJdbcWriter implements ItemWriter<Teacher> {
    @Override
    public void write(List<? extends Teacher> items) throws Exception {
        for (Teacher item : items) {
            System.out.println(item);
        }
    }
}

从普通文件读取数据

关键类：FlatFileItemReader

804,李诚,男,1958-12-02 00:00:00,副教授,计算机系
825,王萍,女,1972-05-05 00:00:00,助教,计算机系
831,刘冰,女,1977-08-14 00:00:00,助教,电子工程系
856,张旭,男,1969-03-12 00:00:00,讲师,电子工程系

    @Bean
    @StepScope
    public FlatFileItemReader<Teacher> fileReader() {
        FlatFileItemReader<Teacher> reader = new FlatFileItemReader<>();
        reader.setResource(new ClassPathResource("test.txt"));
        // 指定跳过第一行
//        reader.setLinesToSkip(1);
        // 解析数据
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        // 指明表头
        tokenizer.setNames("classs","tname","tsex","tbirthday","prof","depart");
        // 把解析出来的一行数据映射成指定对象
        DefaultLineMapper<Teacher> mapper = new DefaultLineMapper<>();
        mapper.setLineTokenizer(tokenizer);
        mapper.setFieldSetMapper(new FieldSetMapper<Teacher>() {
            @Override
            public Teacher mapFieldSet(FieldSet fieldSet) throws BindException {
                Teacher teacher = new Teacher();
                teacher.setClasss(fieldSet.readString("classs"));
                teacher.setTname(fieldSet.readString("tname"));
                teacher.setTsex(fieldSet.readString("tsex"));
                teacher.setTbirthday(fieldSet.readDate("tbirthday"));
                teacher.setProf(fieldSet.readString("prof"));
                teacher.setDepart(fieldSet.readString("depart"));
                return teacher;
            }
        });
        mapper.afterPropertiesSet();
        reader.setLineMapper(mapper);
        return reader;
    }

从xml读取数据

关键类：StaxEventItemReater

1、导入依赖：

<!-- https://mvnrepository.com/artifact/org.springframework/spring-oxm -->
<dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-oxm</artifactId>
    <version>5.3.23</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.thoughtworks.xstream/xstream -->
<dependency>
    <groupId>com.thoughtworks.xstream</groupId>
    <artifactId>xstream</artifactId>
    <version>1.4.19</version>
</dependency>

2、创建xml文件

<teachers>
    <teacher>
        <classs>804</classs>
        <tname>李诚</tname>
        <tsex>男</tsex>
        <tbirthday>1958-12-02 00:00:00</tbirthday>
        <prof>副教授</prof>
        <depart>计算机系</depart>
    </teacher>
    <teacher>
        <classs>825</classs>
        <tname>王萍</tname>
        <tsex>女</tsex>
        <tbirthday>1958-12-02 00:00:00</tbirthday>
        <prof>助教</prof>
        <depart>计算机系</depart>
    </teacher>
</teachers>

3、使用：

 @Bean
    @StepScope
    public StaxEventItemReader<? extends Teacher> xmlReader() {
        StaxEventItemReader<Teacher> reader = new StaxEventItemReader<>();
        reader.setResource(new ClassPathResource("test.xml"));
        // 指定需要处理的跟标签
        reader.setFragmentRootElementName("teacher");
        // 把xml转换为指定对象
        XStreamMarshaller marshaller = new XStreamMarshaller();
        Map<String, Class> map = new HashMap<>();
        // 当对象中存在时间类型时设置格式
        marshaller.setConverters(new DateConverter("yyyy-MM-dd HH:mm:ss",
                new String[]{"yyyy-MM-dd","HH:mm:ss","yyyyMMdd","HHmmss","yyyyMMdd HHmmss","yyyyMMddHHmmss"},
                TimeZone.getTimeZone("Asia/Shanghai")));
        // 设置类型权限，当属性为私有属性时设置
        marshaller.setTypePermissions(AnyTypePermission.ANY);
        // teacher指的是<teacher>标签
        map.put("teacher",Teacher.class);
        marshaller.setAliases(map);
        reader.setUnmarshaller(marshaller);
        return reader;
    }

从多个文件中读取数据

关键类：MultiResourceItemReader

package com.example.demo.config2;

import com.example.demo.po.Teacher;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.FieldSet;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.bind.BindException;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;

import javax.annotation.Resource;

@Configuration
@EnableBatchProcessing
public class MultiFileReaderDemo {
    @Resource
    private JobBuilderFactory jobBuilderFactory;

    @Resource
    private StepBuilderFactory stepBuilderFactory;

    @Resource
    @Qualifier("file1") // 注入指定名字的bean
    private ItemWriter<Teacher> file1;

    @Value("classpath:/*.txt")
    private org.springframework.core.io.Resource[] multiResources;

    @Bean
    public Job multiFileReaderDemoJob() {
        return jobBuilderFactory.get("multiFileReaderDemoJob1").start(multiFileReaderDemoStep()).build();
    }

    public Step multiFileReaderDemoStep() {
        return stepBuilderFactory.get("multiFileReaderDemoStep1")
                .<Teacher,Teacher>chunk(2).reader(multiFileReaderDemoReader()).writer(file1).build();

    }

    @Bean
    public MultiResourceItemReader<? extends Teacher> multiFileReaderDemoReader() {
        MultiResourceItemReader<Teacher> reader = new MultiResourceItemReader<>();
        reader.setDelegate(fileReader1());
        reader.setResources(multiResources);
        return reader;
    }

    @Bean
    @StepScope
    public FlatFileItemReader<Teacher> fileReader1() {
        FlatFileItemReader<Teacher> reader = new FlatFileItemReader<>();
        reader.setResource(new ClassPathResource("test.txt"));
        // 指定跳过第一行
//        reader.setLinesToSkip(1);
        // 解析数据
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        // 指明表头
        tokenizer.setNames("classs","tname","tsex","tbirthday","prof","depart");
        // 把解析出来的一行数据映射成指定对象
        DefaultLineMapper<Teacher> mapper = new DefaultLineMapper<>();
        mapper.setLineTokenizer(tokenizer);
        mapper.setFieldSetMapper(new FieldSetMapper<Teacher>() {
            @Override
            public Teacher mapFieldSet(FieldSet fieldSet) throws BindException {
                Teacher teacher = new Teacher();
                teacher.setClasss(fieldSet.readString("classs"));
                teacher.setTname(fieldSet.readString("tname"));
                teacher.setTsex(fieldSet.readString("tsex"));
                teacher.setTbirthday(fieldSet.readDate("tbirthday"));
                teacher.setProf(fieldSet.readString("prof"));
                teacher.setDepart(fieldSet.readString("depart"));
                return teacher;
            }
        });
        mapper.afterPropertiesSet();
        reader.setLineMapper(mapper);
        return reader;
    }
}

ItemReader处理异常

实现ItemStreamReader

package com.example.demo.config2;

import com.example.demo.po.Teacher;
import org.springframework.batch.item.*;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.FieldSet;
import org.springframework.boot.context.properties.bind.BindException;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;

@Service
public class ItemReaderError implements ItemStreamReader<Teacher> {

    // 读取文件的对象
    private FlatFileItemReader<Teacher> threadFlatFileItemReader = new FlatFileItemReader<>();
    // 记录当前是第几行
    private Long curLine = 0L;
    private boolean restart = false;
    // 向数据库持久化一些信息
    /**
     * 之所以要持久化当前行数，是为了当读取出现异常是能断点续传，当不记录条数时，出现异常重启会重新读取
     */
    private ExecutionContext executionContext;

    public ItemReaderError(){
        threadFlatFileItemReader.setResource(new ClassPathResource("test.txt"));
        // 指定跳过第一行
//        reader.setLinesToSkip(1);
        // 解析数据
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        // 指明表头
        tokenizer.setNames("classs","tname","tsex","tbirthday","prof","depart");
        // 把解析出来的一行数据映射成指定对象
        DefaultLineMapper<Teacher> mapper = new DefaultLineMapper<>();
        mapper.setLineTokenizer(tokenizer);
        mapper.setFieldSetMapper(new FieldSetMapper<Teacher>() {
            @Override
            public Teacher mapFieldSet(FieldSet fieldSet) throws BindException {
                Teacher teacher = new Teacher();
                teacher.setClasss(fieldSet.readString("classs"));
                teacher.setTname(fieldSet.readString("tname"));
                teacher.setTsex(fieldSet.readString("tsex"));
                teacher.setTbirthday(fieldSet.readDate("tbirthday"));
                teacher.setProf(fieldSet.readString("prof"));
                teacher.setDepart(fieldSet.readString("depart"));
                return teacher;
            }
        });
        mapper.afterPropertiesSet();
        threadFlatFileItemReader.setLineMapper(mapper);
    }

    @Override
    public Teacher read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
        Teacher teacher = null;
        this.curLine++;
        if (restart){
            // 设置读取时要跳过的行数
            threadFlatFileItemReader.setLinesToSkip(this.curLine.intValue()-1);
            restart = false;
            System.out.println("start reading from line:"+this.curLine);
        }
        threadFlatFileItemReader.open(this.executionContext);
        teacher = threadFlatFileItemReader.read();
        if (teacher != null && teacher.getClasss().equals("857")){
            throw new RuntimeException("somehing wrong customer id: "+teacher.getTname());
        }
        return teacher;
    }

    /**
     * 在每个step读取之前执行
     * @param executionContext
     * @throws ItemStreamException
     */
    @Override
    public void open(ExecutionContext executionContext) throws ItemStreamException {
        this.executionContext = executionContext;
        if (executionContext.containsKey("curLine")){
            this.curLine = executionContext.getLong("curLine");
            this.restart = true;
        }{
            this.curLine = 0L;
            executionContext.put("curLine",this.curLine);
            System.out.println("start reading from line: "+this.curLine +1);
        }
    }

    /**
     * 每个chunk读完一批执行
     * @param executionContext
     * @throws ItemStreamException
     */
    @Override
    public void update(ExecutionContext executionContext) throws ItemStreamException {
        executionContext.put("curLine",this.curLine);
        System.out.println("currentLine:"+this.curLine);
    }

    /**
     * 读完之后执行
     * @throws ItemStreamException
     */
    @Override
    public void close() throws ItemStreamException {

    }
}

ItemWriter输出到数据库

ItemReader是一个一个读取数据，而ItemWriter是一批一批输出

Neo4jItemWriter
MongoItemWriter
RepositoryItemWriter
HibernateItemWriter
JdbcBatchItemWriter
JpaItemWriter
GemfireItemWriter

@Resource
    private DataSource dataSource;

    @Bean
    public JdbcBatchItemWriter<Teacher> itemWriterDbDemoWriter(){
        JdbcBatchItemWriter<Teacher> writer = new JdbcBatchItemWriter<>();
        writer.setDataSource(dataSource);
        writer.setSql("insert into thread(classs, Tname, Tsex, Tbirthday, Prof, Depart) values"+
                "(:classs, :Tname, :Tsex, :Tbirthday, :Prof, :Depart)");
        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>());
        return writer;
    }

输出数据到普通文件中

关键类：FlatFileItemWriter

@Configuration
public class FileItemWriterConfig {

    @Bean
    public FlatFileItemWriter<Teacher> fileItemWriter() throws Exception {
        // 吧实体对象Teacher转换成某种格式的字符串输出到文件中
        FlatFileItemWriter<Teacher> writer = new FlatFileItemWriter<>();
        // 指定文件路径
        String path = "F:\\test.txt";
        writer.setResource(new FileSystemResource(path));
        writer.setLineAggregator(new LineAggregator<Teacher>() {
            @NotNull
            @Override
            public String aggregate(@NotNull Teacher item) {
                return JSONUtils.toJSONString(item);
            }
        });
        writer.afterPropertiesSet();
        return writer;
    }
}

输出数据到Xml中

关键类： StaxEventItemWriter

@Configuration
public class XmlItemWriterConfig {
    @Bean
    public StaxEventItemWriter<Teacher> staxEventItemWriter() throws Exception {
        StaxEventItemWriter<Teacher> writer = new StaxEventItemWriter<>();
        XStreamMarshaller marshaller = new XStreamMarshaller();
        // 指定父标签，对象中的属性对应子标签
        HashMap<String, Class> map = new HashMap<>();
        map.put("teacher",Teacher.class);
        marshaller.setAliases(map);

        // 设置根标签
        writer.setRootTagName("teachers");
        writer.setMarshaller(marshaller);

        // 指定输入的xml文件路径
        String path = "F\\cus.xml";
        writer.setResource(new FileSystemResource(path));
        writer.afterPropertiesSet();

        return writer;
    }
}

数据输出到多个文件

关键类：CompositeItemWriter

ClassiflerCompositeItemWriter—> 可以将不同的分类读取到不同的文件中

结合上面两种读取方式

@Bean
public CompositeItemWriter<Teacher> multiFileItemWriter() throws Exception {
    CompositeItemWriter<Teacher> writer = new CompositeItemWriter<>();
    // 把输出到普通文件中和xml文件中的ItemWriter
    writer.setDelegates(Arrays.asList(fileItemWriter(),staxEventItemWriter()));
    writer.afterPropertiesSet();
    return writer;
}

@Bean
    public ClassifierCompositeItemWriter<Teacher> multiFileItemWriter2(){
        ClassifierCompositeItemWriter<Teacher> writer = new ClassifierCompositeItemWriter<>();

        // 指定分类方式
        writer.setClassifier(new Classifier<Teacher, ItemWriter<? super Teacher>>() {
            @Override
            public ItemWriter<? super Teacher> classify(Teacher teacher) {
                ItemWriter<Teacher> itemWriter = null;
                try {
                    itemWriter = teacher.getTname().hashCode() % 2 == 0?fileItemWriter():staxEventItemWriter();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return itemWriter;
            }
        });
        return writer;
    }


    @Resource
    @Qualifier("fileItemWriter")
    private ItemStreamWriter<? super Teacher> fileItemWriter;

    @Resource
    @Qualifier("staxEventItemWriter")
    private ItemStreamWriter<? super Teacher> staxEventItemWriter;
    
	@Bean	
    public Step itemWriterDbDemoStep() {
    	return stepBuilderFactory.get("itemWriterDbDemoStep")
                                .<Teacher,Teacher>chunk(2)
                                .reader(itemWriterDbDemoReader())
                                .writer(itemWriterDbDemoWriter)
                                .stream(fileItemWriter)
                                .stream(staxEventItemWriter)
                                .build();

	}

识时务者J

关注

0
点赞
踩
1

收藏

觉得还不错? 一键收藏
0
评论
SpringBatch介绍与使用（中）

例子：从普通文件读取数据从xml读取数据1、导入依赖：2、创建xml文件3、使用：从多个文件中读取数据ItemReader处理异常ItemReader是一个一个读取数据，而ItemWriter是一批一批输出输出数据到普通文件中输出数据到Xml中数据输出到多个文件结合上面两种读取方式
复制链接

扫一扫

专栏目录