SpringBatch

FlatFileItemReader 从普通文件中读取数据

/**
 * FlatFileItemReader 从普通文件中读取数据
 * @author miles
 */
@Slf4j
@Configuration
@EnableBatchProcessing
public class FlatFileItemReaderDemo {

    @Resource
    private JobBuilderFactory jobBuilderFactory;

    @Resource
    private StepBuilderFactory stepBuilderFactory;

    @Bean
    public Job fileItemReaderJob() throws Exception {
        return jobBuilderFactory.get("flatFileReaderJob")
                .start(fileItemReaderStep())
                .build();
    }

    @Bean
    public Step fileItemReaderStep() throws Exception {
        return stepBuilderFactory.get("flatFileReaderStep")
                .<Student, Student>chunk(5)
                .reader(flatFileReader())
                .writer(flatFileWriter())
                .build();
    }

    /**
     * 从普通文件读取数据
     */
    public FlatFileItemReader<Student> flatFileReader() {
        FlatFileItemReader<Student> reader = new FlatFileItemReader<>();
        reader.setResource(new ClassPathResource("student.txt"));
        //1.跳过第一行标题
        reader.setLinesToSkip(1);

        /*2.解析数据*/
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        tokenizer.setNames(new String[]{"id", "name", "age"});

        /*3.把解析出的数据映射为实体对象*/
        DefaultLineMapper<Student> mapper = new DefaultLineMapper<>();
        mapper.setLineTokenizer(tokenizer);
        mapper.setFieldSetMapper(new FieldSetMapper<Student>() {
            @Override
            public Student mapFieldSet(FieldSet fieldSet) throws BindException {
                Student stu = new Student();
                stu.setId(fieldSet.readInt("id"));
                stu.setName(fieldSet.readString("name"));
                stu.setAge(fieldSet.readInt("age"));
                return stu;
            }
        });
        //4.
        mapper.afterPropertiesSet();
        reader.setLineMapper(mapper);
        return reader;
    }

    public FlatFileItemWriter<? super Student> flatFileWriter() throws Exception {
        //1.目标
        FlatFileItemWriter<Student> writer = new FlatFileItemWriter<>();
        String path = "C:\\Users\\miles\\Desktop\\SpringBatch_demo\\src\\main\\resources\\FlatFileItemWriter.txt";
        writer.setResource(new FileSystemResource(path));
        //2. 把对象转换成字符串
        writer.setLineAggregator(new LineAggregator<Student>() {
            final ObjectMapper mapper = new ObjectMapper();
            @Override
            public String aggregate(Student item) {
                String str = null;
                try {
                    str = mapper.writeValueAsString(item);
                } catch (JsonProcessingException e) {
                    e.printStackTrace();
                }
                return str;
            }
        });
        writer.afterPropertiesSet();
        return writer;
    }
}

从数据库中读取数据

 /**
     * 从数据库中读取数据 
     * @StepScope 制定作用域为这个step
     */
    @Bean
    @StepScope
    public ItemReader<? extends Student> jdbcReader() {
        JdbcPagingItemReader<Student> reader = new JdbcPagingItemReader<>();
        reader.setDataSource(dataSource);
        //指定一次读取n个数据
        reader.setFetchSize(2);
        log.info("ItemReader读取一次数据...");
        //把读取到的数据转换成实体对象
        reader.setRowMapper((result, rowNum) -> {
            Student student = new Student();
            student.setId(result.getInt(1));
            student.setName(result.getString(2));
            student.setAge(result.getInt(3));
            return student;
        });
        reader.setRowMapper((result, rowNum) -> {
            Student student = new Student();
            student.setId(result.getInt(1));
            student.setName(result.getString(2));
            student.setAge(result.getInt(3));
            return student;
        });
        /* 给reader指定sql语句 */
        MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider();
        provider.setSelectClause("id, name, age");
        provider.setFromClause("from student");
        //指定排序字段排序，1个字段，map=1
        Map<String, Order> sort = new HashMap<>(1);
        sort.put("age", Order.DESCENDING);
        provider.setSortKeys(sort);
        reader.setQueryProvider(provider);
        return reader;
    }

    /**
     * 把数据写进数据库
     */
    @Bean
    public JdbcBatchItemWriter<? super Student> dbWriter() {
        log.info("JdbcBatchItemWriter, 写数据到数据库...");
        JdbcBatchItemWriter<Student> writer = new JdbcBatchItemWriter<>();
        writer.setDataSource(dataSource);
        writer.setSql("insert into student(id,name,age) values (:id,:name,:age)");
        //属性值的映射
        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>());
        return writer;
    }

MultiFileItemReader 同时从多个文件中读取数据

/**
 * MultiFileItemReader 同时从多个文件中读取数据
 * @author miles
 */
@Slf4j
@Configuration
@EnableBatchProcessing
public class MultiFileReaderDemo {

    @Resource
    private JobBuilderFactory jobBuilderFactory;

    @Resource
    private StepBuilderFactory stepBuilderFactory;

    @Value("classpath:/file*.txt")
    private org.springframework.core.io.Resource[] filesResources;

    @Bean
    public Job multiFileReaderJob() throws Exception {
        return jobBuilderFactory.get("multiFileReaderJob")
                .start(multiFileReaderStep())
                .build();
    }

    @Bean
    public Step multiFileReaderStep() throws Exception {
        return stepBuilderFactory.get("multiFileReaderStep")
                .<Student, Student>chunk(5)
                .reader(multiFileReader())
                .processor(processor())
                .writer(classMultiItemwriter())
                .stream(xmlFileWriter())
                .stream(flatFileWriter())
                .build();
    }
    
    /**
     *  有一个处理的process时直接调用，
     *  有多个process时需要如下写法。
     */
    public CompositeItemProcessor<? super Student,? extends Student> processor() {
        CompositeItemProcessor<Student,Student> processor = new CompositeItemProcessor<>();
        List<ItemProcessor<Student,Student>> delegates = new ArrayList<>();
        //TODO
        //delegates.add();//第一个处理过程，process
        //delegates.add();//第二个处理过程
        processor.setDelegates(delegates);
        return processor;
    }

    /**
     * 从多个文件读取数据
     */
    @Bean
    public MultiResourceItemReader<Student> multiFileReader() {
        MultiResourceItemReader<Student> reader = new MultiResourceItemReader<>();
        reader.setDelegate(flatFileReader());
        reader.setResources(filesResources);
        return reader;
    }

    /**
     * 从普通文件读取数据
     */
    @Bean
    @StepScope
    public FlatFileItemReader<Student> flatFileReader() {
        log.info("读一次数据...");
        FlatFileItemReader<Student> reader = new FlatFileItemReader<>();
        reader.setResource(new ClassPathResource("student.txt"));
        //1.跳过第一行标题
        reader.setLinesToSkip(1);

        /*2.解析数据*/
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        tokenizer.setNames(new String[]{"id", "name", "age"});

        /*3.把解析出的数据映射为实体对象*/
        DefaultLineMapper<Student> mapper = new DefaultLineMapper<>();
        mapper.setLineTokenizer(tokenizer);
        mapper.setFieldSetMapper(fieldSet -> {
            Student stu = new Student();
            stu.setId(fieldSet.readInt("id"));
            stu.setName(fieldSet.readString("name"));
            stu.setAge(fieldSet.readInt("age"));
            return stu;
        });
        //4.
        mapper.afterPropertiesSet();
        reader.setLineMapper(mapper);
        return reader;
    }

    /**
     * 将数据同时写到多个文件中
     */
    @Bean
    public CompositeItemWriter<? super Student> compositeItemWriter() throws Exception {
        CompositeItemWriter<Student> writer = new CompositeItemWriter<>();
        writer.setDelegates(Arrays.asList(flatFileWriter(), xmlFileWriter()));
        writer.afterPropertiesSet();
        return writer;
    }

    /**
     * 将数据分类写到不同文件
     */
    @Bean
    public ClassifierCompositeItemWriter<Student> classMultiItemwriter() {
        ClassifierCompositeItemWriter<Student> writer = new ClassifierCompositeItemWriter<>();
        writer.setClassifier(new Classifier<Student, ItemWriter<? super Student>>() {
            @Override
            public ItemWriter<? super Student> classify(Student student) {
                ItemWriter<Student> itemwriter = null;
                try {
                    //按照Id进行分类
                    itemwriter = student.getId() % 2 == 0 ? flatFileWriter() : xmlFileWriter();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return itemwriter;
            }
        });
        return writer;
    }

    /**
     * 数据写到xml文件
     */
    @Bean
    public StaxEventItemWriter<Student> xmlFileWriter() throws Exception {
        StaxEventItemWriter<Student> writer = new StaxEventItemWriter<>();
        XStreamMarshaller marshaller = new XStreamMarshaller();
        Map<String, Class> map = new HashMap<>();
        map.put("student", Student.class);
        writer.setRootTagName("students");
        writer.setMarshaller(marshaller);
        String path = "C:\\Users\\miles\\Desktop\\SpringBatch_demo\\src\\main\\resources\\ComXml.xml";
        writer.setResource(new FileSystemResource(path));
        writer.afterPropertiesSet();
        return writer;
    }

    /**
     * 数据写到txt文件
     */
    @Bean
    public FlatFileItemWriter<Student> flatFileWriter() throws Exception {
        //1.目标
        FlatFileItemWriter<Student> writer = new FlatFileItemWriter<>();
        String path = "C:\\Users\\miles\\Desktop\\SpringBatch_demo\\src\\main\\resources\\ComJson.txt";
        writer.setResource(new FileSystemResource(path));
        //2. 把对象转换成字符串
        writer.setLineAggregator(new LineAggregator<Student>() {
            final ObjectMapper mapper = new ObjectMapper();

            @Override
            public String aggregate(Student item) {
                String str = null;
                try {
                    str = mapper.writeValueAsString(item);
                } catch (JsonProcessingException e) {
                    e.printStackTrace();
                }
                return str;
            }
        });
        writer.afterPropertiesSet();
        return writer;
    }
}

xml文件读写数据

  private StaxEventItemReader<? extends Student> xmlFileReader() {
        log.info("进入xmlFileReader...");
        StaxEventItemReader reader = new StaxEventItemReader();
        //1.指定数据源
        reader.setResource(new ClassPathResource("student.xml"));
        //2.指定需要处理每个数据项的根标签，这里是<student>
        reader.setFragmentRootElementName("student");
        //3.把xml转换成对象,指定对象
        XStreamMarshaller unmarshaller = new XStreamMarshaller();
        Map<String, Class> map = new HashMap<>();
        map.put("student", Student.class);
        unmarshaller.setAliases(map);
        reader.setUnmarshaller(unmarshaller);
        return reader;
    }

    private StaxEventItemWriter<? super Student> xmlFileWriter() throws Exception {
        StaxEventItemWriter<Student> writer = new StaxEventItemWriter();
        XStreamMarshaller marshaller = new XStreamMarshaller();
        Map<String, Class> map = new HashMap<>();
        map.put("student", Student.class);
        writer.setRootTagName("students");
        writer.setMarshaller(marshaller);
        String path = "C:\\Users\\miles\\Desktop\\SpringBatch_demo\\src\\main\\resources\\XmlFileWriter.xml";
        writer.setResource(new FileSystemResource(path));
        writer.afterPropertiesSet();
        return writer;
    }