使用Spring batch进行批处理

最新推荐文章于 2024-06-11 08:58:09 发布

梦回孤零

最新推荐文章于 2024-06-11 08:58:09 发布

阅读量572

点赞数

分类专栏： java csv 文章标签： spring 批处理数据汇总

本文链接：https://blog.csdn.net/qq_35251019/article/details/76637706

版权

java 同时被 2 个专栏收录

7 篇文章 0 订阅

订阅专栏

csv

1 篇文章 0 订阅

订阅专栏

今日公司Boss提出了新需求：
    使用Spring-batch对数据进行汇总，我在网上找了好多配置方法,都没有配置成功,最后终于还是搞了个基本版出来了，贴出来以免自己忘记了
Spring-batch是一种批量的批处理方式：其工作方式是：
        工作流程为开启一个Job—>开启一个step—>（掉用reader—>调用processor）（循环直到读取到最后一条数据）—>调用writer—>完成一个step—>查看是否有下一个step(next="xxx")—>没有则完成一个Job,有则进入下一个step..
        总结起来就是开启Job重复reader—>processor最后把所有的数据一次性交给writer操作

<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:batch="http://www.springframework.org/schema/batch" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.springframework.org/schema/batch
        http://www.springframework.org/schema/batch/spring-batch-2.1.xsd
        http://www.springframework.org/schema/beans 
        http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
    ">
<bean id="jobLauncher"
        class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
        <property name="jobRepository" ref="jobRepository" />
    </bean>

    <bean id="jobRepository"
        class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
        <property name="validateTransactionState" value="false" />

    </bean>

Job 由于我在Spring 中已经开启了事务，所以便把Spring-batch中的事务关掉了，否则会报已经拥有事务异常

<batch:job id="writerclassJob">
        <batch:step id="stepwriter1" >
            <batch:tasklet>
                <batch:chunk reader="jdbcItemReader" writer="messagesItemWriter"
                    processor="itemProcessor" commit-interval="10">  <!--commit-interval="10"  并行处理的数量-->
                </batch:chunk>
            </batch:tasklet>
        </batch:step>
    </batch:job>

jdbcItemReader 读 ${detail_endtime}从配置文件中读取的数据作为查询的参数

<bean id="jdbcItemReader"
        class="org.springframework.batch.item.database.JdbcCursorItemReader"
        scope="step">
        <property name="dataSource" ref="dataSource" />
        <property name="sql"
            value="select  distinct examineeclassid,classcount from work  where checkdate &gt;'${detail_startime}' and checkdate &lt; '${detail_endtime}' " />
        <property name="rowMapper" ref="ledgerRowMapper">
        </property>
    </bean>

ledgerRowMapper 从数据库读取的数据传过来 ResultSet

package com.yc.batch;

import java.sql.ResultSet;
import java.sql.SQLException;

import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Component;

import com.yc.vo.TeacherWorkdetail;
import com.yc.vo.Workdetail;

@Component("ledgerRowMapper")  
public class LedgerRowMapper implements RowMapper {  
    @Override
    public Object mapRow(ResultSet rs, int rowNum) throws SQLException {  

        Workdetail twd=new Workdetail();
        twd.setClasscount(rs.getInt("classcount"));
        twd.setExamineeclassid(rs.getInt("examineeclassid"));
        return twd;  
    }


}

itemProcessor 操作

<bean id="itemProcessor" class="com.yc.batch.MessagesItemProcessor"
        scope="step">
    </bean>

package com.yc.batch;

import org.hibernate.engine.transaction.jta.platform.internal.SynchronizationRegistryBasedSynchronizationStrategy;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;

import com.yc.vo.Workdetail;


//业务层
@Component("messagesItemProcessor")
public class MessagesItemProcessor implements ItemProcessor<Workdetail, Workdetail> {


//在这类对获取的数据进行操作
    public Workdetail process(Workdetail workdetail) throws Exception {
        return workdetail;
    }

}

messagesItemWriter 写我是讲将数据库中的数据存入csv文件中

package com.yc.batch;

import java.io.InputStream;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

import javax.annotation.Resource;

import org.springframework.batch.item.ItemWriter;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;

import com.yc.biz.ExamineeClassBiz;
import com.yc.biz.WorkBiz;
import com.yc.utils.CsvUtils;
import com.yc.vo.Workdetail;

import net.sf.ehcache.util.PropertyUtil;



//写 
@Component("messagesItemWriter")
public class MessagesItemWriter implements ItemWriter<Workdetail>{

    @Resource(name = "workBiz")
    private WorkBiz workBiz;
    @Resource(name = "examineeClassBiz")
    private ExamineeClassBiz examineeClassBiz;



    public ExamineeClassBiz getExamineeClassBiz() {
        return examineeClassBiz;
    }

    public void setExamineeClassBiz(ExamineeClassBiz examineeClassBiz) {
        this.examineeClassBiz = examineeClassBiz;
    }

    public WorkBiz getWorkBiz() {
        return workBiz;
    }

    public void setWorkBiz(WorkBiz workBiz) {
        this.workBiz = workBiz;
    }

    public void write(List<? extends Workdetail> messages) throws Exception {
        //this.workBiz.updateWorkCheckcount();
        Properties props = new Properties();
        InputStream in= PropertyUtil.class.getClassLoader().getResourceAsStream("connectionConfig.properties");
        props.load(in);
        String startime=props.getProperty("detail_startime");
        String endtime=props.getProperty("detail_endtime");
        String time=props.getProperty("detail_time");
         List<Object> works=new ArrayList<Object>();
        for(Workdetail work:messages){
            Workdetail workdetail=new Workdetail();
            Integer classid=work.getExamineeclassid();
            int num=this.workBiz.getClassWorkNum(classid,startime,endtime);
            String classname=this.examineeClassBiz.findExamineeClassById(classid).getClassName();
            int checkcount=this.workBiz.getWorkCheckcount(classid, startime, endtime);
            Double p=(double) Integer.parseInt(String.valueOf(checkcount/(num*(work.getClasscount()))*100));
            workdetail.setExamineeclassid(classid);
            workdetail.setWorkcount(num);
            workdetail.setClasscount(work.getClasscount());
            workdetail.setClassName(classname);
            workdetail.setCheckcount(checkcount);
            workdetail.setCompletionrate(p);
            works.add(workdetail);
        }
        CsvUtils cu=new CsvUtils();
        String path=this.getClass().getResource("/").getPath();
        //String path=Class.class.getClass().getResource("/").getPath();
        path=path.substring(0,path.lastIndexOf("/"));
        path=path.substring(0,path.lastIndexOf("/"));
        path=path.substring(0,path.lastIndexOf("/"));
        path=path.substring(0,path.lastIndexOf("/"));
        cu.writeCsv(path+"/csv/class_"+time+".csv",works );
    }
}

CsvUtils 类使用jar包为javacsv2.0

package com.yc.utils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;

import net.sf.ehcache.util.PropertyUtil;

public class CsvUtils {

    /** 
     * 读取CSV文件 
     */  
    public List<String[]>  readeCsv(String path){  
        try {      

            ArrayList<String[]> csvList = new ArrayList<String[]>(); //用来保存数据  
            String csvFilePath = path;  
            CsvReader reader = new CsvReader(csvFilePath,',',Charset.forName("GBK"));    //一般用这编码读就可以了      

            //reader.readHeaders(); // 跳过表头   如果需要表头的话，不要写这句。  

            while(reader.readRecord()){ //逐行读入除表头的数据      
                csvList.add(reader.getValues());  
            }              
            reader.close();  

            /*for(int row=0;row<csvList.size();row++){  

                String  cell = csvList.get(row)[0]; //取得第row行第0列的数据  
                System.out.println(cell);  

            }  */
            return csvList;

        }catch(Exception ex){  
            System.out.println(ex);  
        }  

        return null;
    }  

    /** 
     * 写入CSV文件 
     * @throws IOException 
     */  
    public void writeCsv(String path,List<Object> t) throws IOException{  
        String csvFilePath = path;
//      System.out.println(path);
//      System.out.println("=================================");
        String filepath=path.substring(0,path.lastIndexOf("/"));
        File f=new File(filepath);
        if(!f.exists()){
            f.mkdirs();
        }
        File file=new File(path);


        if(!file.exists()){
            file.createNewFile();
        }
        CsvWriter wr =new CsvWriter(csvFilePath,',',Charset.forName("GBK"));
        try { 
            for(Object obj:t){
                String[] contents=obj.toString().split(",");
                //String[] contents = {"aaaaa","bbbbb","cccccc","ddddddddd"};                      
                wr.writeRecord(contents);  
            }
            wr.close();  
        } catch (IOException e) {  
            e.printStackTrace();  
        }  
    }  

}