Java两表数据大数据量比对

使用FutureTask进行多线程数据读取,减少读取时间

 public void checkCompany(){
        //获取数据
        List<CompanyVo> omsList = getOmsList();
        //将要比对数据放入Map
        Map<Long, CompanyVo> omsMap = new HashMap<Long, CompanyVo>();
        for(CompanyVo vo:omsList){
            omsMap.put(vo.getIdUuid(), vo);
        }
        //比较数据
        compareCompany(omsMap);
    }

每2000条数据比对一次,MySQL数据库

public void compareCompany(Map<Long, CompanyVo> omsMap){
		//获取总数据量
    	int count = jdbcTemplate.queryForObject("select count(1) from oms_company", Integer.class);
    	//分页查询
      	int pages = count%2000= 0?(count/2000):(count/2000 + 1);
      	ExecutorService executor = Executors.newFixedThreadPool(10);
      	for(int i = 0; i < pages; i++){
      		int startIndex = i*2000;
      		ProTask task = new ProTask(startIndex, omsMap);
      		FutureTask future = new FutureTask(task,null);
      		executor.execute(future);
      	}
      	executor.shutdown();
      	//判断是否比对完成
      	while(true){
      		if(executor.isTerminated()){
      			break;
      		}
      		try {
				Thread.sleep(1000);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
      	}
    }



获取要比较的总数据,Oracle数据库

public List<CompanyVo> getOmsList(){
    	List<CompanyVo> list = new ArrayList<>();
    	int count = jdbcTemplate.queryForObject("select count(1) from TB_PUBHA", Integer.class);
    	int pages = count%2000 == 0?(count/2000):(count/2000 + 1);
    	ExecutorService executor = Executors.newFixedThreadPool(10);
    	List<FutureTask<List<CompanyVo>>> futureList = new ArrayList<>();
    	for(int i = 0; i < pages; i++){
    		int start = i*2000 + 1;
    		int end = (i+1)*2000;
    		if(end > count) end = count;
    		OmsTask task = new OmsTask(start, end);
    		FutureTask<List<CompanyVo>> future = new FutureTask<>(task);
    		executor.submit(future);
    		futureList.add(future);
    	}
    	executor.shutdown();
    	for(FutureTask<List<CompanyVo>> future:futureList){
    		try {
				List<CompanyVo> listq = future.get();
				if(null != listq && !listq.isEmpty())
					list.addAll(listq);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (ExecutionException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
    	}
    	return list;
    }

OmsTask实现Callable接口,因为需要返回值

class OmsTask implements Callable<List<CompanyVo>>{

    	private int start;
    	
    	private int end;
    	
    	public OmsTask(int start,int end){
    		this.start = start;
    		this.end = end;
    	}
		@Override
		public List<CompanyVo> call() throws Exception {
			String sql = "select * from (select rownum rm, PUBHA_ID id_uuid,UUID company_number,PUBHA003 company_name,PUBHA005 company_address " +
                    ",PUBHA006 post_code,PUBHA008 corporation,PUBHA009 corporation_tel,PUBHA010 manager,PUBHA011 manager_tel,PUBHA012 contract,PUBHA013 contract_tel" +
                    ",PUBHA015 web_url from TB_PUBHA ) where rm between "+start+" and "+end;
			List<Map<String, Object>> mapList = jdbcTemplate.queryForList(sql);
			
			return CompanyVo.getCompany(mapList);
		}
    	
    }

ProTask实现Runnable接口,不需要返回结果,只进行比较

class ProTask implements Runnable{

    	private int startIndex;
    	private Map<Long, CompanyVo> omsMap;
    	
    	public ProTask(int startIndex,Map<Long, CompanyVo> omsMap){
    		this.startIndex = startIndex;
    		this.omsMap = omsMap;
    	}
		@Override
		public void run() {
			List<Map<String, Object>> listMap = jdbcTemplate.queryForList("select id_uuid,company_number,company_name,province_id,city_id,company_address,"
					+ "post_code,corporation,corporation_tel,manager,manager_tel,contract,contract_tel,web_url "
					+ "from oms_company limit "+startIndex+","+ 2000);
			List<CompanyVo> list = CompanyVo.getCompany(listMap);
			if(null != list && !list.isEmpty()){
				List<Object[]> obs = new ArrayList<>();
				for(CompanyVo vo:list){
					CompanyVo oms = omsMap.get(vo.getIdUuid());
					if(null == oms){
						continue;
					}
					if((null == vo.getCompanyNumber() && null != oms.getCompanyNumber())
							|| (null != vo.getCompanyNumber() && !vo.getCompanyNumber().equals(oms.getCompanyNumber()))){
						//结果不同时进行处理
					}
					//对已比对数据进行移除
					omsMap.remove(vo.getIdUuid());
				}
			}
		}
    	
    }

实体类,主要进行查询结果转化

package com.digital.domain;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import com.digital.util.StringUtil;

public class CompanyVo {
	
	private Long idUuid;
	
	private String companyNumber;
	
	
	public Long getIdUuid() {
		return idUuid;
	}

	public void setIdUuid(Long idUuid) {
		this.idUuid = idUuid;
	}

	public String getCompanyNumber() {
		return companyNumber;
	}

	public void setCompanyNumber(String companyNumber) {
		this.companyNumber = companyNumber;
	}

	
	
	public static CompanyVo getCompany(Map<String, Object> map){
		if(null == map) return null;
		CompanyVo vo = new CompanyVo();
		 String idUuid = StringUtil.ObjectToString(map.get("ID_UUID"));
		 if(null != idUuid)
			 vo.setIdUuid(Long.parseLong(idUuid));
         vo.setCompanyNumber(StringUtil.ObjectToString(map.get("COMPANY_NUMBER")));
         
		return vo;
	}
	
	public static List<CompanyVo> getCompany(List<Map<String, Object>> mapList){
		if(null == mapList) return null;
		List<CompanyVo> list = new ArrayList<>();
		for(Map<String, Object> map:mapList){
			CompanyVo vo = getCompany(map);
			if(null != vo)
				list.add(vo);
		}
		return list;
	}

}


发布了12 篇原创文章 · 获赞 7 · 访问量 4万+
展开阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 编程工作室 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览