数据批量导入时，加入队列，分批处理，只是个笔记

1，用到技术点：队列，线程，单例模式，分批处理

2，添加笔记代码：

入口：

if(null != set && set.size() > 0){
	//异步，加入队列
	logger.info(String.format("加入队列，总共  %s 条数据", set.size()));
	TrackBusinessRunner trackBusinessRunner	= TrackBusinessRunner.getInstance();//获取单例
	TrackDataDTO trackDataDTO = new TrackDataDTO();//放入处理好的数据
	trackDataDTO.setParam(param);
	trackDataDTO.setSets(set);
	trackBusinessRunner.putQueueOnload(trackDataDTO);//放入队列
	if(trackBusinessRunner.getThreadTrackBusinessService()==null){
				trackBusinessRunner.setThreadTrackBusinessService(threadTrackBusinessService);
	}
	if(!trackBusinessRunner.isAlive()){
	trackBusinessRunner.start();//判断是否启动状态，如果不是就启动
	}
				
}

单例模式和队列的核心类

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;

import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;

import com.sf.iec.common.util.BatchHandlerInterface;
import com.sf.iec.common.util.BatchHandlerList;
import com.sf.iec.customerbusiness.inquiryorder.dto.TrackDataDTO;
import com.sf.iec.customerbusiness.inquiryorder.service.ThreadTrackBusinessService;

public class TrackBusinessRunner extends Thread {

	private final static Logger LOGGER = Logger.getLogger(TrackBusinessRunner.class);
	private ThreadTrackBusinessService threadTrackBusinessService;
	
	public ThreadTrackBusinessService getThreadTrackBusinessService() {
		return threadTrackBusinessService;
	}
	private LinkedBlockingQueue<TrackDataDTO> blockingQueue = new LinkedBlockingQueue<TrackDataDTO>(300);//队列长度300，非常推荐该队列（put和take好好用）
	private volatile boolean running = true;//开启一个线程
	
	private TrackBusinessRunner(){}
	private static TrackBusinessRunner trackBusinessRunner;
	private static Object obj = new Object();
	//单例模式
	public static TrackBusinessRunner getInstance(){
		if(trackBusinessRunner==null){
			synchronized (obj) {
				if(trackBusinessRunner==null){
					trackBusinessRunner = new TrackBusinessRunner();
				}
			}
		}
		return trackBusinessRunner;
	}
	
	public void putQueueOnload(TrackDataDTO trackDataDTO){
		int i= 0;
		try {
			blockingQueue.put(trackDataDTO);//加入队列
			i = 0;
		} catch (InterruptedException e) {
			LOGGER.error("加入队列信息异常");
			e.printStackTrace();
			if(i < 2){
				putQueueOnload(trackDataDTO);
				i++;
			}
		}
	}

	@Override
	public void run() {
		while (running) {
			try {
				TrackDataDTO trackDataDTO = blockingQueue.take();
				Set<Map<String,Object>> set = trackDataDTO.getSets();
				final Map<String, String> param = trackDataDTO.getParam();
				//处理 插入  	
				List<Map<String,Object>> lst = new ArrayList<Map<String,Object>>();
				CollectionUtils.addAll(lst, set.iterator());
//分批处理，每次取200条
				BatchHandlerList<Map<String, Object>> handler = new BatchHandlerList<Map<String,Object>>(200,lst) {
					@Override
					public void handler(List<Map<String, Object>> subList) {
						// TODO Auto-generated method stub						
						threadTrackBusinessService.saveMainTainTrajectory(subList, param);
						//休眠	
						try {
							Thread.sleep(12000);
						} catch (InterruptedException e) {
							LOGGER.error("batch track handler thread interrupt excption",e);
						}//12秒
					}
				};
				handler.handlerList();
			} catch (Exception e) {
				LOGGER.error("获取队列信息异常",e);
				e.printStackTrace();
			}//取数据，没有的话会等待
		}
	}		
	
	public void setThreadTrackBusinessService(
			ThreadTrackBusinessService threadTrackBusinessService) {
		this.threadTrackBusinessService = threadTrackBusinessService;
	}

}

分批接口

import java.util.List;

public interface BatchHandlerInterface<T> {
	public void handler(List<T> subList);
}

分批处理工具类

import java.util.List;

import org.apache.log4j.Logger;

/**
 * @author  
 * @description 分批调用方法接口
 * */
public abstract class BatchHandlerList<T> implements BatchHandlerInterface<T> {
	
	private static final Logger LOGGER = Logger.getLogger(BatchHandlerList.class);
	//每次处理条数
	private Integer perNum;
	
	private List<T> aylist;

	public BatchHandlerList(Integer perNum, List<T> aylist) {
		super();
		this.perNum = perNum;
		this.aylist = aylist;
	}
	
	/**
	 * 分批调用方法
	 * */
	public void handlerList(){
		try{
			if(aylist!=null && aylist.size() > 0){
				int size = aylist.size();
				int startIndex = 0;
				int endIndex = 1;
				int num = 1;
				if (size > perNum) {
					num = size / perNum;
				}
				for (int i = 1; i <= num; i++) {
					endIndex = (i) * perNum > size ? size : (i) * perNum;
					List<T> subList = aylist.subList(startIndex, endIndex);
					
					startIndex = perNum * i;
					if (subList!=null && subList.size() > 0) {
						handler(subList);
					}
					
					if (num == i && perNum * num < size) {
						//最后一批处理
						subList = aylist.subList(perNum * num, size);
						if (subList.size() > 0) {
							handler(subList);
						}
					}
			   }
			}
		}catch(Throwable e){
			LOGGER.error("batchHandlerList handler exception",e);
			//错误回调方法可以重写
			errorHandler();
		}
	}
	
	public void errorHandler(){};
}