Java中的批量操作实现

package com.h.collection;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by John on 2017/9/11.
 */
public class Test {
    public static void main(String[] args) {
        List<String> list = new ArrayList<String>();
        list.add("1");
        list.add("2");
        list.add("3");
        String str = String.join(",",list);
        System.out.println(str);
    }

    /**
     * 批量查询/插入
     * 查询数据量非常大时,分批查,减轻数据库一次查询的压力
     * @param userIdList 用户ID的集合
     */
    public static void batchSelect(List<String> userIdList){
        int totalSelectCount = userIdList.size();//总查询量
        int perSelectCount = 1000;//每次查询的数据量
        int index = 0;//查询中间量
        List<String> selectList = null;
        List<User> userList = null;
        while (totalSelectCount > index){
            int selectCount = Math.min(totalSelectCount,index + perSelectCount);
            selectList = userIdList.subList(index,selectCount);//查询范围[index,selectCount-1];
            userList = userServiceImpl.batchSelectUserById(String.join(",",selectList));
            for (User user:userList){
                //其他set操作,继续封装
            }
            index += perSelectCount;
        }
    }

    /**
     * 批量查询/插入
     * 查询数据量非常大时,分批查,减轻数据库一次查询的压力
     * @param userIdList
     */
    public static void batchSelect2(List<String> userIdList){
        int totalSelectCount = userIdList.size();//总查询量
        int perSelectCount = 1000;//每次查询的数据量
        int cycleCount = totalSelectCount % perSelectCount == 0 ? totalSelectCount/perSelectCount:totalSelectCount/perSelectCount + 1;//循环查询次数
        List<String> selectList = null;
        List<User> userList = null;
        for (int i=0;i<cycleCount;i++){
            //[0,999] [1000,1999] ... 这里用到了分页的思想
            selectList = userIdList.subList(i*perSelectCount,(i+1)*perSelectCount > totalSelectCount ? totalSelectCount:(i+1)*perSelectCount);//[,)
            userList = userServiceImpl.batchSelectUserById(String.join(",",selectList));
            for (User user:userList){
                //其他set操作,继续封装
            }
        }
    }
}

package com.h.java8;

import java.util.List;
import java.util.stream.Collectors;

/**
 * Created by John on 2018/9/30.
 * 批量操作模板类,如批量查询/插入
 */
public abstract class BatchOperateUtil<T> { //T为操作的实际元素
    //需要操作的数据源
    private List<T> sourceList;

    //批量操作的临界值(每次操作量)
    private int threshold;

    public BatchOperateUtil setSourceList(List<T> sourceList){
        this.sourceList = sourceList;
        return this;
    }

    public BatchOperateUtil setThreshold(int threshold){
        this.threshold = threshold;
        return this;
    }

    public void batchOperate() {
        int size = sourceList.size();
        int fromIndex = 0;
        List<T> list = null;
        int toIndex = 0;
        while (size > fromIndex){
            toIndex = Math.min(size,fromIndex + threshold);
            //注意截取子串时索引越界
            list = sourceList.subList(fromIndex,toIndex);
            operate(list);
            fromIndex += threshold;
        }
    }

    /**
     * 分页的思想实现批量操作
     */
    public void batchOperate2(){
        //总操作量
        int size = sourceList.size();
        //循环操作次数
        int cycleCount = size % threshold == 0 ?  size/threshold :  size/threshold + 1;
        List<T> list = null;
        for (int i=0;i<cycleCount;i++){
            //注意截取子串时索引越界
            list = sourceList.subList(i*threshold,Math.min((i+1)*threshold,size));
            operate(list);
        }
    }

    /**
     * 使用java8的流式操作截取子串(不需要考虑越界问题)
     */
    public void batchOperate3(){
        int size = sourceList.size();
        int fromIndex = 0;
        List<T> list = null;
        while (size > fromIndex){
            list = sourceList.stream().skip(fromIndex).limit(threshold).collect(Collectors.toList());
            operate(list);
            fromIndex += threshold;
        }
    }

    /**
     * 具体的操作实现
     * @param data
     */
    public abstract void operate(List<T> data);
}

package com.h.java8;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by John on 2018/9/23.
 */
public class TestMain {

    public static void main(String[] args) {
        List<String> sourceList = new ArrayList<>();
        sourceList.add("a");
        sourceList.add("b");
        sourceList.add("c");
        sourceList.add("d");
        sourceList.add("e");

        BatchOperateUtil<String> operateUtil = new BatchOperateUtil<String>() {
            @Override
            public void operate(List<String> data) {
                System.out.println(data);
                //可以是批量插入或查询的具体操作
            }
        }.setSourceList(sourceList).setThreshold(3);
        operateUtil.batchOperate();
        System.out.println("==============");
        operateUtil.batchOperate2();
        //使用java8的流式操作
        System.out.println("==============");
        operateUtil.batchOperate3();
    }
}

使用Lambda表达式替代上面的模版方法

public class BatchOperateUtil<T> { //T为操作的实际元素
    //需要操作的数据源
    private List<T> sourceList;

    //批量操作的临界值
    private int threshold;

    public BatchOperateUtil setSourceList(List<T> sourceList){
        this.sourceList = sourceList;
        return this;
    }

    public BatchOperateUtil setThreshold(int threshold){
        this.threshold = threshold;
        return this;
    }

    public void batchOperate(Consumer<List<T>> operater) {
        int size = sourceList.size();
        int fromIndex = 0;
        List<T> list = null;
        while (size > fromIndex){
            list = sourceList.stream().skip(fromIndex).limit(threshold).collect(Collectors.toList());
            operater.accept(list);
            fromIndex += threshold;
        }
    }
}

	    List<String> list = Arrays.asList("a","b","c","d","e");
        BatchOperateUtil<String> batchOperateUtil = new BatchOperateUtil<>();
        batchOperateUtil.setSourceList(list).setThreshold(3).batchOperate(l -> {
            System.out.println(l);//模拟数据库的批量插入或查询操作
        });