多线程高级(实战篇)

• 看到此篇章的小伙伴相信已经看过前面初级,和中级的讲义了。那么这篇博客我们来聊聊如何使用多线程处理百万级的数据?

• 批量插入百万级别的数据,如何做到高效？

百万级数据批量插入我们应该是什么样的设计思路呢？
1.首先我们需要考虑到的是一个高并发的问题,那么处理高并发同步锁或者同步代码块是必不可少的。你可以使用synchronized或者ReentranLock来进行代码块的锁定。
2.其次我们要使用到线程池,每个线程处理多少数据量。
3.使用CountDownLatch计数当所有线程执行完,在执行主线程。

• 代码示例

public class ThreadPoolTool<T> {
	
	//单个线程处理的数据量
    private int singleCount;
	
    private int listSize;
	
    private int runSize;

    private List<T> list;
	
    private CountDownLatch begin,end;
	
    private ExecutorService executorService;

    private CallBack callBack;

    public void setCallBack(CallBack callBack){
        this.callBack = callBack;
    }

    //定义启动线程的数量
    public ThreadPoolTool(int singleCount,List<T> list){
        this.singleCount = singleCount;
        this.list = list;
        if(list!=null){
            this.listSize = list.size();
            //取余如果为整数那么启动的线程数直接相除就行。否则相除+1
            this.runSize = (this.listSize%this.singleCount==0)?
                    this.listSize/this.singleCount:this.listSize/this.singleCount+1;
        }
    }

    public void execute() throws InterruptedException{
        executorService = Executors.newFixedThreadPool(runSize);
        begin = new CountDownLatch(1);
        end = new CountDownLatch(runSize);
        //创建线程
        int startIndex = 0;
        int endIndex = 0;
        List<T> newList = null;
        for (int i = 0; i < runSize; i++) {
            //计算每个线程对应的数据
            if(i<(runSize-1)){
                startIndex = i * singleCount;
                endIndex = (i+1) * singleCount;
                newList = list.subList(startIndex,endIndex);
            }else{
                startIndex = i * singleCount;
                endIndex = listSize;
                newList = list.subList(startIndex,endIndex);
            }
        }

        //创建线程类处理数据
        MyThread<T> myThread = new MyThread<T>(newList,begin,end) {

            @Override
            protected void method(List<T> list) {
                callBack.method(list);
            }
        };

        executorService.execute(myThread);
        //计数器减一
        begin.countDown();
        end.await();
        executorService.shutdown();
    }
}

class UserExecutor{

    public static void main(String[] args) {
        List<User> users = new ArrayList<>();
        for (int i = 0; i < 10000; i++) {
            User user = new User();
            user.setId(i);
            user.setAge(1);
            user.setName("张三" + i);
            users.add(user);
        }

        ThreadPoolTool<String> tool = new ThreadPoolTool(1000, users);
        tool.setCallBack(new CallBack() {

            @Override
            public void method(List list) {
                //执行插入代码 userDao.insertBatchDemo(list);
            }
        });

        try {
            tool.execute();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

}

abstract class MyThread<T> implements Runnable{

    private List<T> list ;
    private CountDownLatch begin,end;

    public MyThread(List<T> list,CountDownLatch begin,CountDownLatch end){
        this.list = list;
        this.begin = begin;
        this.end = end;
    }

    @Override
    public void run() {
        try {
            method(list);
            begin.await();
        }catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            //计数器减一
            end.countDown();
        }
    }

    protected abstract void method(List<T> list);
}

interface CallBack<T> {
    void method(List<T> list);
}

class User{
    private String name;
    private int age;
    private int id;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public int getAge() {
        return age;
    }

    public void setAge(int age) {
        this.age = age;
    }

    public int getId() {
        return id;
    }

    public void setId(int id) {
        this.id = id;
    }
}