java 数据批处理dome

package com.gg;

import com.gg.datasource.DynamicDataSourceManager;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.test.context.junit4.SpringRunner;

import java.util.*;

import java.util.concurrent.*;

@RunWith(SpringRunner.class)
@SpringBootTest
public class OrderTest {

    @Autowired
    private JdbcTemplate jdbcTemplate;

    @Test
    public void batchProcessing(){
        DynamicDataSourceManager.changeManageDataSource();
        long l = System.currentTimeMillis();
//        multithreading();
        batchProcessing1();
        System.out.println("l = " + (System.currentTimeMillis() - l));
    }

    /**
     * 直接循环插入
     */
    private void batchProcessing1(){
        List<Object[]> w = get10W();
        List<Object[]> objects = new ArrayList<>();
        for (Object[] objects1 : w) {
            objects.add(objects1);
        }
        save(objects);
    }


    /**
     * 创建多线程插入
     */
    private void multithreading() {
        //线程数量
        int THREAD_NUM = 5;
        // 线程池
        ExecutorService executorService = Executors.newFixedThreadPool(THREAD_NUM);

        //每个线程处理一万条，处理完后直接保存
        int w_10 = 10000;
        List<Object[]> w = get10W();

        //计算要创建的线程数量
        int ceil = (int)Math.ceil((double) w.size() / w_10);

        //循环创建
        for (int i = 0; i < ceil; i++) {
            // Callable
            int finalI = i;
            executorService.execute( ()-> {
                task(finalI * w_10, finalI+1 == ceil ? w.size() : finalI * w_10 + w_10,w);
            });
        }
        //关闭线程池
        executorService.shutdown();

        try {
            // 等待所有任务完成 trur:成功，false:表示失败
            boolean b = executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
        } catch (Exception e) {
            // 异常处理
            System.out.println("异常...." + e.getMessage());
        } finally {
            // 关闭线程池
            executorService.shutdown();
        }
    }

    /**
     * 实例化数据
     * 使用原生jdbc更快
     */
    private int[] save(List<Object[]> list) {
        long l = System.currentTimeMillis();
        String sql = "insert into act_evt_log(LOG_NR_,TYPE_) values (?,?)";
        int[] ints = jdbcTemplate.batchUpdate(sql, list);
        System.out.println("save ========================= " + (System.currentTimeMillis() - l));
        return ints;
    }


    private void task(int start,int finish,List<Object[]> w){

        System.out.println("task -> start = "+start+";finish = "+finish);

        List<Object[]> objects = new ArrayList<>();
        for (int i = start; i < finish; i++) {
            objects.add(w.get(i));
        }

        int[] save = save(objects);

        String name = Thread.currentThread().getName();
        System.out.println(name+"当前线程执行结束");
    }

    /**
     * 创建测试数据
     */
    private  List<Object[]> get10W() {
        List<Object[]> objects = new ArrayList<>(500000);
        for (int i = 0; i < 500000; i++) {
            objects.add(new Object[]{i,i+"a"});
        }
        return objects;
    }

}