java并发之生产者消费者模式的应用

并发编程作为java语言体系中非常重要的模块，除了有许多概念性的东西需要搞明白之外，还需要多上手实操，才能有更深的体会，更熟练的代码编写。
复习一下并发之常见的概念性名词：happens-before、as-if-serial。学习并发之前基础的概念性理论要明白一点，编译器和处理器为了更好的性能，常常会对程序员编写的代码指令进行指令重排序，也就是实际执行的代码顺序可能跟我们编写代码的顺序并不一致，这样就导致了如果没有一个原则语义来定义编译器和处理器的这种处理逻辑，程序员就崩溃了，因为他感觉自己再也无力去掌控世界了…2333，所以就产生了happens-before原则、as-if-serial语义等概念，编译器和cpu在这种逻辑的束缚下，既提高了性能，又能给程序员一种代码就是自己控制下的1–>2–>3–>4这种的执行，两个字：妙啊…
啰嗦的有点多了，下面到本篇重点，用生产者消费者模式模拟一个简单业务的实现—实现一张10W数据量表的内容复制。
上代码之前，说明下环境相关：
jdk8, mysql8 ，springboot 2.2.6.RELEASE ,
两张数据库表：

直接上代码：

package com.gjn.threadpool.service;

import cn.hutool.core.util.RandomUtil;
import com.gjn.threadpool.model.AdminLog;
import com.gjn.threadpool.model.DBConnection;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.sql.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
 * Created by jn on 2020/5/13
 * 模拟业务：
 *  实现原表（ums_admin_login_log）数据copy到另一张备用表（dgg_admin_login_log），有10W数据量
 * 实现想法：
 * 生产者消费者模式
 * 读数据和写数据用不同的线程，读取的数据放到一个队列里，写数据的线程池先去队列取数据，然后在消费
 */
@Service
@Slf4j
public class ExcuteData {

    @Autowired
    DBConnection dbConnection;

    /**
     * 定义步长
     * 步长推算出需要的id范围
     *      * 1   1  10000
     *      * 2  10001 20000
     *      * 3  20001 30000
     *          .......
     */
    final int step = 10000;
    /**
     * 定义fetchSize，fetchSize有啥用，可以查询jdbc批量读取数据相关博客就懂了
     */
    final int fetchSize = 1000;

    /**
     * 定义线程池处理请求
     */
    ThreadPoolExecutor threadPoolExecutor =
            new ThreadPoolExecutor(5, 15, 10,
                    TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(50));

    /**
     * 读取数据到此队列，多消费端取数据然后X入数据
     */
    LinkedBlockingDeque<List<AdminLog>> linkedBlockingDeque = new LinkedBlockingDeque<>();

    /**
     * 处理数据的入口方法
     */
    public void excuteData() {
        log.info ("获取CPU个数"+Runtime.getRuntime ().availableProcessors () );
        //从数据库中读取数据到阻塞队列
        //1个线程读1W条数据到队列，开启10个任务丢给线程池处理
        for (int i = 0; i < 10; i++) {
            int finalI = i;
            threadPoolExecutor.execute(()->{
               readData(finalI+1);
            });
        }

        //从阻塞队列里读取数据,执行insert到数据库
        //就开个线程池去阻塞队列里读吧
        while(true){
            try {
                //等20s取不出数据，基本说明阻塞队列该读的全读出来了，也没有新的写入了，因此break
                List<AdminLog> list = linkedBlockingDeque.poll(20, TimeUnit.SECONDS);
                if(list == null){
                    break;
                }
                threadPoolExecutor.execute(() -> {
                    writeData(list);
                    log.info("==first==线程池中线程数目：" + threadPoolExecutor.getPoolSize() + "，队列中等待执行的任务数目：" +
                            threadPoolExecutor.getQueue().size() + "，已执行玩别的任务数目：" + threadPoolExecutor.getCompletedTaskCount());
                    log.info("ThreadPoolSize={}, ThreadPoolActiveCount={}", threadPoolExecutor.getPoolSize(), threadPoolExecutor.getActiveCount());
                });
            } catch (InterruptedException e) {
                e.printStackTrace();
            }

        }
    }

    /**
     * 读取数据
     * 每次读取数据的一部分,10000一个步长，10个任务丢给线程池来读取全部10W条数据
     * @param n
     */
    public void readData(int n){
        long start = System.currentTimeMillis();
        //加入队列
        Connection connection = null;
        PreparedStatement ps = null;
        ResultSet rs = null;
        try {
            System.out.println("========" + dbConnection);
            Class.forName(dbConnection.getDriver());
            connection = DriverManager.getConnection(dbConnection.getUrl(), dbConnection.getUsername(), dbConnection.getPassword());
            connection.setAutoCommit(false);
            String sql = "select * from ums_admin_login_log where id between ? and ?";
            ps = connection.prepareStatement(sql);
            ps.setInt(1, (n-1)*step+1);
            ps.setInt(2, n*step);
            //这一步很关键，设置fetchSize会大大提高读取数据的效率
            ps.setFetchSize(fetchSize);
            rs = ps.executeQuery();
            System.out.println("====================ps.get1: "+((n-1)*step+1));
            System.out.println("====================ps.get1: "+n*step);
            List<AdminLog> adminLogs = new ArrayList<>();
            AdminLog adminLog = null;
            while (rs.next()) {
                adminLog = new AdminLog();
                Long id = rs.getLong("id");
                Long adminId = rs.getLong("admin_id");
                Date createTime = rs.getDate("create_time");
                String ip = rs.getString("ip");
                String address = rs.getString("address");
                String userAgent = rs.getString("user_agent");
                adminLog.setId(id);
                adminLog.setAdminId(adminId);
                adminLog.setCreateTime(createTime);
                adminLog.setIp(ip);
                adminLog.setAddress(address);
                adminLog.setUserAgent(userAgent);
                adminLogs.add(adminLog);
            }
            //读出来的数据放入阻塞队列
            linkedBlockingDeque.add(adminLogs);
            long end = System.currentTimeMillis();
            log.info("===执行一次readData方法耗时：=="+(end-start)+"ms");
            //adminLogs.stream().map(a->a.getId()).collect(Collectors.toList())
            //        .stream().forEach(System.out::println);
            //adminLogs.forEach(System.out::println);
            System.out.println("===size=" + adminLogs.size());
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            try {
                if (rs != null) {
                    rs.close();
                }
                if (ps != null) {
                    ps.close();
                }
                if (connection != null) {
                    connection.close();
                }
            } catch (SQLException e){
                
            }
        }
    }

    /**
     * 取队列中的数据并写入数据库
     * @param logs
     */
    public void writeData(List<AdminLog> logs){
        long start = System.currentTimeMillis();
        String sql = "insert into dgg_admin_login_log (id, admin_id, create_time,ip,address,user_agent) " +
                "values(?,?,?,?,?,?)";
        Connection connection = null;
        PreparedStatement ps = null;
        ResultSet rs = null;
        System.out.println("========" + dbConnection);
        try {
            Class.forName(dbConnection.getDriver());
            connection = DriverManager.getConnection(dbConnection.getUrl(), dbConnection.getUsername(), dbConnection.getPassword());
            connection.setAutoCommit(false);
            ps = connection.prepareStatement(sql);
            //int count = 0;
            for (AdminLog log : logs) {
                System.out.println("~~~~~~log:~~~~~~" + log);
                ps.setLong(1, log.getId());
                ps.setLong(2, log.getAdminId());
                ps.setDate(3, (java.sql.Date) log.getCreateTime());
                ps.setString(4, log.getIp());
                ps.setString(5, log.getAddress());
                ps.setString(6, log.getUserAgent());
                ps.addBatch();
                //count++;
                //每1000条记录执行一次批处理(亲测，有点慢，不如放在外层，每一万条做一次批处理提交)
                //if(count%1000==0){
                //    ps.executeBatch();
                //    connection.commit();
                //}
            }
            //处理数据的批处理
            //每一万条提交一次速度更快
            ps.executeBatch();
            //提交事务
            connection.commit();
            long end = System.currentTimeMillis();
            log.info("===执行一次writeData方法耗时：=="+(end-start)+"ms");
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            try {
                if (rs != null) {
                    rs.close();
                }
                if (ps != null) {
                    ps.close();
                }
                if (connection != null) {
                    connection.close();
                }
            } catch (SQLException e){
                
            }
        }

    }

    /**
     * 这里的main方法定义：准备原表的10W条数据
     * @param args
     */
    public static void main(String[] args) {
        long start = System.currentTimeMillis();
        String sql = "insert into ums_admin_login_log (id, admin_id, create_time,ip,address,user_agent) " +
                "values(?,?,?,?,?,?)";
        Connection connection = null;
        PreparedStatement ps = null;
        ResultSet rs = null;

        try {
            Class.forName("com.mysql.cj.jdbc.Driver");
            connection = DriverManager.getConnection("jdbc:mysql://hostIP/testDB?useServerPrepStmts=false&rewriteBatchedStatements=true",
                    "username", "password");
            connection.setAutoCommit(false);
            ps = connection.prepareStatement(sql);

            for (int i=0;i<100000;i++) {

                ps.setLong(1, i+1);
                ps.setLong(2, RandomUtil.randomInt(1,1000));
                ps.setDate(3, new java.sql.Date(System.currentTimeMillis()));
                ps.setString(4, "0:0:0:0:0:0:0:1");
                ps.setString(5, RandomUtil.randomString("ADDRESS:a#$&^%@$$#^%dfsdafjsdljkljsdf", 6));
                ps.setString(6, RandomUtil.randomString("USERAGENT:a#$&^%@$$#^%dfsdafjsdljkj",6));
                ps.addBatch();

                //if(i%1000==0){
                //    ps.executeBatch();
                //    connection.commit();
                //}
            }
            //批处理
            ps.executeBatch();
            //提交事务
            connection.commit();
            long end = System.currentTimeMillis();
            System.out.println("===执行一次writeData方法耗时：=="+(end-start)+"ms");
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            try {
                if (rs != null) {
                    rs.close();
                }
                if (ps != null) {
                    ps.close();
                }
                if (connection != null) {
                    connection.close();
                }
            } catch (SQLException e){
                
            }
        }
    }
}

代码依赖包：

 		<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jdbc</artifactId>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.14</version>

        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.3.2</version>
        </dependency>

相关代码：