多线程分批次查询数据

package cc.mrbird.febs.system.controller;

import cc.mrbird.febs.common.annotation.Log;
import cc.mrbird.febs.common.controller.BaseController;
import cc.mrbird.febs.common.domain.QueryRequest;
import cc.mrbird.febs.common.exception.FebsException;
import cc.mrbird.febs.common.utils.MD5Util;
import cc.mrbird.febs.common.utils.ThredQuery;
import cc.mrbird.febs.system.dao.UserMapper;
import cc.mrbird.febs.system.domain.User;
import cc.mrbird.febs.system.domain.UserConfig;
import cc.mrbird.febs.system.service.UserConfigService;
import cc.mrbird.febs.system.service.UserService;
import cc.mrbird.febs.system.service.impl.UserServiceImpl;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.StringPool;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.google.common.collect.Maps;
import com.wuwenze.poi.ExcelKit;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import javax.validation.constraints.NotBlank;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

@Slf4j
@Validated
@RestController
@RequestMapping("user")
public class UserController extends BaseController {

    private String message;

    @Autowired
    private UserServiceImpl userService;
    @Autowired
    UserMapper userMapper;
    @Autowired
    private UserConfigService userConfigService;
    //创建线程池
    //创建线程 7个参数
    //private static ExecutorService executorService = Executors.newFixedThreadPool(5);
    //1.corePoolSize 线程池种的的线程数量
    //2.maxmumPoolSize 线程池种最大的线程数量
    //3.keepAliveTime 线程池的数量大于线程数量时,多余的线程会在多长时间内销毁 一般设置0
    //4.TimeUnit keepAlive的时间单位 一般设置分钟  TimeUnit.MILLISECONDS
    //5.workQueue:任务队列，被提交但是未被执行的任务 一般设置10
    //6.threadFactory：线程工厂， 一般设置默认值  Executors.defaultThreadFactory(),
    //7.handler:拒绝略，任务太多来不及处理，如何拒绝  也设置 ThreadPoolExecutor.DiscardPolicy()
    //四种拒绝策略分别是:
    //          7.1 AbortPolicy: 丢弃任务并抛出RejectedExecutionException异常。 (默认)
    //          7.2 DiscardPolicy：也是丢弃任务，但是不抛出异常。
    //          7.3 DiscardOldestPolicy：忽略最早的任务（把最早添加任务到队列的任务忽略掉，然后执行当前的任务）
    //          7.4 CallerRunsPolicy：把超出的任务交给当前线程执行
    ExecutorService executorService=new ThreadPoolExecutor(
            5,
            5, 0l,
            TimeUnit.MILLISECONDS,
            new LinkedBlockingDeque<Runnable>(10),
            Executors.defaultThreadFactory(),
            new ThreadPoolExecutor.DiscardPolicy());

   /**
     * 多线程分批次查询用户
     */
    @PostMapping("queryUserByThread")
    public Map<String, Object> queryUserByThread(QueryRequest queryRequest, User user) throws FebsException {
        List<User> result = new ArrayList<>();//返回结果

        try {
            log.error("start....");
            //方法一创建 多线程查询 87ms
            long start = System.currentTimeMillis();
            //查询数据库总数量
            int count = userMapper.countUser();
            int num = 10;//一次查询多少条
            //需要查询的次数
            int times = count / num;
            if (count % num != 0) {
                times = times + 1;
            }
            int bindex = 1;
             
            //Callable用于产生结果
            List<Callable<List<User>>> tasks = new ArrayList<Callable<List<User>>>();
            for (int i = 0; i < times; i++) {
                //1 反射去查询的版本
                Callable<List<User>> qfe = new ThredQuery(user, bindex, num);
                //2.不用反射查询的版本
               // Callable<List<User>> qfe = new ThredQueryWithOutInvok(userService, user, bindex, num);
                tasks.add(qfe);
                bindex += bindex;
            }
            //定义固定长度的线程池  防止线程过多
            ExecutorService executorService = new ThreadPoolExecutor(
                    15,
                    30, 0l,
                    TimeUnit.MILLISECONDS,
                    new LinkedBlockingDeque<Runnable>(10),
                    Executors.defaultThreadFactory(),
                    new ThreadPoolExecutor.DiscardPolicy());
            //Future用于获取结果
            List<Future<List<User>>> futures = executorService.invokeAll(tasks);
            //处理线程返回结果
            if (futures != null && futures.size() > 0) {
                for (Future<List<User>> future : futures) {
                    result.addAll(future.get());
                }
            }
            executorService.shutdown();//关闭线程池
            long end = System.currentTimeMillis();
           System.out.println("方法一创建 多线程查询时:" + (end - start) + "ms");
			//方法二:用流的多线程查询 效率更高 4ms
            result = Collections.synchronizedList(new ArrayList<>());//返回结果
            log.error("start....");
            long start1 = System.currentTimeMillis();
            //查询数据库总数量
            int count1 = userMapper.countUser();
            int num1 = 10;//一次查询多少条
            //需要查询的次数
            int times1 = count1 / num+1;
            List<IPage<User>> searchList=new ArrayList<>();

            for (int i = 0; i < times1; i++) {
                IPage<User> ipage = new Page<>((i+1), num1, false);//分页查询不做总数量查询
                searchList.add(ipage);
            }
            List<User> finalResult1 = result;
            //parallelStream 是jdk源码调用计算机指令开多线程
            searchList.parallelStream().forEach(ipage -> {
                //调用业务方法查询数据，返回集合
                List<User> list = userService.findUserThread2(ipage);
                finalResult1.addAll(list);
            });
            long end1 = System.currentTimeMillis();
            System.out.println("方法二:用流的多线程查询用时 :"+(end1-start1)+"ms");
            
            HashMap<String, Object> map = Maps.newHashMap();
            map.put("list", result);
            return map;
        } catch (Exception e) {
            message = "查询失败";
            log.error(message, e);
            throw new FebsException(message);
        }

    }
}


package cc.mrbird.febs.common.utils;


import cc.mrbird.febs.system.domain.User;
import cc.mrbird.febs.system.service.impl.UserServiceImpl;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.springframework.context.ApplicationContext;

import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;

public class ThredQuery implements Callable<List<User>> {
    /**
     * 通过ApplicationContext获取bean
     */
    private ApplicationContext ac = SpringContextUtil.getApplicationContext();
    private UserServiceImpl userServiceimpl;//需要通过够方法把对应的业务service传进来 实际用的时候把类型变为对应的类型
    private User user;//查询条件 根据条件来定义该类的属性
    private int bindex;//分页 页面
    private int num;//数量


    /**
     * @ClassName: ThredQuery
     * @Description: 多线程查询对象
     * @author: yunkang
     * @date:  2021/11/20
     */
    public ThredQuery(User user, int bindex, int num) {
        this.user = user;
        this.bindex = bindex;
        this.num = num;
    }

    @Override
    public List<User> call() throws Exception {
        // 用反射的原因个人猜测是 因为入参没有 userServiceimpl 接口对象spring框架不会注入 	 userMapper.findUserDetail(page, user)方法
       // 所以要用反射去找 userServiceimpl
      userServiceimpl = (UserServiceImpl) ac.getBean("userServiceImpl");
       Method idMethod = userServiceimpl.getClass().getMethod("findUserThread", Page.class);
       Page page=new Page();
       page.setSize(num);
       page.setCurrent(bindex);
       //调用业务方法查询数据，返回集合
       Object invoke = idMethod.invoke(userServiceimpl, page);
       IPage<User> userPageInof=(IPage<User>)invoke;
       List<User> records = userPageInof.getRecords();
        return records;
    }
}

//不用反射
package cc.mrbird.febs.common.utils;


import cc.mrbird.febs.system.domain.User;
import cc.mrbird.febs.system.service.impl.UserServiceImpl;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.springframework.context.ApplicationContext;

import java.lang.reflect.Method;
import java.util.List;
import java.util.concurrent.Callable;

public class ThredQueryWithOutInvok implements Callable<List<User>> {

    private UserServiceImpl userServiceimpl;//需要通过够方法把对应的业务service传进来 实际用的时候把类型变为对应的类型
    private User user;//查询条件 根据条件来定义该类的属性
    private int bindex;//分页 页面
    private int num;//数量

    /**
     * @ClassName: ThredQuery
     * @Description: 多线程查询对象
     * @author: yunkang
     * @date: 2021/11/20
     */
    public ThredQueryWithOutInvok(UserServiceImpl userServiceimpl, User user, int bindex, int num) {
        this.userServiceimpl = userServiceimpl;
        this.user = user;
        this.bindex = bindex;
        this.num = num;
    }

    @Override
    public List<User> call() throws Exception {
        Page page = new Page();
        page.setSize(num);
        page.setCurrent(bindex);
        //调用业务方法查询数据，返回集合
        IPage<User> list = userServiceimpl.findUserThread(page);
        List<User> records = list.getRecords();
        return records;
    }
}

package cc.mrbird.febs.common.utils;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;

/**
* @ClassName SpringContextUtil
* @Description Spring工具类,用来获取bean
* @projectName **
* @Author yukangyun
* @Date 21/11/21 15:31
* @Version <1.0>
*/
@SuppressWarnings({"unused"})
@Component
public class SpringContextUtil implements ApplicationContextAware {

  public static final Logger logger = LoggerFactory.getLogger(SpringContextUtil.class);

  private static ApplicationContext applicationContext;

  @Override
  public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
      this.applicationContext = applicationContext;
      logger.info("applicationContext:" + SpringContextUtil.applicationContext);
  }

  public static ApplicationContext getApplicationContext() {
      return applicationContext;
  }

  public static Object getBean(String name) {
      return getApplicationContext().getBean(name);
  }

  public static <T> T getBean(Class<T> clazz) {
      return getApplicationContext().getBean(clazz);
  }

  public static <T> T getBean(String name, Class<T> clazz) {
      return getApplicationContext().getBean(name, clazz);
  }

}


package cc.mrbird.febs.system.service;

import cc.mrbird.febs.common.domain.QueryRequest;
import cc.mrbird.febs.system.domain.User;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;

import java.util.List;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;

public interface UserService extends IService<User> {
   
     IPage<User> findUserThread(Page page);
     List<User> findUserThread2(IPage<User> ipage);
}

package cc.mrbird.febs.system.service.impl;

import cc.mrbird.febs.common.domain.FebsConstant;
import cc.mrbird.febs.common.domain.QueryRequest;
import cc.mrbird.febs.common.service.CacheService;
import cc.mrbird.febs.common.utils.SortUtil;
import cc.mrbird.febs.common.utils.MD5Util;
import cc.mrbird.febs.common.utils.ThredQuery;
import cc.mrbird.febs.system.dao.UserMapper;
import cc.mrbird.febs.system.dao.UserRoleMapper;
import cc.mrbird.febs.system.domain.User;
import cc.mrbird.febs.system.domain.UserRole;
import cc.mrbird.febs.system.manager.UserManager;
import cc.mrbird.febs.system.service.UserConfigService;
import cc.mrbird.febs.system.service.UserRoleService;
import cc.mrbird.febs.system.service.UserService;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.StringPool;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import java.util.*;
import java.util.concurrent.*;

@Slf4j
@Service
@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class)
public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements UserService {
    @Override
    public IPage<User> findUserThread(Page page) {
        try {
            User user=new User();
            return this.baseMapper.findUserList(page, user);
        } catch (Exception e) {
            log.error("查询用户异常", e);
            return null;
        }
    }
   @Override
    public List<User> findUserThread2(IPage<User> page) {
        User user=new User();
        List<User> userList = baseMapper.findUserList2(page, user);
        return userList;
    }

}

package cc.mrbird.febs.system.dao;

import cc.mrbird.febs.system.domain.User;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.ibatis.annotations.Param;

import java.util.List;

public interface UserMapper extends BaseMapper<User> {

   IPage<User> findUserList(Page page, @Param("user") User user);
   List<User> findUserList2(IPage page, @Param("user") User user);
}

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cc.mrbird.febs.system.dao.UserMapper">
<select id="findUserList" resultType="user" parameterType="user">
        SELECT
        u.user_id userId,
        u.username,
        u.password,
        u.email,
        u.mobile,
        u. STATUS,
        u.create_time createTime,
        u.ssex,
        u.AVATAR,
        u.DESCRIPTION,
        u.LAST_LOGIN_TIME lastLoginTime
        FROM
        t_user u
       <where>
           <if test="user.username != null and user.username != ''">
               AND u.username = #{user.username}
           </if>
           <if test="user.deptId != null and user.deptId != ''">
               AND d.dept_id = #{user.deptId}
           </if>
           <if test="user.createTimeFrom != null and user.createTimeFrom !=''">
               And u.create_time &gt; #{user.createTimeFrom}
           </if>
           <if test="user.createTimeTo!= null and user.createTimeTo !=''">
               And u.create_time &lt; #{user.createTimeTo}
           </if>
           <if test="user.ssex != null and user.ssex != ''">
               AND u.ssex = #{user.ssex}
           </if>
           <if test="user.status != null and user.status != ''">
               AND u.status = #{user.status}
           </if>
       </where>
    </select>
</mapper>
<select id="findUserList2" resultType="user" parameterType="user">
        SELECT
             u.user_id userId,
             u.username,
             u.password,
             u.email,
             u.mobile,
             u. STATUS,
             u.create_time createTime,
             u.ssex,
             u.AVATAR,
             u.DESCRIPTION,
             u.LAST_LOGIN_TIME lastLoginTime
        FROM
            t_user u
        <where>
            <if test="user.username != null and user.username != ''">
                AND u.username = #{user.username}
            </if>
            <if test="user.deptId != null and user.deptId != ''">
                AND d.dept_id = #{user.deptId}
            </if>
            <if test="user.createTimeFrom != null and user.createTimeFrom !=''">
                And u.create_time &gt; #{user.createTimeFrom}
            </if>
            <if test="user.createTimeTo!= null and user.createTimeTo !=''">
                And u.create_time &lt; #{user.createTimeTo}
            </if>
            <if test="user.ssex != null and user.ssex != ''">
                AND u.ssex = #{user.ssex}
            </if>
            <if test="user.status != null and user.status != ''">
                AND u.status = #{user.status}
            </if>
        </where>
    </select>

package cc.mrbird.febs.system.domain;

import cc.mrbird.febs.common.converter.TimeConverter;
import cc.mrbird.febs.common.domain.RegexpConstant;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.wuwenze.poi.annotation.Excel;
import com.wuwenze.poi.annotation.ExcelField;
import lombok.Data;
import lombok.ToString;

import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.Pattern;
import javax.validation.constraints.Size;
import java.io.Serializable;
import java.util.Date;

@Data
@TableName("t_user")
@Excel("用户信息表")
public class User implements Serializable {

    private static final long serialVersionUID = -4852732617765810959L;
    /**
     * 账户状态
     */
    public static final String STATUS_VALID = "1";

    public static final String STATUS_LOCK = "0";

    public static final String DEFAULT_AVATAR = "default.jpg";

    /**
     * 性别
     */
    public static final String SEX_MALE = "0";

    public static final String SEX_FEMALE = "1";

    public static final String SEX_UNKNOW = "2";

    // 默认密码
    public static final String DEFAULT_PASSWORD = "1234qwer";

    @TableId(value = "USER_ID", type = IdType.AUTO)
    private Long userId;

    @Size(min = 4, max = 10, message = "{range}")
    @ExcelField(value = "用户名")
    private String username;

    private String password;

    private Long deptId;

    @ExcelField(value = "部门")
    private transient String deptName;

    @Size(max = 50, message = "{noMoreThan}")
    @Email(message = "{email}")
    @ExcelField(value = "邮箱")
    private String email;

    @Pattern(regexp = RegexpConstant.MOBILE_REG, message = "{mobile}")
    @ExcelField(value = "手机号")
    private String mobile;

    @NotBlank(message = "{required}")
    @ExcelField(value = "状态", writeConverterExp = "0=锁定,1=有效")
    private String status;

    @ExcelField(value = "创建时间", writeConverter = TimeConverter.class)
    private Date createTime;

    private Date modifyTime;

    @ExcelField(value = "最后登录时间", writeConverter = TimeConverter.class)
    private Date lastLoginTime;

    @NotBlank(message = "{required}")
    @ExcelField(value = "性别", writeConverterExp = "0=男,1=女,2=保密")
    private String ssex;

    @Size(max = 100, message = "{noMoreThan}")
    @ExcelField(value = "个人描述")
    private String description;

    private String avatar;

    @NotBlank(message = "{required}")
    private transient String roleId;
    @ExcelField(value = "角色")
    private transient String roleName;

    // 排序字段
    private transient String sortField;

    // 排序规则 ascend 升序 descend 降序
    private transient String sortOrder;

    private transient String createTimeFrom;
    private transient String createTimeTo;

    private transient String id;

    /**
     * shiro-redis v3.1.0 必须要有 getAuthCacheKey()或者 getId()方法
     * # Principal id field name. The field which you can get unique id to identify this principal.
     * # For example, if you use UserInfo as Principal class, the id field maybe userId, userName, email, etc.
     * # Remember to add getter to this id field. For example, getUserId(), getUserName(), getEmail(), etc.
     * # Default value is authCacheKey or id, that means your principal object has a method called "getAuthCacheKey()" or "getId()"
     *
     * @return userId as Principal id field name
     */
    public Long getAuthCacheKey() {
        return userId;
    }
}