ConcurrentHashMap与ThreadPoolExecutor--demo

 

有一个类似报表的需求，要求小时级更新，sql很简单，但是数据量大概有十几万，需要短时间内能够跑完，写了个菜菜的demo，抽样测试数据没有问题，求指教。。。。。。

设计如下：

任务的最终目的是拼接出一个ConcurrentHashMap<String,List>，并把ConcurrentHashMap存库

task_1:必须首先执行，产生一个List<map>,将每一个map的key存为ConcurrentHashMap的key值，并把value存入ConcurrentHashMap的内层List

task_2,task_3,task_4:必须在task_1之后执行，每个任务产生一个list<map>，对于每一个map，如果map的key与当前ConcurrentHashMap的key相等就把value拼入该key对应的内层List

task_5：用于进行一些额外的计算和存库，必须在task_2,task_3,task_4执行完之后再执行，ConcurrentHashMap分成多组，起多个线程，同时处理

实现如下：

​
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

@Controller
@RequestMapping("/Test")
public class TestTask {
    private static final String DATE="2018-09-05";
    private static ThreadPoolExecutor threadPool = new ThreadPoolExecutor(5, 12, 30,
            TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(10), new ThreadPoolExecutor.AbortPolicy());
    private final ConcurrentHashMap<String,ConcurrentHashMap<String,String>> resultMap=new ConcurrentHashMap<>();
    private Logger logger = LoggerFactory.getLogger(TestTask.class);
    @Autowired
    private TaskService taskService;
    /**
     * @author
     * @description demo
     * @date 2018/9/5 12:21
     */
    @RequestMapping("/task.do")
    @ResponseBody
    public synchronized Object task(HttpServletRequest request){
        try{
            long startTime = System.currentTimeMillis(); //程序开始记录时间
            //清空上次执行数据
            resultMap = new ConcurrentHashMap<>();
            //构建任务task_1和任务列表tasks_select(包含三个任务task_2,task_3,task_4)
            Callable<List<Map>> task_1=task_1();
            List<Callable<List<Map>>> tasks_select=new ArrayList<>();
            tasks_select.add(task_2());
            tasks_select.add(task_3());
            tasks_select.add(task_4());
            List<Future<List<Map>>> futures_select=null;
            //触发task_1执行，用get()方法阻塞，直至task_1执行结束，然后触发task_select
            Future<List<Map>> future= threadPool.submit(task_1);
            future.get();
            futures_select= threadPool.invokeAll(tasks_select);
            //阻塞task_select,直至完成
            for(Future ele : futures_select) {
                ele.get();
            }
            //分组执行task_5(计算，拼接，存库)
            int total=resultMap.size();
            int i=0;
            ConcurrentHashMap<String,ConcurrentHashMap<String,String>> subMap_0=new ConcurrentHashMap();
            ConcurrentHashMap<String,ConcurrentHashMap<String,String>> subMap_1=new ConcurrentHashMap();
            ConcurrentHashMap<String,ConcurrentHashMap<String,String>> subMap_2=new ConcurrentHashMap();
            ConcurrentHashMap<String,ConcurrentHashMap<String,String>> subMap_3=new ConcurrentHashMap();
            ConcurrentHashMap<String,ConcurrentHashMap<String,String>> subMap_4=new ConcurrentHashMap();
            for (String key : resultMap.keySet()) {
                if(i%5==0){
                    subMap_0.put(key, resultMap.get(key));
                    i++;
                    continue;
                }
                if(i%5==1){
                    subMap_1.put(key, resultMap.get(key));
                    i++;
                    continue;
                }
                if(i%5==2){
                    subMap_2.put(key, resultMap.get(key));
                    i++;
                    continue;
                }
                if(i%5==3){
                    subMap_3.put(key, resultMap.get(key));
                    i++;
                    continue;
                }
                if(i%5==4){
                    subMap_4.put(key, resultMap.get(key));
                    i++;
                    continue;
                }
            }
            List<Callable<Integer>> tasks_insert=new ArrayList<>();
            tasks_insert.add(task_5(subMap_0));
            tasks_insert.add(task_5(subMap_1));
            tasks_insert.add(task_5(subMap_2));
            tasks_insert.add(task_5(subMap_3));
            tasks_insert.add(task_5(subMap_4));
            List<Future<Integer>> future_insert=null;
            future_insert=threadPool.invokeAll(tasks_insert);
            for(Future ele : future_insert) {
                ele.get();
            }
            long endTime = System.currentTimeMillis(); //程序结束时间
            logger.info("任务执行开始时间：" + startTime + "执行结束时间：" + endTime + "共计耗时：" + (endTime - startTime)+"共计产生"+resultMap.size()+"条数据");
            return resultMap;
        }catch(Exception e){
            logger.error("任务执行失败",e);
            return null;
        }
    }
    private Callable task_5(final ConcurrentHashMap<String,ConcurrentHashMap<String,String>> map)throws Exception{
        Callable<Integer> task_5 = new Callable<Integer>() {
            public Integer call()throws Exception{
                try{
                    long startTime = System.currentTimeMillis(); //程序开始记录时间
                    
                    //省略具体业务逻辑
                    //... 

                    long endTime = System.currentTimeMillis(); //程序开始结束时间
                    System.out.println("task_5任务执行开始时间：" + startTime + "执行结束时间：" + endTime + "共计耗时：" + (endTime - startTime));
                    System.out.println("核心线程数" + threadPool.getCorePoolSize() + "线程池数" + threadPool.getPoolSize() + "队列任务数" + threadPool.getQueue().size());
                    return null;
                }catch(Exception e){
                    e.printStackTrace();
                    return null;
                }
            }
        };
        return task_5;
    }

    //...
    //省略task_1()方法,task_2()方法,task_3()方法,task_4()方法

   }

​

 测试结果如下：

   当需拼接ConcurrentHashMap的size为10时

   单线程情况下耗时773ms

   INFO  [2018-09-07 19:45:02,843] [TestTask:55] - Fri Sep 07 19:45:02 CST 2018标注结果统计任务执行开始时间：1536320702070执行结束时间：1536320702843共计耗时：773共计产生10条数据

   多线程情况下耗时595ms

   INFO  [2018-09-07 19:48:00,203] [TestTask:136] - Fri Sep 07 19:48:00 CST 2018标注结果统计任务执行开始时间：1536320879608执行结束时间：1536320880203共计耗时：595共计产生10条数据