定义自己的并发池
@Component public class ThreadPoolConfig { @Bean(name = "commonThreadPool") public ThreadPoolTaskExecutor commonThreadPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(6); executor.setMaxPoolSize(6); executor.setQueueCapacity(30000); executor.setKeepAliveSeconds(60); executor.setThreadNamePrefix("commonThreadPool"); executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); executor.initialize(); return executor; } }
代码中实际调用:
由于poiinfo是个对象,需要做一定的预处理,先处理为List<Map>形式
考虑tmps可能过大,使用Lists.partition进行分组,每组20000个传入并发
List<Map> tmps = new ArrayList<>(); while (poiinfo.next()) { Map part = accessToPart.dealGdSwPre(poiinfo); //预处理 tmps.add(part); } logger.info("当前城市共有数据:" + tmps.size()); List<List<Map>> partition = Lists.partition(tmps, 20000); for (List<Map> maps : partition) { threadPoolConfig.commonThreadPool().execute(() -> { int count = atomicInteger.incrementAndGet(); if (count % 50000 == 0){ logger.info("当前城市已处理:" + count); } String result = null; for (Map maptmp : maps) { try{ result = streamCore.streamCore(maptmp); //准聚融流程 // result = getAcmpServer.getAcmpServer(part); Map resMap = JSON.parseObject(result); if (!"0".equalsIgnoreCase(resMap.get("status").toString())){ logger.info("当前数据处理失败,具体数据是:" + poiinfo + ", 失败原因是:" + result); } } catch (Exception e){ e.printStackTrace(); logger.info("当前数据处理失败,具体数据是:" + poiinfo + ", 失败原因是:" + result); } } }); }