android多线程爬分页数据,分页获取数据+多线程下+CountDownLatch遇到的问题

问题说明

每20条分页获取完数据后，提交线程池进行任务处理，由于返回每页的条数小于20条，极端情况下每页只返回1条，这样每次只能提交一个任务，导致多线程弱化为单线程，任务执行时间剧增。

代码分析

原代码

Integer pageSize = 20;

for (int i = 1; i > 0; i++) {

//获取户型数据

HousePageDO layoutsByPage = getHousePageDO(startId,pageSize);

List list = layoutsByPage.getList();

log.info(LOG_PRE + "提交第" + i + "页数据，共" + list.size() + "条数据");

//注意当这里的list每次只有1条数据的时候，这时整个任务就弱化为单线程了

//提交任务

submitTasks(list,taskId,POOL);

//如果maxId为null 停止

if (layoutsByPage.getMaxId() == null) {

log.info(LOG_PRE + "获取到maxId为null,结束获取户型信息");

break;

}

//获取下一页开始id

startId = layoutsByPage.getMaxId();

}

private void submitTasks(List list,Long taskId,ExecutorService POOL) throws InterruptedException {

if(CollectionUtils.isNotEmpty(list)){

CountDownLatch begin = new CountDownLatch(list.size());

list.forEach(house -> {

//设置大区，业务组

resblockDictionaryHelper.setBusInfo(house,resblockMap);

POOL.execute(() -> {

try {

call(house);

}catch (Exception e){

e.printStackTrace();

TLayoutPriceTaskError error = TLayoutPriceTaskError.builder()

.createDatetime(new Date())

.layoutId(house.getLayoutId())

.msg(e.toString())

.param(house.getLayoutId() + "--" + house.getBeforeVersion() + "--" + house.getAfterVersion())

.taskTag(taskId).build();

layoutPriceTaskErrorMapper.insert(error);

}finally {

begin.countDown();

}

});

});

begin.await();

}

}

改后代码

//分页获取户型数据

Integer pageSize = 20;

List list = new ArrayList<>();

for (int i = 1; i > 0; i++) {

//获取户型数据

HousePageDO layoutsByPage = getHousePageDO(startId,pageSize);

log.info(LOG_PRE + "提交第" + i + "页数据，共" + layoutsByPage.getList().size() + "条数据");

if(!CollectionUtils.isEmpty(layoutsByPage.getList())){

list.addAll(layoutsByPage.getList());

}

//保证每次调价的任务大于一千

if(list.size() < 1000 && layoutsByPage.getMaxId() != null){

startId = layoutsByPage.getMaxId();

continue;

}

//提交任务

submitTasks(list,taskId,POOL);

//如果maxId为null 停止

if (layoutsByPage.getMaxId() == null) {

log.info(LOG_PRE + "获取到maxId为null,结束获取户型信息");

break;

}

//获取下一页开始id

startId = layoutsByPage.getMaxId();

list = new ArrayList<>();

}

原文：https://www.cnblogs.com/fangcheng2018/p/12898833.html