//抓取XXXX 车型任务
public void executeModel() {
long startTime = System.currentTimeMillis();
final int nThreads = 12;
ExecutorService executor = Executors.newFixedThreadPool(nThreads);
List<CarSeriesMatchBean> list = CrawlManageService.getInstance().getCrawl12Month();
List<CarSeriesMatchBean> tempList = null;
final List<CarSeriesMatchBean>sendEmailList=new ArrayList<CarSeriesMatchBean>();
int sum=list.size();
int totalPage = (sum - 1) / nThreads + 1;//页码,逢1进1
for(int i=0;i<totalPage;i++){
int pageNo=i+1;
tempList=sublistCarSeries(totalPage,pageNo,list);
tempList.size();
final List<CarSeriesMatchBean> subList = tempList;
final int row=i;
executor.execute(new Runnable() {
public void run() {
System.out.println("线程"+row+"抓取"+subList.size()+"个车系下的车型~~~:");
for (CarSeriesMatchBean carSeriesMatchBean : subList) {
//验证缓存是否失效
if(redisFactory.isEnable()){
String redisKey = RedisUtils.JAVA+RedisUtils.STRING+RedisUtils.QI_TA+"AutohomeGrabJobExecuteModel;"+carSeriesMatchBean.getAutohomeUri();
redisKey = StringUtils.upperCase(redisKey);
if(redisFactory.getRedisOperator().exists(redisKey)){continue;}
}
//抓取XXXX车型
boolean result=processCrawlModel(carSeriesMatchBean);
// 抓取后加入缓存,保存7个工作日
if(result&&redisFactory.isEnable()){
String redisKey = RedisUtils.JAVA+RedisUtils.STRING+RedisUtils.QI_TA+"AutohomeGrabJobExecuteModel;"+carSeriesMatchBean.getAutohomeUri();
redisKey = StringUtils.upperCase(redisKey);
redisFactory.getRedisOperator().setex(redisKey, RedisUtils.TIME_7, String.valueOf(carSeriesMatchBean.getAutohomeUri()));
}
}
}
});
}
executor.shutdown();
long endTime = System.currentTimeMillis();
log.info("usetime:"+(endTime-startTime));
while (true) {
if(executor.isTerminated()){
// long endTime = System.currentTimeMillis();
log.info("测试---------------------usetime:"+(endTime-startTime));
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
检查线程是否关闭
最新推荐文章于 2024-08-01 21:18:46 发布