使用线程池和CountDownLatch计数器来进行抓取的简单事例2

import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.log4j.Logger;
import util.ProxyUtil;
import util.StringUtil;
public class DictEnToChar implements Runnable{
static Logger log = Logger.getLogger(DictEnToChar.class);
static int threadnum = 5;
public static void main(String[] args) {
	if(args.length==1){
		try{
			threadnum = Integer.parseInt(args[0]);
		}catch(Exception e){
			e.printStackTrace();
			System.out.println("请输入线程个数.为整型参数....");
		}
	}
	DictEnToChar dic = new DictEnToChar();
	for(char c='A'; c<='Z'; c++){
		dic.forTotalPage("http://www.*****.com/hotproduct_list_"+c+"_(*).html",c);
	}
	DictEn.main(args);
}
/**
 * 对每个列表页进行循环
 */
static int totalPage;//每个cas下标的总页数
static int startpage;//每个cas下标的开始页
static String threadurl;//列表页的地址
static CountDownLatch cdl = new CountDownLatch(threadnum);//计数器,每个cas的下标全部抓完才能抓取下一个下标
static ExecutorService threadpool = Executors.newCachedThreadPool();//线程池
public void forTotalPage(String url,int index){
	String content = ProxyUtil.getProxyStr(StringUtil.changeURL(url, 1), "utf-8", "", "CAS Number");
	String maxstr = StringUtil.getFirststr(content, "Page 1 of", "<span", 1).trim();
	try{
		totalPage = Integer.parseInt(maxstr);
	}catch(Exception e){
		e.printStackTrace();
		System.out.println(index+"获得最大页数失败............");
	}
	System.out.println("产品名称的下标为"+index+"的最大页数是:"+totalPage);
	
	cdl = new CountDownLatch(threadnum);//计数重置
	startpage=0;//赋初值
	threadurl = url;//赋初值
	DictEnToChar dicrun = new DictEnToChar();
	for(int num=0; num<threadnum; num++){
		threadpool.execute(dicrun);//在线程池中启动线程
	}
	try {
		cdl.await();//如果不使用await,那么这个方法便会执行结束
	} catch (InterruptedException e) {
		e.printStackTrace();}
	System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
}
//线程执行方法
public void run(){
	while(startpage < totalPage){
		synchronized(DictEn.class){
			try {
				Thread.sleep(50);//进行睡眠，防止下面的线程不安全
			} catch (InterruptedException e) {
				e.printStackTrace();}
			startpage++;
			System.out.println(Thread.currentThread().getName()+"#页数#"+startpage);
		}//在这里会有线程安全问题，故在上面的锁内进行睡眠
		getLinkList(StringUtil.changeURL(threadurl, startpage));
	}
	cdl.countDown();
	System.out.println(Thread.currentThread().getName()+"-----------------线程结束-----------------还有线程个数："+cdl.getCount());
}
/**
 * 得到每个列表页的链接
 */
public void getLinkList(String url){
	System.out.println(Thread.currentThread().getName()+"#当前列表页:"+url);
	log.info("lookchem列表:"+url);
	
	String content=ProxyUtil.getProxyStr(url, "utf-8", "", "html");
	List<String> list=StringUtil.getalltagslist(content, "<table width=\"100%\" border=\"0\" cellspacing=\"0\"", "</table>");
	for(String s:list){
		String cas = StringUtil.getFirststr(s, "<td width=\"120\" valign=\"middle\" >", "</td>", 1).trim();
		//在这里先判断数据库中是否存在该数据
		if(!StringUtil.isCasNo(cas) || !DictEnDao.checkCas(cas)){
			System.out.println(cas+":Cas重复..............");
			continue;
		}
		String u = StringUtil.getFirststr(s, "href=\"", "\"", 1);
		getContent(u);//访问详细页面
	}
}