用webmagic抓文章列表和详细页

webMaigc文档：http://webmagic.io

public class ForumPageprocess implements PageProcessor {

private Sitesite= Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
privatestatic String username="qq_14955245";
private String domian="http://blog.csdn.net/";//网站首页
private int size=0;//抓取到的文章数量
public Site getSite() {
return site;
}

public void process(Pagepage) {
if(!page.getUrl().regex(""+domian+""+username+"/article/details/\\d+").match()) {
page.addTargetRequests(page.getHtml().xpath("//*[@id=\"article_toplist\"]").regex(""+username+"/article/details/\\d+").replace(""+username+"",domian+username+"").all());
page.addTargetRequests(page.getHtml().xpath("//*[@id=\"article_list\"]").regex(""+username+"/article/details/\\d+").replace(""+username+"",domian+username+"").all());
 
}
else {
 
  page.putField("title:",page.getHtml().xpath("//*[@id=\"article_details\"]/div[1]/h1/span/").replace("<[^>]*>",""));
  page.putField("原创:",page.getHtml().regex("ico_type_(Original)",1).get());
  page.putField("阅读次数:",page.getHtml().xpath("//*[@id=\"article_details\"]/div[2]/div[2]/span[2]").regex("(\\d+)人阅读", 1));
  page.putField("评论次数:",page.getHtml().xpath("//*[@id=\"article_details\"]/div[2]/div[2]/span[3]").regex("(\\d+)", 1));
  page.putField("发布时间:",page.getHtml().css("span.link_postdate","text"));
  page.putField("标签:",page.getHtml().xpath("//*[@id=\"article_details\"]/div[2]/div[1]/span").regex("<a[^>]*>(.*?)</a>",1).all();
  page.putField("分类:",page.getHtml().xpath("//*[@id=\"article_details\"]/div[3]/div[2]/label/span/text()"));
  page.putField("文章内容:",page.getHtml().xpath("//*[@id=\"article_content\"]").replace("<[^>]*>",""));
  System.out.println("抓取第"+size+"文章");

 
}

public static void main(String[] args) {
		long startTime,endTime;
		System.out.println("爬虫开始请大家耐心等待");
		startTime =System.currentTimeMillis();
		System.out.println(startTime);
		 // 从用户博客首页开始抓，开启2个线程，启动爬虫
		Spider.create(new ForumPageprocess()).addPipeline(new ConsolePipeline()) .addUrl("http://blog.csdn.net/"+username+"").thread(2).run();
		endTime=System.currentTimeMillis();
		System.out.println("总共用时"+((endTime-startTime)/1000)+"秒");
		
	}

运行结果：