webmagic原始实例

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.List;


import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
public class ganjitest implements PageProcessor{


private Site site=Site.me().setSleepTime(3000).setRetryTimes(2);


List<Request> list=null; //取得当前内存中存放的相关的网页链接
int num=0; //取得所有的链接
public void Show(Page page){

String addressUrl=null;
String url=page.getUrl().toString();
if(url.trim()!="http://www.xinhuanet.com/" || !url.equals("http://www.xinhuanet.com/")){
try{
page.putField("address",addressUrl=page.getUrl().regex("(http://news\\.xinhuanet\\.com/politics/.*)").toString());
}catch(Exception e){
System.out.println("url not contains");
}
}else{
page.putField("address", addressUrl=page.getUrl().toString());
}
page.putField("name", page.getHtml().xpath("//title/text()").toString());
//提取meta中的content的值
String str=page.getHtml().xpath("//meta[@name='keywords']").toString();
int a=str.lastIndexOf("content=\"");
int b=str.lastIndexOf("\"");
page.putField("meta", str.substring(a+9, b));
//提取最后刷新时间
URL hp = null;
URLConnection hpCon;
try {
   SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 
hp = new URL(addressUrl);
hpCon = hp.openConnection();
//转化为本地时间
page.putField("lasttimes",sdf.format(new Date(hpCon.getLastModified())));
} catch (Exception e) {
e.printStackTrace();
} 


if(page.getResultItems().get("name")==null){
page.setSkip(true);
}
}
public void process(Page page) {

//存放到内存中的网页
page.addTargetRequests(page.getHtml().links().regex("(http://news.xinhuanet.com/politics/.*)").all());
list= page.getTargetRequests();
num+=list.size();
System.out.println(num);
if(num>100){
return ;
}else{
Show(page);
}

}
public Site getSite() {
return site;
}
public static void main(String[] args) {
//初始化进入的首页
System.out.println("开始");
//从哪里开始抓
Spider.create(new ganjitest()).addUrl("http://www.xinhuanet.com/").thread(5).run();
System.out.println("结束");

}
}