package com.tvs.webmgic;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
public class MyWebmagic implements PageProcessor {
// 抓取网站的相关配置,包括:编码、抓取间隔、重试次数等
private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
@Override
public Site getSite() {
return site;
}
@Override
public void process(Page page) {
page.addTargetRequests(page.getHtml().css("div#page").links().all());
/*String title = page.getHtml().xpath("//*[@id=\"main\"]/div[1]/div["+i+"]/dl/dd/a//text()").toString();
System.out.println(title);*/
if(page.getUrl().regex("http://www.bjnews.com.cn/opinion/[?page=\\d{0,}]{0,1}").match()){
page.addTargetRequests(page.getHtml().css("div.news").links().all());
}
if(page.getUrl().regex("http://www.bjnews.com.cn/opinion/2017/\\d{2}/\\d{2}/\\d{6}.html").match()){
String author = page.getHtml().xpath("//*[@id=\"author_baidu\"]//text()").toString();
System.out.println(author);
String title = page.getHtml().xpath("//*[@id=\"main\"]/div[1]/h1//text()").toString();
System.out.println(title);
}
}
public static void main(String[] args) {
Spider.create(new MyWebmagic()).addUrl("http://www.bjnews.com.cn/opinion/").thread(5).run();
}
}