Java爬虫③WebMagic 爬取案例

Java爬虫①HttpClient
Java爬虫②Jsoup
WebMagic官方文档
WebMagic
是一个简单灵活的Java爬虫框架。基于WebMagic，你可以快速开发出一个高效、易维护的爬虫。

特性：

1. 简单的API，可快速上手
2. 模块化的结构，可轻松扩展
3. 提供多线程和分布式支持

导入maven依赖`

<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-core</artifactId>
    <version>0.7.3</version>
</dependency>
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-extension</artifactId>
    <version>0.7.3</version>
</dependency>

demo

package com.sihi.crawler.webmagic;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class Demo {
    public static void main(String[] args) {
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        Spider.create(new MyPageProcessor())
                .addPipeline(new MyPipeline())
                .setDownloader(httpClientDownloader)
                .addUrl("http://www.sikiedu.com")
                .run();
    }
}

class MyPageProcessor implements PageProcessor{
    public void process(Page page) {
        Document document = page.getHtml().getDocument();
        Element span = document.select(".subtitle").get(0).previousElementSibling().select("span").get(0);
        String text = span.text();
        page.putField("title",text);
    }

    public Site getSite() {
        return new Site();
    }
}

class MyPipeline implements Pipeline{
    public void process(ResultItems resultItems, Task task) {
        String title = resultItems.get("title").toString();
        System.out.println("拿到标签"+title);
    }
}

使用Selectable抽取元素，链接发现，配置Site

package com.sihi.crawler.webmagic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

public class TestPageProcessor implements PageProcessor {
    public void process(Page page) {
        Html html = page.getHtml();
        //用xpath获取IP
        List<String> ipList = html.xpath("//table[@id='ip_list']//tbody//tr//td[2]//text()").all();
        System.out.println(ipList);
        //用css选择器获取端口号
        Selectable portList = html.css("#ip_list tbody tr td:nth-child{3}", "text");
        System.out.println(portList);
        //通过css选择器获取服务器地址,链接的发现
        List<String> links = html.css("#ip_list tbody tr td:nth-child(4)").links().regex(".*/2018.*").all();
        System.out.println(links);

        //进入连接循环调用这个方法
        page.addTargetRequests(links);
    }

//    对站点本身的一些配置信息，例如编码、HTTP头、超时时间、重试策略等、代理等，都可以通过设置Site对象
    private Site site = Site.me()
            .setCharset("UTF-8")
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75")
            .setTimeOut(3000)
            .setRetryTimes(3)
            .setSleepTime(5000)
            .addHeader("cookie","__root_domain_v=.zdaye.com; _qddaz=QD.azbaun.g9ezu7.kjtn2fn1; acw_tc=76b20f6a16107