webmagic爬虫demo

package com.ray.webmagic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

public class CompanyInfoSpider implements PageProcessor {

// 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
private Site site = Site.me().setRetryTimes(Integer.MAX_VALUE).setSleepTime(1000).setUserAgent("Chrome/75.0.3770.142").setCycleRetryTimes(Integer.MAX_VALUE).setCharset("UTF-8");
private static int count = 0;
private static int provinceNum = 0;
private static int cityNum = 0;

private static String startUrl = "https://gongshang.mingluji.com/list";
private String provinceUrlRegex = "https://gongshang.mingluji.com/[a-z]+/diqu$";
private String cityUrlRegex = "https://gongshang.mingluji.com/[a-z]+/diqu/[0-9A-Z%]+";
private String companyRegex = "https://gongshang.mingluji.com/[a-z]+/name/[0-9A-Z%]+";


public Site getSite() {
    return site;
}

public void process(Page page) {

    Html html = page.getHtml();
    List<String> links;
    Selectable url = page.getUrl();

    //爬取的url为入口
    if (url.regex(startUrl).match()) {
        links = html.xpath("//td[@class='views-field-field-province']/a/@href").all();
        for (int i = 0; i < links.size(); i++) {
            links.set(i, links.get(i) + "/diqu");
        }

        page.addTargetRequests(links);
        //page.setSkip(true);
    } else if (url.regex(provinceUrlRegex).match()) {

        provinceNum++;
        links = html.xpath("//div[@class='views-field-city']/span/a/@href").all();

        page.addTargetRequests(links);
        //page.setSkip(true);
    } else if (url.regex(cityUrlRegex).match()) {
        cityNum++;
        links = html.xpath("//td[@class='views-field-name']/a/@href").all();
        page.addTargetRequests(links);

        links = html.xpath("//li[@class='pager-next']/a/@href").all();
        page.addTargetRequests(links);
        //page.setSkip(true);
    } else if (url.regex(companyRegex).match()) {

        page.putField("name", html.xpath("//span[@itemprop='name']/a/text()").get());
        page.putField("address", html.xpath("//span[@itemprop='address']/text()").get());
        page.putField("code", html.xpath("//span[@itemprop='identifier']/a/text()").get());
        count++;
    } else {
        System.out.println("error---");
    }


}

private void save() {

}

public static void main(String[] args) {

    System.out.println("开始爬取...");

    Spider.create(new CompanyInfoSpider()).addUrl(startUrl).addPipeline(new ConsolePipeline()).thread(2).run();


    System.out.println("抓取了" + count + "条记录");
    System.out.println(provinceNum);
    System.out.println(cityNum);

}
}