使用WebMagic编写一个网络爬虫

最新推荐文章于 2022-04-13 11:57:52 发布

梦尘啊~

最新推荐文章于 2022-04-13 11:57:52 发布

阅读量143

点赞数

文章标签： java 爬虫

本文链接：https://blog.csdn.net/weixin_43932582/article/details/110083404

版权

使用WebMagic编写一个网络爬虫

引入依赖

<dependency>
        <groupId>us.codecraft</groupId>
        <artifactId>webmagic-core</artifactId>
        <version>0.7.4</version>
    </dependency>
    <dependency>
        <groupId>us.codecraft</groupId>
        <artifactId>webmagic-extension</artifactId>
        <version>0.7.4</version>
    </dependency>

编写测试类

public class NewPage implements PageProcessor {


    //配置爬虫的各种配置
    private Site site = Site
            .me()
            .setUserAgent(
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36");

    /**
     * 处理页面，提取网址以提取，提取数据并存储
     * */
    @Override
    public void process(Page page)
    {
        //根据xpath获取网页的元素
        page.putField("title", page.getHtml().xpath("//h1[@class='content-heading']/text()").get());
        page.putField("time", page.getHtml().xpath("//div[@class='container']/p/text()").get());
        page.putField("name", page.getHtml().xpath("//div[@class='card-inner']").get());
        if (page.getResultItems().get("name")==null){
            //skip this page
            page.setSkip(true);
        }
    }
    @Override
    public Site getSite() {
        return site;
    }
    public static void main(String[] args) {

        for (int nums = 1000;nums< 4895;nums++) {
            //启动爬虫addPipeline添加持久化配置
            Spider.create(new NewPage()).addUrl("https://acm.taifua.com/bzoj/p/"+nums+".html").addPipeline(new TestPipeline(nums)).run();
        }
    }
}

编写持久化类

public class TestPipeline  implements Pipeline {
    private  int index;


    public int getIndex() {
        return index;
    }

    public TestPipeline(int index) {
        this.setIndex(index);
    }
/**
 * 配置爬虫的持久化等】
 *resultItems爬虫爬取的数据
 * */
    @Override
    public void process(ResultItems resultItems, Task task) {


        try {
            //获取文章标题title
            String title = resultItems.get("title");
            PrintWriter printWriter = new PrintWriter(new FileWriter("D:\\text\\"+this.getIndex()+ title+".json"));
            printWriter.write(JSON.toJSONString(resultItems.getAll()));
            printWriter.close();
        } catch (IOException var5) {


        }

    }

    public TestPipeline setIndex(int index) {
        this.index = index;
        return this;
    }
}