Webmagic爬虫--②爬自己的CSDN博客列表

最新推荐文章于 2023-11-03 13:52:14 发布

撸码小丑

最新推荐文章于 2023-11-03 13:52:14 发布

阅读量2.6k

点赞数 1

分类专栏： Webmagic爬虫

本文链接：https://blog.csdn.net/weixin_35852328/article/details/78625067

版权

Webmagic爬虫专栏收录该内容

2 篇文章 0 订阅

订阅专栏

1.创建Maven项目，添加依赖

<dependency>
  <groupId>us.codecraft</groupId>
  <artifactId>webmagic-core</artifactId>
  <version>0.7.3</version>
</dependency>
<dependency>
  <groupId>us.codecraft</groupId>
  <artifactId>webmagic-extension</artifactId>
  <version>0.7.3</version>
</dependency>

2.PageProcessor代码

package com.xt;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * Created by XT on 2017/11/24.
 */
public class MyCSND implements PageProcessor {
    //http://blog.csdn.net/weixin_35852328/article/list/1
    public static final String URL_LIST = "http://blog\\.csdn\\.net/weixin_35852328/article/list/\\d{1}";

    //http://blog.csdn.net/weixin_35852328/article/details/78144353
    public static final String URL_POST = "http://blog\\.csdn\\.net/weixin_35852328/article/details/\\d{8}";

    private static int count = 0;

    private Site site = Site
            .me()
            .setDomain("blog.csdn.net")
            .setRetryTimes(3).setSleepTime(100);

    public void process(Page page) {
        //列表页
        if (page.getUrl().regex(URL_LIST).match()) {
            page.addTargetRequests(page.getHtml().xpath("//span[@class=link_title]/a/@href").all());
            page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());
            //文章页
        } else {
            count++;
            page.putField("title", page.getHtml().xpath("//span[@class='link_title']/a/text()"));
            page.putField("view", page.getHtml().xpath("//span[@class='link_view']/text()"));
            page.putField("date",
                    page.getHtml().xpath("//span[@class='link_postdate']/text()"));
        }
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        long startTime, endTime;
        System.out.println("开始爬取...");
        startTime = System.currentTimeMillis();
        Spider.create(new MyCSND()).addUrl("http://blog.csdn.net/weixin_35852328/article/list/2").thread(5).run();
        endTime = System.currentTimeMillis();
        System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了" + count + "条记录");


    }
}