Spring boot+webMagic实现自动化爬取网站内容

本文摘自：https://funyan.cn/p/6861.html

前言

在网站中，内容才是王道，如果你的网站功能很多也很好看，但是没有内容，那么将毫无意义，但是要是靠站长自己写的话那将是一段非常漫长的过程，所以这时候就需要借鉴其他站的内容，来丰富自己的网站，所以爬虫就出现了，那么今天就教大家如何使用Spring boot+webMagic实现自动化爬取网站内。

开发过程

第一步：引入webMagic

<dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-extension</artifactId>
            <version>0.7.3</version>
</dependency>

第二步：编写工具类

package cn.funyan.blog.task;

import cn.funyan.blog.Article;
import cn.funyan.blog.ArticleSpider;
import cn.funyan.blog.service.ArticleSpiderService;
import cn.funyan.utils.SpringUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/** 
* @Description: 自动化爬虫工具类 
* @Author: Chris.Ren
* @Date: 2021-06-29 
*/

@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@Slf4j
@Component
public class WebMagicUtil implements PageProcessor
{

	//id
	private Integer id;
	//url
	private String url;
	//文章列表dom值
	private String listDom;
	//文章分页dom值
	private String pageDom;
	//文章详情dom值
	private String articleDom;
	//文章标题dom值
	private String headDom;
	//文章内容dom值
	private String contentDom;
	//文章标签dom值
	private String tagDom;
	//文章标签类型
	private Integer tagType;
	//文章题图
	private String img;
        
        //这里需要autowird，因为我们在创建爬虫的时候需要用到这个保存
      	@Autowired
	private SaveArticle saveArticle;


	public WebMagicUtil(WebMagicUtil webMagicUtil) {
		this.url = webMagicUtil.getUrl();
		this.listDom = webMagicUtil.getListDom();
		this.pageDom = webMagicUtil.getPageDom();
		this.articleDom = webMagicUtil.getArticleDom();
		this.headDom = webMagicUtil.getHeadDom();
		this.contentDom = webMagicUtil.getContentDom();
		this.tagDom = webMagicUtil.getTagDom();
		this.tagType = webMagicUtil.getTagType();
		this.img = webMagicUtil.getImg();
		saveArticle = SpringUtil.getBean(SaveArticle.class);
		articleSpiderService = SpringUtil.getBean(ArticleSpiderService.class);
	}

	@Override
	public void process(Page page) {

		Html html = page.getHtml();

		//获取详情页链接
		List<Selectable> linkList = html.css(this.listDom).nodes();
		if (linkList.size() <= 1) {
			//保存文章
			preSaveArticle(page);
		} else {
			//遍历文章列表，获取详情
			for (Selectable link : linkList) {
				//详情页图片
				String artLink = link.css(this.articleDom).links().toString();

				page.addTargetRequest(artLink);
			}

			//找到下一页
			String bkUrl = html.css(this.pageDom).links().toString();
			if (!StringUtils.isEmpty(bkUrl)) {
				page.addTargetRequest(bkUrl);
			}
		}
	}

	//获取文章详情
	private void preSaveArticle(Page page) {
		try {
			Html html = page.getHtml();
			//到这里已经有15个详情页
			Article article = new Article();
			article.setTitlePhoto(this.img);
			//标题
			article.setHead(html.css(this.headDom, "text").toString());
			log.info("正在解析文章:{}", article.getHead());

			//内容
			String content = html.css(this.contentDom).toString();
			article.setContent(content);

			//标签
			String tag = "";
			if (tagType.equals(1)) {
				List<Selectable> tagList = html.css(this.tagDom, "text").nodes();
				for (Selectable tags : tagList) {
					tag = tag + tags.toString() + ",";
				}
				article.setTags(tag.substring(0, tag.lastIndexOf(",")));
			} else {
				tag = this.tagDom;
				article.setTags(tag);
			}

			//类型
			article.setArticleClassify(3);
			article.setCheckStatus(0);
			article.setArticleType(2);
			article.setDescription(article.getHead());
			article.setUserId((int) (Math.random() * 5 + 1));

			page.putField("artInfo", article);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	private Site site = new Site()
			.setCharset("utf8")
			.setTimeOut(10000)
			.setRetrySleepTime(3000)
			.setRetryTimes(3);

	@Override
	public Site getSite() {
		return site;
	}


	public void processor(WebMagicUtil webMagicUtil) {
		this.url = webMagicUtil.getUrl();
		log.info("正在解析网站：{}", url);

		Spider.create(new WebMagicUtil(webMagicUtil))
				.addUrl(url)
				.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
				.thread(10)
				.addPipeline(saveArticle)
				.run();
	}

	public static void main(String[] args) {
		WebMagicUtil wb = new WebMagicUtil();
		wb.processor(WebMagicUtil
				.builder()
				.articleDom("111")
				.contentDom("www")
				.url("https://funyan.cn")
				.build());
	}

}

第三步：编写持久化类，就是把你爬取的东西保存下来

package cn.funyan.blog.task;

import cn.funyan.blog.Article;
import cn.funyan.blog.service.ArticleService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

/**
 *
 **/
@Slf4j
@Component
public class SaveArticle implements Pipeline
{

	@Autowired
	private ArticleService articleService;


	@Override
	public void process(ResultItems resultItems, Task task) {

		Article article=resultItems.get("artInfo");

		if(article!=null) {
			log.info("正在保存文章：{}",article.getHead());
			articleService.sendNewArticle(article);
		}
	}
}

第四步：调用，因为我们把工具类交给spring管理，所以我们只需要注入到需要的地方即可

        @Autowired
	private WebMagicUtil webMagicUtil;


	@PostMapping("spiderArticle")
	public Res spiderArticle(Integer id){
		//查看是否爬过了
		ArticleSpider as = service.getById(id);
		if(as==null){
			return new Res().failed();
		}

		if(as.getStatus().equals(1)){
			return new Res().failed().setMsg("该文章已爬取过，请勿重复爬取");
		}
		webMagicUtil.processor(WebMagicUtil
				.builder()
				.id(as.getId())
				.url(as.getUrl())
				.img(as.getImg())
				.listDom(as.getListDom())
				.articleDom(as.getArticleDom())
				.pageDom(as.getPageDom())
				.headDom(as.getHeadDom())
				.contentDom(as.getContentDom())
				.tagDom(as.getTagDom())
				.tagType(as.getTagType())
				.build());

		return new Res();
	}

这样就完成了Spring boot+webMagic实现自动化爬取网站，后期就可以在后台管理系统调用接口，输入网站dom名实现自动化的爬取内容了

本文摘自：https://funyan.cn/p/6861.html