爬虫日记
先讲下做这个的来由吧,项目要求做一个文章栏目,而文章的主要来源是经过文章链接转换(其实就是微信文章);
话不多说直接上代码:
在这里插入代码片/**
private static String fileUrl;
* 微信公众号文章链接转换
* @param url
* @return
* @throws IOException
*/
public static Article getWXArticleByUrl(String url) throws IOException {
if (StringUtils.isNull(url)) {
return null;
}
Article article = new Article();
article.setUrl(url);
Document doc = Jsoup.connect(url).get();
// 获取文章标题
String title = doc.head().select("meta[property=og:title]").attr("content");
article.setTitle(title);
// 获取文章缩略图
String image = doc.head().select("meta[property=og:image]").attr("content");
String thumbnail = fileUrl + FileUploadUtils.upload(GcsConfig.getArticleUpload(), image);
article.setThumbnail(thumbnail);
// 获取文章内容
Element js_content = doc.body().select("div#js_content").removeAttr("style").get(0);
// 处理文章内容图片
for (Element el: js_content.select("img")) {
String imgUrl = el.attr("data-src");
String newImgPath = fileUrl + FileUploadUtils.upload(GcsConfig.getArticleUpload(), imgUrl);
el.removeAttr("data-src").attr("src", newImgPath);
}
String html = js_content.outerHtml().replaceAll("\r|\n|\t", "").replaceAll("> <|> <","><");
article.setContent(html);
return article;
}
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
说明:该项目用的是springboot框架,fileUrl为服务地址