实现思路:
- 获取博客列表
- 获取博客详情
- 解析博客详情html,找出 #article_content部分
- 通过jsoup解析博客内容,转成md格式文件
依赖
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>2.0.33</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-core</artifactId>
<version>5.8.21</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.3</version>
</dependency>
java实现代码
package com.example.iotdbdemo.config;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.file.FileWriter;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import java.io.IOException;
/**
* @author WANG
*/
@Slf4j
public class Csdn2Hexo {
private static final String LIST_URL = "https://blog.csdn.net/community/home-api/v1/get-business-list?page=2&size=10000&businessType=blog&orderby=&noMore=false&year=&month=&username=%s";
private static final String DETAIL_URL = "https://blog.csdn.net/%s/article/details/%s";
private static final String USER_ID = "your-userId";
private static String getHttp(String url) {
String responseStr = null;
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpPost = new HttpGet(url);
RequestConfig config = RequestConfig.custom()
.build();
httpPost.setConfig(config);
try (CloseableHttpResponse response2 = httpclient.execute(httpPost)){
HttpEntity entity2 = response2.getEntity();
responseStr = EntityUtils.toString(entity2);
} catch (IOException e) {
e.printStackTrace();
}
return responseStr;
}
private static void parseHtml(String html, String title, String description, String date) {
Document document = Jsoup.parse(html);
Element articleContentDiv = document.select("#article_content").first();
assert articleContentDiv != null;
Document doc = Jsoup.parse(articleContentDiv.html());
StringBuilder markdown = new StringBuilder();
markdown.append("---").append("\r\n")
.append("title: ").append(title).append("\r\n")
.append("date: ").append(date).append("\r\n")
.append("---").append("\r\n")
.append(description)
.append("\r\n");
Element element = doc.body().children().get(0);
for (Node node : element.childNodes()) {
if (node instanceof TextNode textNode) {
markdown.append(textNode.text());
} else if (node instanceof Element childElement) {
switch (childElement.tagName()) {
case "h1", "h2", "h3", "h4", "h5", "h6" -> {
int level = Integer.parseInt(childElement.tagName().substring(1));
markdown.append("\n").append("#".repeat(level)).append(" ").append(childElement.text()).append("\n\n");
}
case "p" -> markdown.append("\n").append(childElement.text()).append("\n\n");
case "ul" -> markdown.append(convertListToMarkdown(childElement, false));
case "ol" -> markdown.append(convertListToMarkdown(childElement, true));
case "pre" -> markdown.append("\n```\n").append(childElement.text()).append("\n```\n\n");
default -> markdown.append("\r\n");
}
}
}
title = title.replaceAll("\\.+", "-");
title = title.replaceAll(":+", "-");
title = title.replaceAll("\\|+", "-");
title = title.replaceAll(">+", "-");
FileWriter writer = new FileWriter("E:\\gitee\\html\\" + title + ".md");
writer.write(markdown.toString());
log.info("Write {} success", writer.getFile().getAbsolutePath());
}
private static String convertListToMarkdown(Element list, boolean ordered) {
StringBuilder markdown = new StringBuilder();
Elements items = list.children();
int count = 1;
for (Element item : items) {
String prefix = ordered ? (count++ + ". ") : "* ";
markdown.append(prefix).append(item.text()).append("\n");
}
return markdown.append("\n").toString();
}
public static void main(String[] args) {
String listUrl = String.format(LIST_URL, USER_ID);
String listJson = getHttp(listUrl);
JSONObject jsonObject = JSONObject.parseObject(listJson);
Integer code = jsonObject.getInteger("code");
if(code == 200){
JSONArray jsonArray = jsonObject.getJSONObject("data").getJSONArray("list");
if(CollUtil.isNotEmpty(jsonArray)){
for (Object o : jsonArray) {
if(o instanceof JSONObject item){
Integer articleId = item.getInteger("articleId");
String title = item.getString("title");
String description = item.getString("description");
String date = item.getString("postTime");
String detailUrl = String.format(DETAIL_URL, USER_ID, articleId);
String html = getHttp(detailUrl);
parseHtml(html, title, description, date);
}
}
}
}
}
}