Java 爬取网页并保存到本地

所用技术

1. java
2. jsoup
3. io

package com.cron.cron.test;

import lombok.SneakyThrows;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;

/**
 * ClassName: downloadWeb
 * Description:
 * date: 2022/2/28 10:46
 *
 * @author robotname a
 * @author dev whz
 * @since JDK 1.8
 */

public class downloadWeb {

    @SneakyThrows
    public static void main(String[] args){
        String url = "https://www.liaoxuefeng.com/wiki/1252599548343744";
        long l = System.currentTimeMillis();
        //链接到目标地址
        Connection connect = Jsoup.connect(url);
        //设置useragent,设置超时时间，并以get请求方式请求服务器
        Document document = null;
        {
            try {
                document = connect.userAgent("Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)").
                        timeout(6000).ignoreContentType(true).get();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        Element elementById = document.getElementById("x-sidebar-left");
        Element elementById1 = elementById.getElementById("1252599548343744");
        Elements a = elementById1.getElementsByTag("a");
        for (int i = 0; i < a.size(); i++) {
            String href = a.get(i).attr("href");
            try {
                url = "https://www.liaoxuefeng.com" + href;
                Connection connects = Jsoup.connect(url);
                document = connects.userAgent("Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)").
                        timeout(6000).ignoreContentType(true).get();
                BufferedWriter writer = new BufferedWriter(new FileWriter(a.get(i).text()+".html"));
                writer.write(document.toString());
                writer.newLine();
            } catch (Exception e) {
                Thread.sleep(6000);
            }
        }
        System.err.println("下载用时"+(System.currentTimeMillis() - l));
    }
}

看效果图

还不是很完善 样式并没有拉取 今天先写到这 明天继续 记录一下 加油

后续会把css搞定 然后转成md文档 或者pdf的

不再让生活苟且
不再与过去和解