jsoup深度遍历html中所有的子链接

最新推荐文章于 2021-06-25 22:18:15 发布

无名小卒菜

最新推荐文章于 2021-06-25 22:18:15 发布

阅读量1.8k

点赞数 2

分类专栏： Java 文章标签： JSOUP 深度遍历递归

本文链接：https://blog.csdn.net/weixin_39428938/article/details/80982696

版权

Java 专栏收录该内容

52 篇文章 1 订阅

订阅专栏

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;

public class testDOM  {


    public static void main(String[] args) {
        String url = "https://mirrors.edge.kernel.org/pub/software/libs/glibc/hjl/";
        getAllUrl(url);
    }



    public static List getAllUrl(String url){

        List list = new ArrayList();
       try {
           Document doc = (Document) Jsoup.connect(url).get();
           Elements links = ((Element) doc).select("a[href]");
           for (Element link : links) {
               String linkHref = link.attr("abs:href");
               String linkText = link.text();
               String test = linkHref+"............................"+linkText;
               System.out.println(test);
               list.add(linkHref);
               if(!linkText.endsWith("../")&&linkText.endsWith("/")){
                    getAllUrl(linkHref);
               }
           }

       } catch (IOException e) {
           // TODO Auto-generated catch block
           e.printStackTrace();
       }
        return list;
   }




}

改进版如下：

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;

public class testDOM  {
    private  static List   list;

    public static void main(String[] args) {
        list = new ArrayList();
        String url = "https://mirrors.edge.kernel.org/pub/software/libs/glibc/hjl/";
         List list2 = getAllUrl(url);
        for (int i = 0; i <list2.size() ; i++) {
            System.out.println(list2.get(i));
        }
    }


    public static List getAllUrl(String url){
       try {
           Document doc = (Document) Jsoup.connect(url).get();
           Elements links = ((Element) doc).select("a[href]");
           for (Element link : links) {
               String linkHref = link.attr("abs:href");
               String linkText = link.text();
               String test = linkHref+"............................"+linkText;
//               System.out.println(test);
               if(!linkText.endsWith("../")&&linkText.endsWith("/")){
                    getAllUrl(linkHref);
               }
               list.add(linkHref);
           }

       } catch (IOException e) {
           // TODO Auto-generated catch block
           e.printStackTrace();
       }
        return list;
   }
}

所需jar包链接：密码: 9ir5