jsoup爬虫发送get、post请求、解析html、获取json

逻辑峰

已于 2023-11-21 08:57:43 修改

阅读量2.4k

点赞数

分类专栏：爬虫文章标签： java 开发语言

于 2022-12-07 10:15:09 首次发布

本文链接：https://blog.csdn.net/weixin_43684214/article/details/128214895

版权

文章目录

1 简介
2 发送get请求
3 发送post请求
4 Document的使用

1 简介

jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据

依赖

        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.8.3</version>
        </dependency>

官网地址：https://jsoup.org/apidocs/org/jsoup/nodes/Element.html

2 发送get请求

2.1设置请求头和cookie

    public static void main(String[] args) throws IOException {
   
        doGet("","");
    }

    public static void doGet(String url ,String cookie) throws IOException {
   
        url = "https://www.baidu.com/";
        cookie = "BIDUPSID=56C4FFBF72723876ACBD5B2EA2C75AD8; PSTM=1669362017; BD_UPN=12314753; BAIDUID=F4A7FCF34D70EE033317D31778290FE0:FG=1; ZFY=bMkQzdDwo03XIz75dDC8:AwpdCLhCOaBeWmYRxguNzHQ:C; BAIDUID_BFESS=F4A7FCF34D70EE033317D31778290FE0:FG=1; baikeVisitId=a61698de-af11-451a-8f37-7a0efc41ef7c; __bid_n=184d169de2c1b652e44207; RT=\"z=1&dm=baidu.com&si=7ao2cg4efas&ss=lb63h3q0&sl=3&tt=2hm&bcn=https://fclog.baidu.com/log/weirwood?type=perf&ld=31p&ul=18ia&hd=18ka\"; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; COOKIE_SESSION=439025_2_8_9_12_10_0_0_8_6_0_0_459518_56535_3_0_1670314644_1669853798_1670314641|9#56556_3_1669853666|2; BA_HECTOR=24a484ah0k84258h0h852gj41hovnrf1g; ab_sr=1.0.1_ODA0NTY3OGYzMmViNzMyNGQ1YWU1MzhjZDc1YjM2ZjQ0MjlkZjlhOTI2ZjZjMjk1N2NlODAzNjg5YjRlNDQxNjUzZjQ2M2JkZDYwOWQ2Y2Q1MDI4NzRhNjRkM2RjYmQxOGNiNjZhMTk0YjFhZThmMmE4ZjljN2MyMzJmOWFhOWNlMTk3MmZjNTcyNjNkN2RhYjQ3M2Y1MjNlMjViNjcyOQ==; BDRCVFR[PGnakqNNAQT]=mk3SLVN4HKm; BD_HOME=1; H_PS_PSSID=37856_36560_36920_37835_37841_37871_37765_37797_37760_37853_26350_22160_37881; delPer=0; BD_CK_SAM=1; PSINO=1; H_PS_645EC=35d9T+b4lE+jHT8VNaqK1aghA5CXp4Yf4fqDSH/hRVMSETsPPBQsOh069iA";
        //获取请求连接
        Connection con = Jsoup.connect(url);
        //请求头设置，特别是cookie设置
        con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
        con.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36");
        con.header("Host","www.baidu.com");
        con.header("Referer","https://www.baidu.com/link?url=Su25Siv6Fi0gZJgDQoDcxse30NxOOLSSqQcNy0k6FU3&wd=&eqid=d8ed532b000af73400000003638feddc");
        con.header("Cookie", cookie);