jsoup爬取基金数据，http://gs.amac.org.cn/amac-infodisc/res/pof/fund/index.html

分析

1.打开网址 http://gs.amac.org.cn/amac-infodisc/res/pof/fund/index.html

2.用fiddler抓取数据：

url:

http://gs.amac.org.cn/amac-infodisc/api/pof/fund?rand=0.8173862954864839&page=0&size=20

headers:

POST http://gs.amac.org.cn/amac-infodisc/api/pof/fund?rand=0.8173862954864839&page=0&size=20 HTTP/1.1
Host: gs.amac.org.cn
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0
Accept: application/json, text/javascript, */*; q=0.01
Accept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2
Accept-Encoding: gzip, deflate
Referer: http://gs.amac.org.cn/amac-infodisc/res/pof/fund/index.html
Content-Type: application/json
X-Requested-With: XMLHttpRequest
Content-Length: 2
Connection: keep-alive


HTTP/1.1 200
Server: nginx
Date: Mon, 18 Feb 2019 05:50:31 GMT
Content-Type: application/json;charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive

 

3.先用postman尝试

总是报错

 4.postman修改错误

参考网址https://ask.csdn.net/questions/699862?sort=votes_count

得出可能出错的地方："POST请求这个地址，POST发送的参数内容为{}，要不会报错，发送{}就可以了"

重新尝试，成功：

5.编写代码

package com;

import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.entity.Fund;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.IOException;
import java.util.List;

public class ATest {
    public static void main(String[] args) {
        //获取请求连接
        Connection con = Jsoup.connect("http://gs.amac.org.cn/amac-infodisc/api/pof/fund");
        //请求头设置
        con.header("Host", "gs.amac.org.cn");
        con.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0");
        con.header("Accept-Encoding", "gzip, deflate");
        con.header("Accept", "application/json, text/javascript, */*; q=0.01");
        con.header("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        con.header("Referer", "http://gs.amac.org.cn/amac-infodisc/res/pof/fund/index.html");
        con.header("Content-Type", "application/json");
        con.header("X-Requested-With", "XMLHttpRequest");
        con.header("Content-Length", "2");
        con.header("Connection", "keep-alive");


        
        con.data("page", String.valueOf(1));//第几页
        con.data("size", String.valueOf(100));//一页显示几行
        con.data("rand", "0.45045581928993783");

        con.requestBody("{}");//POST发送的参数内容为{}，要不会报错，发送{}就可以了

        try {
            Document doc = con.ignoreContentType(true).post();
            String body = doc.select("body").text();
            ObjectMapper mapper = new ObjectMapper();
            JsonNode rootNode = mapper.readTree(body);
            JsonNode totalPagesNode = rootNode.path("totalPages");
            String totalPagestr = mapper.writeValueAsString(totalPagesNode);
            int totalPages = Integer.parseInt(totalPagestr);
            System.out.println("totalPages="+totalPages);
            JsonNode recordsNode = rootNode.path("content");//内容
            String recodesJson = mapper.writeValueAsString(recordsNode);

            JavaType recordType = mapper.getTypeFactory().constructParametricType(List.class, Fund.class);
            List<Fund> list = mapper.readValue(recodesJson, recordType);//得到具体的内容
            if (list != null) {
                for (Fund fund : list) {
                    System.out.println(fund.getFundName());
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}