使用代理调用http接口

最新推荐文章于 2024-05-27 14:15:00 发布

我是真的想笑

最新推荐文章于 2024-05-27 14:15:00 发布

阅读量242

点赞数

文章标签： http 网络协议网络

本文链接：https://blog.csdn.net/qq_43588909/article/details/131377277

版权

package com.inmyshow;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class MainTest {

    public static void main(String[] args) throws IOException {

        HttpRequest request = HttpUtil.createGet("https://bigdata-pdf.weiq.com/xingtu/test_uid_list.xlsx");
        HttpResponse httpResponse = request.execute();
        List<ReadData2> readDataList = EasyExcel.read(httpResponse.bodyStream()).head(ReadData2.class).ignoreEmptyRow(false).sheet().doReadSync();
        httpResponse.close();

        List<WriteData2> writeDataList = new ArrayList<>();

        int count = 0;

        TimeInterval timer = DateUtil.timer();

        for (ReadData2 readData2 : readDataList) {

            timer.restart();

            count++;
            System.out.println("count = " + count);

            WriteData2 writeData2 = new WriteData2();

            BeanUtil.copyProperties(readData2, writeData2);

            try {
                String response;
                while (true) {

                    response = HttpRequest.get("https://weibo.com/ajax/profile/detail?uid=" + readData2.getUid()).setHttpProxy("8.140.193.161", 9976)
                            .header("authority", "weibo.com")
                            .header("accept", "application/json, text/plain, */*")
                            .header("accept-language", "zh-CN,zh;q=0.9")
                            .header("client-version", "v2.40.69")
                            .header("cookie", "SINAGLOBAL=3800303113230.0835.1678328293211; UOR=,,login.sina.com.cn; SSOLoginState=1686302106; _s_tentry=weibo.com; Apache=7137011272638.041.1686302123137; ULV=1686302123143:4:1:1:7137011272638.041.1686302123137:1681908308850; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhP_aSkKzQcFfXLqgksbbN45JpX5KMhUgL.FoMfSK.R1hzfSh.2dJLoI0qLxKMLB.zL1-BLxKBLBonL1KqLxKML1-2L1hBLxKnLBozL1hzLxK-LBo5L12qLxK-LBK.LBo-t; ALF=1689424306; SCF=An53GtvALMV5QP_zt3sbEz119kG5MS325qvDo2T_aODc27g38LwqljBulqxhxCxq-mk3FKQ3AKFcNTH8IJBJsZo.; SUB=_2A25Jj3TiDeRhGeFL7lsZ-CzJzzWIHXVq_eEqrDV8PUNbmtAGLXnkkW9NffH7j5LXC1kCB77EqO08G3dbP_KalIwz")
                            .header("referer", "https://weibo.com/u/5471347537")
                            .header("sec-ch-ua", "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"")
                            .header("sec-ch-ua-mobile", "?0")
                            .header("sec-ch-ua-platform", "\"Windows\"")
                            .header("sec-fetch-dest", "empty")
                            .header("sec-fetch-mode", "cors")
                            .header("sec-fetch-site", "same-origin")
                            .header("server-version", "v2023.06.15.2")
                            .header("traceparent", "00-e80026a121df5e0475d54b8b9d8bed2c-62895f936f2cdcfc-00")
                            .header("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
                            .header("x-requested-with", "XMLHttpRequest")
                            .header("x-xsrf-token", "mOcVp3myReMYeep3qW0VN2Mg").execute().body();


                    if (response.contains("414 Request-URI Too Large")) {

                        System.out.println("请求数量过大，等待重试");

                        Thread.sleep(20000);

                        return;

                    } else {
                        break;
                    }
                }

                JSONObject resObj = JSONObject.parseObject(response);

                JSONObject data = resObj.getJSONObject("data");
                if (ObjectUtil.isNotNull(data)) {
                    JSONObject mcnDesc = data.getJSONObject("mcn_desc");
                    if (ObjectUtil.isNotNull(mcnDesc)) {
                        String desc = mcnDesc.getString("desc");
                        writeData2.setJiGou(desc);
                    }
                }

                System.out.println("data = " + JSON.toJSONString(writeData2));

            } catch (Exception e) {
                System.out.println("抓取有误 uid = " + readData2.getUid());
            }

            System.out.println("耗时 " + timer.intervalMs() + " ms");

            writeDataList.add(writeData2);

        }

        String fileName = "C:\\Users\\EDZ\\Desktop\\0615test2.xlsx";
        EasyExcel.write(fileName, WriteData2.class)
                .sheet("模板")
                .doWrite(writeDataList);

    }

}