HttpURLConnection使用

最新推荐文章于 2023-03-11 01:07:36 发布
最新推荐文章于 2023-03-11 01:07:36 发布
阅读量183 收藏 1
点赞数
文章标签： HttpURLConnection
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_26475401/article/details/82108428
版权 
import com.google.common.collect.Lists;
import com.meiyunji.spider.contant.HttpConnectContant;
import com.meiyunji.spider.crawl.response.HeaderVo;
import com.meiyunji.spider.crawl.response.ResponseObj;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.*;
import java.io.*;
import java.net.*;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;

/**
 * @author ldj
 * @date 2018/6/25.
 * <p>
 * 使用 httpUrlConnection请求信息
 */
public class HttpConnectUtil {

    private static final Logger logger = LoggerFactory.getLogger(HttpConnectUtil.class);

    /**
     * Get 方法
     *
     * @param url        请求连接
     * @param ip         本次请求使用的ip
     * @param headers    请求头, 为空会填写默认头部信息
     * @param parameters 请求体,
     * @return ResponseObj 返回结果
     **/
    public static ResponseObj urlGet(String url, String ip, Map<String, String> headers, Map<String, String> parameters) {
        return urlBaseGet(url, ip, null, headers, parameters);
    }

    /**
     * 代理请求
     *
     * @param url        请求连接
     * @param ip         代理ip
     * @param port       代理端口
     * @param headers    请求头, 为空会填写默认头部信息
     * @param parameters 请求体,
     * @return ResponseObj 返回结果
     */
    public static ResponseObj urlPorxyGet(String url, String ip, Integer port, Map<String, String> headers, Map<String, String> parameters) {
        return urlBaseGet(url, ip, port, headers, parameters);
    }

    /**
     * Get 请求
     */
    private static ResponseObj urlBaseGet(String url, String ip, Integer port, Map<String, String> headers, Map<String, String> parameters) {

        ResponseObj responseObj;
        try {
            url = url.trim();
            if (parameters != null) {
                StringBuilder sbd = new StringBuilder(url);
                parameters.forEach((key, value) -> sbd.append(StringUtils.containsIgnoreCase(sbd, "?") ? "?" : "&").append(key).append("=").append(value));
                url = sbd.toString();
            }
            URL requestUrl = new URL(url);
            if (HttpConnectContant.CONNECT_TYPE_HTTPS.equalsIgnoreCase(requestUrl.getProtocol())) {
                responseObj = httpsConnectionGet(requestUrl, ip, port, headers, parameters);
            } else {
                responseObj = httpConnectionGet(requestUrl, ip, port, headers, parameters);
            }
        } catch (IOException e) {
            responseObj = new ResponseObj();
            responseObj.setStatusCode(-1);
            responseObj.setReasonPhrase("Crawl Error : URL DealHtml error, the url is not support request.");
        }
        responseObj.setIp(ip);
        responseObj.setUrl(url);

        logger.info("url: {}", url);
        logger.info("statusCode: {}", responseObj.getStatusCode());
        logger.info("message: {}", responseObj.getReasonPhrase());
        return responseObj;
    }

    /**
     * Https Get 方法
     *
     * @param  requestUrl 请求连接
     * @param ip 本次请求使用的ip
     * @param headers 请求头, 为空会填写默认头部信息
     * @param parameters 请求体,
     * @return  ResponseObj 返回结果
     * String ua,
     * String html,
     * String statusCode,
     * String ip;
     * String url;
     * String userAgent;
     */
    private static ResponseObj httpsConnectionGet(URL requestUrl, String ip, Integer port, Map<String, String> headers, Map<String, String> parameters) {
        InputStreamReader reader = null;
        GZIPInputStream gzipInputStream = null;
        InputStream inputStream = null;
        HttpsURLConnection httpsURLConnection = null;
        ResponseObj responseObj = new ResponseObj();
        try {
            // 获取 URLConnection对象
            httpsURLConnection = (HttpsURLConnection) getUrlConnection(requestUrl, ip, port);
            setDefaultProperties(httpsURLConnection);
            httpsURLConnection.setRequestMethod(HttpConnectContant.GET);

            if (headers == null || headers.size() == 0) {
                setDefaultHeader(httpsURLConnection);
            } else {
                setCustomizedHeader(httpsURLConnection, headers);
            }

            /// 设置绕过https验证
            trustAllHosts(httpsURLConnection);
            httpsURLConnection.setHostnameVerifier(DO_NOT_VARIFY);

            // 打开链接
            httpsURLConnection.connect();

            // 获取结果
            if (httpsURLConnection.getResponseCode() == HttpStatus.SC_OK || httpsURLConnection.getResponseCode() == HttpStatus.SC_CREATED) {
                inputStream = httpsURLConnection.getInputStream();
            } else {
                inputStream = httpsURLConnection.getErrorStream();
            }
            String contentEncoding = httpsURLConnection.getContentEncoding();
            if (contentEncoding != null && "gzip".equals(contentEncoding.trim().toLowerCase())) {
                gzipInputStream = new GZIPInputStream(inputStream);
                reader = new InputStreamReader(gzipInputStream);
            } else {
                reader = new InputStreamReader(inputStream);
            }

            StringBuilder sbfs = new StringBuilder();
            char[] c = new char[1024];
            int len;
            while (0 < (len = reader.read(c))) {
                sbfs.append(c, 0, len);
            }
            List<HeaderVo> headerList = Lists.newArrayList();
            Map<String, List<String>> responseHeaderFields = httpsURLConnection.getHeaderFields();
            if (responseHeaderFields != null && responseHeaderFields.size() > 0) {
                responseHeaderFields.forEach((key, value) -> headerList.add(new HeaderVo(key, JsonUtil.objectToJson(value))));
            }

            responseObj.setStatusCode(httpsURLConnection.getResponseCode());
            responseObj.setContent(sbfs.toString());
            responseObj.setResponseHeader(headerList);
            responseObj.setUserAgent(httpsURLConnection.getRequestProperty("user-agent"));

        } catch (IOException e) {
            e.printStackTrace();
            responseObj.setStatusCode(-1);
            responseObj.setReasonPhrase("Crawl Error : HttpsUrlConnection IOException .");
        } finally {
            try {
                if (reader != null) {
                    reader.close();
                }
                if (gzipInputStream != null) {
                    gzipInputStream.close();
                }
                if (inputStream != null) {
                    inputStream.close();
                }
                if (httpsURLConnection != null) {
                    httpsURLConnection.disconnect();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return responseObj;
    }

    /**
     * Http Get 方法
     *
     * @param  requestUrl 请求连接
     * @param ip 本次请求使用的ip
     * @param headers 请求头, 为空会填写默认头部信息
     * @param parameters 请求体,
     * @return  ResponseObj 返回结果
     * String ua,
     * String html,
     * String statusCode,
     * String ip;
     * String url;
     * URL requestUrl;
     * String userAgent;
     */
    private static ResponseObj httpConnectionGet(URL requestUrl, String ip, Integer port, Map<String, String> headers, Map<String, String> parameters) {
        InputStreamReader reader = null;
        GZIPInputStream gzipInputStream = null;
        InputStream inputStream = null;
        HttpURLConnection httpUrlConnection = null;
        ResponseObj responseObj = new ResponseObj();
        try {
            httpUrlConnection = (HttpURLConnection) getUrlConnection(requestUrl, ip, port);
            setDefaultProperties(httpUrlConnection);
            httpUrlConnection.setRequestMethod(HttpConnectContant.GET);
            if (headers == null || headers.size() <= 0) {
                setDefaultHeader(httpUrlConnection);
            } else {
                setCustomizedHeader(httpUrlConnection, headers);
            }

            httpUrlConnection.connect();

            if (httpUrlConnection.getResponseCode() == HttpStatus.SC_OK || httpUrlConnection.getResponseCode() == HttpStatus.SC_CREATED) {
                inputStream = httpUrlConnection.getInputStream();
            } else {
                inputStream = httpUrlConnection.getErrorStream();
            }
            String contentEncoding = httpUrlConnection.getContentEncoding();
            if (contentEncoding != null && "gzip".equals(contentEncoding.trim().toLowerCase())) {
                gzipInputStream = new GZIPInputStream(inputStream);
                reader = new InputStreamReader(gzipInputStream);
            } else {
                reader = new InputStreamReader(inputStream);
            }
            StringBuilder sbfs = new StringBuilder();
            char[] c = new char[1024];
            int len;
            while (0 < (len = reader.read(c))) {
                sbfs.append(c, 0, len);
            }

            List<HeaderVo> headerList = Lists.newArrayList();
            httpUrlConnection.getHeaderFields().forEach((key, value) -> headerList.add(new HeaderVo(key, JsonUtil.objectToJson(value))));
            responseObj.setStatusCode(httpUrlConnection.getResponseCode());
            responseObj.setContent(sbfs.toString());
            responseObj.setResponseHeader(headerList);
            responseObj.setUrl(requestUrl.getProtocol() + "://" + requestUrl.getHost() + requestUrl.getPath());
            if (headers != null) {
                responseObj.setUserAgent(headers.get("user_agent"));
            }

        } catch (IOException e) {
            responseObj.setStatusCode(-1);
            responseObj.setReasonPhrase("Crawl Error : HttpsUrlConnection IOException .");
        } finally {
            try {
                if (reader != null) {
                    reader.close();
                }
                if (gzipInputStream != null) {
                    gzipInputStream.close();
                }
                if (inputStream != null) {
                    inputStream.close();
                }
                if (httpUrlConnection != null) {
                    httpUrlConnection.disconnect();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return responseObj;
    }

    /**
     * POST 请求方式
     * 默认以JSON的形式传递数据
     */
    public static ResponseObj urlPost(String url, String ip, Map<String, String> headers, Map<String, String> bodyMap) {
        return urlAdvancedPost(url, ip, headers, bodyMap, HttpConnectContant.POST_BODY_TYPE_JSON);
    }

    /**
     * POST 走代理请求
     */
    public static ResponseObj urlProxyPost(String url, String ip, Integer port, Map<String, String> headers, Map<String, String> bodyMap) {
        return urlAdvancedPost(url, ip, headers, bodyMap, HttpConnectContant.POST_BODY_TYPE_JSON);
    }

    /**
     * POST请求方式
     *
     * @param url       请求连接
     * @param ip        请求ip
     * @param headers   请求头
     * @param bodyMap   请求体
     * @param submiType 提交数据方式  format / json
     */
    public static ResponseObj urlAdvancedPost(String url, String ip, Map<String, String> headers, Map<String, String> bodyMap, String submiType) {
        return urlBasePost(url, ip, null, headers, bodyMap, submiType);
    }

    /**
     * POST 代理请求方式
     *
     * @param url        请求连接
     * @param ip         代理请求ip
     * @param port       代理请求端口
     * @param headers    请求头
     * @param bodyMap    请求体
     * @param submitType 提交数据方式  format / json
     */
    public static ResponseObj urlAdvancedProxyPost(String url, String ip, Integer port, Map<String, String> headers, Map<String, String> bodyMap, String submitType) {
        return urlBasePost(url, ip, port, headers, bodyMap, submitType);
    }

    private static ResponseObj urlBasePost(String url, String ip, Integer port, Map<String, String> headers, Map<String, String> bodyMap, String submitType) {
        ResponseObj responseObj;
        try {
            URL requestUrl = new URL(url);
            if (HttpConnectContant.CONNECT_TYPE_HTTPS.equalsIgnoreCase(requestUrl.getProtocol())) {
                responseObj = httpsConnectionPost(requestUrl, ip, port, headers, bodyMap, submitType);
            } else {
                responseObj = httpConnectionPost(requestUrl, ip, port, headers, bodyMap, submitType);
            }
        } catch (IOException e) {
            responseObj = new ResponseObj();
            responseObj.setStatusCode(-1);
            responseObj.setReasonPhrase("Crawl Error : URL DealHtml error, the url is not support request.");
        }
        responseObj.setIp(ip);
        responseObj.setUrl(url);
        logger.info("url: {}", url);
        logger.info("statusCode: {}", responseObj.getStatusCode());
        logger.info("message: {}", responseObj.getReasonPhrase());
        return responseObj;
    }

    /**
     * https POST方法
     */
    private static ResponseObj httpsConnectionPost(URL requestUrl, String ip, Integer port, Map<String, String> headers, Map<String, String> bodyMap, String submitType) {
        Reader reader = null;
        GZIPInputStream gzipInputStream = null;
        InputStream inputStream = null;
        HttpsURLConnection httpsURLConnection = null;
        ResponseObj responseObj = new ResponseObj();
        try {
            httpsURLConnection = (HttpsURLConnection) getUrlConnection(requestUrl, ip, port);
            httpsURLConnection.setRequestMethod(HttpConnectContant.POST);

            // 设置默认请求属性
            setDefaultProperties(httpsURLConnection);
            httpsURLConnection.setUseCaches(false);

            // 绕过 https 验证
            trustAllHosts(httpsURLConnection);
            httpsURLConnection.setHostnameVerifier(DO_NOT_VARIFY);

            // 拼接请求头
            if (headers == null || headers.size() <= 0) {
                setDefaultHeader(httpsURLConnection);
            } else {
                setCustomizedHeader(httpsURLConnection, headers);
            }

            // 拼接请求体
            String parameterStr = getPostBody(bodyMap, submitType);

            byte[] writeBytes = parameterStr.getBytes();
            httpsURLConnection.setRequestProperty("Content-Length", String.valueOf(writeBytes.length));

            // 发起请求
            OutputStream outputStream = httpsURLConnection.getOutputStream();
            outputStream.write(writeBytes);
            outputStream.flush();
            outputStream.close();

            if (httpsURLConnection.getResponseCode() == HttpStatus.SC_OK || httpsURLConnection.getResponseCode() == HttpStatus.SC_CREATED) {
                inputStream = httpsURLConnection.getInputStream();
            } else {
                inputStream = httpsURLConnection.getErrorStream();
            }
            String contentEncoding = httpsURLConnection.getContentEncoding();
            if (contentEncoding != null && HttpConnectContant.RESPONSE_CODE_GZIP.equals(contentEncoding.trim().toLowerCase())) {
                gzipInputStream = new GZIPInputStream(inputStream);
                reader = new InputStreamReader(gzipInputStream);
            } else {
                reader = new InputStreamReader(inputStream);
            }

            StringBuilder sbd = new StringBuilder();
            char[] c = new char[1024];
            int len;
            while (0 < (len = reader.read(c))) {
                sbd.append(c, 0, len);
            }

            List<HeaderVo> headerList = Lists.newArrayList();
            Map<String, List<String>> responseHeaderFields = httpsURLConnection.getHeaderFields();
            if (responseHeaderFields != null && responseHeaderFields.size() > 0) {
                responseHeaderFields.forEach((key, value) -> headerList.add(new HeaderVo(key, JsonUtil.objectToJson(value))));
            }

            responseObj.setStatusCode(httpsURLConnection.getResponseCode());
            responseObj.setContent(sbd.toString());
            responseObj.setResponseHeader(headerList);
            responseObj.setUserAgent(httpsURLConnection.getRequestProperty("user-agent"));

        } catch (IOException e) {
            e.printStackTrace();
            responseObj.setStatusCode(-1);
            responseObj.setReasonPhrase("Crawl Error : HttpsUrlConnection IOException .");
        } finally {
            try {
                if (reader != null) {
                    reader.close();
                }
                if (gzipInputStream != null) {
                    gzipInputStream.close();
                }
                if (inputStream != null) {
                    inputStream.close();
                }
                if (httpsURLConnection != null) {
                    httpsURLConnection.disconnect();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return responseObj;
    }

    /**
     * https POST方法
     */
    private static ResponseObj httpConnectionPost(URL requestUrl, String ip, Integer port, Map<String, String> headers, Map<String, String> bodyMap, String submitType) {
        Reader reader = null;
        GZIPInputStream gzipInputStream = null;
        InputStream inputStream = null;
        HttpURLConnection httpURLConnection = null;
        ResponseObj responseObj = new ResponseObj();
        try {
            httpURLConnection = (HttpURLConnection) getUrlConnection(requestUrl, ip, port);
            httpURLConnection.setRequestMethod(HttpConnectContant.POST);

            // 设置默认请求属性
            setDefaultProperties(httpURLConnection);
            httpURLConnection.setUseCaches(false);

            // 拼接请求头
            if (headers == null || headers.size() <= 0) {
                setDefaultHeader(httpURLConnection);
            } else {
                setCustomizedHeader(httpURLConnection, headers);
            }

            // 拼接请求体
            String parameterStr = getPostBody(bodyMap, submitType);

            byte[] writeBytes = parameterStr.getBytes();
            httpURLConnection.setRequestProperty("Content-Length", String.valueOf(writeBytes.length));

            // 发起请求
            OutputStream outputStream = httpURLConnection.getOutputStream();
            outputStream.write(writeBytes);
            outputStream.flush();
            outputStream.close();

            if (httpURLConnection.getResponseCode() == HttpStatus.SC_OK || httpURLConnection.getResponseCode() == HttpStatus.SC_CREATED) {
                inputStream = httpURLConnection.getInputStream();
            } else {
                inputStream = httpURLConnection.getErrorStream();
            }
            String contentEncoding = httpURLConnection.getContentEncoding();
            if (contentEncoding != null && HttpConnectContant.RESPONSE_CODE_GZIP.equals(contentEncoding.trim().toLowerCase())) {
                gzipInputStream = new GZIPInputStream(inputStream);
                reader = new InputStreamReader(gzipInputStream);
            } else {
                reader = new InputStreamReader(inputStream);
            }

            StringBuilder sbd = new StringBuilder();
            char[] c = new char[1024];
            int len;
            while (0 < (len = reader.read(c))) {
                sbd.append(c, 0, len);
            }

            List<HeaderVo> headerList = Lists.newArrayList();
            Map<String, List<String>> responseHeaderFields = httpURLConnection.getHeaderFields();
            if (responseHeaderFields != null && responseHeaderFields.size() > 0) {
                responseHeaderFields.forEach((key, value) -> headerList.add(new HeaderVo(key, JsonUtil.objectToJson(value))));
            }

            responseObj.setStatusCode(httpURLConnection.getResponseCode());
            responseObj.setContent(sbd.toString());
            responseObj.setResponseHeader(headerList);
            responseObj.setUserAgent(httpURLConnection.getRequestProperty("user-agent"));

        } catch (IOException e) {
            e.printStackTrace();
            responseObj.setStatusCode(-1);
            responseObj.setReasonPhrase("Crawl Error : HttpsUrlConnection IOException .");
        } finally {
            try {
                if (reader != null) {
                    reader.close();
                }
                if (gzipInputStream != null) {
                    gzipInputStream.close();
                }
                if (inputStream != null) {
                    inputStream.close();
                }
                if (httpURLConnection != null) {
                    httpURLConnection.disconnect();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return responseObj;
    }

    /**
     * 拼接 POST 请求 请求体
     */
    private static String getPostBody(Map<String, String> bodyMap, String submitType) {
        String parameterStr;
        StringBuilder sbd = new StringBuilder();
        if (bodyMap != null && bodyMap.size() > 0 && HttpConnectContant.POST_BODY_TYPE_FORMAT.equals(submitType)) {
            //todo 数组类型数据不合适,需要优化
            bodyMap.forEach((key, value) -> sbd.append(key).append("=").append(value).append("&"));
            parameterStr = sbd.toString();
        } else {
            parameterStr = JsonUtil.objectToJson(bodyMap);
        }
        if (StringUtils.isBlank(parameterStr)) {
            parameterStr = "";
        }
        return parameterStr;
    }

//region---------------------------------------设置头部信息-----------------------------------------------------

    /**
     * 先设置默认头部信息
     * 然后用新的头部信息覆盖默认信息
     */
    private static void setCustomizedHeader(URLConnection urlConnection, Map<String, String> headerMap) {
        setDefaultHeader(urlConnection);
        for (Map.Entry<String, String> entry : headerMap.entrySet()) {
            urlConnection.setRequestProperty(entry.getKey(), entry.getValue());
        }
    }

    /**
     * 设置默认头部信息
     */
    private static void setDefaultHeader(URLConnection urlConnection) {
        urlConnection.setRequestProperty("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        urlConnection.setRequestProperty("accept-encoding", "gzip, deflate, br");
        urlConnection.setRequestProperty("accept-language", "en-US,en;q=0.9,fr-FR;q=0.8,fr-CA;q=0.7,fr;q=0.6,de;q=0.5,zh-CN;q=0.4,zh;q=0.3,en-AU;q=0.2");
        urlConnection.setRequestProperty("cache-control", "max-age=0");
        urlConnection.setRequestProperty("user-agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36");
        urlConnection.setRequestProperty("connection", "keep-alive");
    }

    /**
     * 设置默认属性
     */
    private static void setDefaultProperties(URLConnection urlConnection) {
        urlConnection.setDoOutput(true);
        urlConnection.setDoInput(true);
        urlConnection.setConnectTimeout(HttpConnectContant.CONNECT_TIME_OUT_TIME);

        // 系统超时设置, 防止网络异常情况下, 可能会导致程序僵死而不继续往下执行
        System.setProperty("sun.net.client.defaultConnectionTimeOut", String.valueOf(HttpConnectContant.CONNECT_TIME_OUT_TIME));
        System.setProperty("sun.net.client.defaultReadTimeout", String.valueOf(HttpConnectContant.CONNECT_TIME_OUT_TIME));
    }

    /**
     * 打开连接,获取连接对象
     */
    private static URLConnection getUrlConnection(URL requestUrl, String ip, Integer port) throws IOException {
        Proxy proxy = null;
        if (StringUtils.isNotBlank(ip) && port != null && port > 0) {
            proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ip, port));
        }
        if (proxy == null) {
            return requestUrl.openConnection();
        }
        return requestUrl.openConnection(proxy);
    }


//endregion-------------------------------------------------------------------------------------------------------------

//region ------------------------------------- 绕过 https 验证 -----------------------------------------------------------
    /**
     * 覆盖java默认的证书验证
     */
    private static final TrustManager[] TRUST_ALL_CERTS = new TrustManager[]{
            new X509TrustManager() {
                @Override
                public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {

                }

                @Override
                public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {

                }

                @Override
                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[]{};
                }
            }
    };

    /**
     * 设置不验证主机
     */
    private static final HostnameVerifier DO_NOT_VARIFY = new HostnameVerifier() {
        @Override
        public boolean verify(String s, SSLSession sslSession) {
            return true;
        }
    };

    /**
     * 信任所有的链接
     */
    private static SSLSocketFactory trustAllHosts(HttpsURLConnection connection) {
        SSLSocketFactory sslSocketFactory = connection.getSSLSocketFactory();
        try {
            SSLContext tls = SSLContext.getInstance("TLS");
            tls.init(null, TRUST_ALL_CERTS, new SecureRandom());
            SSLSocketFactory socketFactory = tls.getSocketFactory();
            connection.setSSLSocketFactory(socketFactory);
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (KeyManagementException e) {
            e.printStackTrace();
        }
        return sslSocketFactory;
    }
// endregion -----------------------------------------------------------------------------------------------------------


}
LiDejie34
关注
netty 实现tomcat
weixin_43931625的博客
06-02 696
netty 实现tomcat
java http 发送json数据_HttpUrlConnection使用Post发送Json数据
weixin_30227565的博客
03-04 550
private String post(String reqUrl, String json) {String result = "";HttpURLConnection httpURLConnection = null;BufferedReader reader = null;try {URL url = new URL(reqUrl);httpURLConnection = (HttpURLC...
Java网页爬虫--基于URLConnection的网页爬虫工具类
pengjunlee的博客
12-26 4万+
在这个数据为王的时代，爬虫应用地越来越广泛，对于一个萌新程序员来说如果你要做爬虫，那么Python是你的不二之选。但是对于那些老腊肉的Java程序员（亦或者你是程序媛）想使用Java做爬虫也不是不行，只是没有Python那么方便。身为一块Java老腊肉的我在此记录一下自己在使用Java做网络爬虫使用的工具类。 在pom.xml文件中引入commons-lang3 依赖： <depe...
Http请求-HttpURLConnection、HttpClient
weixin_40017062的博客
03-11 1107
常见的Http请求方式
connect()链接不成功问题总结
菜鸟进化史
09-23 8344
1.信号槽参数不一致； 2.信号槽参数只能写类型不能写变量名； 3.qt程序没有写Q_OBJECT宏
使用jsoup/HTTPConnection 访问页面失败 ，但在浏览器中可以打开页面
shaochong047的博客
03-21 8994
             我在做一个爬虫项目的时候遇到过这种情况，本身可以正确访问的某个网页，用HPPTConnention后台访问确报错。         错误很明确，404，                     发现问题：          经过研究我发现，其实用浏览器访问该页面的时候也是404，只不过资源都可以正产加载，所以页面可以正常显示。           解决方法：       ...
Java HttpURLConnection使用方法详解
08-28
Java HttpURLConnection 使用方法详解 Java HttpURLConnection 是 Java 中用于发送 HTTP 请求的类，通过使用 HttpURLConnection，可以发送 GET 请求和 POST 请求，並获取服务器的响应结果。下面是使用 Java ...
HttpURLConnection使用总结示例源码
06-27
本篇文章将深入探讨HttpURLConnection的使用方法、特性以及一些关键的示例代码。 一、HttpURLConnection简介 HttpURLConnection继承自URLConnection，它提供了对HTTP协议的直接支持。相比于HttpClient，...
Java HttpURLConnection使用详解：GET与POST请求
"Android平台上的 HttpURLConnection 使用详解" 在 Android 开发中，`HttpURLConnection` 是一个常用的网络通信组件，尤其在处理 HTTP 请求时。本摘要将深入解析 `HttpURLConnection` 的核心概念、请求类型以及参数...
HttpUrlConnection使用示例
09-13
本篇文章将深入探讨如何使用`HttpURLConnection`以POST方式提交请求。 ### 1. 创建连接 首先，我们需要获取到`HttpURLConnection`实例。这通常通过调用`URL`对象的`openConnection()`方法实现，然后强制转换为`...
HttpURLConnection.getHeaderField
yhtppp的专栏
10-28 8385
使用HttpURLConnection.getHeaderField，获取Header中某项的值， 有时写成：httpUrlConnection.getHeaderField("Location") 有时写成：httpUrlConnection.getHeaderField("location") 都是可以的，Java自带的HttpURLConnection，httpclient，okh
由于HttpURLConnection的getContentEncoding处理错误的问题
jimmee的专栏
09-17 1118
今天犯了一个弱智的错误，我使用java.net.HttpURLConnection从网络上读取输入， 由于我没有预览判断返回给我的数据的内容编码格式，导致了我读取数据时出错。 我就直接当作普通的InputStream来读取，但是实际上返回的内容可能是gzip压缩过的。。。 所以需要先调用getContentEncoding来判断是否是“gzip”的压缩格式，再使用GZIPInpu...
X509证书信任管理器类的实现及应用
legend_x的专栏
05-20 3291
X509证书信任管理器类的实现及应用   (2011-09-20 10:27:18) 转载▼ 标签：  杂谈   在JSSE中，证书信任管理器类就是实现了接口X509TrustManager的类。我们可以自己实现该接口，让它信任我们指定的证书。 　　接口X509TrustManager有下述三个公有的方法需要我们实现：
（转）HttpURLConnection中设置网络超时
06-17 1888
转：http://www.xd-tech.com.cn/blog/article.asp?id=37 Java中可以使用HttpURLConnection来请求WEB资源。HttpURLConnection对象不能直接构造，需要通过URL.openConnection()来获得HttpURLConnection对象，示例代码如下：String urlStr= www.jaddy.org...
HttpURLConnection小记
懒人记录处
01-08 576
1.HttpURLConnection可以发送https请求，并非一定要HttpsURLConnection 2.获取输出流时会先进行连接 3.注意setConnectTimeout以及setReadTimeout，不然等一辈子。 httpConn.setConnectTimeout(毫秒); httpConn.setReadTimeout(毫秒); 连接超时貌似不会报错，只是继续往下执行而已，这次项目超时没有报错，导致误以为HttpURLConnection不能发送https请求，实际是
Java中发送Http请求之HttpURLConnection
ABestRookie的博客
11-20 5362
Java中发送Http请求之HttpURLConnection1 HttpURLConnection1 准备一个SpringBoot项目环境2 添加一个控制器3 添加一个发送请求的工具类4 添加测试工具类5 测试结果 Java中发送Http请求的方式有很多,记录一下相关的请求方式,本次记录Jdk自带的HttpURLConnection,简单简洁,使用简单. 1 HttpURLConnection HttpURLConnection是Jdk自带的请求工具,不用依赖第三方jar包,适用简单的场景使用. 使用
详解HttpURLConnection
热门推荐
清箫的专栏
01-29 18万+
请求响应流程设置连接参数的方法setAllowUserInteractionsetDoInputsetDoOutputsetIfModifiedSincesetUseCachessetDefaultAllowUserInteractionsetDefaultUseCaches设置请求头或响应头HTTP请求允许一个key带多个用逗号分开的values，但是HttpURLConnection只提供了单
org.apache.catalina.loader.DevLoader overrides final method start.()V
iteye_9972的博客
05-05 243
参照http://blog.csdn.net/eclipser1987/article/details/6371288，eclipse上配置web直接发布到tomcat下，启动tomcat7，出现如题异常，不解，换tomcat6，正常，看来的tomcat7将其方法start.()定义成final了。 后来发现另一作者的解决方案： 解决方案：将项目的tomcat设置中 activeDevLoad...
HttpURLConnection 使用案例
最新发布
06-13
以下是一个使用Java标准库中的HttpURLConnection发送HTTP GET请求的简单案例： ```java import java.net.*; import java.io.*; public class HttpGetExample { public static void main(String[] args) { try { // 创建URL对象 URL url = new URL("http://example.com/api/data"); // 打开连接 HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // 设置请求方法为GET conn.setRequestMethod("GET"); // 设置连接超时时间和读取超时时间 conn.setConnectTimeout(5000); conn.setReadTimeout(5000); // 获取响应码 int responseCode = conn.getResponseCode(); System.out.println("Response Code : " + responseCode); // 读取响应内容 BufferedReader in = new BufferedReader( new InputStreamReader(conn.getInputStream())); String inputLine; StringBuffer response = new StringBuffer(); while ((inputLine = in.readLine()) != null) { response.append(inputLine); } in.close(); // 输出响应内容 System.out.println(response.toString()); } catch (Exception e) { e.printStackTrace(); } } } ``` 在该案例中，我们发送了一个HTTP GET请求，并读取了响应内容。我们首先创建了一个URL对象，然后使用HttpURLConnection打开连接。我们设置了请求方法为GET，并设置了连接超时时间和读取超时时间。然后，我们获取响应码以及响应内容，并将响应内容输出到控制台。
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值