【末世旅行之Java】HttpClient工具类，包含POST和GET请求，使用了连接池和代理

最新推荐文章于 2024-10-17 08:42:20 发布

晚风摇叶

最新推荐文章于 2024-10-17 08:42:20 发布

阅读量887

点赞数 1

分类专栏： Java 文章标签： HttpClient 工具类 Http 爬数据 java

本文链接：https://blog.csdn.net/q1425603211/article/details/78178311

版权

Java 专栏收录该内容

7 篇文章 0 订阅

订阅专栏

包含POST和GET请求

使用了连接池和代理

用了饿汉式单例设计模式，可多线程

主要用于从网上爬数据，若使用Fiddler抓包，将useFiddler置为true

代理proxy字段格式：127.0.0.1:80

要获取图片等，使用getInputStream方法获取流

引用的包：

httpclient-4.5.3.jar

httpcore-4.4.6.jar

commons-logging-1.2.jar

jar包下载地址：http://download.csdn.net/download/q1425603211/10011373

工程文件下载地址：http://download.csdn.net/download/q1425603211/10011376

Fiddler中文pj版下载地址：http://download.csdn.net/download/q1425603211/10011382

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

public class HttpClientUtil {

	private static PoolingHttpClientConnectionManager cm;

	public static String CHARSET = "UTF-8";
	public static boolean useFiddler = false;
	public static String proxy = null;

	public static final String CONTENT = "content";
	public static final String STATUSCODE = "statusCode";

	private static HttpClientUtil httpUtil = new HttpClientUtil();

	private HttpClientUtil() {
	}

	public static HttpClientUtil getInstance() {
		return httpUtil;
	}

	private void init() {
		if (cm == null) {
			cm = new PoolingHttpClientConnectionManager();
			cm.setMaxTotal(50);// 整个连接池最大连接数
			cm.setDefaultMaxPerRoute(5);// 每路由最大连接数，默认值是5
		}
	}

	/**
	 * 通过连接池获取HttpClient
	 * 
	 * @return
	 */
	private CloseableHttpClient getHttpClient() {
		init();
		return HttpClients.custom().setConnectionManager(cm).build();
	}

	/**
	 * 发送GET请求
	 * 
	 * @param url
	 *            URL
	 * @param params
	 *            请求参数
	 * @param headers
	 *            请求头
	 * @return 返回map集合，响应体内容key为content，响应头key为headers
	 * @throws URISyntaxException
	 */
	public Map<String, String> httpGET(String url, Map<String, String> params, Map<String, String> headers)
			throws URISyntaxException {
		URIBuilder ub = new URIBuilder(url);

		if (null != params) {
			ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
			ub.setParameters(pairs);
		}

		HttpGet httpGet = new HttpGet(ub.build());
		if (null != headers) {
			for (Map.Entry<String, String> param : headers.entrySet()) {
				httpGet.addHeader(param.getKey(), String.valueOf(param.getValue()));
			}
		}
		return getResult(httpGet);
	}

	/**
	 * 发送POST请求
	 * 
	 * @param url
	 *            URL
	 * @param params
	 *            请求参数
	 * @param headers
	 *            请求头
	 * @return 返回map集合，响应体内容key为content
	 * @throws UnsupportedEncodingException
	 */
	public Map<String, String> httpPOST(String url, Map<String, String> params, Map<String, String> headers)
			throws UnsupportedEncodingException {
		HttpPost httpPost = new HttpPost(url);
		if (headers != null) {
			for (Map.Entry<String, String> param : headers.entrySet()) {
				httpPost.addHeader(param.getKey(), String.valueOf(param.getValue()));
			}
		}

		if (params != null) {
			ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
			httpPost.setEntity(new UrlEncodedFormEntity(pairs, CHARSET));
		}
		Map<String, String> restlt = getResult(httpPost);

		return restlt;
	}

	/**
	 * 获取去返回的输入流
	 * 
	 * @param url
	 *            URL
	 * @param params
	 *            请求参数
	 * @param headers
	 *            请求头
	 * @return 返回map集合，InputStream内容key为content
	 * @throws URISyntaxException
	 */
	public Map<String, Object> getInputStream(String url, Map<String, String> params, Map<String, String> headers)
			throws URISyntaxException {
		URIBuilder ub = new URIBuilder(url);

		if (null != params) {
			ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
			ub.setParameters(pairs);
		}

		HttpGet httpGet = new HttpGet(ub.build());
		for (Map.Entry<String, String> param : headers.entrySet()) {
			httpGet.addHeader(param.getKey(), String.valueOf(param.getValue()));
		}
		return getInputStreamResult(httpGet);
	}

	private ArrayList<NameValuePair> covertParams2NVPS(Map<String, String> params) {
		ArrayList<NameValuePair> pairs = new ArrayList<NameValuePair>();
		for (Map.Entry<String, String> param : params.entrySet()) {
			pairs.add(new BasicNameValuePair(param.getKey(), String.valueOf(param.getValue())));
		}

		return pairs;
	}

	/**
	 * 处理HTTP请求
	 * 
	 * @param request
	 * @return
	 */
	private Map<String, String> getResult(HttpRequestBase request) {
		CloseableHttpClient httpClient = getHttpClient();
		// Fiddler代理
		if (useFiddler) {
			proxy = "127.0.0.1:8888";
		}
		// 使用代理
		if (proxy != null) {
			String[] split = null;
			try {
				split = proxy.split(":");
				HttpHost proxy = new HttpHost(split[0], Integer.parseInt(split[1]));
				RequestConfig config = RequestConfig.custom().setProxy(proxy).build();
				request.setConfig(config);
			} catch (Exception e) {
				e.printStackTrace();
			}

		}
		try {
			CloseableHttpResponse response = httpClient.execute(request);
			String statusCode = response.getStatusLine().getStatusCode() + "";
			HttpEntity entity = response.getEntity();
			if (entity != null) {
				String content = EntityUtils.toString(entity, CHARSET);
				Map<String, String> result = new HashMap<String, String>();
				result.put(CONTENT, content);
				result.put(STATUSCODE, statusCode);
				result.putAll(getResponseHeaders(response));
				response.close();
				return result;
			}
		} catch (ClientProtocolException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
		}
		return null;
	}

	/**
	 * 处理HTTP请求
	 * 
	 * @param request
	 * @return
	 */
	private Map<String, Object> getInputStreamResult(HttpRequestBase request) {
		CloseableHttpClient httpClient = getHttpClient();
		// Fiddler代理
		if (useFiddler) {
			proxy = "127.0.0.1:8888";
		}
		// 使用代理
		if (proxy != null) {
			String[] split = null;
			try {
				split = proxy.split(":");
				HttpHost proxy = new HttpHost(split[0], Integer.parseInt(split[1]));
				RequestConfig config = RequestConfig.custom().setProxy(proxy).build();
				request.setConfig(config);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		CloseableHttpResponse response = null;
		try {
			response = httpClient.execute(request);
			String statusCode = response.getStatusLine().getStatusCode() + "";
			HttpEntity entity = response.getEntity();
			if (entity != null) {
				InputStream content = entity.getContent();
				Map<String, Object> result = new HashMap<String, Object>();
				result.put(CONTENT, content);
				result.put(STATUSCODE, statusCode);
				result.putAll(getResponseHeaders(response));

				return result;
			}
		} catch (ClientProtocolException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
		}
		return null;
	}

	/**
	 * 解析header
	 * 
	 * @param response
	 * @return
	 */
	private Map<String, String> getResponseHeaders(CloseableHttpResponse response) {
		Map<String, String> headers = new HashMap<String, String>();
		StringBuffer cookie = new StringBuffer();
		boolean haveCookie = false;

		Header[] headerArr = response.getAllHeaders();
		for (int i = 0; i < headerArr.length; i++) {
			Header header = headerArr[i];
			if ("Set-Cookie".equals(header.getName())) {
				haveCookie = true;
				cookie.append(header.getValue());
				continue;
			}
			headers.put(header.getName(), header.getValue());
		}
		if (haveCookie) {
			headers.put("Set-Cookie", cookie.toString());
		}
		return headers;
	}

	public static void main(String[] args) {
		String url = "http://www.csdn.net/";
		Map<String, String> request = null;
		try {
			request = HttpClientUtil.getInstance().httpGET(url, null, null);
		} catch (URISyntaxException e) {
			e.printStackTrace();
		}
		String content = request.get(HttpClientUtil.CONTENT);
		System.out.println(content);
	}

}