import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.Args;
import org.apache.http.util.EntityUtils;
import org.apache.http.util.TextUtils;
import com.qianbao.http.common.CharsetEncodingEnum;
import com.qianbao.http.common.MimeTypeEnum;
import com.qianbao.http.exception.HttpStatusException;
import com.thinkgem.jeesite.common.web.container.HttpContainer;
import com.zrj.base.common.log.AppLogger;
public class HttpClientFB {
private static ThreadLocal<CloseableHttpResponse> threadLocal4Response = new ThreadLocal<>();
private static ThreadLocal<HttpEntity> threadLocal4HttpEntity = new ThreadLocal<>();
private final static String CHARACTER_ENCODING = "ISO-8859-1";
private final static String MIME_TYPE = "text/plain";
private ContentType contentType;
private CloseableHttpClient httpClient;
private AppLogger logger = new AppLogger(HttpContainer.class);
/**
* 默认请求<br>
* mimeType:text/plain<br>
* characterEncoding:ISO-8859-1<br>
* 需要修改编码格式或者mimeType,请使用有参构造
*/
public HttpClientFB(boolean usePool) {
contentType = ContentType.create(MIME_TYPE, CHARACTER_ENCODING);
createHttpClient(usePool);
Map<String, Object> logMap = new HashMap<>();
logMap.put("MIME_TYPE", MIME_TYPE);
logMap.put("CHARACTER_ENCODING", CHARACTER_ENCODING);
logMap.put("usePool", usePool);
logger.doInfo("create httpClient mimeType:{},characterEnoding:{},usePool:{}", logMap);
}
/**
* @param characterEncoding 请求编码格式
* @param mimeType 请求数据类型
*/
public HttpClientFB(CharsetEncodingEnum characterEncoding, MimeTypeEnum mimeType, boolean usePool) {
String type = (Args.notBlank(mimeType.getValue(), "MIME type")).toLowerCase(Locale.ROOT);
Args.check(valid(type), "MIME type may not contain reserved characters");
contentType = ContentType.create(type, characterEncoding.getValue());
createHttpClient(usePool);
Map<String, Object> logMap = new HashMap<>();
logMap.put("MIME_TYPE", MIME_TYPE);
logMap.put("CHARACTER_ENCODING", CHARACTER_ENCODING);
logMap.put("usePool", usePool);
logger.doInfo("create httpClient mimeType:{},characterEnoding:{},usePool:{}", logMap);
}
/**
* get方式
*
* @param uri 请求链接 请勿使用? 拼接参数
* @param headers 消息头信息集合
* @param paramMap 参数信息集合
* @param connectTimeout 链接超时时间
* @return httpEntity相应结果
*/
public HttpEntity doGet(String uri, Map<String, String> headers, Map<String, String> paramMap, int connectTimeout) throws IOException {
String url = notUri(uri, "uri");
String paramStr = "";
List<NameValuePair> params = new ArrayList<>();
if (paramMap != null) {
for (String paramName : paramMap.keySet()) {
params.add(new BasicNameValuePair(paramName, paramMap.get(paramName)));
}
logger.doInfo("[doGet] params assemble success,params size:{}", params.size());
}
// 参数转义
try {
if (!params.isEmpty()) {
paramStr = EntityUtils.toString(new UrlEncodedFormEntity(params, contentType.getCharset()));
logger.doInfo("[doGet] params assemble and to string ,paramStr:{}", paramStr);
}
} catch (IOException e) {
logger.error("UrlEncodedFormEntity exception", e);
throw e;
}
// 连接超时时间
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.build();
String getUrl = TextUtils.isBlank(paramStr) ? url : String.format("%s%s%s", url, "?", paramStr);
HttpGet httpget = new HttpGet(getUrl);
httpget.setConfig(requestConfig);
// 消息头
if (headers != null) {
for (String header : headers.keySet()) {
httpget.setHeader(header, headers.get(header));
}
}
HttpClientContext context = HttpClientContext.create();
HttpEntity entity = getHttpEntity(httpget, context);
return entity;
}
/**
* post方式
*
* @param uri 请求链接
* @param headers 消息头信息集合
* @param formParamsMap form表单参数信息集合
* @param connectTimeout 链接超时时间
* @return httpEntity相应结果
*/
public HttpEntity doPost(String uri, Map<String, String> headers, Map<String, String> formParamsMap, int connectTimeout) throws IOException {
String url = notUri(uri, "uri");
HttpPost httppost = new HttpPost(url);
// form请求参数
List<NameValuePair> params = new ArrayList<>();
if (formParamsMap != null) {
for (String paramName : formParamsMap.keySet()) {
params.add(new BasicNameValuePair(paramName, formParamsMap.get(paramName)));
}
logger.doInfo("[doPost] params assemble success,params size:{}", params.size());
}
if (!params.isEmpty()) {
UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(params, contentType.getCharset());
httppost.setEntity(urlEncodedFormEntity);
}
// 连接超时时间
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.build();
httppost.setConfig(requestConfig);
// 消息头
if (headers != null) {
for (String header : headers.keySet()) {
httppost.setHeader(header, headers.get(header));
}
}
HttpClientContext context = HttpClientContext.create();
HttpEntity entity = getHttpEntity(httppost, context);
return entity;
}
/**
* post方式
*
* @param uri 请求链接
* @param headers 消息头信息集合
* @param bodyData body类型方式数据信息
* @param paramMap 表单类型参数
* @param connectTimeout 链接超时时间
* @return httpEntity相应结果
*/
public HttpEntity doPost(String uri, Map<String, String> headers, String bodyData, Map<String, String> paramMap, int connectTimeout) throws IOException {
String url = notUri(uri, "uri");
List<NameValuePair> params = new ArrayList<>();
String paramStr = "";
if (paramMap != null) {
for (String paramName : paramMap.keySet()) {
params.add(new BasicNameValuePair(paramName, paramMap.get(paramName)));
}
logger.doInfo("[doPost] params assemble success,params size:{}", params.size());
}
// 参数转义
try {
if (!params.isEmpty()) {
paramStr = EntityUtils.toString(new UrlEncodedFormEntity(params, contentType.getCharset()));
logger.doInfo("[doPost] params assemble and to string ,paramStr:{}", paramStr);
}
} catch (IOException e) {
logger.error("[doPost] UrlEncodedFormEntity exception", e);
}
String postUrl = TextUtils.isBlank(paramStr) ? url : String.format("%s%s%s", url, "?", paramStr);
HttpPost httppost = new HttpPost(postUrl);
StringEntity stringEntity = new StringEntity(bodyData, contentType);
httppost.setEntity(stringEntity);
// 连接超时时间
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.build();
httppost.setConfig(requestConfig);
// 消息头
if (headers != null) {
for (String header : headers.keySet()) {
httppost.setHeader(header, headers.get(header));
}
}
HttpClientContext context = HttpClientContext.create();
HttpEntity entity = getHttpEntity(httppost, context);
return entity;
}
/**
* put方式
*
* @param uri 请求链接
* @param headers 消息头信息集合
* @param bodyData body类型方式数据信息
* @param paramMap 表单类型参数
* @param connectTimeout 链接超时时间
* @return httpEntity相应结果
*/
public HttpEntity doPut(String uri, Map<String, String> headers, String bodyData, Map<String, String> paramMap, int connectTimeout) throws IOException {
String url = notUri(uri, "uri");
List<NameValuePair> params = new ArrayList<>();
String paramStr = "";
if (paramMap != null) {
for (String paramName : paramMap.keySet()) {
params.add(new BasicNameValuePair(paramName, paramMap.get(paramName)));
}
logger.doInfo("[doPut] params assemble success,params size:{}", params.size());
}
// 参数转义
try {
if (!params.isEmpty()) {
paramStr = EntityUtils.toString(new UrlEncodedFormEntity(params, contentType.getCharset()));
logger.doInfo("[doPut] params assemble and to string ,paramStr:{}", paramStr);
}
} catch (IOException e) {
logger.error("[doPut] UrlEncodedFormEntity exception", e);
throw e;
}
String putUrl = TextUtils.isBlank(paramStr) ? url : String.format("%s%s%s", url, "?", paramStr);
HttpPut httpPut = new HttpPut(putUrl);
// 参数数据
StringEntity stringEntity = new StringEntity(bodyData, contentType);
stringEntity.setChunked(true);
httpPut.setEntity(stringEntity);
// 连接超时时间
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.build();
httpPut.setConfig(requestConfig);
// 消息头
if (headers != null) {
for (String header : headers.keySet()) {
httpPut.setHeader(header, headers.get(header));
}
}
HttpClientContext context = HttpClientContext.create();
HttpEntity entity = getHttpEntity(httpPut, context);
return entity;
}
/**
* delete方式
*
* @param uri 请求链接
* @param headers 消息头信息集合
* @param paramMap 表单类型参数
* @param connectTimeout 链接超时时间
* @return httpEntity相应结果
*/
public HttpEntity doDelete(String uri, Map<String, String> headers, Map<String, String> paramMap, int connectTimeout) throws IOException {
String url = notUri(uri, "uri");
List<NameValuePair> params = new ArrayList<>();
String paramStr = "";
if (paramMap != null) {
for (String paramName : paramMap.keySet()) {
params.add(new BasicNameValuePair(paramName, paramMap.get(paramName)));
}
logger.doInfo("[doDelete] params assemble success,params size:{}", params.size());
}
// 参数转义
try {
if (!params.isEmpty()) {
paramStr = EntityUtils.toString(new UrlEncodedFormEntity(params, contentType.getCharset()));
}
logger.doInfo("[doDelete] params assemble and to string ,paramStr:{}", paramStr);
} catch (IOException e) {
logger.error("[doDelete] UrlEncodedFormEntity exception", e);
throw e;
}
String deleteUrl = TextUtils.isBlank(paramStr) ? url : String.format("%s%s%s", url, "?", paramStr);
HttpDelete httpDelete = new HttpDelete(deleteUrl);
// 连接超时时间
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.build();
httpDelete.setConfig(requestConfig);
// 消息头
if (headers != null) {
for (String header : headers.keySet()) {
httpDelete.setHeader(header, headers.get(header));
}
}
HttpClientContext context = HttpClientContext.create();
HttpEntity entity = getHttpEntity(httpDelete, context);
return entity;
}
/**
* 按照相应方式提交
*
* @param httpRequestBase HttpPost,HttpGet,HttpPut,HttpDelete
*/
private HttpEntity getHttpEntity(HttpRequestBase httpRequestBase, HttpClientContext context) throws IOException {
HttpEntity entity = null;
CloseableHttpResponse response = null;
try {
response = httpClient.execute(
httpRequestBase, context);
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
throw new HttpStatusException(
statusLine.getStatusCode(),
statusLine.getReasonPhrase());
}
entity = response.getEntity();
} catch (ClientProtocolException ex) {
logger.error("ClientProtocolException exception", ex);
throw ex;
} catch (IOException ex) {
logger.error("IOException exception", ex);
throw ex;
} finally {
if (response != null) {
threadLocal4Response.set(response);
}
if (entity != null) {
threadLocal4HttpEntity.set(entity);
}
}
return entity;
}
/**
* 关闭并释放资源
*/
public void closeAllResouces() throws IOException {
CloseableHttpResponse httpResponse = threadLocal4Response.get();
HttpEntity entity = threadLocal4HttpEntity.get();
if (entity != null) {
EntityUtils.consume(entity);
}
if (httpResponse != null) {
httpResponse.close();
}
if (httpClient != null) {
httpClient.close();
}
}
/**
* 关闭并释放资源
*/
public void closeSelfResouces() throws IOException {
CloseableHttpResponse httpResponse = threadLocal4Response.get();
HttpEntity entity = threadLocal4HttpEntity.get();
if (entity != null) {
EntityUtils.consume(entity);
}
if (httpResponse != null) {
httpResponse.close();
}
}
public void closeClientResouces() throws IOException {
if (httpClient != null) {
httpClient.close();
}
}
private static boolean valid(String s) {
for (int i = 0; i < s.length(); ++i) {
char ch = s.charAt(i);
if (ch == 34 || ch == 44 || ch == 59) {
return false;
}
}
return true;
}
public static String notUri(final String uri, final String name) {
if (uri == null) {
throw new IllegalArgumentException(name + " may not be null");
}
if (TextUtils.isBlank(uri)) {
throw new IllegalArgumentException(name + " may not be blank");
}
if (uri.contains("?")) {
throw new IllegalArgumentException(name + " must not contain '?' ");
}
return uri;
}
private void createHttpClient(boolean usePool) {
if (usePool) {
logger.info("[createHttpClient] PoolingHttpClient create begin");
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
httpClient = HttpClients.custom()
.setConnectionManager(cm)
.build();
logger.info("[createHttpClient] PoolingHttpClient create end, success create");
} else {
logger.info("[createHttpClient] createDefaultHttpClient create begin");
httpClient = HttpClients.createDefault();
logger.info("[createHttpClient] createDefaultHttpClient create end , success create");
}
}
}
参考代码:
public class PoolTest {
private static void config(HttpRequestBase httpRequestBase) {
httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");
httpRequestBase.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5");
httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7");
// 配置请求的超时设置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(3000)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.build();
httpRequestBase.setConfig(requestConfig);
}
public static void main(String[] args) {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
// 将最大连接数增加到200
cm.setMaxTotal(200);
// 将每个路由基础的连接增加到20
cm.setDefaultMaxPerRoute(20);
// 将目标主机的最大连接数增加到50
HttpHost localhost = new HttpHost("http://blog.csdn.net/gaolu",80);
cm.setMaxPerRoute(new HttpRoute(localhost), 50);
//请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception,int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler)
.build();
// URL列表数组
String[] urisToGet = {
"http://blog.csdn.net/gaolu/article/details/48466059",
"http://blog.csdn.net/gaolu/article/details/48243103",
"http://blog.csdn.net/gaolu/article/details/47656987",
"http://blog.csdn.net/gaolu/article/details/47055029",
"http://blog.csdn.net/gaolu/article/details/46400883",
"http://blog.csdn.net/gaolu/article/details/46359127",
"http://blog.csdn.net/gaolu/article/details/46224821",
"http://blog.csdn.net/gaolu/article/details/45305769",
"http://blog.csdn.net/gaolu/article/details/43701763",
"http://blog.csdn.net/gaolu/article/details/43195449",
"http://blog.csdn.net/gaolu/article/details/42915521",
"http://blog.csdn.net/gaolu/article/details/41802319",
"http://blog.csdn.net/gaolu/article/details/41045233",
"http://blog.csdn.net/gaolu/article/details/40395425",
"http://blog.csdn.net/gaolu/article/details/40047065",
"http://blog.csdn.net/gaolu/article/details/39891877",
"http://blog.csdn.net/gaolu/article/details/39499073",
"http://blog.csdn.net/gaolu/article/details/39314327",
"http://blog.csdn.net/gaolu/article/details/38820809",
"http://blog.csdn.net/gaolu/article/details/38439375",
};
long start = System.currentTimeMillis();
try {
int pagecount = urisToGet.length;
ExecutorService executors = Executors.newFixedThreadPool(pagecount);
CountDownLatch countDownLatch = new CountDownLatch(pagecount);
for(int i = 0; i< pagecount;i++){
HttpGet httpget = new HttpGet(urisToGet[i]);
config(httpget);
//启动线程抓取
executors.execute(new GetRunnable(httpClient,httpget,countDownLatch));
}
countDownLatch.await();
executors.shutdown();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成,开始进入下一步!");
}
long end = System.currentTimeMillis();
System.out.println("consume -> " + (end - start));
}
static class GetRunnable implements Runnable {
private CountDownLatch countDownLatch;
private final CloseableHttpClient httpClient;
private final HttpGet httpget;
public GetRunnable(CloseableHttpClient httpClient, HttpGet httpget, CountDownLatch countDownLatch){
this.httpClient = httpClient;
this.httpget = httpget;
this.countDownLatch = countDownLatch;
}
@Override
public void run() {
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpget,HttpClientContext.create());
HttpEntity entity = response.getEntity();
System.out.println(EntityUtils.toString(entity, "utf-8")) ;
EntityUtils.consume(entity);
} catch (IOException e) {
e.printStackTrace();
} finally {
countDownLatch.countDown();
try {
if(response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
pom配置
<!-- httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.4</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.5.2</version>
</dependency>