java http连接池使用_HttpClient4.5 使用http连接池发送http请求深度示例

public classPoolTest {private static voidconfig(HttpRequestBase httpRequestBase) {

httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");

httpRequestBase.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");

httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5");

httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7");//配置请求的超时设置

RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(3000).setConnectTimeout(3000).setSocketTimeout(3000).build();

httpRequestBase.setConfig(requestConfig);

}public static voidmain(String[] args) {

ConnectionSocketFactory plainsf=PlainConnectionSocketFactory.getSocketFactory();

LayeredConnectionSocketFactory sslsf=SSLConnectionSocketFactory.getSocketFactory();

Registry registry = RegistryBuilder. create().register("http", plainsf).register("https", sslsf).build();

PoolingHttpClientConnectionManager cm= newPoolingHttpClientConnectionManager(registry);//将最大连接数增加到200

cm.setMaxTotal(200);//将每个路由基础的连接增加到20

cm.setDefaultMaxPerRoute(20);//将目标主机的最大连接数增加到50

HttpHost localhost = new HttpHost("http://blog.csdn.net/gaolu", 80);

cm.setMaxPerRoute(new HttpRoute(localhost), 50);//请求重试处理

HttpRequestRetryHandler httpRequestRetryHandler = newHttpRequestRetryHandler() {public boolean retryRequest(IOException exception, intexecutionCount, HttpContext context) {if (executionCount >= 5) {//如果已经重试了5次，就放弃

return false;

}if (exception instanceof NoHttpResponseException) {//如果服务器丢掉了连接，那么就重试

return true;

}if (exception instanceof SSLHandshakeException) {//不要重试SSL握手异常

return false;

}if (exception instanceof InterruptedIOException) {//超时

return false;

}if (exception instanceof UnknownHostException) {//目标服务器不可达

return false;

}if (exception instanceof ConnectTimeoutException) {//连接被拒绝

return false;

}if (exception instanceof SSLException) {//ssl握手异常

return false;

}

HttpClientContext clientContext=HttpClientContext.adapt(context);

HttpRequest request=clientContext.getRequest();//如果请求是幂等的，就再次尝试

if (!(request instanceofHttpEntityEnclosingRequest)) {return true;

}return false;

}

};

CloseableHttpClient httpClient=HttpClients.custom().setConnectionManager(cm)

.setRetryHandler(httpRequestRetryHandler).build();//URL列表数组

String[] urisToGet ={"http://AAAAAAAAAAAAAAAAAA/48466059","http://AAAAAAAAAAAAAAAAAA/48243103","http://AAAAAAAAAAAAAAAAAA/47656987","http://AAAAAAAAAAAAAAAAAA/47055029","http://AAAAAAAAAAAAAAAAAA/46400883","http://AAAAAAAAAAAAAAAAAA/46359127","http://AAAAAAAAAAAAAAAAAA/46224821","http://AAAAAAAAAAAAAAAAAA/45305769","http://AAAAAAAAAAAAAAAAAA/43701763","http://AAAAAAAAAAAAAAAAAA/43195449","http://AAAAAAAAAAAAAAAAAA/42915521","http://AAAAAAAAAAAAAAAAAA/41802319","http://AAAAAAAAAAAAAAAAAA/41045233","http://AAAAAAAAAAAAAAAAAA/40395425","http://AAAAAAAAAAAAAAAAAA/40047065","http://AAAAAAAAAAAAAAAAAA/39891877","http://AAAAAAAAAAAAAAAAAA/39499073","http://AAAAAAAAAAAAAAAAAA/39314327","http://AAAAAAAAAAAAAAAAAA/38820809","http://AAAAAAAAAAAAAAAAAA/38439375", };long start =System.currentTimeMillis();try{int pagecount =urisToGet.length;

ExecutorService executors=Executors.newFixedThreadPool(pagecount);

CountDownLatch countDownLatch= newCountDownLatch(pagecount);for (int i = 0; i < pagecount; i++) {

HttpGet httpget= newHttpGet(urisToGet[i]);

config(httpget);//启动线程抓取

executors.execute(newGetRunnable(httpClient, httpget, countDownLatch));

}

countDownLatch.await();

executors.shutdown();

}catch(InterruptedException e) {

e.printStackTrace();

}finally{

System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！");

}long end =System.currentTimeMillis();

System.out.println("consume -> " + (end -start));

}static class GetRunnable implementsRunnable {privateCountDownLatch countDownLatch;private finalCloseableHttpClient httpClient;private finalHttpGet httpget;publicGetRunnable(CloseableHttpClient httpClient, HttpGet httpget, CountDownLatch countDownLatch) {this.httpClient =httpClient;this.httpget =httpget;this.countDownLatch =countDownLatch;

}

@Overridepublic voidrun() {

CloseableHttpResponse response= null;try{

response=httpClient.execute(httpget, HttpClientContext.create());

HttpEntity entity=response.getEntity();

System.out.println(EntityUtils.toString(entity,"utf-8"));

EntityUtils.consume(entity);

}catch(IOException e) {

e.printStackTrace();

}finally{

countDownLatch.countDown();try{if (response != null)

response.close();

}catch(IOException e) {

e.printStackTrace();

}

}

}

}

}