httpClinet

httpClient多线程请求




使用httpClient可模拟请求Url获取资源，使用单线程的请求速度上会有一定的限制，参考了Apache给出的例子，自己做了测试实现多线程并发请求，以下代码需要HttpClient 4.2的包，可以在http://hc.apache.org/downloads.cgi下载

1、并发请求



复制代码

1 package generate.httpclient;
2
3 import java.util.List;
4 import java.util.concurrent.ExecutorService;
5 import java.util.concurrent.Executors;
6
7 import org.apache.http.HttpEntity;
8 import org.apache.http.HttpResponse;
9 import org.apache.http.client.HttpClient;
10 import org.apache.http.client.methods.HttpGet;
11 import org.apache.http.conn.ClientConnectionManager;
12 import org.apache.http.conn.params.ConnManagerParams;
13 import org.apache.http.conn.scheme.PlainSocketFactory;
14 import org.apache.http.conn.scheme.Scheme;
15 import org.apache.http.conn.scheme.SchemeRegistry;
16 import org.apache.http.impl.client.DefaultHttpClient;
17 import org.apache.http.impl.conn.PoolingClientConnectionManager;
18 import org.apache.http.params.BasicHttpParams;
19 import org.apache.http.params.HttpConnectionParams;
20 import org.apache.http.params.HttpParams;
21 import org.apache.http.protocol.BasicHttpContext;
22 import org.apache.http.protocol.HttpContext;
23 import org.apache.http.util.EntityUtils;
24
25 public class ThreadPoolHttpClient {
26 // 线程池
27 private ExecutorService exe = null;
28 // 线程池的容量
29 private static final int POOL_SIZE = 20;
30 private HttpClient client = null;
31 String[] urls=null;
32 public ThreadPoolHttpClient(String[] urls){
33 this.urls=urls;
34 }
35 public void test() throws Exception {
36 exe = Executors.newFixedThreadPool(POOL_SIZE);
37 HttpParams params =new BasicHttpParams();
38 /* 从连接池中取连接的超时时间 */
39 ConnManagerParams.setTimeout(params, 1000);
40 /* 连接超时 */
41 HttpConnectionParams.setConnectionTimeout(params, 2000);
42 /* 请求超时 */
43 HttpConnectionParams.setSoTimeout(params, 4000);
44 SchemeRegistry schemeRegistry = new SchemeRegistry();
45 schemeRegistry.register(
46 new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
47
48 //ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
49 PoolingClientConnectionManager cm=new PoolingClientConnectionManager(schemeRegistry);
50 cm.setMaxTotal(10);
51 final HttpClient httpClient = new DefaultHttpClient(cm,params);
52
53 // URIs to perform GETs on
54 final String[] urisToGet = urls;
55 /* 有多少url创建多少线程，url多时机子撑不住
56 // create a thread for each URI
57 GetThread[] threads = new GetThread[urisToGet.length];
58 for (int i = 0; i < threads.length; i++) {
59 HttpGet httpget = new HttpGet(urisToGet[i]);
60 threads[i] = new GetThread(httpClient, httpget);
61 }
62 // start the threads
63 for (int j = 0; j < threads.length; j++) {
64 threads[j].start();
65 }
66
67 // join the threads，等待所有请求完成
68 for (int j = 0; j < threads.length; j++) {
69 threads[j].join();
70 }
71 使用线程池*/
72 for (int i = 0; i < urisToGet.length; i++) {
73 final int j=i;
74 System.out.println(j);
75 HttpGet httpget = new HttpGet(urisToGet[i]);
76 exe.execute( new GetThread(httpClient, httpget));
77 }
78
79
80 //创建线程池，每次调用POOL_SIZE
81 /*
82 for (int i = 0; i < urisToGet.length; i++) {
83 final int j=i;
84 System.out.println(j);
85 exe.execute(new Thread() {
86 @Override
87 public void run() {
88 this.setName("threadsPoolClient"+j);
89
90 try {
91 this.sleep(100);
92 System.out.println(j);
93 } catch (InterruptedException e) {
94 // TODO Auto-generated catch block
95 e.printStackTrace();
96 }
97
98 HttpGet httpget = new HttpGet(urisToGet[j]);
99 new GetThread(httpClient, httpget).get();
100 }
101
102
103
104 });
105 }
106
107 */
108 //exe.shutdown();
109 System.out.println("Done");
110 }
111 static class GetThread extends Thread{
112
113 private final HttpClient httpClient;
114 private final HttpContext context;
115 private final HttpGet httpget;
116
117 public GetThread(HttpClient httpClient, HttpGet httpget) {
118 this.httpClient = httpClient;
119 this.context = new BasicHttpContext();
120 this.httpget = httpget;
121 }
122 @Override
123 public void run(){
124 this.setName("threadsPoolClient");
125 try {
126 Thread.sleep(5000);
127 } catch (InterruptedException e) {
128 // TODO Auto-generated catch block
129 e.printStackTrace();
130 }
131 get();
132 }
133
134 public void get() {
135 try {
136 HttpResponse response = this.httpClient.execute(this.httpget, this.context);
137 HttpEntity entity = response.getEntity();
138 if (entity != null) {
139 System.out.println(this.httpget.getURI()+": status"+response.getStatusLine().toString());
140 }
141 // ensure the connection gets released to the manager
142 EntityUtils.consume(entity);
143 } catch (Exception ex) {
144 this.httpget.abort();
145 }finally{
146 httpget.releaseConnection();
147 }
148 }
149 }
150 }

复制代码



2、多线程异步请求



复制代码

1 package generate.httpclient;
2
3 import java.io.ByteArrayOutputStream;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.util.ArrayList;
7 import java.util.HashMap;
8 import java.util.List;
9 import java.util.Map;
10 import java.util.concurrent.CountDownLatch;
11
12 import org.apache.http.HttpResponse;
13 import org.apache.http.client.methods.HttpGet;
14 import org.apache.http.concurrent.FutureCallback;
15 import org.apache.http.impl.nio.client.DefaultHttpAsyncClient;
16 import org.apache.http.nio.client.HttpAsyncClient;
17 import org.apache.http.nio.reactor.IOReactorException;
18
19 public class AsynClient{
20 /**
21 * @param args
22 * @throws IOReactorException
23 * @throws InterruptedException
24 */
25 private List<String> urls;
26 private HandlerFailThread failHandler;
27 public AsynClient(List<String> list){
28 failHandler=new HandlerFailThread();
29 urls=list;
30 }
31 public Map<String,String> asynGet() throws IOReactorException,
32 InterruptedException {
33 final HttpAsyncClient httpclient = new DefaultHttpAsyncClient();
34 httpclient.start();
35 int urlLength=urls.size();
36 HttpGet[] requests = new HttpGet[urlLength];
37 int i=0;
38 for(String url : urls){
39 requests[i]=new HttpGet(url);
40 i++;
41 }
42 final CountDownLatch latch = new CountDownLatch(requests.length);
43 final Map<String, String> responseMap=new HashMap<String, String>();
44 try {
45 for (final HttpGet request : requests) {
46 httpclient.execute(request, new FutureCallback<HttpResponse>() {
47
48 public void completed(final HttpResponse response) {
49 latch.countDown();
50 responseMap.put(request.getURI().toString(), response.getStatusLine().toString());
51 try {
52 System.out.println(request.getRequestLine() + "->"
53 + response.getStatusLine()+"->");
54 //+readInputStream(response.getEntity().getContent())
55
56 } catch (IllegalStateException e) {
57 failHandler.putFailUrl(request.getURI().toString(),
58 response.getStatusLine().toString());
59 e.printStackTrace();
60 } catch (Exception e) {
61 failHandler.putFailUrl(request.getURI().toString(),
62 response.getStatusLine().toString());
63 e.printStackTrace();
64 }
65 }
66
67 public void failed(final Exception ex) {
68 latch.countDown();
69 ex.printStackTrace();
70 failHandler.putFailUrl(request.getURI().toString(),
71 ex.getMessage());
72 }
73
74 public void cancelled() {
75 latch.countDown();
76 }
77
78 });
79 }
80 System.out.println("Doing...");
81 } finally {
82 latch.await();
83 httpclient.shutdown();
84 }
85 System.out.println("Done");
86 failHandler.printFailUrl();
87 return responseMap;
88 }
89 private String readInputStream(InputStream input) throws IOException{
90 byte[] buffer = new byte[128];
91 int len = 0;
92 ByteArrayOutputStream bytes = new ByteArrayOutputStream();
93 while((len = input.read(buffer)) >= 0) {
94 bytes.write(buffer, 0, len);
95 }
96 return bytes.toString();
97 }
98 /**
99 * Test
100 * @param args
101 */
102 public static void main(String[] args) {
103 List<String> urls=new ArrayList<String>();
104 urls.add("http://127.0.0.1/examples/servlets/");
105 urls.add("http://127.0.0.1/examples/servlets/");
106 urls.add("http://127.0.0.1/examples/servlets/");
107 for(int i=0;i<10;i++){
108 urls.addAll(urls);
109 }
110 System.out.println(urls.size());
111 AsynClient client=new AsynClient(urls);
112 try {
113 client.asynGet();
114 } catch (IOReactorException e) {
115 e.printStackTrace();
116 } catch (InterruptedException e) {
117 e.printStackTrace();
118 }
119 System.out.println("done");
120 }
121 }

复制代码

创建一个线程记录失败的请求





复制代码

1 package generate.httpclient;
2
3 import java.util.HashMap;
4 import java.util.Map;
5
6 public class HandlerFailThread extends Thread{
7 Map<String, String> failUrl=new HashMap<String, String>();
8 public void putFailUrl(String url,String status){
9 synchronized (failUrl) {
10 failUrl.put(url,status);
11 }
12 }
13 @Override
14 public void run() {
15 while(true){
16
17 }
18 }
19 public void printFailUrl(){
20 for(Map.Entry<String, String> m: failUrl.entrySet()){
21 System.out.println("****fail:url:"+m.getKey()+ " code :"+m.getValue());
22 }
23 }
24 }

复制代码

异步请求，也可通过pool管理，例如

ConnectingIOReactor nio=new DefaultConnectingIOReactor();
PoolingClientAsyncConnectionManager manager=new PoolingClientAsyncConnectionManager(nio);
manager.setMaxTotal(1000);
manager.setDefaultMaxPerRoute(100);
HttpParams params=new BasicHttpParams();
/* 连接超时 */
HttpConnectionParams.setConnectionTimeout(params, 10000);
/* 请求超时 */
HttpConnectionParams.setSoTimeout(params, 60*1000);
DefaultHttpAsyncClient.setDefaultHttpParams(params);
final HttpAsyncClient httpclient = new DefaultHttpAsyncClient(manager);
httpclient.start();