httpClient多线程请求

 

使用httpClient可模拟请求Url获取资源，使用单线程的请求速度上会有一定的限制，参考了Apache给出的例子，自己做了测试实现多线程并发请求，以下代码需要HttpClient 4.2的包，可以在http://hc.apache.org/downloads.cgi下载

1、并发请求

View Code

  1  package generate.httpclient;
  2 
  3  import java.util.List;
  4  import java.util.concurrent.ExecutorService;
  5  import java.util.concurrent.Executors;
  6 
  7  import org.apache.http.HttpEntity;
  8  import org.apache.http.HttpResponse;
  9  import org.apache.http.client.HttpClient;
 10  import org.apache.http.client.methods.HttpGet;
 11  import org.apache.http.conn.ClientConnectionManager;
 12  import org.apache.http.conn.params.ConnManagerParams;
 13  import org.apache.http.conn.scheme.PlainSocketFactory;
 14  import org.apache.http.conn.scheme.Scheme;
 15  import org.apache.http.conn.scheme.SchemeRegistry;
 16  import org.apache.http.impl.client.DefaultHttpClient;
 17  import org.apache.http.impl.conn.PoolingClientConnectionManager;
 18  import org.apache.http.params.BasicHttpParams;
 19  import org.apache.http.params.HttpConnectionParams;
 20  import org.apache.http.params.HttpParams;
 21  import org.apache.http.protocol.BasicHttpContext;
 22  import org.apache.http.protocol.HttpContext;
 23  import org.apache.http.util.EntityUtils;
 24 
 25  public  class ThreadPoolHttpClient {
 26      //  线程池
 27       private ExecutorService exe =  null;
 28      //  线程池的容量
 29       private  static  final  int POOL_SIZE = 20;
 30      private HttpClient client =  null;
 31     String[] urls= null;
 32      public ThreadPoolHttpClient(String[] urls){
 33          this.urls=urls;
 34     }
 35      public  void test()  throws Exception {
 36         exe = Executors.newFixedThreadPool(POOL_SIZE);
 37         HttpParams params = new BasicHttpParams();
 38          /*  从连接池中取连接的超时时间  */ 
 39         ConnManagerParams.setTimeout(params, 1000);
 40          /*  连接超时  */ 
 41         HttpConnectionParams.setConnectionTimeout(params, 2000); 
 42          /*  请求超时  */
 43         HttpConnectionParams.setSoTimeout(params, 4000);
 44         SchemeRegistry schemeRegistry =  new SchemeRegistry();
 45         schemeRegistry.register(
 46                  new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
 47 
 48          // ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
 49          PoolingClientConnectionManager cm= new PoolingClientConnectionManager(schemeRegistry);
 50         cm.setMaxTotal(10);
 51          final HttpClient httpClient =  new DefaultHttpClient(cm,params);
 52 
 53          //  URIs to perform GETs on
 54           final String[] urisToGet = urls;
 55          /* 有多少url创建多少线程，url多时机子撑不住
 56          // create a thread for each URI
 57          GetThread[] threads = new GetThread[urisToGet.length];
 58          for (int i = 0; i < threads.length; i++) {
 59              HttpGet httpget = new HttpGet(urisToGet[i]);
 60              threads[i] = new GetThread(httpClient, httpget);            
 61          }
 62          // start the threads
 63          for (int j = 0; j < threads.length; j++) {
 64              threads[j].start();
 65          }
 66 
 67          // join the threads，等待所有请求完成
 68          for (int j = 0; j < threads.length; j++) {
 69              threads[j].join();
 70          }
 71           使用线程池*/
 72          for ( int i = 0; i < urisToGet.length; i++) {
 73              final  int j=i;
 74             System.out.println(j);
 75             HttpGet httpget =  new HttpGet(urisToGet[i]);
 76             exe.execute(  new GetThread(httpClient, httpget));
 77         }
 78         
 79         
 80          // 创建线程池，每次调用POOL_SIZE
 81           /*
 82          for (int i = 0; i < urisToGet.length; i++) {
 83              final int j=i;
 84              System.out.println(j);
 85              exe.execute(new Thread() {
 86                  @Override
 87                  public void run() {
 88                      this.setName("threadsPoolClient"+j);
 89                      
 90                          try {
 91                              this.sleep(100);
 92                              System.out.println(j);
 93                          } catch (InterruptedException e) {
 94                              // TODO Auto-generated catch block
 95                              e.printStackTrace();
 96                          }
 97                          
 98                          HttpGet httpget = new HttpGet(urisToGet[j]);
 99                          new GetThread(httpClient, httpget).get();
100                      }
101                      
102                      
103                  
104              });
105          }
106          
107           */
108          // exe.shutdown();
109          System.out.println("Done");
110     }
111      static  class GetThread  extends Thread{
112         
113          private  final HttpClient httpClient;
114          private  final HttpContext context;
115          private  final HttpGet httpget;
116         
117          public GetThread(HttpClient httpClient, HttpGet httpget) {
118              this.httpClient = httpClient;
119              this.context =  new BasicHttpContext();
120              this.httpget = httpget;
121         }
122         @Override
123          public  void run(){
124              this.setName("threadsPoolClient");
125              try {
126                 Thread.sleep(5000);
127             }  catch (InterruptedException e) {
128                  //  TODO Auto-generated catch block
129                  e.printStackTrace();
130             }
131             get();
132         }
133         
134          public  void get() {
135              try {
136                 HttpResponse response =  this.httpClient.execute( this.httpget,  this.context);
137                 HttpEntity entity = response.getEntity();
138                  if (entity !=  null) {
139                     System.out.println( this.httpget.getURI()+": status"+response.getStatusLine().toString());
140                 }
141                  //  ensure the connection gets released to the manager
142                  EntityUtils.consume(entity);
143             }  catch (Exception ex) {
144                  this.httpget.abort();
145             } finally{
146                 httpget.releaseConnection();
147             }
148         }
149     }
150 }

 

2、多线程异步请求

View Code

  1  package generate.httpclient;
  2 
  3  import java.io.ByteArrayOutputStream;
  4  import java.io.IOException;
  5  import java.io.InputStream;
  6  import java.util.ArrayList;
  7  import java.util.HashMap;
  8  import java.util.List;
  9  import java.util.Map;
 10  import java.util.concurrent.CountDownLatch;
 11 
 12  import org.apache.http.HttpResponse;
 13  import org.apache.http.client.methods.HttpGet;
 14  import org.apache.http.concurrent.FutureCallback;
 15  import org.apache.http.impl.nio.client.DefaultHttpAsyncClient;
 16  import org.apache.http.nio.client.HttpAsyncClient;
 17  import org.apache.http.nio.reactor.IOReactorException;
 18 
 19  public  class AsynClient{
 20      /**
 21       *  @param  args
 22       *  @throws  IOReactorException
 23       *  @throws  InterruptedException
 24        */
 25      private List<String> urls;
 26      private HandlerFailThread failHandler;
 27      public AsynClient(List<String> list){
 28         failHandler= new HandlerFailThread();
 29         urls=list;
 30     }
 31      public Map<String,String> asynGet()  throws IOReactorException,
 32             InterruptedException {
 33          final HttpAsyncClient httpclient =  new DefaultHttpAsyncClient();
 34         httpclient.start();
 35          int urlLength=urls.size();
 36         HttpGet[] requests =  new HttpGet[urlLength];
 37          int i=0;
 38          for(String url : urls){
 39             requests[i]= new HttpGet(url);
 40             i++;
 41         }
 42          final CountDownLatch latch =  new CountDownLatch(requests.length);
 43          final Map<String, String> responseMap= new HashMap<String, String>();
 44          try {
 45              for ( final HttpGet request : requests) {
 46                 httpclient.execute(request,  new FutureCallback<HttpResponse>() {
 47 
 48                      public  void completed( final HttpResponse response) {
 49                         latch.countDown();
 50                         responseMap.put(request.getURI().toString(), response.getStatusLine().toString());
 51                          try {
 52                             System.out.println(request.getRequestLine() + "->"
 53                                     + response.getStatusLine()+"->");
 54                              // +readInputStream(response.getEntity().getContent())
 55                              
 56                         }  catch (IllegalStateException e) {
 57                             failHandler.putFailUrl(request.getURI().toString(),
 58                                     response.getStatusLine().toString());
 59                             e.printStackTrace();
 60                         }  catch (Exception e) {
 61                             failHandler.putFailUrl(request.getURI().toString(),
 62                                     response.getStatusLine().toString());
 63                             e.printStackTrace();
 64                         }
 65                     }
 66 
 67                      public  void failed( final Exception ex) {
 68                         latch.countDown();
 69                         ex.printStackTrace();
 70                         failHandler.putFailUrl(request.getURI().toString(),
 71                                 ex.getMessage());
 72                     }
 73 
 74                      public  void cancelled() {
 75                         latch.countDown();
 76                     }
 77 
 78                 });
 79             }
 80             System.out.println("Doing...");
 81         }  finally {
 82             latch.await();
 83             httpclient.shutdown();
 84         }
 85         System.out.println("Done");
 86         failHandler.printFailUrl();
 87          return responseMap;
 88     }
 89      private String readInputStream(InputStream input)  throws IOException{
 90          byte[] buffer =  new  byte[128];
 91          int len = 0;
 92         ByteArrayOutputStream bytes =  new ByteArrayOutputStream();
 93          while((len = input.read(buffer)) >= 0) {
 94             bytes.write(buffer, 0, len);
 95         }
 96          return bytes.toString();
 97     }
 98      /**
 99       * Test
100       *  @param  args
101        */
102      public  static  void main(String[] args) {
103         List<String> urls= new ArrayList<String>();
104         urls.add("http://127.0.0.1/examples/servlets/");
105         urls.add("http://127.0.0.1/examples/servlets/");
106         urls.add("http://127.0.0.1/examples/servlets/");
107          for( int i=0;i<10;i++){
108             urls.addAll(urls);
109         }
110         System.out.println(urls.size());
111         AsynClient client= new AsynClient(urls);
112          try {
113             client.asynGet();
114         }  catch (IOReactorException e) {
115             e.printStackTrace();
116         }  catch (InterruptedException e) {
117             e.printStackTrace();
118         }
119         System.out.println("done");
120     }
121 }

创建一个线程记录失败的请求

 

View Code

 1  package generate.httpclient;
 2 
 3  import java.util.HashMap;
 4  import java.util.Map;
 5 
 6  public  class HandlerFailThread   extends Thread{
 7     Map<String, String> failUrl= new HashMap<String, String>();
 8      public  void putFailUrl(String url,String status){
 9          synchronized (failUrl) {
10             failUrl.put(url,status);
11         }
12     }
13     @Override
14      public  void run() {
15          while( true){
16             
17         }
18     }
19      public  void printFailUrl(){
20          for(Map.Entry<String, String> m: failUrl.entrySet()){
21             System.out.println("****fail:url:"+m.getKey()+ "  code :"+m.getValue());
22         }
23     }
24 }

  异步请求，也可通过pool管理，例如

 ConnectingIOReactor nio=new DefaultConnectingIOReactor();
  PoolingClientAsyncConnectionManager manager=new PoolingClientAsyncConnectionManager(nio);
  manager.setMaxTotal(1000);
  manager.setDefaultMaxPerRoute(100);
  HttpParams params=new BasicHttpParams();
  /* 连接超时 */
  HttpConnectionParams.setConnectionTimeout(params, 10000);
  /* 请求超时 */
  HttpConnectionParams.setSoTimeout(params, 60*1000);
  DefaultHttpAsyncClient.setDefaultHttpParams(params);
  final HttpAsyncClient httpclient = new DefaultHttpAsyncClient(manager);
  httpclient.start();

 

HttpClient相关可参看，里面有很多说明与例子

http://hc.apache.org/httpcomponents-client-ga/tutorial/html/connmgmt.html

 

转载于:https://www.cnblogs.com/wasp520/archive/2012/06/28/2568897.html