HTTP请求同步和异步

最新推荐文章于 2024-06-14 11:11:05 发布

lerist

最新推荐文章于 2024-06-14 11:11:05 发布

阅读量1.7k

点赞数

分类专栏：爬虫文章标签： java 异步线程 http请求

爬虫专栏收录该内容

1 篇文章 0 订阅

订阅专栏

今天看到论文中提到异步Http请求在并行环境中能够提高网页获取速率，因此上网查找了下相关的资料，分别对同步请求和异步请求相同数量的urls进行了测试，发现异步请求url的速率确实比同步快很多，但是不知道在负责的Url请求下异步性能如何，先放在这里，之后再好好研究Httpclient的异步请求类HttpAsyncClient 。代码是在http://www.2cto.com/kf/201405/300720.html和http://www.cnblogs.com/wasp520/archive/2012/06/28/2568897.html这两篇文章上获取的。

同步请求：

import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

public class SynHttpClient {
    // 线程池
    private ExecutorService exe = null;
    // 线程池的容量
    private static final int POOL_SIZE = 20;
    private HttpClient client = null;
    String[] urls=null;
    public SynHttpClient(String[] urls){
        this.urls=urls;
    }
    public void test() throws Exception {
        exe = Executors.newFixedThreadPool(POOL_SIZE);
        HttpParams params =new BasicHttpParams();
        /* 从连接池中取连接的超时时间 */
        ConnManagerParams.setTimeout(params, 1000);
        /* 连接超时 */
        HttpConnectionParams.setConnectionTimeout(params, 2000);
        /* 请求超时 */
        HttpConnectionParams.setSoTimeout(params, 4000);
        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(
                new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

        //ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
        PoolingClientConnectionManager cm=new PoolingClientConnectionManager(schemeRegistry);
        cm.setMaxTotal(10);
        final HttpClient httpClient = new DefaultHttpClient(cm,params);

        // URIs to perform GETs on
        final String[] urisToGet = urls;
        //有多少url创建多少线程，url多时机子撑不住
        // create a thread for each URI
        GetThread[] threads = new GetThread[urisToGet.length];
        for (int i = 0; i < threads.length; i++) {
            HttpGet httpget = new HttpGet(urisToGet[i]);
            threads[i] = new GetThread(httpClient, httpget);
        }
        // start the threads
        for (int j = 0; j < threads.length; j++) {
            threads[j].start();
        }

        // join the threads，等待所有请求完成
        for (int j = 0; j < threads.length; j++) {
            threads[j].join();
        }
        //使用线程池
//        for (int i = 0; i < urisToGet.length; i++) {
//            final int j=i;
//            System.out.println(j);
//            HttpGet httpget = new HttpGet(urisToGet[i]);
//            exe.execute( new GetThread(httpClient, httpget));
//        }
//
//
           System.out.println("Done");
    }
    static class GetThread extends Thread{

        private final HttpClient httpClient;
        private final HttpContext context;
        private final HttpGet httpget;

        public GetThread(HttpClient httpClient, HttpGet httpget) {
            this.httpClient = httpClient;
            this.context = new BasicHttpContext();
            this.httpget = httpget;
        }
        @Override
        public void run(){
            this.setName("threadsPoolClient");
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            get();
        }

        public void get() {
            try {
                HttpResponse response = this.httpClient.execute(this.httpget, this.context);
                HttpEntity entity = response.getEntity();
                if (entity != null) {
                    System.out.println(this.httpget.getURI()+": status"+response.getStatusLine().toString());
                }
                // ensure the connection gets released to the manager
                EntityUtils.consume(entity);
            } catch (Exception ex) {
                this.httpget.abort();
            }finally{
                httpget.releaseConnection();
            }
        }
    }


    public static void main(String[] args) throws Exception {
        String[] urls = {"http://blog.csdn.net/neweastsun/article/details/41307683","http://blog.csdn.net/turkeyzhou/article/details/5510960",
                "http://blog.csdn.net/morethinkmoretry/article/details/5791258","http://blog.csdn.net/longerandlonger/article/details/7255357",
                "http://www.oschina.net/p/httpasyncclient"};
        SynHttpClient client = new SynHttpClient(urls);
        long start = System.currentTimeMillis();
        client.test();
        long end = System.currentTimeMillis();
        System.out.println(end-start);
    }
}

异步请求：

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.CountDownLatch;

import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.DefaultHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.nio.client.HttpAsyncClient;
import org.apache.http.nio.reactor.IOReactorException;

public class AsynClient{
    /**
     * @param args
     * @throws IOReactorException
     * @throws InterruptedException
     */
    private List<String> urls;
    private HandlerFailThread failHandler;
    public AsynClient(List<String> list){
        failHandler=new HandlerFailThread();
        urls=list;
    }
    public Map<String,String> asynGet() throws IOReactorException,
            InterruptedException {
        final Map<String, String> responseMap=new HashMap<String, String>();
        CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault();
        try {
            httpclient.start();
            int urlLength=urls.size();
            HttpGet[] requests = new HttpGet[urlLength];
            int i=0;
            for(String url : urls){
                requests[i]=new HttpGet(url);
                i++;
            }
            final CountDownLatch latch = new CountDownLatch(requests.length);
//            int k =0;
            for (final HttpGet request : requests) {
//                System.out.println(k++);
                httpclient.execute(request, new FutureCallback<HttpResponse>() {

                    public void completed(final HttpResponse response) {
                        latch.countDown();
                        responseMap.put(request.getURI().toString(), response.getStatusLine().toString());
                        try {
                            System.out.println(request.getRequestLine() + "->"
                                    + response.getStatusLine()+"->");
                            //+readInputStream(response.getEntity().getContent())

                        } catch (IllegalStateException e) {
                            failHandler.putFailUrl(request.getURI().toString(),
                                    response.getStatusLine().toString());
                            e.printStackTrace();
                        } catch (Exception e) {
                            failHandler.putFailUrl(request.getURI().toString(),
                                    response.getStatusLine().toString());
                            e.printStackTrace();
                        }
                    }

                    public void failed(final Exception ex) {
                        latch.countDown();
                        ex.printStackTrace();
                        failHandler.putFailUrl(request.getURI().toString(),
                                ex.getMessage());
                    }

                    public void cancelled() {
                        latch.countDown();
                    }

                });
            }
            latch.await();
            System.out.println("Shutting down");
        } finally {
            try {
                httpclient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        System.out.println("Done");
        failHandler.printFailUrl();
        return responseMap;
    }
    private String readInputStream(InputStream input) throws IOException{
        byte[] buffer = new byte[128];
        int len = 0;
        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
        while((len = input.read(buffer)) >= 0) {
            bytes.write(buffer, 0, len);
        }
        return bytes.toString();
    }
    /**
     * Test
     * @param args
     */
    public static void main(String[] args) throws IOReactorException, InterruptedException {
        String[] urlArray = {"http://blog.csdn.net/neweastsun/article/details/41307683","http://blog.csdn.net/turkeyzhou/article/details/5510960",
                "http://blog.csdn.net/morethinkmoretry/article/details/5791258","http://blog.csdn.net/longerandlonger/article/details/7255357",
                "http://www.oschina.net/p/httpasyncclient"};
        List<String> urls= Arrays.asList(urlArray);
//        urls.add("http://blog.csdn.net/neweastsun/article/details/41307683");
//        urls.add("http://blog.csdn.net/neweastsun/article/details/41307683");
//        urls.add("http://blog.csdn.net/neweastsun/article/details/41307683");
//        for(int i=0;i<10;i++){
//            urls.addAll(urls);
//        }
        System.out.println(urls.size());
        AsynClient client=new AsynClient(urls);
        long start = System.currentTimeMillis();
        client.asynGet();
        System.out.println("done");
        long end = System.currentTimeMillis();
        System.out.println(end-start);
    }
}

异步请求中创建了一个线程(HandlerFailThread)记录失败请求：

import java.util.HashMap;
import java.util.Map;

public class HandlerFailThread  extends Thread{
    Map<String, String> failUrl=new HashMap<String, String>();
    public void putFailUrl(String url,String status){
        synchronized (failUrl) {
            failUrl.put(url,status);
        }
    }
    @Override
    public void run() {
        while(true){

        }
    }
    public void printFailUrl(){
        for(Map.Entry<String, String> m: failUrl.entrySet()){
            System.out.println("****fail:url:"+m.getKey()+ "  code :"+m.getValue());
        }
    }
}

里面需要用到 httpasyncclient-4.0.1.jar，httpclient-4.3.2.jar,httpcore-4.3.2.jar,httpcore-nio-4.3.2.jar,commons-logging-1.1.3.jar版本不一致可能会导致运行出错。

先保存着，以后好好研究下