今天看到论文中提到异步Http请求在并行环境中能够提高网页获取速率,因此上网查找了下相关的资料,分别对同步请求和异步请求相同数量的urls进行了测试,发现异步请求url的速率确实比同步快很多,但是不知道在负责的Url请求下异步性能如何,先放在这里,之后再好好研究Httpclient的异步请求类HttpAsyncClient 。代码是在http://www.2cto.com/kf/201405/300720.html和http://www.cnblogs.com/wasp520/archive/2012/06/28/2568897.html这两篇文章上获取的。
同步请求:
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
public class SynHttpClient {
// 线程池
private ExecutorService exe = null;
// 线程池的容量
private static final int POOL_SIZE = 20;
private HttpClient client = null;
String[] urls=null;
public SynHttpClient(String[] urls){
this.urls=urls;
}
public void test() throws Exception {
exe = Executors.newFixedThreadPool(POOL_SIZE);
HttpParams params =new BasicHttpParams();
/* 从连接池中取连接的超时时间 */
ConnManagerParams.setTimeout(params, 1000);
/* 连接超时 */
HttpConnectionParams.setConnectionTimeout(params, 2000);
/* 请求超时 */
HttpConnectionParams.setSoTimeout(params, 4000);
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(
new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
//ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
PoolingClientConnectionManager cm=new PoolingClientConnectionManager(schemeRegistry);
cm.setMaxTotal(10);
final HttpClient httpClient = new DefaultHttpClient(cm,params);
// URIs to perform GETs on
final String[] urisToGet = urls;
//有多少url创建多少线程,url多时机子撑不住
// create a thread for each URI
GetThread[] threads = new GetThread[urisToGet.length];
for (int i = 0; i < threads.length; i++) {
HttpGet httpget = new HttpGet(urisToGet[i]);
threads[i] = new GetThread(httpClient, httpget);
}
// start the threads
for (int j = 0; j < threads.length; j++) {
threads[j].start();
}
// join the threads,等待所有请求完成
for (int j = 0; j < threads.length; j++) {
threads[j].join();
}
//使用线程池
// for (int i = 0; i < urisToGet.length; i++) {
// final int j=i;
// System.out.println(j);
// HttpGet httpget = new HttpGet(urisToGet[i]);
// exe.execute( new GetThread(httpClient, httpget));
// }
//
//
System.out.println("Done");
}
static class GetThread extends Thread{
private final HttpClient httpClient;
private final HttpContext context;
private final HttpGet httpget;
public GetThread(HttpClient httpClient, HttpGet httpget) {
this.httpClient = httpClient;
this.context = new BasicHttpContext();
this.httpget = httpget;
}
@Override
public void run(){
this.setName("threadsPoolClient");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
get();
}
public void get() {
try {
HttpResponse response = this.httpClient.execute(this.httpget, this.context);
HttpEntity entity = response.getEntity();
if (entity != null) {
System.out.println(this.httpget.getURI()+": status"+response.getStatusLine().toString());
}
// ensure the connection gets released to the manager
EntityUtils.consume(entity);
} catch (Exception ex) {
this.httpget.abort();
}finally{
httpget.releaseConnection();
}
}
}
public static void main(String[] args) throws Exception {
String[] urls = {"http://blog.csdn.net/neweastsun/article/details/41307683","http://blog.csdn.net/turkeyzhou/article/details/5510960",
"http://blog.csdn.net/morethinkmoretry/article/details/5791258","http://blog.csdn.net/longerandlonger/article/details/7255357",
"http://www.oschina.net/p/httpasyncclient"};
SynHttpClient client = new SynHttpClient(urls);
long start = System.currentTimeMillis();
client.test();
long end = System.currentTimeMillis();
System.out.println(end-start);
}
}
异步请求:
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.DefaultHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.nio.client.HttpAsyncClient;
import org.apache.http.nio.reactor.IOReactorException;
public class AsynClient{
/**
* @param args
* @throws IOReactorException
* @throws InterruptedException
*/
private List<String> urls;
private HandlerFailThread failHandler;
public AsynClient(List<String> list){
failHandler=new HandlerFailThread();
urls=list;
}
public Map<String,String> asynGet() throws IOReactorException,
InterruptedException {
final Map<String, String> responseMap=new HashMap<String, String>();
CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault();
try {
httpclient.start();
int urlLength=urls.size();
HttpGet[] requests = new HttpGet[urlLength];
int i=0;
for(String url : urls){
requests[i]=new HttpGet(url);
i++;
}
final CountDownLatch latch = new CountDownLatch(requests.length);
// int k =0;
for (final HttpGet request : requests) {
// System.out.println(k++);
httpclient.execute(request, new FutureCallback<HttpResponse>() {
public void completed(final HttpResponse response) {
latch.countDown();
responseMap.put(request.getURI().toString(), response.getStatusLine().toString());
try {
System.out.println(request.getRequestLine() + "->"
+ response.getStatusLine()+"->");
//+readInputStream(response.getEntity().getContent())
} catch (IllegalStateException e) {
failHandler.putFailUrl(request.getURI().toString(),
response.getStatusLine().toString());
e.printStackTrace();
} catch (Exception e) {
failHandler.putFailUrl(request.getURI().toString(),
response.getStatusLine().toString());
e.printStackTrace();
}
}
public void failed(final Exception ex) {
latch.countDown();
ex.printStackTrace();
failHandler.putFailUrl(request.getURI().toString(),
ex.getMessage());
}
public void cancelled() {
latch.countDown();
}
});
}
latch.await();
System.out.println("Shutting down");
} finally {
try {
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
System.out.println("Done");
failHandler.printFailUrl();
return responseMap;
}
private String readInputStream(InputStream input) throws IOException{
byte[] buffer = new byte[128];
int len = 0;
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
while((len = input.read(buffer)) >= 0) {
bytes.write(buffer, 0, len);
}
return bytes.toString();
}
/**
* Test
* @param args
*/
public static void main(String[] args) throws IOReactorException, InterruptedException {
String[] urlArray = {"http://blog.csdn.net/neweastsun/article/details/41307683","http://blog.csdn.net/turkeyzhou/article/details/5510960",
"http://blog.csdn.net/morethinkmoretry/article/details/5791258","http://blog.csdn.net/longerandlonger/article/details/7255357",
"http://www.oschina.net/p/httpasyncclient"};
List<String> urls= Arrays.asList(urlArray);
// urls.add("http://blog.csdn.net/neweastsun/article/details/41307683");
// urls.add("http://blog.csdn.net/neweastsun/article/details/41307683");
// urls.add("http://blog.csdn.net/neweastsun/article/details/41307683");
// for(int i=0;i<10;i++){
// urls.addAll(urls);
// }
System.out.println(urls.size());
AsynClient client=new AsynClient(urls);
long start = System.currentTimeMillis();
client.asynGet();
System.out.println("done");
long end = System.currentTimeMillis();
System.out.println(end-start);
}
}
异步请求中创建了一个线程(HandlerFailThread)记录失败请求:
import java.util.HashMap;
import java.util.Map;
public class HandlerFailThread extends Thread{
Map<String, String> failUrl=new HashMap<String, String>();
public void putFailUrl(String url,String status){
synchronized (failUrl) {
failUrl.put(url,status);
}
}
@Override
public void run() {
while(true){
}
}
public void printFailUrl(){
for(Map.Entry<String, String> m: failUrl.entrySet()){
System.out.println("****fail:url:"+m.getKey()+ " code :"+m.getValue());
}
}
}
里面需要用到
httpasyncclient-4.0.1.jar,httpclient-4.3.2.jar,httpcore-4.3.2.jar,httpcore-nio-4.3.2.jar,commons-logging-1.1.3.jar版本不一致可能会导致运行出错。
先保存着,以后好好研究下