import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.NameValuePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author changya.dai
*
*/
public class HTTPClient {
private Logger logger = LoggerFactory.getLogger(HTTPClient.class);
private static HttpClient hc = new HttpClient(new MultiThreadedHttpConnectionManager());
static {
hc.getHttpConnectionManager().getParams().setConnectionTimeout(2000);
hc.getHttpConnectionManager().getParams().setSoTimeout(5000);
hc.getHttpConnectionManager().getParams().setDefaultMaxConnectionsPerHost(256);
hc.getHttpConnectionManager().getParams().setMaxTotalConnections(1024);
}
public Map<String,String> multiGetHtml(Map<String,String> urls) {
return multiGetHtml(urls,"UTF-8");
}
public String getHtml(String url) {
return getHtml(url,"UTF-8");
}
/**
* 多个Url 用get方式
*/
public Map<String,String> multiGetHtml(Map<String,String> urls,String charSet) {
Map<String,String> ret = new LinkedHashMap<String,String>();
GetMethod multiGetMethod = null;
if (Strings.isEmpty(charSet)) {
charSet = "UTF-8";
}
int status = -1;
try {
multiGetMethod = new GetMethod();
for (String path : urls.keySet()) {
handelGetMethod(multiGetMethod,urls.get(path),charSet);
status = hc.executeMethod(multiGetMethod);
String result = getStringFromResponseStream(multiGetMethod.getResponseBodyAsStream(),charSet);
if (status == 200) {
ret.put(path, result);
// logger.info(status + " " + result.length() + " " + urls.get(path));
} else {
// logger.error(status + " " + result.length() + " " + urls.get(path));
}
}
} catch (UnsupportedEncodingException e) {
logger.error("UnsupportedEncodingException status:" + status, e);
return null;
} catch (HttpException e){
logger.error("HttpException status:" + status, e);
return null;
} catch (IOException e) {
logger.error("IOException status:" + status, e);
return null;
} catch (Exception e) {
logger.error("status:" + status, e);
return null;
} finally {
if (multiGetMethod != null){
multiGetMethod.releaseConnection();
}
}
return ret;
}
public Map<String,String> multiPostHtml(Map<String, Map.Entry<String, Map<String, String>>> urls) {
Map<String,String> ret = new LinkedHashMap<String,String>();
PostMethod multiPostMethod = null;
int status = -1;
try {
for (Map.Entry<String, Map.Entry<String, Map<String, String>>> entry : urls.entrySet()){
status = -1;
multiPostMethod = new PostMethod();
String key = entry.getKey();
Map.Entry<String, Map<String, String>> request = entry.getValue();
multiPostMethod.setRequestHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
multiPostMethod.setRequestHeader("Connection", "Keep-Alive");
multiPostMethod.setRequestHeader("Accept-Language","zh-cn");
multiPostMethod.setRequestHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; InfoPath.2)");
multiPostMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
multiPostMethod.setURI(new URI(request.getKey(), true));
multiPostMethod.setRequestBody(mapToNameValuePairs(request.getValue()));
status = hc.executeMethod(multiPostMethod);
String result = getStringFromResponseStream(multiPostMethod.getResponseBodyAsStream(),"utf-8");
if (status == 200) {
ret.put(key, result);
}
}
} catch (UnsupportedEncodingException e) {
logger.error("UnsupportedEncodingException status:" + status, e);
return null;
} catch (HttpException e){
logger.error("HttpException status:" + status, e);
return null;
} catch (IOException e) {
logger.error("IOException status:" + status, e);
e.printStackTrace();
return null;
} catch (Exception e) {
logger.error("status:" + status, e);
return null;
} finally {
if (multiPostMethod != null){
multiPostMethod.releaseConnection();
}
}
return ret;
}
public String getHtml(String url,String charSet) {
if (Strings.isEmpty(charSet)){
charSet = "UTF-8";
}
String result = null;
GetMethod getMethod = null;
int status = -1;
try {
getMethod = new GetMethod();
handelGetMethod(getMethod,url,charSet);
status = hc.executeMethod(getMethod);
if (status == 200){
result = getStringFromResponseStream(getMethod.getResponseBodyAsStream(),charSet);
// logger.info(status + " " + result.length() + " " + url);
} else {
// logger.error(status + " " + result.length() + " " + url);
}
} catch (UnsupportedEncodingException e) {
logger.error("UnsupportedEncodingException status:" + status+" " + url, e);
}catch (IOException e) {
logger.error("IOException status:" + status+" " + url, e);
}catch (Exception e) {
logger.error("Exception status:" + status+" " + url, e);
} finally {
if (getMethod != null) {
getMethod.releaseConnection();
}
}
return result;
}
public String postHtml(String url,Map<String,String> param) {
String paramstr = "";
for (String key : param.keySet()) {
paramstr += "&" + key+"=" + Strings.urlEncode(param.get(key));
}
if (paramstr.length() > 0) {
paramstr = paramstr.substring(1,paramstr.length());
}
return postHtml(url,paramstr);
}
public String postHtml(String url,String param){
String result = null;
try{
URL postUrl = new URL(url);
HttpURLConnection connection = (HttpURLConnection) postUrl.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
connection.setRequestMethod("POST");
connection.setUseCaches(false);
connection.setInstanceFollowRedirects(true);
connection.setRequestProperty("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
connection.connect();
DataOutputStream out = new DataOutputStream(connection.getOutputStream());
String content = param;
out.writeBytes(content);
out.flush();
out.close();
result = getStringFromResponseStream(connection.getInputStream(),"utf-8");
connection.disconnect();
} catch (IOException e) {
logger.error("url:"+url +" param:"+param+"访问异常",e);
return null;
}
return result;
}
/**
* 给getMethod添加参数 getMetod不能为空
*/
private void handelGetMethod(GetMethod getMethod,String url,String charSet)
throws URIException{
try {
getMethod.setURI(new URI(url,true));
getMethod.getParams().setContentCharset(charSet);
getMethod.setRequestHeader("Accept-Language","zh-cn");
getMethod.setRequestHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; InfoPath.2)");
getMethod.setRequestHeader("Connection", "Keep-Alive");
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
} catch (URIException e) {
logger.error("error url:" + url,e);
}
}
private String getStringFromResponseStream(InputStream is,String charSet)
throws java.io.IOException, java.io.UnsupportedEncodingException {
char[] buffer = new char[4096];
if (is != null) {
StringBuilder sb = new StringBuilder(4096);
InputStreamReader reader = new InputStreamReader(is,charSet);
while (true) {
int len = reader.read(buffer);
if (len <= 0) {
break;
}
sb.append(buffer,0,len);
}
return sb.toString();
}
return null;
}
private NameValuePair[] mapToNameValuePairs(Map<String, String> params) {
NameValuePair[] ret = null;
if (params != null) {
ret = new NameValuePair[params.size()];
int i = 0;
for (Map.Entry<String, String> e : params.entrySet()) {
ret[i++] = new NameValuePair(e.getKey(), e.getValue());
}
}
return ret;
}
}
HTTPClient
最新推荐文章于 2022-09-23 13:49:49 发布