HttpUtils.java
package com.test.http.util;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.RedirectStrategy;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpCoreContext;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import com.test.http.HttpResult;
import com.test.http.util.StringUtils;
/**
* http工具类
* http请求头大全:http://tools.jb51.net/table/http_header
* https解决方案:http://www.cnblogs.com/yangmengdx3/p/6322380.html
* @author youxiang
*
*/
public class HttpUtils {
private static Logger log = Logger.getLogger(HttpUtils.class);
/**
* User-Agent 浏览器用户标识
*/
public static final String USER_AGENT_VALUE="Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36";
/**
* http请求参数编码
*/
public static final String HTTP_PARAMS_CODING="UTF-8";
private static final String HTTP = "http";
private static final String HTTPS = "https";
private static SSLConnectionSocketFactory sslFactory = null;
private static PoolingHttpClientConnectionManager connectionManager = null;
private static SSLContextBuilder builder = null;
private static RequestConfig requestConfig = null;
private static RedirectStrategy redirectStrategy = null;
static {
try {
builder = new SSLContextBuilder();
// HTTPS全部信任 不做身份鉴定
builder.loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
});
sslFactory = new SSLConnectionSocketFactory(builder.build(), new String[]{"SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.2"}, null, NoopHostnameVerifier.INSTANCE);
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register(HTTP, new PlainConnectionSocketFactory()).register(HTTPS, sslFactory).build();
connectionManager = new PoolingHttpClientConnectionManager(registry);
connectionManager.setMaxTotal(300);//连接池最大并发连接数
connectionManager.setDefaultMaxPerRoute(100);//单路由的最大并发连接数
/**
* 设置请求和.设置从connect Manager获取Connection 超时时间.传输超时时间
* .setConnectTimeout(5000) 建立连接的时间
* .setConnectionRequestTimeout(5000) 从连接池中获取连接的时间
* .setSocketTimeout(10000) 数据传输处理时间
*/
requestConfig = RequestConfig.custom().setConnectTimeout(20000).build();
//解决post重定向问题
redirectStrategy = new LaxRedirectStrategy();
} catch (Exception e) {
log.error("设置httpclient参数失败", e);
}
}
private HttpUtils() {
throw new IllegalStateException("Utility class");
}
/**
* 获取HttpClient对象
* @return
*/
public static HttpClient getHttpClient() {
return HttpClients.custom().setRedirectStrategy(redirectStrategy).setSSLSocketFactory(sslFactory).setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).setConnectionManagerShared(true).build();
}
/**
* 获取网页内容
* @param response
* @return
* @throws IOException
* @throws Exception
*/
public static HttpResult getHtml(HttpResponse response) throws IOException {
HttpResult result = new HttpResult();
String html = "";
String charsetStr = HttpUtils.HTTP_PARAMS_CODING;
HttpEntity entity = response.getEntity();
if(entity != null){
byte [] bytes = EntityUtils.toByteArray(entity);
if(bytes!=null && bytes.length>0){
Charset charset = null;
// 如果头部Content-Type中包含了编码信息,那么我们可以直接在此处获取
ContentType contentType = ContentType.getOrDefault(entity);
charset = contentType.getCharset();
//charSet = EntityUtils.getContentCharSet(entity);//方法已过时
// 如果头部中没有,那么我们需要 查看页面源码,这个方法虽然不能说完全正确,因为有些粗糙的网页编码者没有在页面中写头部编码信息
if (charset==null) {
try{
String regEx = "(?=<meta).*?(?<=charset=[\\'|\\\"]?)([a-zA-Z0-9-]*)";
//获取页面中header部分的编码
String charsetName = HttpUtils.getStrings(new String(bytes), regEx, 2, 1);
charset = Charset.forName(charsetName);
}catch(Exception e){
log.error("未找到编码 error:"+e.getMessage());
}
}
if(charset!=null) {
charsetStr = charset.name();
}
log.debug("charSet:"+charsetStr);
// 至此,我们可以将原byte数组按照正常编码专成字符串输出(如果找到了编码的话)
html = new String(bytes, charsetStr);
log.debug(html);
}
}
result.setResponseBody(html);
result.setCharSet(charsetStr);
return result;
}
/**
* 获取响应信息
* @param httpContext
* @return
* @throws IOException
*/
public static HttpResult getHttpInfo(HttpClientContext httpContext) throws IOException {
HttpResult result = new HttpResult();
HttpUriRequest request = (HttpUriRequest) httpContext.getAttribute(HttpCoreContext.HTTP_REQUEST);
//获取请求链接
StringBuilder requestUrl = new StringBuilder();
List<URI> uriList = httpContext.getRedirectLocations();
if(uriList==null) {
HttpHost currentHost = (HttpHost) httpContext.getAttribute(HttpCoreContext.HTTP_TARGET_HOST);
requestUrl.append((request.getURI().isAbsolute()) ? request.getURI().toString() : (currentHost.toURI() + request.getURI()));
}else {
for(URI uri:uriList) {
requestUrl.append(uri.toString()).append("\r\n");
}
}
result.setRequestUrl(requestUrl.toString());
//获取请求头
StringBuilder requestHeaderStr = new StringBuilder ();
Header[] requestHeaders = request.getAllHeaders();
if(requestHeaders!=null) {
for(Header requestHeader:requestHeaders) {
requestHeaderStr.append(requestHeader.toString()).append("\r\n");
}
}
result.setRequestHeader(requestHeaderStr.toString());
//获取响应头
HttpResponse response = (HttpResponse) httpContext.getAttribute(HttpCoreContext.HTTP_RESPONSE);
StringBuilder responseHeaderStr = new StringBuilder ();
Header[] responseHeaders = response.getAllHeaders();
if(responseHeaders!=null) {
for(Header responseHeader:responseHeaders) {
responseHeaderStr.append(responseHeader.toString()).append("\r\n");
}
}
result.setResponseHeader(responseHeaderStr.toString());
//获取网页内容
HttpResult htmlResult = HttpUtils.getHtml(response);
result.setResponseBody(htmlResult.getResponseBody());
result.setCharSet(htmlResult.getCharSet());
int statusCode = response.getStatusLine().getStatusCode();
result.setStatusCode(statusCode);
if(statusCode>=400){
//输入信息
log.info(result.toString());
throw new CustomException(requestUrl+" Error:请求失败,返回码->"+statusCode);
}
return result;
}
/**
* 根据正则表达式获取文本
* @param html 文本
* @param regex 正则表达式
* @param template 模板 (指定第几个模板)从1开始
* @param match_no 匹配数字(获取第几个) 0表示随机选取,1表示选取第一个
*/
public static String getStrings(String html, String regex, int template, int matchNo) {
String result="";
try{
Pattern p = Pattern.compile(regex, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(html);
ArrayList<String> strs = new ArrayList<>();
while (m.find()) {
strs.add(m.group(template-1));
}
if(matchNo==0){
int index = new Random().nextInt(strs.size());
result=strs.get(index);
}else if(matchNo>0){
result=strs.get(matchNo-1);
}
}catch(Exception e){
throw new CustomException("找不到值!正则表达式:"+regex+";模板:"+template+";匹配数字:"+matchNo+";异常:"+e.getMessage());
}
return result;
}
/**
* 通过前后边界获取字符
* @param html 文本
* @param LB 左边开始字符
* @param RB 右边结束字符
*/
public static String getStrings(String html, String leftBefore, String rightBefore){
String result="";
try{
int index = 0;
if(StringUtils.isNotEmpty(leftBefore)) {
index = html.indexOf(leftBefore);
if(index<0) {
throw new CustomException("返回数据找不到LB值!LB:"+leftBefore);
}
index = index + leftBefore.length();
}
String cache = html.substring(index, html.length());
int end = cache.length();
if(StringUtils.isNotEmpty(rightBefore)) {
end = cache.indexOf(rightBefore);
if(end<0) {
throw new CustomException("返回数据找不到RB值!RB:"+rightBefore);
}
}
result = cache.substring(0, end);
}catch(Exception e){
throw new CustomException("找不到值!LB:"+leftBefore+";RB:"+rightBefore+";异常:"+e.getMessage());
}
return result;
}
}
}
}
HttpResult.java
package com.test.http;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import org.apache.log4j.Logger;
import com.test.http.util.StringUtils;
public class HttpResult implements Serializable {
private static Logger log = Logger.getLogger(HttpResult.class);
private static final long serialVersionUID = 1L;
/**
* 请求过程中的链接
*/
private String requestUrl;
/**
* 请求头信息
*/
private String requestHeader;
/**
* 响应头信息
*/
private String responseHeader;
/**
* 响应正文
*/
private String responseBody;
/**
* 响应状态码
*/
private int statusCode;
/**
* 响应内容编码
*/
private String charSet;
public String getRequestUrl() {
return requestUrl;
}
public void setRequestUrl(String requestUrl) {
this.requestUrl = requestUrl;
}
public String getRequestHeader() {
return requestHeader;
}
public void setRequestHeader(String requestHeader) {
this.requestHeader = requestHeader;
}
public String getResponseHeader() {
return responseHeader;
}
public void setResponseHeader(String responseHeader) {
this.responseHeader = responseHeader;
}
public String getResponseBody() {
return responseBody;
}
public void setResponseBody(String responseBody) {
this.responseBody = responseBody;
}
public int getStatusCode() {
return statusCode;
}
public void setStatusCode(int statusCode) {
this.statusCode = statusCode;
}
public String getCharSet() {
return charSet;
}
public void setCharSet(String charSet) {
this.charSet = charSet;
}
public String toString() {
StringBuilder str = new StringBuilder();
str.append("请求连接:\r\n").append(this.requestUrl);
str.append("\r\n请求头:\r\n").append(this.requestHeader);
str.append("\r\n响应头:\r\n").append(this.responseHeader);
str.append("\r\n状态码:").append(this.statusCode);
str.append("\r\n响应内容: \r\n").append(this.responseBody);
return str.toString();
}
/**
* 内容输出到日志
* @param debugLogPath 日志文件夹路径
* @param charSet html文件编码
* @throws IOException
*/
public void log(String debugLogPath) {
String coding = "UTF-8";
if(StringUtils.isNotEmpty(this.charSet)) {
coding = this.charSet;
}
log.info("请求连接:\r\n"+this.requestUrl);
log.info("请求头:\r\n"+this.requestHeader);
log.info("响应头:\r\n"+this.responseHeader);
log.info("状态码:"+this.statusCode);
if(StringUtils.isNotEmpty(debugLogPath) && responseBody.indexOf("<html")!=-1) {
long timeMillis = System.currentTimeMillis();
File dir = new File(debugLogPath);
if(!dir.exists()) {
dir.mkdirs();
}
String filePath = debugLogPath + File.separator + timeMillis + ".html";
log.info("响应内容路径:" + filePath);
File file = new File(filePath);
try(OutputStream outputStream = new FileOutputStream(file, false);
OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream, coding)){
outputStreamWriter.append(this.responseBody);
outputStreamWriter.flush();
}catch (Exception e) {
log.info("响应内容:\r\n" + this.responseBody);
log.info("日志目录:" + debugLogPath);
log.error("响应内容输出到文件失败", e);
}
}else {
log.info("响应内容:\r\n" + this.responseBody);
log.info("日志目录:" + debugLogPath);
}
}
}
Request.java
package com.test.http;
import java.io.Serializable;
import java.util.Map;
import org.apache.http.client.HttpClient;
/**
* Http请求实体类
* @author youxiang
*
*/
public class Request implements Serializable {
private static final long serialVersionUID = 1L;
/**
* HttpClient对象
*/
private HttpClient httpClient;
/**
* 请求链接
*/
private String url;
/**
* 请求头
*/
private Map<String, String> headers;
/**
* 请求参数
*/
private Map<String, String> values;
/**
* 请求body内容
*/
private String bodyData;
public HttpClient getHttpClient() {
return httpClient;
}
public void setHttpClient(HttpClient httpClient) {
this.httpClient = httpClient;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Map<String, String> getHeaders() {
return headers;
}
public void setHeaders(Map<String, String> headers) {
this.headers = headers;
}
public Map<String, String> getValues() {
return values;
}
public void setValues(Map<String, String> values) {
this.values = values;
}
public String getBodyData() {
return bodyData;
}
public void setBodyData(String bodyData) {
this.bodyData = bodyData;
}
public static long getSerialversionuid() {
return serialVersionUID;
}
}
PostRequest.java
package com.test.http;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicNameValuePair;
import org.apache.log4j.Logger;
import com.test.http.util.HttpUtils;
import com.test.http.util.StringUtils;
public class PostRequest {
private static Logger log = Logger.getLogger(PostRequest.class);
private PostRequest() {
throw new IllegalStateException("Utility class");
}
/**
* POST请求
* @param request
* {
* httpClient : HttpClient对象
* url : 请求链接
* headers : 请求头
* values : 请求参数
* bodyData : body请求参数
* }
* @return HttpClientContext
* @throws IOException
*/
public static HttpClientContext execute(Request request) throws IOException {
HttpClientContext httpContext = HttpClientContext.create();
HttpClient httpClient = request.getHttpClient();
if(httpClient==null) {
log.info("HttpClient对象为空,自动获取新HttpClient对象");
httpClient = HttpUtils.getHttpClient();
}
HttpPost post = new HttpPost(request.getUrl());
//设置请求头
PostRequest.setHeader(post, request.getHeaders());
PostRequest.setValue(post, request.getValues());
PostRequest.setBodyValue(post, request.getBodyData());
httpClient.execute(post, httpContext);
return httpContext;
}
/**
* post设置请求头
* @param post
* @param headers
* @throws Exception
*/
private static void setHeader(HttpPost post, Map<String, String> headers) {
if (headers != null && !headers.isEmpty()) {
if(!headers.containsKey("User-Agent")){
post.addHeader("User-Agent", HttpUtils.USER_AGENT_VALUE);
}
for(Map.Entry<String, String> header_entry : headers.entrySet()){
post.addHeader(header_entry.getKey(), header_entry.getValue());
}
}
}
/**
* post设置数据
* @param post
* @param values
* @throws UnsupportedEncodingException
* @throws Exception
*/
private static void setValue(HttpPost post, Map<String, String> values) throws UnsupportedEncodingException {
if (values != null && !values.isEmpty()) {
List<NameValuePair> nvps = new ArrayList<>();
for(Map.Entry<String, String> value_entry : values.entrySet()){
nvps.add(new BasicNameValuePair(value_entry.getKey(), value_entry.getValue()));
}
post.setEntity(new UrlEncodedFormEntity(nvps, HttpUtils.HTTP_PARAMS_CODING));
}
}
/**
* postBody设置数据
* @param post
* @param bodyValue
* @throws Exception
*/
private static void setBodyValue(HttpPost post, String bodyValue) {
if(StringUtils.isNotEmpty(bodyValue)) {
post.addHeader("Accept", "application/json");
post.addHeader("Content-Type", "application/json; charset="+HttpUtils.HTTP_PARAMS_CODING);
StringEntity entity = new StringEntity(bodyValue, Charset.forName(HttpUtils.HTTP_PARAMS_CODING));
entity.setContentEncoding(HttpUtils.HTTP_PARAMS_CODING);
entity.setContentType("application/json");// 设置为 json数据
post.setEntity(entity);
}
}
}
GetRequest.java
package com.test.http;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.log4j.Logger;
import com.test.http.util.HttpUtils;
public class GetRequest {
private static Logger log = Logger.getLogger(GetRequest.class);
private GetRequest() {
throw new IllegalStateException("Utility class");
}
/**
* GET请求
* @param request
* {
* httpClient : HttpClient对象
* url : 请求链接
* headers : 请求头
* values : 请求参数
* }
* @return HttpClientContext
* @throws IOException
*/
public static HttpClientContext execute(Request request) throws IOException {
HttpClientContext httpContext = HttpClientContext.create();
HttpClient httpClient = request.getHttpClient();
if(httpClient==null) {
log.info("HttpClient对象为空,自动获取新HttpClient对象");
httpClient = HttpUtils.getHttpClient();
}
Map<String, String> values = request.getValues();
String url=GetRequest.setValue(request.getUrl(), values);
HttpGet get = new HttpGet(url);
//设置请求头
GetRequest.setHeader(get, request.getHeaders());
httpClient.execute(get, httpContext);
return httpContext;
}
/**
* get设置请求头
* @param get
* @param headers
* @throws Exception
*/
private static void setHeader(HttpGet get, Map<String, String> headers) {
if (headers != null && !headers.isEmpty()) {
if(!headers.containsKey("User-Agent")){
get.addHeader("User-Agent", HttpUtils.USER_AGENT_VALUE);
}
for(Map.Entry<String, String> header_entry : headers.entrySet()){
get.addHeader(header_entry.getKey(), header_entry.getValue());
}
}
}
/**
* get设置数据
* @param get
* @param values
* @throws UnsupportedEncodingException
* @throws Exception
*/
private static String setValue(String url, Map<String, String> values) throws UnsupportedEncodingException {
StringBuilder param = new StringBuilder(url);
if (values != null && !values.isEmpty()) {
for(Map.Entry<String, String> value_entry : values.entrySet()){
if(param.length()==0 || param.indexOf("?") == -1){
param.append("?"+URLEncoder.encode(value_entry.getKey(), HttpUtils.HTTP_PARAMS_CODING)+"="+URLEncoder.encode(value_entry.getValue(), HttpUtils.HTTP_PARAMS_CODING));
}else{
param.append("&"+URLEncoder.encode(value_entry.getKey(), HttpUtils.HTTP_PARAMS_CODING)+"="+URLEncoder.encode(value_entry.getValue(), HttpUtils.HTTP_PARAMS_CODING));
}
}
}
return param.toString();
}
}
测试方法
public static void main(String[] args) throws IOException {
HttpClient httpclient = HttpUtils.getHttpClient();
//String url = "http://192.168.30.122:8080/risk-war";
String url = "https://www.baidu.com";
Map<String, String> headers=new HashMap<String, String>();
//headers.put("Cookie", "JSESSIONID=50CA8625B613A4A335D971B8A3500D17");
Map<String, String> values=new HashMap<String, String>();
values.put("username", "youxiang");
values.put("password", "123456");
values.put("referer", "");
Request request = new Request();
request.setHeaders(headers);
request.setHttpClient(httpclient);
request.setUrl(url);
request.setValues(values);
HttpClientContext httpContext = GetRequest.execute(request);
HttpResult result = HttpUtils.getHttpInfo(httpContext);
System.out.println(result.toString());
}