spider工具类

package com.util;


import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.List;
import javax.net.ssl.SSLContext;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;


/**
 * httpclient4.3 爬虫工具类
 * @author ssn
 *
 */
public class SpiderUtil {
public static final int CONNECT_TIMEOUT = 10* 1000;// 连接时间
public static final int READ_TIMEOUT = 60 * 1000;// 读1分钟秒超时



/**
* 获取正文内容
* @param url 请求地址
* @param charset 字符集编码
* @return
*/
public static String getContentByPostWithRef(String url,String charset,String cookies, List<NameValuePair> nvps,String ref){
//创建HttpClientBuilder  
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();  
//HttpClient  
CloseableHttpClient closeableHttpClient = httpClientBuilder.build();  
//创建httpget 请求
HttpPost httpPost = new HttpPost(url);


//设置表单提交编码为UTF-8
try {
if(nvps!=null)
httpPost.setEntity(new UrlEncodedFormEntity(nvps, HTTP.UTF_8));
} catch (UnsupportedEncodingException e1) {
e1.printStackTrace();
}
httpPost.setHeader("Content-Type","application/x-www-form-urlencoded");
httpPost.setHeader("Host","www.tracepartsonline.net");
httpPost.setHeader("Origin","www.tracepartsonline.net");
httpPost.setHeader("X-Requested-With","XMLHttpRequest");
httpPost.setHeader("Connection", "keep-alive");
httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1653.0 Safari/537.36");
httpPost.setHeader("Referer",ref);
if(cookies!=null){
//头部信息
httpPost.setHeader("Cookie", cookies);
}
//创建字符串 接受返回内容
String content ="";
try {
//执行get请求
HttpResponse httpResponse = closeableHttpClient.execute(httpPost);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
System.err.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
content = EntityUtils.toString(entity,charset);
}
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
closeableHttpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}

/**
* 获取正文内容
* @param url 请求地址
* @param charset 字符集编码
* @return
*/
public static String getContentByPost(String url,String charset,String cookies, List<NameValuePair> nvps){
//创建HttpClientBuilder  
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();  
//HttpClient  
CloseableHttpClient closeableHttpClient = httpClientBuilder.build();  
//创建httpget 请求
HttpPost httpPost = new HttpPost(url);


//设置表单提交编码为UTF-8
try {
if(nvps!=null)
httpPost.setEntity(new UrlEncodedFormEntity(nvps, HTTP.UTF_8));
} catch (UnsupportedEncodingException e1) {
e1.printStackTrace();
}
httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1653.0 Safari/537.36");
if(cookies!=null){
//头部信息
httpPost.setHeader("Cookie", cookies);
}
//创建字符串 接受返回内容
String content ="";
try {
//执行get请求
HttpResponse httpResponse = closeableHttpClient.execute(httpPost);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
System.err.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
content = EntityUtils.toString(entity,charset);
}
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
closeableHttpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}






/**
* 获取正文内容
* @param url 请求地址
* @param charset 字符集编码
* @return
*/
public static String getContentByPost_1(String url,String charset,String cookies,String ip,int host, List<NameValuePair> nvps){


HttpHost proxy = new HttpHost(ip, host);
DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);  
CloseableHttpClient closeableHttpClient = HttpClients.custom().setRoutePlanner(routePlanner).build();  




HttpPost httpPost = new HttpPost(url);


//设置表单提交编码为UTF-8
try {
httpPost.setEntity(new UrlEncodedFormEntity(nvps, HTTP.UTF_8));
} catch (UnsupportedEncodingException e1) {
e1.printStackTrace();
}
httpPost.setHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");


RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(10000).setConnectTimeout(10000).build();
httpPost.setConfig(requestConfig);
httpPost.setHeader("Connection", "close"); 
httpPost.setHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");
if(cookies!=null){
//头部信息
httpPost.setHeader("Cookie", cookies);
}
//创建字符串 接受返回内容
String content ="";
try {
//执行get请求
HttpResponse httpResponse = closeableHttpClient.execute(proxy,httpPost);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
// System.out.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
content = EntityUtils.toString(entity,charset);
}
} catch (Exception e) {
System.out.println("代理失效，换下一个代理");
}finally{
try {
closeableHttpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}


/**
* 获取正文内容
* @param url 请求地址
* @param charset 字符集编码
* @return
*/
public static String getContentByGet(String url,String charset,String cookies,String ip,int host){


HttpHost proxy = new HttpHost(ip, host);
DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);  
CloseableHttpClient closeableHttpClient = HttpClients.custom().setRoutePlanner(routePlanner).build();  


HttpGet httpGet = new HttpGet(url);


RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(10000).setConnectTimeout(10000).build();
httpGet.setConfig(requestConfig);
httpGet.setHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");
// httpGet.setHeader("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
// httpGet.setHeader("Accept-Language", "zh-cn,zh;q=0.5");
// httpGet.setHeader("Accept","text ml,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// httpGet.setHeader("Accept-Encoding", "gzip,deflate");


if(cookies!=null){
//头部信息
httpGet.setHeader("cookie", cookies);
}
//创建字符串 接受返回内容
String content ="";
try {
//执行get请求
HttpResponse httpResponse = closeableHttpClient.execute(proxy,httpGet);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
// System.out.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
content = EntityUtils.toString(entity,charset);
}
} catch (Exception e) {
System.out.println("代理失效，换下一个代理");
}finally{
try {
closeableHttpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}






public static String getContentByGet(String url,String charset,String cookies){
//创建HttpClientBuilder  
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();  
//HttpClient  
CloseableHttpClient closeableHttpClient = httpClientBuilder.build();  
HttpGet httpGet = new HttpGet(url);
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(15*1000).setConnectTimeout(15*1000).build();
httpGet.setConfig(requestConfig);
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1653.0 Safari/537.36");


if(cookies!=null){
//头部信息
httpGet.setHeader("cookie", cookies);
}
//创建字符串 接受返回内容
String content ="";
try {
//执行get请求
HttpResponse httpResponse = closeableHttpClient.execute(httpGet);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
System.out.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
content = EntityUtils.toString(entity,charset);
}
} catch (Exception e) {
e.printStackTrace();
content = "";
}finally{
try {
closeableHttpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}


public static String getContentByGetWithReferer(String url,String charset,String cookies,String referer){
//创建HttpClientBuilder  
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();  
//HttpClient  
CloseableHttpClient closeableHttpClient = httpClientBuilder.build();  
HttpGet httpGet = new HttpGet(url);
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(10000).setConnectTimeout(10000).build();
httpGet.setConfig(requestConfig);
httpGet.setHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");


if(cookies!=null){
//头部信息
httpGet.setHeader("cookie", cookies);
}
httpGet.setHeader("Referer",referer);
// httpGet.setHeader("host","www.china-weldnet.com");
//创建字符串 接受返回内容
String content ="";
try {
//执行get请求
HttpResponse httpResponse = closeableHttpClient.execute(httpGet);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
System.err.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
content = EntityUtils.toString(entity,charset);
}
} catch (Exception e) {
e.printStackTrace();
content = "";
}finally{
try {
closeableHttpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}


/**
* httpclient4 绕过https证书验证方法
* @return
*/
public static CloseableHttpClient createSSLClientDefault(RequestConfig globalConfig,CookieStore cookieStore ){
try {
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] chain,
String authType) throws CertificateException {
return true;
}
}).build();
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
return HttpClients.custom().setDefaultRequestConfig(globalConfig).setDefaultCookieStore(cookieStore).setSSLSocketFactory(sslsf).build();
} catch (KeyManagementException e) {
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
} catch (KeyStoreException e) {
e.printStackTrace();
}
return  HttpClients.createDefault();
}


public static String getCookiesGet(String url) throws Exception{
CookieStore cookieStore = new BasicCookieStore();
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(cookieStore);
RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();
CloseableHttpClient httpClient = SpiderUtil.createSSLClientDefault(globalConfig,cookieStore);


String cookies ="";
HttpGet httpGet = new HttpGet(url);


//设置表单提交编码为UTF-8


httpGet.setHeader("Connection", "close"); 
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1653.0 Safari/537.36");
//创建字符串 接受返回内容
try {
//执行get请求
HttpResponse httpResponse = httpClient.execute(httpGet);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
System.out.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
List<Cookie> cookiesList= cookieStore.getCookies();
for (int i = 0; i < cookiesList.size(); i++) {
cookies = cookies +cookiesList.get(i).getName()+"="+cookiesList.get(i).getValue()+";";
}  
cookies = cookies+"CFID=36";
}
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return cookies;


}

public static String getCookies(String url,List<NameValuePair> nvps) throws Exception{
CookieStore cookieStore = new BasicCookieStore();
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(cookieStore);
RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();
CloseableHttpClient httpClient = SpiderUtil.createSSLClientDefault(globalConfig,cookieStore);


String cookies ="";
HttpPost httpPost = new HttpPost(url);


//设置表单提交编码为UTF-8
try {
httpPost.setEntity(new UrlEncodedFormEntity(nvps, HTTP.UTF_8));
} catch (UnsupportedEncodingException e1) {
e1.printStackTrace();
}
httpPost.setHeader("Connection", "close"); 
httpPost.setHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");
//创建字符串 接受返回内容
try {
//执行get请求
HttpResponse httpResponse = httpClient.execute(httpPost);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
//响应状态
// System.out.println("status:"+httpResponse.getStatusLine());
//判断响应实体是否为空
if(entity != null){
List<Cookie> cookiesList= cookieStore.getCookies();
for (int i = 0; i < cookiesList.size(); i++) {
cookies = cookies +cookiesList.get(i).getName()+"="+cookiesList.get(i).getValue()+";";
}  
}
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return cookies;
}
public static void downloadPDF(String url,String ip,int host,String file_path,String file_name) throws Exception{  


HttpHost proxy = new HttpHost(ip, host);
DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);  
CloseableHttpClient closeableHttpClient = HttpClients.custom().setRoutePlanner(routePlanner).build();  


HttpGet httpGet = new HttpGet(url);


RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(10000).setConnectTimeout(10000).build();
httpGet.setConfig(requestConfig);
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();  


HttpResponse httpResponse = closeableHttpClient.execute(proxy,httpGet);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
InputStream in = entity.getContent();
File file = new File(file_path+"/"+file_name);
try {
FileOutputStream fout = new FileOutputStream(file);
int l = -1;
byte[] tmp = new byte[1024];
while ((l = in.read(tmp)) != -1) {
fout.write(tmp, 0, l);
// 注意这里如果用OutputStream.write(buff)的话，图片会失真，大家可以试试
}
fout.flush();
fout.close();
} finally {
// 关闭低层流。
in.close();
}
closeableHttpClient.close();
}  


public static void download_file(String url,String file_path,String file_name) throws Exception{  


HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();  
//HttpClient  
CloseableHttpClient closeableHttpClient = httpClientBuilder.build();  


HttpGet httpGet = new HttpGet(url);
HttpResponse httpResponse = closeableHttpClient.execute(httpGet);
//获取响应消息实体
HttpEntity entity = httpResponse.getEntity();
InputStream in = entity.getContent();
File file = new File(file_path+"/"+file_name);
try {
FileOutputStream fout = new FileOutputStream(file);
int l = -1;
byte[] tmp = new byte[1024];
while ((l = in.read(tmp)) != -1) {
fout.write(tmp, 0, l);
// 注意这里如果用OutputStream.write(buff)的话，图片会失真，大家可以试试
}
fout.flush();
fout.close();
} finally {
// 关闭低层流。
in.close();
}
closeableHttpClient.close();
}  
}