HttpUtils
public class HttpUtils {
//声明httpclient管理器对象()
// private static PoolingHttpClientConnectionManager cm;
private static List<String> userAgentlist = null;
private static RequestConfig config =null;
static {
//cm = new PoolingHttpClientConnectionManager();
//cm.setMaxTotal(200);
//cm.setDefaultMaxPerRoute(20);
config =RequestConfig.custom()
.setSocketTimeout(10000)
.setConnectionRequestTimeout(10000)
.setConnectionRequestTimeout(10000)
.build();
userAgentlist =new ArrayList<>();
userAgentlist.add("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
userAgentlist.add("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763");
userAgentlist.add("Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
userAgentlist.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:65.0) Gecko/20100101 Firefox/65.0");
userAgentlist.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36");
}
public static String getHtml(String url){
CloseableHttpClient httpClient = SSLClient.createSSLClient();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(config);
httpGet.setHeader("User-Agent",userAgentlist.get(new Random().nextInt(userAgentlist.size())));
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200){
String html ="";
if (response.getEntity() !=null) {
html = EntityUtils.toString(response.getEntity(), "UTF-8");
}
return html;
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}
跳过SSL证书访问
public class SSLClient {
public static CloseableHttpClient createSSLClient(){
SSLContext sslContext = null;
try {
sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
//通过所有证书
return true;
}
}).build();
SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, new HostnameVerifier() {
@OverrideSS
public boolean verify(String s, SSLSession sslSession) {
//不验证hostname
return true;
}
});
return HttpClients.custom().setSSLSocketFactory(sslSocketFactory).build();
//如果产生了异常,就创建普通的client
}catch (Exception e){
e.printStackTrace();
}
return HttpClients.createDefault();
}
}
所使用的依赖
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.22</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.10.3</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
</dependency>