java 爬取ssl加密页面_一个爬取https和http通用的工具类(JDK自带的URL的用法)

packagecn.qlq.craw.Jsoup;importjava.io.File;importjava.io.FileWriter;importjava.io.Writer;importjava.net.MalformedURLException;importjava.net.URL;importjava.security.SecureRandom;importjava.security.cert.CertificateException;importjava.security.cert.X509Certificate;importjava.util.Map;importjavax.net.ssl.HostnameVerifier;importjavax.net.ssl.HttpsURLConnection;importjavax.net.ssl.SSLContext;importjavax.net.ssl.SSLSession;importjavax.net.ssl.X509TrustManager;importorg.jsoup.Connection;importorg.jsoup.Jsoup;importorg.jsoup.helper.HttpConnection;importorg.jsoup.nodes.Document;public classHttpCommonUtil {public static voidtrustEveryone() {try{

HttpsURLConnection.setDefaultHostnameVerifier(newHostnameVerifier() {public booleanverify(String hostname, SSLSession session) {return true;

}

});

SSLContext context= SSLContext.getInstance("TLS");

context.init(null, new X509TrustManager[] { newX509TrustManager() {public void checkClientTrusted(X509Certificate[] chain, String authType) throwsCertificateException {

}public void checkServerTrusted(X509Certificate[] chain, String authType) throwsCertificateException {

}publicX509Certificate[] getAcceptedIssuers() {return new X509Certificate[0];

}

} },newSecureRandom());

HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());

}catch(Exception e) {

e.printStackTrace();

}

}public static Object getHttpHeaders(URL url, inttimeout) {try{

trustEveryone();

Connection conn=HttpConnection.connect(url);

conn.timeout(timeout);

conn.header("Accept-Encoding", "gzip,deflate,sdch");

conn.header("Connection", "close");

conn.get();//String result=conn.response().body();

Map result =conn.response().headers();

result.put("title", conn.response().parse().title());returnresult;

}catch(Exception e) {

e.printStackTrace();

}return null;

}public static Object getHttpBody(URL url, inttimeout) {try{

trustEveryone();

Connection conn=HttpConnection.connect(url);

conn.timeout(timeout);

conn.header("Accept-Encoding", "gzip,deflate,sdch");

conn.header("Connection", "close");

conn.get();//String result=conn.response().body();//String result = conn.response().body();

String result =conn.response().body();

File file= new File("C:\\Users\\liqiang\\Desktop\\实习\\python\\javaCrawPicture\\tianmao.html");if(!file.exists()){

file.createNewFile();

}else{

file.delete();

}

file.createNewFile();

Writer fileWriter= newFileWriter(file);

fileWriter.write(result);

fileWriter.close();returnresult;

}catch(Exception e) {

e.printStackTrace();

}return null;

}public static voidmain(String[] args) {try{

URL url= new URL("http", "www.tmall.com", -1, "");

System.out.println(getHttpBody(url,100000));

}catch(MalformedURLException e) {

e.printStackTrace();

}

}

}