今天在java爬取天猫的时候因为ssl报错,所以从网上找了一个可以爬取https和http通用的工具类。但是有的时候此工具类爬到的数据不全,此处不得不说python爬虫很厉害。
package cn.qlq.craw.Jsoup; import java.io.File; import java.io.FileWriter; import java.io.Writer; import java.net.MalformedURLException; import java.net.URL; import java.security.SecureRandom; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.Map; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import javax.net.ssl.X509TrustManager; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.helper.HttpConnection; import org.jsoup.nodes.Document; public class HttpCommonUtil { public static void trustEveryone() { try { HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { public boolean verify(String hostname, SSLSession session) { return true; } }); SSLContext context = SSLContext.getInstance("TLS"); context.init(null, new X509TrustManager[] { new X509TrustManager() { public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { } public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { } public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } } }, new SecureRandom()); HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory()); } catch (Exception e) { e.printStackTrace(); } } public static Object getHttpHeaders(URL url, int timeout) { try { trustEveryone(); Connection conn = HttpConnection.connect(url); conn.timeout(timeout); conn.header("Accept-Encoding", "gzip,deflate,sdch"); conn.header("Connection", "close"); conn.get(); //String result=conn.response().body(); Map<String, String> result = conn.response().headers(); result.put("title", conn.response().parse().title()); return result; } catch (Exception e) { e.printStackTrace(); } return null; } public static Object getHttpBody(URL url, int timeout) { try { trustEveryone(); Connection conn = HttpConnection.connect(url); conn.timeout(timeout); conn.header("Accept-Encoding", "gzip,deflate,sdch"); conn.header("Connection", "close"); conn.get(); //String result=conn.response().body(); // String result = conn.response().body(); String result = conn.response().body(); File file = new File("C:\\Users\\liqiang\\Desktop\\实习\\python\\javaCrawPicture\\tianmao.html"); if(!file.exists()){ file.createNewFile(); }else{ file.delete(); } file.createNewFile(); Writer fileWriter = new FileWriter(file); fileWriter.write(result); fileWriter.close(); return result; } catch (Exception e) { e.printStackTrace(); } return null; } public static void main(String[] args) { try { URL url = new URL("http", "www.tmall.com", -1, ""); System.out.println(getHttpBody(url, 100000)); } catch (MalformedURLException e) { e.printStackTrace(); } } }