一个爬取https和http通用的工具类(JDK自带的URL的用法)

 

　　今天在java爬取天猫的时候因为ssl报错，所以从网上找了一个可以爬取https和http通用的工具类。但是有的时候此工具类爬到的数据不全，此处不得不说python爬虫很厉害。

package cn.qlq.craw.Jsoup;

import java.io.File;
import java.io.FileWriter;
import java.io.Writer;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Map;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.X509TrustManager;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.helper.HttpConnection;
import org.jsoup.nodes.Document;  
  
public class HttpCommonUtil {  
      
    public static void trustEveryone() {   
        try {    
            HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {    
                public boolean verify(String hostname, SSLSession session) {    
                    return true;    
                }    
            });    
    
            SSLContext context = SSLContext.getInstance("TLS");    
            context.init(null, new X509TrustManager[] { new X509TrustManager() {    
                public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {    
                }    
    
                public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {    
                }    
    
                public X509Certificate[] getAcceptedIssuers() {    
                    return new X509Certificate[0];    
                }    
            } }, new SecureRandom());    
            HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());    
        } catch (Exception e) {    
            e.printStackTrace();    
        }    
    }    
    
    public static Object getHttpHeaders(URL url, int timeout) {    
        try {    
            trustEveryone();   
            Connection conn = HttpConnection.connect(url);    
            conn.timeout(timeout);    
            conn.header("Accept-Encoding", "gzip,deflate,sdch");    
            conn.header("Connection", "close");    
            conn.get();    
            //String result=conn.response().body();  
            Map<String, String> result = conn.response().headers();    
            result.put("title", conn.response().parse().title());    
            return result;   
        } catch (Exception e) {    
            e.printStackTrace();    
        }    
        return null;    
    }    
    
    public static Object getHttpBody(URL url, int timeout) {    
        try {    
            trustEveryone();   
            Connection conn = HttpConnection.connect(url);    
            conn.timeout(timeout);    
            conn.header("Accept-Encoding", "gzip,deflate,sdch");    
            conn.header("Connection", "close");    
            conn.get();    
            //String result=conn.response().body();  
//            String result = conn.response().body();    
            String result = conn.response().body();    
            File file = new File("C:\\Users\\liqiang\\Desktop\\实习\\python\\javaCrawPicture\\tianmao.html");
            if(!file.exists()){
                file.createNewFile();
            }else{
                file.delete();
            }
            file.createNewFile();
            Writer fileWriter = new FileWriter(file); 
            fileWriter.write(result);
            fileWriter.close();
            return result;   
        } catch (Exception e) {    
            e.printStackTrace();    
        }    
        return null;    
    }    
    
    
    
    public static void main(String[] args) {    
        try {    
            URL url = new URL("http", "www.tmall.com", -1, "");   
            System.out.println(getHttpBody(url, 100000));
        } catch (MalformedURLException e) {    
            e.printStackTrace();    
        }    
    }    
}