学习笔记4

获取请求链接的服务器头链接，无协议，给用户加协议，tomcat下获取webApp路径，传入文件名，删除不是同年月日的所有文件，判断网页的编码，爬虫完美绕过服务器反爬检查代码

获取请求链接的服务器头链接：String hostPort = request.getScheme() + “://” + request.getServerName() + “:” + request.getServerPort();

// 无协议，给用户加协议
        if (!url.contains("http")) {
            try {
                URL url0 = new URL("https://"+url);
                url0.openStream();
                url="https://"+url;
            } catch (Exception e) {
                    url="http://"+url;
            }
        }

tomcat下获取webApp路径：
nowpath=System.getProperty(“user.dir”);
tempdir=nowpath.replace(“bin”, “webapps”); //把bin 文件夹变到 webapps文件里面
删除文件夹下的文件

/**
     * 传入文件名，删除不是同年月日的所有文件
     * 
     * @param fileName
     */
    public void deleteOutDateJsp(String fileName) {
        String nowpath; //当前tomcat的bin目录的路径 如 D:\java\software\apache-tomcat-6.0.14\bin
        String tempdir;
        nowpath=System.getProperty("user.dir");
        tempdir=nowpath.replace("bin", "webapps"); //把bin 文件夹变到 webapps文件里面
        String date = fileName.substring(0, 10);
//      File folder = new File("src/main/webapp/static/temp/");//jetty
        File folder = new File(tempdir+"/datacrawl/static/temp/");//tomcat
        File[] files = folder.listFiles();
        for (int i = 0; i < files.length; i++) {
            if (files[i]!=null&&(files[i].getName().contains(date) || files[i].getName().equals("get_xpath.html"))) {
                continue;
            } else if (files[i]!=null){
                files[i].delete();
            }
        }
    }

判断网页的编码：

判断网页的编码：
public String findCharset(String htmlfileName) {
        BufferedReader bufReader = null;
        try {
            bufReader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(htmlfileName))));
            for (String tmp1 = null; (tmp1 = bufReader.readLine()) != null; tmp1 = null) {
                String tmp = new String(tmp1.toString());
                if (tmp.contains("meta") && tmp.contains("charset") && (tmp.contains("gbk") || tmp.contains("GBK"))) {
                    return "GBK";
                }
                if (tmp.contains("meta") && tmp.contains("charset")
                        && (tmp.contains("utf-8") || tmp.contains("UTF-8"))) {
                    return "UTF-8";
                }
                if (tmp.contains("meta") && tmp.contains("charset")
                        && (tmp.contains("gb2312") || tmp.contains("GB2312"))) {
                    return "GB2312";
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                bufReader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return "";
    }

爬虫完美绕过服务器反爬检查代码

HostnameVerifier hv = new HostnameVerifier() {  
                public boolean verify(String urlHostName, SSLSession session) {  
                    System.out.println("Warning: URL Host: " + urlHostName + " vs. "  
                                       + session.getPeerHost());  
                    return true;  
                }  
            }; 
            trustAllHttpsCertificates();
            HttpsURLConnection.setDefaultHostnameVerifier(hv);  
        // 插入取到的html代码
        try {
            URL url = new URL(string);
            URLConnection connection = url.openConnection();
            connection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
             connection.connect();
            /*URLConnection conn = url.openConnection();
            conn.setRequestProperty("User-Agent",
                    "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");*/
            isr=new InputStreamReader(connection.getInputStream(),Charset.forName("UTF-8")); 
            //isr=new InputStreamReader(conn.getInputStream(), "UTF-8");
            bufr = new BufferedReader(isr);
        } catch (Exception e) {
            e.printStackTrace();
        }
private void trustAllHttpsCertificates() {
        javax.net.ssl.TrustManager[] trustAllCerts = new javax.net.ssl.TrustManager[1];  
        javax.net.ssl.TrustManager tm = new miTM();  
        trustAllCerts[0] = tm;  
        javax.net.ssl.SSLContext sc = null;
        try {
            sc = javax.net.ssl.SSLContext  
                    .getInstance("SSL");
        } catch (NoSuchAlgorithmException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        }  
        try {
            sc.init(null, trustAllCerts, null);
        } catch (KeyManagementException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }  
        javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc  
                .getSocketFactory());

    }
     static class miTM implements javax.net.ssl.TrustManager,  
     javax.net.ssl.X509TrustManager {  
 public java.security.cert.X509Certificate[] getAcceptedIssuers() {  
     return null;  
 }  

 public boolean isServerTrusted(  
         java.security.cert.X509Certificate[] certs) {  
     return true;  
 }  

 public boolean isClientTrusted(  
         java.security.cert.X509Certificate[] certs) {  
     return true;  
 }  

 public void checkServerTrusted(  
         java.security.cert.X509Certificate[] certs, String authType)  
         throws java.security.cert.CertificateException {  
     return;  
 }  

 public void checkClientTrusted(  
         java.security.cert.X509Certificate[] certs, String authType)  
         throws java.security.cert.CertificateException {  
     return;  
 }  
}