java爬虫绕过登录页面

转自：http://blog.csdn.net/jiangsanfeng1111/article/details/51282966

当我们在写爬虫的时候，往往会遇到很多反爬的问题。

    比如：在登录页面设置验证码、扫描二维码登录、滑动鼠标登录、手机短信验证码登录等等。这里介绍一种个人已经实现的方法——绕过登录页面。这里的绕过不是说真的可以绕过登录，除非这个系统本来就有问题，这是这个系统天大的bug。这里说的绕过登录是指登录一次记住cookie信息，下次登录的时候就直接跳过了登录的页面。废话不多说，直接上代码：

[java] view plain copy

1. package com.xiaojiang.spidertest;  
2.   
3. import java.io.ByteArrayInputStream;  
4. import java.io.ByteArrayOutputStream;  
5. import java.io.IOException;  
6. import java.net.HttpCookie;  
7. import java.util.zip.GZIPInputStream;  
8.   
9. import org.apache.commons.io.IOUtils;  
10. import org.apache.http.Header;  
11. import org.apache.http.HttpResponse;  
12. import org.apache.http.client.ClientProtocolException;  
13. import org.apache.http.client.methods.HttpGet;  
14. import org.apache.http.impl.client.DefaultHttpClient;  
15. import org.apache.http.message.BasicHeader;  
16. import org.apache.http.util.EntityUtils;  
17. import org.jsoup.Jsoup;  
18. import org.jsoup.nodes.Document;  
19.   
20. import com.xiaojiang.exception.DataTaskException;  
21. import com.xiaojiang.httpclient.HttpUserAgent;  
22.   
23. public class CookieTest {  
24.   
25.     public static void main(String[] args) throws Exception, IOException {  
26.           
27.         DefaultHttpClient client = new DefaultHttpClient();  
28.         HttpResponse response = null;  
29.           
30.         String newUrl = "http://www.dajie.com/home";  
31.         HttpGet httpGet = new HttpGet(newUrl);  
32.           
33.         //在页面控制台执行document.cookie  
34.         String cookie = "DJ_RF=empty; DJ_EU=http%3A%2F%2Fwww.dajie.com%2Fhome; DJ_UVID=MTQ2MTkwNzk3NDU4MTg1NjQ2; dj_cap=0564c054acc1ce12402998471ae0af54; regSucceedType=email; dj_auth_v3=MrZrP3TGNRNXCNiOpQY7Ggscf4kjfEEsJzFPDzu3iwi5XtG9tS3Sw-WgChC2DVKL; uchome_loginuser=35375099; USER_ACTION=\"request^AProfessional^AREG^Aregm:crt0^A-\"; send_verify_mail=961254858%40qq.com; login_email=961254858%40qq.com; inbound_tag=true";  
35.         httpGet.addHeader(new BasicHeader("Cookie", cookie));  
36.           
37.         httpGet.setHeader("Accept-Language", "zh-cn,zh;q=0.5");  
38.         httpGet.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");  
39.         httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");  
40.         httpGet.setHeader("Accept-Encoding", "gzip, deflate");  
41.         httpGet.setHeader("User-Agent", HttpUserAgent.get());   
42.           
43.         response = client.execute(httpGet);  
44.         String html = formatResponse(response);  
45.         Document doc = Jsoup.parse(html);  
46.         String text = doc.select(".feed-header").get(0).text();  
47.           
48.         System.out.println(text);  
49.           
50.         httpGet.releaseConnection();  
51.           
52.     }  
53.       
54.     private static String formatResponse(HttpResponse response) throws Exception {  
55.           
56.         ByteArrayInputStream bis = null;  
57.         Header contentEncoding = response.getFirstHeader("Content-Encoding");  
58.           
59.         if(contentEncoding == null){  
60.             return EntityUtils.toString(response.getEntity(),"UTF-8");  
61.         } else {  
62.               
63.             String charset = "utf-8";  
64.             Header contentType = response.getFirstHeader("Content-Type");  
65.               
66.             if(contentType != null){  
67.                 String contentTypeStr = contentType.getValue();  
68.                 if(contentTypeStr != null && !"".equals(contentTypeStr)){  
69.                     charset = contentTypeStr.substring(contentTypeStr.indexOf("=") + 1,contentTypeStr.length());  
70.                 }  
71.             }  
72.             String contentEncodingType = contentEncoding.getValue();  
73.             if(contentEncodingType.equalsIgnoreCase("gzip")){  
74.                 if(response.toString().contains("soufun"))  
75.                     charset = "gb2312";  
76.                   
77.                 byte[] bytes = IOUtils.toByteArray(response.getEntity().getContent());  
78.                 bis = new ByteArrayInputStream(bytes);  
79.                   
80.                 return uncompress(bis ,charset);  
81.             }  
82.               
83.         }  
84.         return null;  
85.     }  
86.   
87.     /**  
88.      * GZIP解压  
89.      */  
90.     private static String uncompress(ByteArrayInputStream in, String charset) {  
91.   
92.         ByteArrayOutputStream out = new ByteArrayOutputStream();  
93.           
94.         try {  
95.             GZIPInputStream gunzip = new GZIPInputStream(in);  
96.             byte[] buffer = new byte[256];  
97.             int n;  
98.             while((n = gunzip.read(buffer)) >=0 ){  
99.                 out.write(buffer, 0, n);  
100.             }  
101.             return out.toString(charset);  
102.               
103.         } catch (IOException e) {  
104.             e.printStackTrace();  
105.         }  
106.         return null;  
107.     }  
108.       
109. }