httpclient结合http报文查看工具的一个实例

from:http://blog.chinaunix.net/u2/62093/showart_484572.html

使用httpclient下载需要登录的网页续

上次写的 使用httpclient下载需要登录的网页 只是对最普通的登录页面进行模拟登陆的代码。网络上有些网站在处理登录时，为了保证安全经常把对登录过程的处理写入到js脚本中，上次的代码对于这样的网站时无能为力的。         为了解决个问题，我请教了网络上的一个朋友，终于得到了解决的方法。         要对这类网站进行模拟登陆，可以先使用一款查看HTTP报文信息工具（例如ieHttpHeaders, WSockExpert）来对一次正常的登录过程进行分析，看看在正常的登录过程中，客户端都向服务器端发送了哪些数据，服务器端返回了哪些信息，对于服务器端返回的信息应该如何处理。然后使用httpclient对正常的登录过程进行模拟就可以了。        下面我将以模拟登陆 www.fane.cn为例，具体讲讲应该如何做。         首先使用ieHttpHeaders得到登录 www.fane.cn时，客户端和服务器端是如何进行交互的。    POST /login.asp HTTP/1.1 Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint, */* Referer: http://www.fane.cn/ Accept-Language: zh-cn Content-Type: application/x-www-form-urlencoded Accept-Encoding: gzip, deflate User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) Host: www.fane.cn Content-Length: 65 Connection: Keep-Alive Cache-Control: no-cache Cookie: __utma=195664059.2123330255.1175915188.1176193830.1176253918.5; __utmz=195664059.1175915188.1.1.utmccn=(direct)|utmcsr=(direct)|utmcmd=(none); __utmb=195664059; __utmc=195664059; cms%5F2=cookies%5Ftrue=yes; ASPSESSIONIDCCCQSSQB=HMFPKNPBPCALFAFBLNDEIIOK chk=yes&re_log=yes&username=dongle2001&password=******&x=26&y=8 HTTP/1.1 302 Object moved Date: Wed, 11 Apr 2007 01:12:55 GMT Server: Microsoft-IIS/6.0 Location: http://www.fane.cn/ Content-Length: 140 Content-Type: text/html Set-Cookie: cms%5F2=user%5Flast%5Ftit=%D3%C3%BB%A7%B5%C7%C2%BD&user%5Flast%5Ftim=2007%2D04%2D11+09%3A12%3A55&iscookies=&guest%5Fname=&login%5Fusername=dongle2001&time%5Fload=2007%2D4%2D11+9%3A12%3A50&login%5Fpassword=c322fdf149b3dfca&login%5Fid=57305&cookies%5Ftrue=yes; path=/ Cache-control: private GET / HTTP/1.1 Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint, */* Referer: http://www.fane.cn/ Accept-Language: zh-cn Accept-Encoding: gzip, deflate User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) Host: www.fane.cn Connection: Keep-Alive Cache-Control: no-cache Cookie: __utma=195664059.2123330255.1175915188.1176193830.1176253918.5; __utmz=195664059.1175915188.1.1.utmccn=(direct)|utmcsr=(direct)|utmcmd=(none); __utmb=195664059; __utmc=195664059; cms%5F2=user%5Flast%5Ftit=%D3%C3%BB%A7%B5%C7%C2%BD&user%5Flast%5Ftim=2007%2D04%2D11+09%3A12%3A55&iscookies=&guest%5Fname=&login%5Fusername=dongle2001&time%5Fload=2007%2D4%2D11+9%3A12%3A50&login%5Fpassword=c322fdf149b3dfca&login%5Fid=57305&cookies%5Ftrue=yes; ASPSESSIONIDCCCQSSQB=HMFPKNPBPCALFAFBLNDEIIOK HTTP/1.1 200 OK Date: Wed, 11 Apr 2007 01:12:56 GMT Server: Microsoft-IIS/6.0 Content-Length: 67527 Content-Type: text/html Cache-control: private      有了这些信息就可以使用程序进行模拟登录了。我将把我写的java程序和网络上的朋友给我做参考的C#程序都贴在下面供大家参考。要说明的是，这种方法不一定对任何的网站都有效，由于网站设计不同，安全性不同，它对于登录过程的保护程度也不同，所以具体的问题还需要具体分析。 C#程序    //第一部分是获取cookie但是没有成功 //HttpWebRequest request = HttpWebRequest.Create "http://www.fane.cn/") as HttpWebRequest; //request.Method = "GET"; //request.Accept = "* /*"; //request.Headers.Add("Accept-Language", "en-CA,zh-cn;q=0.5"); //request.Headers.Add("UA-CPU", "x86"); //request.Headers.Add("Accept-Encoding", "gzip, deflate"); //request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727)"; //HttpWebResponse response = request.GetResponse() as HttpWebResponse; //CookieCollection cookies = response.Cookies; //response.Close(); //第二部分是提交，模拟我的IE7的提交过程，被注释掉的部分不是必需的，剩下的都是必须的 request = HttpWebRequest.Create("http://www.fane.cn/login.asp") as HttpWebRequest; request.Method = "POST"; //request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint, */*"; request.Referer = http://www.fane.cn/; //request.Headers.Add("Accept-Language", "en-CA,zh-cn;q=0.5"); request.ContentType = "application/x-www-form-urlencoded"; //request.Headers.Add("UA-CPU", "x86"); //request.Headers.Add("Accept-Encoding", "gzip, deflate"); //request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727)"; request.ContentLength = 63; //request.KeepAlive = true; request.CookieContainer = new CookieContainer(); request.CookieContainer.Add(new Cookie("cms%5F2", "user%5Flast%5Ftit=%B7%AD%D2%EB%D6%D0%B9%FA&user%5Flast%5Ftim=2007%2D04%2D12+11%3A05%3A03&iscookies=day&guest%5Fname=%D3%CE%BF%CD534970473&login%5Fusername=&login%5Fpassword=&login%5Fid=&cookies%5Ftrue=yes", "/", "www.fane.cn")); Stream stream = request.GetRequestStream(); Encoding encoding = new ASCIIEncoding(); byte[] bytes = encoding.GetBytes("chk=yes&re_log=yes&username=jiang925&password=********&x=27&y=9"); stream.Write(bytes, 0, bytes.Length); response = request.GetResponse() as HttpWebResponse; stream = response.GetResponseStream() ; //第三部分显示结果 StreamReader sr = new StreamReader(stream, Encoding.Default); txtResponse.Text = sr.ReadToEnd(); java程序   //模拟登陆   HttpClient client = new HttpClient();   client.getHostConfiguration().setHost( "www.fane.cn" , 80, "http" );   client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);   PostMethod post = new PostMethod( "/login.asp" );   NameValuePair chk = new NameValuePair( "chk" , "yes" );   NameValuePair re = new NameValuePair( "re_log" , "yes" );   NameValuePair name = new NameValuePair( "username" , "dongle2001" );   NameValuePair pass = new NameValuePair( "password" , "1a9i8h1a" );   NameValuePair x = new NameValuePair( "x" , "26" );   NameValuePair y = new NameValuePair( "y" , "8" );   post.setRequestHeader("Referer", "http://www.fane.cn/");   post.setRequestHeader("ContentType", "application/x-www-form-urlencoded");   post.setRequestHeader("Content-Length", "65");   post.setRequestHeader("Cookie", "cms%5F2=user%5Flast%5Ftit=%D3%C3%BB%A7%B5%C7%C2%BD&user%5Flast%5Ftim=2007%2D04%2D11+09%3A12%3A55&iscookies=&guest%5Fname=&login%5Fusername=&time%5Fload=2007%2D4%2D13+9%3A12%3A50&login%5Fpassword=&login%5Fid=&cookies%5Ftrue=yes; path=/");   post.setRequestBody( new NameValuePair[]{chk,re,name,pass,x,y/*,submit*/});   int status = client.executeMethod(post);   if (status == HttpStatus.SC_MOVED_TEMPORARILY || status == HttpStatus.SC_MOVED_PERMANENTLY) {             Header locationHeader = post.getResponseHeader("location");             if (locationHeader != null) {                 String redirectUri = locationHeader.getValue();                 if (redirectUri == null || "".equals(redirectUri)) {                     redirectUri = "/";                 }                 GetMethod get = new GetMethod(redirectUri);                 client.executeMethod(get);                 System.err.println("get redirect:");                 Header[] headers = get.getResponseHeaders();                 for (int i = 0; i < headers.length; ++i) {                     System.err.println(headers[i].toString());                 }                 BufferedReader bf = new BufferedReader(new InputStreamReader(get.getResponseBodyAsStream()));                 String s = null;                 while ((s=bf.readLine()) != null)                  System.out.println(s);                 get.releaseConnection();             }         }   //查看 cookie 信息   CookieSpec cookiespec = CookiePolicy.getDefaultSpec();   Cookie[] cookies = cookiespec.match("www.fane.cn", 80, "/" , false , client.getState().getCookies());   if (cookies.length == 0) {      System.out.println( "None" );    } else {    for ( int i = 0; i < cookies.length; i++) {       System.out.println(cookies[i].toString());    }   }