java如果去请求一个网页的数据

我们可以通过在java程序中模拟浏览器一样，把数据抓下来，具体方法是在java程序中set header和cookie,下面是一个例子:

public class NetConnection {

	public static final int MAX_HOTWORDS_FILE_SIZE = 256 * 1024;
	public static void main(String[] args) {
		send("http://tuan.aibang.com/shenzhen/new_2033549.html");
	}
	
	static  void send(String _url){
		HttpClient http = new HttpClient();
		http.getHttpConnectionManager().getParams().setConnectionTimeout(
				100000);
		GetMethod get = new GetMethod(_url);
		get.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 100000);
		get.setFollowRedirects(false);
		InputStream is;
		String host = "www.aibang.com";
		  try {
	    	  get.setRequestHeader("Host",host);
	   			get.setRequestHeader(
	       			"user-agent",
	       			"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8");
				get.setRequestHeader(
				"Accept",
				"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
	                	get.setRequestHeader("Accept-Language","zh-cn,zh;q=0.5");
	               		get.setRequestHeader("Accept-Charset","GB2312,utf-8;q=0.7,*;q=0.7");
				get.setRequestHeader("Connection","Keep-Alive");  
				get.setRequestHeader("Cookie","582081171805; cy=1; __utma=205923334.3209590505032285000.1256126987.1269858466.1270605495.11; __utmz=205923334.1256126987.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ano=0snUeoHWygEkAAAAOTAyZjM3ZjQtNjA2NC00NWYzLWIxNWYtMjRlMDliZjAzYTM3jnk83_pKoSEk-9gwcIFx8jFOXVM1; sid=no4tpkmvedoj3gycvfz1s055; lb.dp=100729098.20480.0000; __utmb=205923334.1.10.1270605495; __utmc=205923334");
	    	  int er = 0;
	    	  er = http.executeMethod(get);
	    	  System.out.println("er="+er);
	    	  if (er == 200) {
	    		  try {
						is = get.getResponseBodyAsStream();
						ReadFile2(is);
					} catch (Exception e) {
						System.out.println("download error="+e);
					}
	    	  }
	    	  
		  }catch(Exception ex){
			  ex.printStackTrace();
		  }
	}
	
	//解析inputStream里的数据
	static void ReadFile2(InputStream inputStream){
		byte[] buffer = new byte[1024];
		StringBuffer bab = new StringBuffer();
		int nTotal = 0;
		
		try {
		do{
			int nBytesRead = inputStream.read(buffer);
			System.out.println("nBytesRead="+nBytesRead);
			if(nBytesRead > 0){
				bab.append(new String(buffer,0,nBytesRead));
				nTotal += nBytesRead;
			}else{
				System.out.println(String.format("download end. file size=%d", nTotal));
				break;
			}
			
		}while(nTotal < MAX_HOTWORDS_FILE_SIZE);
		System.out.println("bab = "+bab);
		
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
}

这里要引入这几个包

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;