使用URLConnection获得网页内容

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class HttpClient {

	public static final String METHOD_GET = "GET";

	public static final String METHOD_POST = "POST";

	private String url;

	private String cookie;

	HttpURLConnection httpConn = null;

	private boolean connected = false;

	private Map<String, String> requestHeader = new HashMap<String, String>();

	private static final String USER_AGENT = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)";

	public HttpClient(String url) {
		this(url, null);
	}

	public HttpClient(String url, String cookie) {
		this.url = url;
		this.cookie = cookie;
	}

	public void addRequestHeader(String key, String value) {
		this.requestHeader.put(key, value);
	}

	public void connect(String method) throws IOException {
		try {
			URLConnection con = new URL(url).openConnection();
			if (con instanceof HttpURLConnection) {
				httpConn = (HttpURLConnection) con;
			}
			if ("POST".equals(method)) {
				httpConn.setDoInput(true);
				httpConn.setDoOutput(true);
				httpConn.setRequestProperty("Content-Type",
						"application/x-www-form-urlencoded");
			}
			httpConn.setInstanceFollowRedirects(false);
			httpConn.setRequestMethod(method);
			if (cookie != null) {
				httpConn.addRequestProperty("Cookie", cookie);
			}
			httpConn.setRequestProperty("User-Agent", USER_AGENT);
			httpConn.setUseCaches(false);

			if (!this.requestHeader.isEmpty()) {
				for (String key : this.requestHeader.keySet()) {
					httpConn.setRequestProperty(key, this.requestHeader
							.get(key));
				}
			}

			httpConn.connect();

			connected = true;
		} catch (MalformedURLException e) {
			throw new IOException(e);
		}
	}

	public boolean followRedirects() throws IOException {
		int resultCode = httpConn.getResponseCode();
		return (resultCode >= 300 && resultCode < 400);
	}

	public String getLocation() {
		String location = httpConn.getHeaderField("Location");
		if (location == null) {
			location = httpConn.getHeaderField("location");
		}
		return location;
	}

	public InputStream getInputStream() throws IOException {
		return httpConn.getInputStream();
	}

	public String getBodyString() throws IOException {
		return getBodyString(null);
	}

	public String getBodyString(String charset) throws IOException {
		BufferedReader reader;
		if (charset == null) {
			reader = new BufferedReader(new InputStreamReader(getInputStream()));
		} else {
			reader = new BufferedReader(new InputStreamReader(getInputStream(),
					charset));
		}
		StringBuffer sb = new StringBuffer();
		String inputLine = null;
		while ((inputLine = reader.readLine()) != null) {
			sb.append(inputLine);
		}
		return sb.toString();
	}

	public void post(List<String> dataList) throws IOException {
		PrintWriter out = new PrintWriter(httpConn.getOutputStream());
		StringBuffer postData = new StringBuffer();
		for (String data : dataList) {
			postData.append(data).append("&");
		}
		out.write(postData.toString());
		out.flush();
		out.close();
	}

	public String getCookie() {
		String key = null;
		String cookie = "";
		for (int i = 1; (key = httpConn.getHeaderFieldKey(i)) != null; i++) {
			if ("set-cookie".equalsIgnoreCase(key)) {
				String cookieVal = httpConn.getHeaderField(i);
				cookieVal = cookieVal.substring(0, cookieVal.indexOf(";"));
				cookie = cookie + cookieVal + ";";
			}
		}
		return cookie;
	}

	public void disconnect() {
		if (connected) {
			httpConn.disconnect();
			connected = false;
		}
	}

	public static void main(String[] args) throws IOException {
	    String url="http://renmai.china.alibaba.com/personal/index.html";
	    String cookie="******";
	    HttpClient hc = new HttpClient(url,cookie);
	    hc.connect(METHOD_GET);
	    System.out.println(hc.getBodyString());
	    System.out.println(hc.getCookie());
	    hc.disconnect();
	}

}

 在这个页面不需要登录的时候，这个方法是非常简单的，cookie可以不设置，但是如果需要登录的话，就比较麻烦，这时可以采用fiddler等web代理工具，在打开此页面时将header中的值抓取出来，拷贝到程序中，则问题即解决，如附件图所示：