java利用url解析网页内容并模拟手动form提交数据

package com.test;



import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

/**
 * 
 * 
 * @author: ZhouQiShan
 * @since: 2011-06-09
 */

public class CookieUtil {

	public final static String CONTENT_TYPE = "Content-Type";
	public static void setProxy(String host, String port) {  
        System.setProperty("proxySet", "true");  
        System.setProperty("proxyHost", host);  
        System.setProperty("proxyPort", port);  
    }  
	public static Content curl(String method, String sUrl,
			Map<String, String> paramMap, Map<String, String> requestHeaderMap,
			boolean isOnlyReturnHeader,String path) {
		Content content = null;
		HttpURLConnection httpUrlConnection = null;
		InputStream in = null;
		 setProxy("172.16.55.51", "808"); 
		try {
			URL url = new URL(sUrl);
			boolean isPost = "POST".equals(method);
			if (method == null
					|| (!"GET".equalsIgnoreCase(method) && !"POST"
							.equalsIgnoreCase(method))) {
				method = "POST";
			}
			URL resolvedURL = url;
			if ("GET".equals(method) && paramMap != null) {
				boolean firstParam = true;
				StringBuffer newUrlBuffer = new StringBuffer(url
						.toExternalForm());
				if (url.getQuery() == null) {
					newUrlBuffer.append("?");
				} else {
					newUrlBuffer.append("&");
				}
				for (Map.Entry<String, String> entry : paramMap.entrySet()) {
					String encName = URLEncoder.encode(entry.getKey(), "UTF-8");
					if (firstParam) {
						firstParam = false;
					} else {
						newUrlBuffer.append("&");
					}
					String encValue = URLEncoder.encode(entry.getValue(),
							"UTF-8");
					newUrlBuffer.append(encName);
					newUrlBuffer.append("=");
					newUrlBuffer.append(encValue);
				}
				resolvedURL = new java.net.URL(newUrlBuffer.toString());
			}
			URLConnection urlConnection = resolvedURL.openConnection();
			httpUrlConnection = (HttpURLConnection) urlConnection;
			httpUrlConnection.setRequestMethod(method);
			httpUrlConnection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
			// Do not follow redirects, We will handle redirects ourself
			httpUrlConnection.setInstanceFollowRedirects(false);
			urlConnection.setDoOutput(true);
			urlConnection.setDoInput(true);
			urlConnection.setConnectTimeout(5000);
			urlConnection.setReadTimeout(5000);
			urlConnection.setUseCaches(false);
			urlConnection.setDefaultUseCaches(false);
			// set request header
			if (requestHeaderMap != null) {
				for (Map.Entry<String, String> entry : requestHeaderMap
						.entrySet()) {
					String key = entry.getKey();
					String val = entry.getValue();					
					if (key != null && val != null) {
						urlConnection.setRequestProperty(key, val);
					}
				}
			}
			if (isPost) {
				urlConnection.setDoOutput(true);
				ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
				boolean firstParam = true;
				for (Map.Entry<String, String> entry : paramMap.entrySet()) {
					String encName = URLEncoder.encode(entry.getKey(), "UTF-8");
					if (firstParam) {
						firstParam = false;
					} else {
						bufOut.write((byte) '&');
					}
					String encValue = URLEncoder.encode(entry.getValue(),
							"UTF-8");
					bufOut.write(encName.getBytes("UTF-8"));
					bufOut.write((byte) '=');
					bufOut.write(encValue.getBytes("UTF-8"));
				}
				byte[] postContent = bufOut.toByteArray();
				if (urlConnection instanceof HttpURLConnection) {
					((HttpURLConnection) urlConnection)
							.setFixedLengthStreamingMode(postContent.length);
				}
				OutputStream postOut = urlConnection.getOutputStream();
				postOut.write(postContent);
				postOut.flush();
				postOut.close();
			}
			httpUrlConnection.connect();
			int responseCode = httpUrlConnection.getResponseCode();
			
			// We handle redirects ourself
			if (responseCode == HttpURLConnection.HTTP_MOVED_PERM
					|| responseCode == HttpURLConnection.HTTP_MOVED_TEMP) {
				String location = httpUrlConnection.getHeaderField("Location");
				URL newAction = new URL(url, location);
				// Recurse
				StringBuffer newUrlSb = new StringBuffer(newAction
						.getProtocol()
						+ "://" + newAction.getHost());
				if (newAction.getPort() != -1) {
					newUrlSb.append(":" + newAction.getPort());
				}
				if (newAction.getPath() != null) {
					newUrlSb.append(newAction.getPath());
				}
				if (newAction.getQuery() != null) {
					newUrlSb.append("?" + newAction.getQuery());
				}
				if (newAction.getRef() != null) {
					newUrlSb.append("#" + newAction.getRef());
				}
				
				return curl("POST", newUrlSb.toString(), paramMap, requestHeaderMap,
						isOnlyReturnHeader,path);
			} else if (responseCode == HttpURLConnection.HTTP_OK
					|| responseCode == HttpURLConnection.HTTP_CREATED) {
				byte[] bytes = new byte[0];
				if (!isOnlyReturnHeader) {
					if(isPost){
					in = httpUrlConnection.getInputStream();
					ByteArrayOutputStream bout = new ByteArrayOutputStream();
					byte[] buf = new byte[1024];
					while (true) {
						int rc = in.read(buf);
						if (rc <= 0) {
							break;
						} else {
							bout.write(buf, 0, rc);
						}
					}
					bytes = bout.toByteArray();
					in.close();
					}else{					
						 DataInputStream ins = new DataInputStream(httpUrlConnection
							      .getInputStream());
						 //验证码的位置
							    DataOutputStream out = new DataOutputStream(new FileOutputStream(
							      path+"/code.bmp"));
							    byte[] buffer = new byte[4096];
							    int count = 0;
							    while ((count = ins.read(buffer)) > 0) {
							     out.write(buffer, 0, count);
							    }
							    
							    out.close();
							    ins.close();
					}
				}
				// only fetch Content-Length and Last-Modified header
				String encoding = null;
				if (encoding == null) {
					encoding = getEncodingFromContentType(httpUrlConnection
							.getHeaderField(CONTENT_TYPE));
				}				
				
				content = new Content(sUrl, new String(bytes, encoding),
						httpUrlConnection.getHeaderFields());
			}
		} catch (Exception e) {
			return null;
		} finally {
			if (httpUrlConnection != null) {
				httpUrlConnection.disconnect();
			}
		}
		return content;
	}

	public static String getEncodingFromContentType(String contentType) {
		String encoding = null;
		if (contentType == null) {
			return null;
		}
		StringTokenizer tok = new StringTokenizer(contentType, ";");
		if (tok.hasMoreTokens()) {
			tok.nextToken();
			while (tok.hasMoreTokens()) {
				String assignment = tok.nextToken().trim();
				int eqIdx = assignment.indexOf('=');
				if (eqIdx != -1) {
					String varName = assignment.substring(0, eqIdx).trim();
					if ("charset".equalsIgnoreCase(varName)) {
						String varValue = assignment.substring(eqIdx + 1)
								.trim();
						if (varValue.startsWith("\"")
								&& varValue.endsWith("\"")) {
							// substring works on indices
							varValue = varValue.substring(1,
									varValue.length() - 1);
						}
						if (Charset.isSupported(varValue)) {
							encoding = varValue;
						}
					}
				}
			}
		}
		if (encoding == null) {
			return "UTF-8";
		}
		return encoding;
	}

	public static void main(String[] args) {
		// login
		//System.out.println(getHtmlReadLine("http://www.zhaopin.com/"));	
		//验证码的位置
		Content content = curl("GET", "http://www.haoshijia.com.cn/register/index/verify", null, null, false,"d:/");
		
		// build request headers & do rate of user review
		List<String> lsit = content.getHeaders().get("Set-Cookie");
		Map<String, String> resmap = new HashMap<String, String>();		
		if (lsit != null) {
			StringBuffer sb = new StringBuffer();
			boolean isLast = false;
			int i = 0;
			for (String val : lsit) {
				i++;
				if (i == lsit.size()) {
					isLast = true;
				}
				int pos = val.indexOf("=");
				if (pos != -1) {
					String cookieName = val.substring(0, pos);
					String cookieVal = val.substring(pos + 1);
					System.out.println(cookieName+":"+cookieVal);
					cookieVal = cookieVal.split(";")[0];
					if (isLast) {
						sb.append(cookieName + "=" + cookieVal);
					} else {
						sb.append(cookieName + "=" + cookieVal + ";");
					}
				}
			}			System.out.println(sb.toString());			
			resmap.put("Cookie", sb.toString());
		}
		String a="";
		  System.out.print("请输入验证码：");
		  BufferedReader strin=new BufferedReader(new InputStreamReader(System.in));
		  try {
			a=strin.readLine();
		}  catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		  System.out.println("输入的数是："+a);
		 
		String email = "xsl0218";
		String pass = "zqs021823";
		String loginUrl = "http://www.haoshijia.com.cn/register/index/logincheck";
		String rateReviewUrl = "http://www.haoshijia.com.cn/member/index/index";
		Map<String, String> paramMap = new HashMap<String, String>();
		paramMap.put("login_name", email);
		paramMap.put("login_password", pass);
		//paramMap.put("login", "1");
		paramMap.put("login_code", a+"");
		content = curl("POST", loginUrl, paramMap, resmap, false,"");
		//System.out.println(content.getBody());
		// build request headers & do rate of user review		
		paramMap = new HashMap<String, String>();

		content = curl("POST", rateReviewUrl, paramMap, resmap, false,"");
		inFile(content.getBody(), "D:/sss.txt");
		System.out.println(content.getBody());
	}

	// 这个是输出
	public static boolean inFile(String content, String path) {
		PrintWriter out = null;
		File file = new File(path);
		try {
			if (!file.exists()) {
				file.createNewFile();
			}
			out = new PrintWriter(new FileWriter(file));

			out.write(content);
			out.flush();
			return true;
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			out.close();
		}
		return false;
	}

	public static String getHtmlReadLine(String httpurl){
		String CurrentLine="";
		String TotalString="";
		InputStream urlStream;
		String content="";

		try {
			URL url = new URL(httpurl);

			HttpURLConnection connection = (HttpURLConnection)url.openConnection();

			connection.connect();
			System.out.println(connection.getResponseCode());
			urlStream = connection.getInputStream();

			BufferedReader reader = new BufferedReader(

			new InputStreamReader(urlStream,"utf-8"));

			while ((CurrentLine = reader.readLine()) != null) {

				TotalString += CurrentLine+"\n";
			}

			content = TotalString;			

		} catch (Exception e) {}

		return content;		
	}
}

class Content {
	private String url;
	private String body;
	private Map<String, List<String>> m_mHeaders = new HashMap<String, List<String>>();

	public Content(String url, String body, Map<String, List<String>> headers) {
		this.url = url;
		this.body = body;
		this.m_mHeaders = headers;
	}

	public String getUrl() {
		return url;
	}

	public String getBody() {
		return body;
	}

	public Map<String, List<String>> getHeaders() {
		return m_mHeaders;
	}

}