httpclient/jsoup模拟登陆人人网

                                  

• HttpClient(DefaultHttpClient)代表了一个会话，在同一个会话中，HttpClient对cookie自动进行管理(当然，也可以在程序中进行控制)。
• 在同一个会话中，当使用post或是get发起一个新的请求时，一般需要对调用前一个会话的abort()方法，否则会抛出异常。
• 有些网站登录成功后会重定向(302, 303)，比如这里的人人网。如果发出的是post请求，需要从响应头中取出location，并再次向网站发送请求，以获取最终数据。
• 抓取程序不要运行地过于频繁，大部分站点都有抵制刷网站机制。人人网访问过于频繁会锁账号。

代码如下:

import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.SSLException;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

public class ClientConnect {

	DefaultHttpClient httpclient = new DefaultHttpClient();
	HttpPost httppost = new HttpPost();
	SaveFiile savefile = new SaveFiile();
	HttpResponse response;
	HttpEntity entity;
	HttpRequestRetryHandler retryhandler;
        String username=" ... ", password=" ..." ; //你自己的登陆名称和密码进行初始化
	String url;

	public ClientConnect(String url) {
		this.url = url;
	}

	private boolean login() {
		List<NameValuePair> formparams = new ArrayList<NameValuePair>();

		formparams.add(new BasicNameValuePair("email", username));
		formparams.add(new BasicNameValuePair("password", password));
		formparams.add(new BasicNameValuePair("origURL",
				"http://www.renren.com/home"));
		formparams.add(new BasicNameValuePair("domain", "renren.com"));
		formparams.add(new BasicNameValuePair("key_id", "1"));
		formparams.add(new BasicNameValuePair("captcha_type", "web_login"));
		UrlEncodedFormEntity Urlentity;
		try {
			Urlentity = new UrlEncodedFormEntity(formparams, "UTF-8");

			httppost = new HttpPost(url);
			httppost.setEntity(Urlentity);
			response = httpclient.execute(httppost);

			this.retryconnection(); // 调用连接错误处理函数

		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		} finally {
			httppost.abort();
		}
		return true;

	}

	// 登陆后页面会重定向.因此需从响应头中取出定向后的url.
	private String getRedirectLocation() {
		Header locationHeader = response.getFirstHeader("Location");
		if (locationHeader == null) {
			return null;
		}
		return locationHeader.getValue();
	}

	// 获取从定向后的html内容
	private String getText(String redirection) {
		HttpGet httpget = new HttpGet(redirection);

		retryconnection();

		String html = "";

		try {
			response = httpclient.execute(httpget);
			int statu = response.getStatusLine().getStatusCode();

			if (statu == HttpStatus.SC_OK) {

				entity = response.getEntity();
				if (entity != null) {
					html = EntityUtils.toString(entity);
				}
			}
		} catch (ClientProtocolException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		} catch (IOException e2) {
			e2.printStackTrace();
		} finally {
			httpclient.getConnectionManager().shutdown();
		}

		return html;
	}

	private void retryconnection() {
		// to set request retry handler

		retryhandler = new HttpRequestRetryHandler() {

			@Override
			public boolean retryRequest(IOException exception,
					int exceutionCount, HttpContext context) {
				// TODO Auto-generated method stub

				if (exceutionCount >= 5) {
					// Do not retry if over max retry count
					return false;
				}
				if (exception instanceof InterruptedIOException) {
					// Timeout
					return false;
				}
				if (exception instanceof UnknownHostException) {
					// unknown host
					return false;
				}
				if (exception instanceof ConnectException) {
					// connection refused
					return false;
				}
				if (exception instanceof SSLException) {
					// SSL handshake excetion
					return false;
				}
				HttpRequest request = (HttpRequest) context
						.getAttribute(ExecutionContext.HTTP_REQUEST);
				boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);

				if (idempotent) {
					// Retry if the request is considered idempotent
					return true;
				}
				return false;
			}

		};
		httpclient.setHttpRequestRetryHandler(retryhandler);

	}

	public void clientConnetion() {
		String html = "";

		if (login()) {
			String redirection = getRedirectLocation();
			if (redirection != null) {
				// System.out.println(getText(redirection));
				html = getText(redirection);
				savefile.saveToFile(url, html);
			}
		}

	}
}

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;

public class SaveFiile {

	public void saveToFile(String url, String htmlentity) {

		String filename = url.substring(11);
		filename = filename.replace("/", ".") + ".html";

		try {
			BufferedWriter write = new BufferedWriter(new FileWriter(filename));

			write.write(htmlentity);
			write.flush();
			write.close();

			System.out.println("file save successfully!");

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}
}

public class LoadRenRen {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		ClientConnect clientconnect = new ClientConnect(
				"http://www.renren.com/PLogin.do");
		clientconnect.clientConnetion();
	}

}