指定URL的网络爬虫。。

 

package Spider;

import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.*;

import java.io.*;
public class RetrivePage2 {

	/**
	 * @param args
	 */
	
	private static HttpClient httpClient=new HttpClient();
	//设置代理服务器
	static{
		//设置代理服务器的IP地址和端口
		 httpClient.getHostConfiguration().setProxy("162.168.2.112", 8080);
	}
	public static boolean downloadePage(String path) throws HttpException,IOException{
		InputStream input=null;
		OutputStream output=null;
		//得到post方法
		PostMethod postMethod=new PostMethod(path);
		//设置post方法的参数
		NameValuePair[] postData=new NameValuePair[2];
		postData[0]=new NameValuePair("name","lietu");
		postData[1]=new NameValuePair("password","*****");
		postMethod.addParameters(postData);
		//执行，返回状态码
		int statusCode=httpClient.executeMethod(postMethod);
		System.out.println(statusCode);
		//针对状态码进行处理（简单起见，只处理返回值为200的状态码）
		if(statusCode==HttpStatus.SC_OK){
			input=postMethod.getResponseBodyAsStream();
			//得到文件名
			String filename=path.substring(path.lastIndexOf('/')+1);
			//获得文件输出流
			output=new FileOutputStream(filename);
			//输出到文件
			int tempByte=-1;
			while((tempByte=input.read())>0){
				output.write(tempByte);
			}
			//关闭输入输出流
			if(input!=null){
				input.close();
			}
			if(output!=null){
				output.close();
			}
			return true;
		}
		return false;
	}
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		
		//抓起lietu首页，输出
		try{
			RetrivePage2.downloadePage("http://www.baidu.com/");
			
		}catch(HttpException e){
			e.printStackTrace();
		}catch(IOException e){
			e.printStackTrace();
		}
	}

}