java中利用开源HttpClient包抓取网页

1.用到的jar包：

commons-logging.jar

commons-httpclient.jar(此包的版本为3.1)

commons-codec.jar

2.源码：

package test;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;

public class Test2 {
	private static HttpClient httpclient=new HttpClient();
	
	//设置代理服务器
	/*static {
		httpclient.getHostConfiguration().setProxy("192.168.0.1", 8080);
	}*/
	
	public static boolean downloadPage(String path) throws HttpException, IOException{
		
		
		//得到post方法
		PostMethod postmethod=new PostMethod(path);
		
		//设置post方法的参数
		NameValuePair[] postData=new NameValuePair[1];
		postData[0]=new NameValuePair("name","google");
		//postData[1]=new NameValuePair("password","000");
		postmethod.addParameters(postData);
		
		//返回状态执行码
		int statusCode =httpclient.executeMethod(postmethod);
		System.out.print(statusCode);
		//针对状态码进行处理
		if(statusCode==HttpStatus.SC_OK){
			
			//得到文件名(将http：//去掉并将“/”替换为空)
			String str=path.substring(7).replaceAll("/", "");
		
			//判断文件是否存在，若存在删除重新建立
			File file =new File("d:"+File.separator+"page"+File.separator+str+".html");
			if(file.exists()){
				file.delete();
				file.createNewFile();
			}
			
			//创建输入流
			InputStream input=postmethod.getResponseBodyAsStream();
			
			//文件输出流
			DataOutputStream dos=new DataOutputStream(new FileOutputStream(file));
			
			//将文件写入
			byte[] buffer=new byte[1024];
			int len=0;
			while((len=input.read(buffer))>0){
				dos.write(buffer, 0, len);
			}
			
			
			//关闭流
			if(input!=null){
				input.close();
			}
			if(dos!=null){
				dos.close();
			}
			
			//提示
			System.err.println("抓取页面成功");
			return true;
		}
		return false;
	}
	
	public static void main(String []args){
		try{
			Test2.downloadPage("http://lietu.com/");
		}catch(HttpException e){
			e.printStackTrace();
		}catch(IOException e){
			e.printStackTrace();
		}
	}
}