抓取网页并存储在指定的文件中

package HttpClient;


import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;


import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;


/*
 * 将抓取的网页存在本地的目录下
 * */
public class GetSample {


public static void main(String[] args){
BufferedReader br=null;
BufferedWriter bw=null;
String url="http://www.ibm.com/developerworks/cn/opensource/os-cn-crawler/";
//创建客户端
HttpClient  httpClient=new HttpClient();
//创建GetMethod
GetMethod getMethod=new GetMethod(url);
//设置回复的默认策略
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
//返回服务器的编码
System.out.println(getMethod.getResponseCharSet());
//执行execute方法
try {
int statusCode=httpClient.executeMethod(getMethod);
if(statusCode!=HttpStatus.SC_OK){
System.out.println("method failed!");
}
System.out.println(getMethod.getResponseHeader("Content-Type")
.getValue());
Header[] headers=getMethod.getResponseHeaders();
for(Header  h:  headers)
       System.out.println(h.getName()+" "+h.getValue());

byte[] responseBody=getMethod.getResponseBody();
String result=new String(responseBody,"UTF-8");
System.out.println(result);
InputStream response = getMethod.getResponseBodyAsStream();
br=new BufferedReader(new InputStreamReader(response,"UTF-8"));
String path="d://yy.log";
FileOutputStream fos=new FileOutputStream(path);
OutputStreamWriter osw=new OutputStreamWriter(fos);
bw=new BufferedWriter(osw);
String line="";
while((line=br.readLine())!=null){
bw.write(line);
}
} catch (HttpException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
//释放连接
getMethod.releaseConnection();
try {
br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}