基于GET请求的网页请求并保存
package cn.pzh.test;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.eclipse.jetty.http.HttpStatus;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
public class GetTrainInfo {
// 创建一个HttpClient打开一个代理
private static HttpClient httpClient = new HttpClient();
static{
// 设置代理服务器的IP地址和端口号
httpClient.getHostConfiguration().setProxy("58.222.40.26", 80);
// 设置HTTP链接超时5s
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
}
/**
* 保存页面
* @param path
* @return
* @throws HttpException
* @throws IOException
*/
public static boolean downPage(String path) throws HttpException, IOException{
// 输入流读取
InputStream is = null;
// 输出流保存
OutputStream os = null;
// 1、封装post地址
GetMethod getMethod = new GetMethod(path);
// 设置请求超时时间
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
// 设置重试处理
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
// 3、执行抓取操作
int statusCode = httpClient.executeMethod(getMethod);
// 4、判定是否成功
if(statusCode == HttpStatus.OK_200){
// 获取输入流
is = getMethod.getResponseBodyAsStream();
// 获取输出流
os = new FileOutputStream("C:\\12306.html");
// 输出到文件
int tempByte = -1;
while((tempByte = is.read()) > 0){
os.write(tempByte);
}
//关闭流
if(null != is){
is.close();
}
if(null != os){
os.close();
}
return true;
}
return false;
}
public static void main(String[] args) throws HttpException, IOException {
downPage("http://dynamic.12306.cn/mapping/kfxt/zwdcx/LCZWD/cx.jsp?cz=%C7%FA%B8%B7%B6%AB&cc=G243&cxlx=0&rq=2018-01-16&czEn=-E6-9B-B2-E9-98-9C-E4-B8-9C&tp=1516167414232");
}
}
基于POST请求的网页请求并保存
package cn.pzh.test;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.cxf.common.util.StringUtils;
import org.eclipse.jetty.http.HttpStatus;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
public class HttpClientDemo {
// 创建一个HttpClient打开一个代理
private static HttpClient httpClient = new HttpClient();
static{
// 设置代理服务器的IP地址和端口号
httpClient.getHostConfiguration().setProxy("127.0.0.1", 8080);
// 设置HTTP链接超时5s
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
}
/**
* 保存页面
* @param path
* @return
* @throws HttpException
* @throws IOException
*/
public static boolean downPage(String path) throws HttpException, IOException{
// 输入流读取
InputStream is = null;
// 输出流保存
OutputStream os = null;
// 1、封装post地址
PostMethod postMethod = new PostMethod(path);
// 设置请求超时时间
postMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
// 设置重试处理
postMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
// 2、配置post参数信息
NameValuePair[] postData = {new NameValuePair("wd","adminxu")};
postMethod.addParameters(postData);
// 3、执行抓取操作
int statusCode = httpClient.executeMethod(postMethod);
// 如果是重定向页面则重定向
if((statusCode == HttpStatus.MULTIPLE_CHOICES_300) ||
(statusCode == HttpStatus.MOVED_PERMANENTLY_301) ||
(statusCode == HttpStatus.FOUND_302) ||
(statusCode == HttpStatus.SEE_OTHER_303) ||
(statusCode == HttpStatus.NOT_MODIFIED_304)){
Header header = postMethod.getResponseHeader("location");
if(null != header){
String newUrl = header.getValue();
if(!StringUtils.isEmpty(newUrl)){
postMethod = new PostMethod(newUrl);
}
}
}
// 4、判定是否成功
if(statusCode == HttpStatus.OK_200){
// 获取输入流
is = postMethod.getResponseBodyAsStream();
// 获取输出流
os = new FileOutputStream("C:\\baidu.html");
// 输出到文件
int tempByte = -1;
while((tempByte = is.read()) > 0){
os.write(tempByte);
}
//关闭流
if(null != is){
is.close();
}
if(null != os){
os.close();
}
return true;
}
return false;
}
public static void main(String[] args) throws HttpException, IOException {
downPage("http://localhost:8080/prm/security/loginInit.action");
}
}