1.利用HttpURLConnect去访问网页的工具类
那么在Java中,我们要去访问一个网页,就得用到Jdk自带的HttpUrlConnection
HttpURLConnection conn = null;
URL url = new URL("www.baidu.com");
conn = (HttpURLConnection)url.openConnection();
关于HttpURLConnection的使用方法可以自行度娘
帮助我们封装一个请求去访问网页
那么本节介绍的是一个我自己写的关于在抓包过程中比较好用的HttpUrlConnection工具类,此工具类可以帮助我们自动处理抓包过程中遇到的各类细节问题:
1.可以自动处理302转发
2.自行判断是临时地址方式或者cookie形式,如果是cookie形式,会自动从第一请求中的set-cookie头中去除cookie并设置到第二次的请求头中
3.把请求头和返回头都对象化,使得更加适配多种情况
/**
*
* @author Mr.W
*一个请求体对应bean
*/
public class NetSendData {
//请求地址
private String url;
//完成这个请求后得到头部里面的content内容
private String content;
//头部
private Map<String,String> headers = new HashMap<String,String>();
//参数,跟下面一样
private Map<String,String> parmars = new HashMap<String,String>();
//字符串型参数
private String pars;
//服务器主地址
private String host;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getPars() {
return pars;
}
public void setPars(String pars) {
this.pars = pars;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public Map<String, String> getHeaders() {
return headers;
}
public void setHeaders(Map<String, String> headers) {
this.headers = headers;
}
public Map<String, String> getParmars() {
return parmars;
}
public void setParmars(Map<String, String> parmars) {
this.parmars = parmars;
}/**
* 浏览器返回体对应的bean
* @author Mr.W
*
*/
public class NetReceiverData {
//返回码
private int stateCode;
//如果是302转发,那么转发的地址
private String location;
//头部content内容
private byte[] content;
//来自哪个网页跳转过来的(对于临时地址形式的记录登录状态,这个参数有用,后面会讲到)
private String fromUrl;
//头部
private Map<String,String> headers = new HashMap<String,String>();
public Map<String, String> getHeaders() {
return headers;
}
public void setHeaders(Map<String, String> headers) {
this.headers = headers;
}
public String getFromUrl() {
return fromUrl;
}
public void setFromUrl(String fromUrl) {
this.fromUrl = fromUrl;
}
public int getStateCode() {
return stateCode;
}
public void setStateCode(int stateCode) {
this.stateCode = stateCode;
}
public String getLocation() {
return location;
}
public void setLocation(String location) {
this.location = location;
}
public byte[] getContent() {
return content;
}
public void setContent(byte[] content) {
this.content = content;
}
public String getContent2String(){
try {
return new String(content,"gb2312");
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return new String(content);
}
}
上面就是俩个请求对应的bean,先不要纠结里面的成员变量各有什么用,等看完一整章你再回头来看(其实也是lz封装得不好哈哈哈哈).其实我是建议自己再进行二次修改会更好,接下来高能,具体工具类:
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.io.output.ByteArrayOutputStream;
public class MyHttpUtil {
public static NetReceiverData sendGet(NetSendData sendData) {
// 先实例化返回体bean
NetReceiverData returnData = new NetReceiverData();
URL url = null;
HttpURLConnection conn = null;
InputStream is = null;
ByteArrayOutputStream baos = null;
try {
url = new URL(sendData.getUrl());
conn = (HttpURLConnection) url.openConnection();
conn.setReadTimeout(20000);
conn.setConnectTimeout(20000);
conn.setRequestMethod("GET");
if (sendData.getHeaders() != null
&& !sendData.getHeaders().isEmpty()) {
for (Map.Entry<String, String> entry : sendData.getHeaders()
.entrySet()) {
conn.setRequestProperty(entry.getKey(), entry.getValue());
}
}
// conn.getResponseCode(),这句代码意味着发送请求,然后判断返回码
if (conn.getResponseCode() == 200) {
processResponseHeader(conn, returnData.getHeaders(),
sendData.getHeaders());
// 获取输入流
is = conn.getInputStream();
// 读取流
ByteArrayOutputStream bos = new ByteArrayOutputStream();
byte[] bs = new byte[1024];
int len = -1;
while ((len = is.read(bs)) != -1) {
bos.write(bs, 0, len);
}
byte b[] = bos.toByteArray();
bos.close();
returnData.setFromUrl(sendData.getUrl());
returnData.setContent(b);
returnData.setStateCode(200);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (is != null)
is.close();
} catch (IOException e) {
}
try {
if (baos != null)
baos.close();
} catch (IOException e) {
}
if (conn != null) {
conn.disconnect();
conn = null;
}
}
return returnData;
}
public static NetReceiverData sendPost(NetSendData sendData) {
NetReceiverData returnData = new NetReceiverData();
PrintWriter out = null;
BufferedReader in = null;
String result = "";
HttpURLConnection conn = null;
try {
URL realUrl = new URL(sendData.getUrl());
conn = (HttpURLConnection) realUrl.openConnection();
conn.setRequestMethod("POST");
conn.setUseCaches(false);
conn.setDoOutput(true);
conn.setDoInput(true);
conn.setReadTimeout(8000);
conn.setConnectTimeout(8000);
conn.setInstanceFollowRedirects(false);
if (sendData.getHeaders() != null
&& !sendData.getHeaders().isEmpty()) {
for (Map.Entry<String, String> entry : sendData.getHeaders()
.entrySet()) {
conn.setRequestProperty(entry.getKey(), entry.getValue());
}
}
if (sendData.getParmars() != null
&& !sendData.getParmars().isEmpty()) {
String params = "";
int i = 0;
for (Map.Entry<String, String> entry : sendData.getParmars()
.entrySet()) {
if (i == 0) {
params = entry.getKey() + "=" + entry.getValue();
} else {
params += "&" + entry.getKey() + "=" + entry.getValue();
}
i++;
}
// 获取URLConnection对象对应的输出流
out = new PrintWriter(conn.getOutputStream());
out.print(params);
// flush输出流的缓冲
out.flush();
} else if (sendData.getPars() != null) {
out = new PrintWriter(conn.getOutputStream());
// 发送请求参数
out.print(sendData.getPars());
// flush输出流的缓冲
out.flush();
} else {
conn.setRequestProperty("Content-Length", "0");
}
// 定义BufferedReader输入流来读取URL的响应流
in = new BufferedReader(new InputStreamReader(
conn.getInputStream(), "gb2312"));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
if (conn.getResponseCode() == 200) {
processResponseHeader(conn, returnData.getHeaders(),
sendData.getHeaders());
returnData.setFromUrl(sendData.getUrl());
returnData.setContent(result.getBytes("gb2312"));
returnData.setStateCode(200);
} else if (conn.getResponseCode() == 302) {
NetSendData Data302 = new NetSendData();
processResponseHeader(conn, Data302.getHeaders(),
sendData.getHeaders());
Data302.getHeaders().put("Referer", sendData.getUrl());
returnData.setFromUrl(sendData.getUrl());
String temp = conn.getHeaderField("Location");
Data302.setUrl(sendData.getHost() + temp);
return sendGet(Data302);
}
} catch (Exception e) {
e.printStackTrace();
}
finally {
try {
if (out != null) {
out.close();
}
if (in != null) {
in.close();
}
} catch (IOException ex) {
// System.out.println("ioexception");
ex.printStackTrace();
}
if (conn != null) {
conn.disconnect();
}
}
return returnData;
}
/**
* 处理服务器返回的conn中的头部包含Set-Cookie的情况 要取出并把它封装到returnData的header中，即inMap
*
* @param http
* @param returnDataHeader
* @param sendDataHeader
* @throws UnsupportedEncodingException
*/
private static void processResponseHeader(HttpURLConnection http,
Map<String, String> returnDataHeader,
Map<String, String> sendDataHeader)
throws UnsupportedEncodingException {
if (sendDataHeader.containsKey("Cookie")) {
returnDataHeader.put("Cookie", sendDataHeader.get("Cookie"));
}
Map<String, String> header = getHttpResponseHeader(http);
for (Map.Entry<String, String> entry : header.entrySet()) {
String key = entry.getKey() != null ? entry.getKey() : "";
if (key.equals("Set-Cookie")) {
returnDataHeader.put(
"Cookie",
returnDataHeader.get("Cookie") == null ? entry
.getValue() : returnDataHeader.get("Cookie")
+ ";" + entry.getValue() + ";");
}
}
}
private static Map<String, String> getHttpResponseHeader(
HttpURLConnection http) throws UnsupportedEncodingException {
Map<String, String> header = new LinkedHashMap<String, String>();
for (int i = 0;; i++) {
String mine = http.getHeaderField(i);
if (mine == null)
break;
header.put(http.getHeaderFieldKey(i), mine);
}
return header;
}
}
我的建议是大家按照这个思路来写属于你自己的工具类,主要还是为了解决我上面说的那些问题,因为你不可能每个请求都去处理类似302,400这些返回码情况,所以尽量都抽取出来
795
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
