java抓取http指定内容_Java实现多种方式的http数据抓取

packagecom.yeezhao.common.http;importjava.io.BufferedReader;importjava.io.InputStream;importjava.io.InputStreamReader;importjava.net.HttpURLConnection;importjava.net.URL;importorg.apache.commons.httpclient.HttpClient;importorg.apache.commons.httpclient.HttpMethod;importorg.apache.commons.httpclient.methods.GetMethod;importorg.apache.commons.io.IOUtils;importorg.jsoup.Jsoup;/*** http工具对比

*

*@authorAdministrator -> junhong

*

* 2016年12月27日*/

public classHttpFetchUtil {/*** 获取访问的状态码

*@paramrequest

*@return*@throwsException*/

public static int getResponseCode(String request) throwsException {

URL url= newURL(request);

HttpURLConnection conn=(HttpURLConnection) url.openConnection();returnconn.getResponseCode();

}/*** 1)JDK自带HTTP连接，获取页面或Json

*@paramrequest

*@paramcharset

*@return*@throwsException*/

public static String JDKFetch(String request, String charset) throwsException {

URL url= newURL(request);

HttpURLConnection conn=(HttpURLConnection) url.openConnection();//模拟浏览器参数

conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36"

+ " (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");if (conn.getResponseCode() ==HttpURLConnection.HTTP_OK) {

InputStream input=conn.getInputStream();

StringBuffer sb= newStringBuffer();

BufferedReader reader= new BufferedReader(newInputStreamReader(input, charset));

String s;while ((s = reader.readLine()) != null) {

sb.append(s+ "\n");

}

input.close();

conn.disconnect();returnsb.toString();

}return "";

}/*** 2) JDK自带URL连接，获取页面或Json

*@paramrequest

*@paramcharset

*@return*@throwsException*/

public static String URLFetch(String request, String charset) throwsException {

URL url= newURL(request);returnIOUtils.toString(url.openStream());

}/*** 3)HttpClient Get工具，获取页面或Json

*@paramurl

*@paramcharset

*@return*@throwsException*/

public static String httpClientFetch(String url, String charset) throwsException {//GET

HttpClient httpClient = newHttpClient();

httpClient.getParams().setContentCharset(charset);

HttpMethod method= newGetMethod(url);

httpClient.executeMethod(method);returnmethod.getResponseBodyAsString();

}/*** 4)commons-io工具，获取页面或Json

*@paramurl

*@paramcharset

*@return*@throwsException*/

public static String commonsIOFetch(String url, String charset) throwsException {return IOUtils.toString(newURL(url), charset);

}/*** 5) Jsoup工具(通常用于html字段解析)，获取页面,非Json返回格式

*@paramurl

*@return*@throwsException*/

public static String jsoupFetch(String url) throwsException {return Jsoup.parse(new URL(url), 2 * 1000).html();

}

}