Java发送Http请求工具类之Jsoup

引入maven依赖

<dependency>
    <groupId>org.jsoup</groupId>
    <artifactId>jsoup</artifactId>
    <version>1.11.3</version>
</dependency>

HttpUtil

package com.zhl.common.utils;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;

import javax.net.ssl.*;
import java.io.*;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * Http发送post请求工具,兼容http和https两种请求类型
 */
public class HttpUtil {

	/**
	 * 请求超时时间 10s
	 */
	private static final int TIME_OUT = 10000;

	/**
	 * Https请求
	 */
	private static final String HTTPS = "https";

	/**
	 * Content-Type
	 */
	private static final String CONTENT_TYPE = "Content-Type";

	/**
	 * 表单提交方式Content-Type
	 */
	private static final String FORM_TYPE = "application/x-www-form-urlencoded;charset=UTF-8";

	/**
	 * JSON提交方式Content-Type
	 */
	private static final String JSON_TYPE = "application/json;charset=UTF-8";

	/**
	 * 发送Get请求
	 * 
	 * @param url
	 *            请求URL
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response get(String url) throws IOException {
		return get(url, null);
	}

	/**
	 * 发送Get请求
	 * 
	 * @param url
	 *            请求URL
	 * @param headers
	 *            请求头参数
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response get(String url, Map<String, String> headers) throws IOException {
		if (null == url || url.isEmpty()) {
			throw new RuntimeException("The request URL is blank.");
		}

		// 如果是Https请求
		if (url.startsWith(HTTPS)) {
			getTrust();
		}
		Connection connection = Jsoup.connect(url);
		connection.method(Method.GET);
		connection.timeout(TIME_OUT);
		connection.ignoreHttpErrors(true);
		connection.ignoreContentType(true);
		
		if (null != headers) {
			connection.headers(headers);
		}

		Response response = connection.execute();
		return response;
	}

	/**
	 * 发送JSON格式参数POST请求
	 * 
	 * @param url
	 *            请求路径
	 * @param params
	 *            JSON格式请求参数
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response post(String url, String params) throws IOException {
		return doPostRequest(url, null, params);
	}

	/**
	 * 发送JSON格式参数POST请求
	 * 
	 * @param url
	 *            请求路径
	 * @param headers
	 *            请求头中设置的参数
	 * @param params
	 *            JSON格式请求参数
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response post(String url, Map<String, String> headers, String params) throws IOException {
		return doPostRequest(url, headers, params);
	}

	/**
	 * 字符串参数post请求
	 * 
	 * @param url
	 *            请求URL地址
	 * @param paramMap
	 *            请求字符串参数集合
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response post(String url, Map<String, String> paramMap) throws IOException {
		return doPostRequest(url, null, paramMap, null);
	}

	/**
	 * 带请求头的普通表单提交方式post请求
	 * 
	 * @param headers
	 *            请求头参数
	 * @param url
	 *            请求URL地址
	 * @param paramMap
	 *            请求字符串参数集合
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response post(Map<String, String> headers, String url, Map<String, String> paramMap)
			throws IOException {
		return doPostRequest(url, headers, paramMap, null);
	}

	/**
	 * 带上传文件的post请求
	 * 
	 * @param url
	 *            请求URL地址
	 * @param paramMap
	 *            请求字符串参数集合
	 * @param fileMap
	 *            请求文件参数集合
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response post(String url, Map<String, String> paramMap, Map<String, File> fileMap)
			throws IOException {
		return doPostRequest(url, null, paramMap, fileMap);
	}

	/**
	 * 带请求头的上传文件post请求
	 * 
	 * @param url
	 *            请求URL地址
	 * @param headers
	 *            请求头参数
	 * @param paramMap
	 *            请求字符串参数集合
	 * @param fileMap
	 *            请求文件参数集合
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	public static Response post(String url, Map<String, String> headers, Map<String, String> paramMap,
			Map<String, File> fileMap) throws IOException {
		return doPostRequest(url, headers, paramMap, fileMap);
	}

	/**
	 * 执行post请求
	 * 
	 * @param url
	 *            请求URL地址
	 * @param headers
	 *            请求头
	 * @param jsonParams
	 *            请求JSON格式字符串参数
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	private static Response doPostRequest(String url, Map<String, String> headers, String jsonParams)
			throws IOException {
		if (null == url || url.isEmpty()) {
			throw new RuntimeException("The request URL is blank.");
		}

		// 如果是Https请求
		if (url.startsWith(HTTPS)) {
			getTrust();
		}

		Connection connection = Jsoup.connect(url);
		connection.method(Method.POST);
		connection.timeout(TIME_OUT);
		connection.ignoreHttpErrors(true);
		connection.ignoreContentType(true);
		connection.maxBodySize(0);

		if (null != headers) {
			connection.headers(headers);
		}

		connection.header(CONTENT_TYPE, JSON_TYPE);
		connection.requestBody(jsonParams);

		Response response = connection.execute();
		return response;
	}

	/**
	 * 普通表单方式发送POST请求
	 * 
	 * @param url
	 *            请求URL地址
	 * @param headers
	 *            请求头
	 * @param paramMap
	 *            请求字符串参数集合
	 * @param fileMap
	 *            请求文件参数集合
	 * @return HTTP响应对象
	 * @throws IOException
	 *             程序异常时抛出,由调用者处理
	 */
	private static Response doPostRequest(String url, Map<String, String> headers, Map<String, String> paramMap,
			Map<String, File> fileMap) throws IOException {
		if (null == url || url.isEmpty()) {
			throw new RuntimeException("The request URL is blank.");
		}

		// 如果是Https请求
		if (url.startsWith(HTTPS)) {
			getTrust();
		}

		Connection connection = Jsoup.connect(url);
		connection.method(Method.POST);
		connection.timeout(TIME_OUT);
		connection.ignoreHttpErrors(true);
		connection.ignoreContentType(true);
		connection.maxBodySize(0);

		if (null != headers) {
			connection.headers(headers);
		}

		// 收集上传文件输入流,最终全部关闭
		List<InputStream> inputStreamList = null;
		try {
			// 添加文件参数
			if (null != fileMap && !fileMap.isEmpty()) {
				inputStreamList = new ArrayList<InputStream>();
				InputStream in = null;
				File file = null;

				for (Map.Entry<String, File> e : fileMap.entrySet()) {
					file = e.getValue();
					in = new FileInputStream(file);
					inputStreamList.add(in);
					connection.data(e.getKey(), file.getName(), in);
				}
			}

			// 普通表单提交方式
			else {
				connection.header(CONTENT_TYPE, FORM_TYPE);
			}

			// 添加字符串类参数
			if (null != paramMap && !paramMap.isEmpty()) {
				connection.data(paramMap);
			}

			Response response = connection.execute();
			return response;
		}

		// 关闭上传文件的输入流
		finally {
			closeStream(inputStreamList);
		}
	}

	/**
	 * 关流
	 * 
	 * @param streamList
	 *            流集合
	 */
	private static void closeStream(List<? extends Closeable> streamList) {
		if (null != streamList) {
			for (Closeable stream : streamList) {
				try {
					stream.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	/**
	 * 获取服务器信任
	 */
	private static void getTrust() {
		try {
			HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
				public boolean verify(String hostname, SSLSession session) {
					return true;
				}
			});
			SSLContext context = SSLContext.getInstance("TLS");
			context.init(null, new X509TrustManager[] { new X509TrustManager() {
				public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
				}
				public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
				}
				public X509Certificate[] getAcceptedIssuers() {
					return new X509Certificate[0];
				}
			} }, new SecureRandom());
			HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

}

 

  • 0
    点赞
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
Jsoup+httpclient 模拟登陆和抓取页面 package com.app.html; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.text.SimpleDateFormat; import java.util.Date; import org.apache.commons.httpclient.Cookie; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.cookie.CookieSpec; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.params.HttpMethodParams; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.app.comom.FileUtil; public class HttpClientHtml { private static final String SITE = "login.goodjobs.cn"; private static final int PORT = 80; private static final String loginAction = "/index.php/action/UserLogin"; private static final String forwardURL = "http://user.goodjobs.cn/dispatcher.php/module/Personal/?skip_fill=1"; private static final String toUrl = "d:\\test\\"; private static final String css = "http://user.goodjobs.cn/personal.css"; private static final String Img = "http://user.goodjobs.cn/images"; private static final String _JS = "http://user.goodjobs.cn/scripts/fValidate/fValidate.one.js"; /** * 模拟等录 * @param LOGON_SITE * @param LOGON_PORT * @param login_Action * @param params * @throws Exception */ private static HttpClient loginHtml(String LOGON_SITE, int LOGON_PORT,String login_Action,String ...params) throws Exception { HttpClient client = new HttpClient(); client.getHostConfiguration().setHost(LOGON_SITE, LOGON_PORT); // 模拟登录页面 PostMethod post = new PostMethod(login_Action); NameValuePair userName = new NameValuePair("memberName",params[0] ); NameValuePair password = new NameValuePair("password",params[1] ); post.setRequestBody(new NameValuePair[] { userName, password }); client.executeMethod(post); post.releaseConnection(); // 查看cookie信息 CookieSpec cookiespec = CookiePolicy.getDefaultSpec(); Cookie[] cookies = cookiespec.match(LOGON_SITE, LOGON_PORT, "/", false, client.getState().getCookies()); if (cookies != null) if (cookies.length == 0) { System.out.println("Cookies is not Exists "); } else { for (int i = 0; i < cookies.length; i++) { System.out.println(cookies[i].toString()); } } return client; } /** * 模拟等录 后获取所需要的页面 * @param client * @param newUrl * @throws Exception */ private static String createHtml(HttpClient client, String newUrl) throws Exception { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); String filePath = toUrl + format.format(new Date() )+ "_" + 1 + ".html"; PostMethod post = new PostMethod(newUrl); client.executeMethod(post); //设置编码 post.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "GBK"); String content= post.getResponseBodyAsString(); FileUtil.write(content, filePath); System.out.println("\n写入文件成功!"); post.releaseConnection(); return filePath; } /** * 解析html代码 * @param filePath * @param random * @return */ private static String JsoupFile(String filePath, int random) { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); File infile = new File(filePath); String url = toUrl + format.format(new Date()) + "_new_" + random+ ".html"; try { File outFile = new File(url); Document doc = Jsoup.parse(infile, "GBK"); String html="<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'>"; StringBuffer sb = new StringBuffer(); sb.append(html).append("\n"); sb.append("<html>").append("\n"); sb.append("<head>").append("\n"); sb.append("<title>欢迎使用新安人才网个人专区</title>").append("\n"); Elements meta = doc.getElementsByTag("meta"); sb.append(meta.toString()).append("\n"); ////////////////////////////body////////////////////////// Elements body = doc.getElementsByTag("body"); ////////////////////////////link////////////////////////// Elements links = doc.select("link");//对link标签有href的路径都作处理 for (Element link : links) { String hrefAttr = link.attr("href"); if (hrefAttr.contains("/personal.css")) { hrefAttr = hrefAttr.replace("/personal.css",css); Element hrefVal=link.attr("href", hrefAttr);//修改href的属性值 sb.append(hrefVal.toString()).append("\n"); } } ////////////////////////////script////////////////////////// Elements scripts = doc.select("script");//对script标签 for (Element js : scripts) { String jsrc = js.attr("src"); if (jsrc.contains("/fValidate.one.js")) { String oldJS="/scripts/fValidate/fValidate.one.js";//之前的css jsrc = jsrc.replace(oldJS,_JS); Element val=js.attr("src", jsrc);//修改href的属性值 sb.append(val.toString()).append("\n").append("</head>"); } } ////////////////////////////script////////////////////////// Elements tags = body.select("*");//对所有标签有src的路径都作处理 for (Element tag : tags) { String src = tag.attr("src"); if (src.contains("/images")) { src = src.replace("/images",Img); tag.attr("src", src);//修改src的属性值 } } sb.append(body.toString()); sb.append("</html>"); BufferedReader in = new BufferedReader(new FileReader(infile)); Writer out = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(outFile), "gbk")); String content = sb.toString(); out.write(content); in.close(); System.out.println("页面已经爬完"); out.close(); } catch (IOException e) { e.printStackTrace(); } return url; } public static void main(String[] args) throws Exception { String [] params={"admin","admin123"}; HttpClient client = loginHtml(SITE, PORT, loginAction,params); // 访问所需的页面 String path=createHtml(client, forwardURL); System.out.println( JsoupFile(path,1)); } }

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值