HttpClient和jsoup基础API使用

最新推荐文章于 2023-07-09 21:26:23 发布

hikktn

最新推荐文章于 2023-07-09 21:26:23 发布

阅读量202

点赞数

分类专栏：网络爬虫

本文链接：https://blog.csdn.net/qq_41520636/article/details/116379554

版权

网络爬虫专栏收录该内容

2 篇文章 1 订阅

订阅专栏

准备配置

pom

    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.24</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.10</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.13.1</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.3</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.4</version>
        </dependency>

    </dependencies>

log4j.properties

log4j.rootLogger=DEBUG,A1
log4j.logger.com.hikktn = DEBUG

log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

准备SpringDataJpa

一个springdatajpa简单项目，进行下面的测试。

https://blog.csdn.net/qq_41520636/article/details/115472103

启动相应的环境。

完成springdatajpa入门程序后，需要简单更改一下。

在UserController.java中增加或修改接口

    /*
		查询用户
	 */
	@GetMapping("/getOne")
	public UserEntity getUserInfo(@Param("userId") int userId) {
		Optional<UserEntity> optional = userRepository.findById(userId);
		return optional.orElseGet(UserEntity::new);
	}

	/*
		分页查询多个用户
	 */
	@GetMapping("/list")
	public Page<UserEntity> pageQuery(@RequestParam(value = "pageNum", defaultValue = "1") Integer pageNum,
	                                  @RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
		return userRepository.findAll(PageRequest.of(pageNum - 1, pageSize));
	}

	@PostMapping(path = "/testPostParam")
	public String getPostUser(@RequestParam("name") String name){
		return "测试POST请求："+name;
	}

	@PostMapping(path = "/testPost")
	public String getUser(){
		return "测试POST请求：无参数";
	}

HttpClient

基础入门程序 get方式

package com.hikktn.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @ClassName CrawlerFirst
 * @Description TODO
 * @Author lisonglin
 * @Date 2021/5/2 18:33
 * @Version 1.0
 */
public class CrawlerFirst {

	public static void main(String[] args) throws IOException {
		// 创建httpclient对象
		CloseableHttpClient httpClient = HttpClients.createDefault();
		// 创建httpget对象，设置url访问地址
		HttpGet httpGet = new HttpGet("http://localhost:8080/user/all");

		// 使用httpclient发送请求，获取响应数据
		CloseableHttpResponse response = httpClient.execute(httpGet);

		// 解析响应
		if (response.getStatusLine().getStatusCode() == 200) {
			HttpEntity entity = response.getEntity();
			String result = EntityUtils.toString(entity, "utf8");
			System.out.println(result);
		}
	}
}

输出

get有参方式

package com.hikktn.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URISyntaxException;

/**
 * @ClassName HttpGetTest
 * @Description TODO
 * @Author lisonglin
 * @Date 2021/5/2 20:18
 * @Version 1.0
 */
public class HttpGetTest {

	public static void main(String[] args){
		// 创建httpclient对象
		CloseableHttpClient httpClient = HttpClients.createDefault();
		HttpGet httpGet =null;
		// 设置请求地址 https://www.bilibili.com/bangumi/play/ss38234?spm_id_from=333.851.b_62696c695f7265706f72745f616e696d65.61
		// 创建URIBuilder
		try {
			URIBuilder uriBuilder = new URIBuilder("http://localhost:8080/user/getOne");
			// 设置参数
			uriBuilder.setParameter("userId","1");

			httpGet = new HttpGet(uriBuilder.build());

		} catch (URISyntaxException e) {
			e.printStackTrace();
		}

		// 使用httpclient发送请求，获取响应数据
		CloseableHttpResponse response = null;
		try {
			// 配置信息
			RequestConfig requestConfig = RequestConfig.custom()
					// 设置连接超时时间(单位毫秒)
					.setConnectTimeout(5000)
					// 设置请求超时时间(单位毫秒)
					.setConnectionRequestTimeout(5000)
					// socket读写超时时间(单位毫秒)
					.setSocketTimeout(5000)
					// 设置是否允许重定向(默认为true)
					.setRedirectsEnabled(true).build();

			// 将上面的配置信息 运用到这个Get请求里
			httpGet.setConfig(requestConfig);

			response = httpClient.execute(httpGet);
			// 解析响应
			if (response.getStatusLine().getStatusCode() == 200) {
				HttpEntity entity = response.getEntity();
				String result = EntityUtils.toString(entity, "utf8");
				System.out.println(result);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}finally {
			try {
				response.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

	}
}

输出

post无参方式

package com.hikktn.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URISyntaxException;

/**
 * @ClassName HttpGetTest
 * @Description TODO
 * @Author lisonglin
 * @Date 2021/5/2 20:18
 * @Version 1.0
 */
public class HttpPostTest {

	public static void main(String[] args) {
		// 创建httpclient对象
		CloseableHttpClient httpClient = HttpClients.createDefault();
		HttpPost httpPost = new HttpPost("http://localhost:8080/user/testPost");


		// 使用httpclient发送请求，获取响应数据
		CloseableHttpResponse response = null;
		try {
			response = httpClient.execute(httpPost);
			// 解析响应
			if (response.getStatusLine().getStatusCode() == 200) {
				String result = EntityUtils.toString(response.getEntity(), "utf8");
				System.out.println(result);
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				response.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

	}
}

输出

post有参方式

package com.hikktn.test;

import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

/**
 * @ClassName HttpGetTest
 * @Description TODO
 * @Author lisonglin
 * @Date 2021/5/2 20:18
 * @Version 1.0
 */
public class HttpPostParamTest {

	public static void main(String[] args) {
		// 创建httpclient对象
		CloseableHttpClient httpClient = HttpClients.createDefault();
		HttpPost httpPost = new HttpPost("http://localhost:8080/user/testPostParam");

		// 使用list集合封装参数
		List<NameValuePair> params = new ArrayList<>();
		params.add(new BasicNameValuePair("name","hikktn"));
		try {
			// 创建表单的entity对象
			UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "utf8");
			// 设置formEntity对象给post对象
			httpPost.setEntity(formEntity);
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		// httpPost.setHeader("Content-Type","application/json;charset=utf8");
		// 使用httpclient发送请求，获取响应数据
		CloseableHttpResponse response = null;
		try {
			response = httpClient.execute(httpPost);
			// 解析响应
			if (response.getStatusLine().getStatusCode() == 200) {
				String result = EntityUtils.toString(response.getEntity(), "utf8");
				System.out.println(result);
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				response.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

	}
}

输出

Jsoup

基础入门程序

解析文档 HTTPS协议

package com.hikktn.test;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.X509TrustManager;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;

/**
 * @ClassName SSLHelper
 * @Description TODO
 * @Author lisonglin
 * @Date 2021/5/3 13:53
 * @Version 1.0
 */
public class SSLHelper {

	public static String USER_AGENT = "Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 5.0)";

	static public void init() {
		try {
			SSLContext context = SSLContext.getInstance("TLS");
			context.init(null, new X509TrustManager[]{new X509TrustManager() {
				public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
				}

				public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
				}

				public X509Certificate[] getAcceptedIssuers() {
					return new X509Certificate[0];
				}
			}}, new SecureRandom());
			HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
		} catch (NoSuchAlgorithmException e) {
		} catch (KeyManagementException e) {
		}
	}
}

public class JsoupTest {

	public static String USER_AGENT = "Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 5.0)";
	@Test
	public void testUrl(){
		SSLHelper.init();
		try {
			// 解析url
			// Document document = Jsoup.parse(new URL("https://www.bilibili.com/"), 1000);
			Connection connect = Jsoup.connect("https://www.bilibili.com/").userAgent(USER_AGENT);
			connect.header("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
			connect.header("Accept-Encoding", "gzip, deflate, sdch");
			connect.header("Accept-Language", "zh-CN,zh;q=0.8");
			connect.timeout(3000);
			connect.ignoreHttpErrors(true);
			Document document = connect.get();
			// 获取title内容
			Element title = document.getElementsByTag("title").first();
			System.out.println(title.text());
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

输出

准备HTML

<html>
 <head> 
  <title>哔哩哔哩 (゜-゜)つロ 干杯~-bilibili</title> 
 </head> 
 <body>
	<div class="city">
		<h3 id="city_bj">北京中心</h3>
		<div class="city_in">
			<div class="city_con" style="display: none;">
				<ul>
					<li id="test" class="class_a class_b">
						<a href="https://www.bilibili.com/" target="_blank">
							<span class="s_name">北京</span>
						</a>
					</li>
					<li>
						<a href="" target="_blank">
							<span class="s_name">上海</span>
						</a>
					</li>
					<li>
						<a href="" target="_blank">
							<span abc="123" class="s_name">广州</span>
						</a>
					</li>
					<ul>
						<li>天津</li>
					</ul>					
				</ul>
			</div>
		</div>
	</div>
 </body>
</html>

读取HTML文件的标题

	@Test
	public void testStringTitle() throws IOException {
		// 获取字符串
		String html = FileUtils.readFileToString(new File("C:\\Users\\12134\\Desktop\\test.html"), "utf-8");

		Document doc = Jsoup.parse(html);

		String title = doc.getElementsByTag("title").first().text();
		System.out.println(title);
	}

输出

读取HTML文件的body

	@Test
	public void testStringBody() throws IOException {
		// 获取字符串
		String html = FileUtils.readFileToString(new File("C:\\Users\\12134\\Desktop\\test.html"), "utf-8");

		Document doc = Jsoup.parseBodyFragment(html);

		Element body = doc.body();

		System.out.println(body);

	}

输出：

操作dom

    @Test
	public void testDom() throws IOException {
		// 获取字符串
		String html = FileUtils.readFileToString(new File("C:\\Users\\12134\\Desktop\\test.html"), "utf-8");

		Document doc = Jsoup.parse(html);

		// jQuery选择器
		Elements el1 = doc.select("#city_bj");
		System.out.println(el1);
		System.out.println("===========");

		// id 选择器
		Element test = doc.getElementById("test");
		System.out.println(test);

		// 获取id
		String id = test.id();
		System.out.println("id:"+id);

		// 获取class名字
		String className = test.className();
		System.out.println("className:"+className);

		// 获取标签内的属性
		String id1 = test.attr("id");
		System.out.println("attr_id:"+id1);

		// 获取标签内所有的属性
		String all = test.attributes().toString();
		System.out.println("all"+all);

		// 获取文本内容
		String text = test.text();
		System.out.println(text);
		System.out.println("===========");

		// CSS样式选择器
		Elements s_name = doc.getElementsByClass("class_a");
		System.out.println(s_name);

		System.out.println("===========");

		// 标签选择器
		Elements span = doc.getElementsByTag("span");
		System.out.println(span);

		System.out.println("===========");

		// 自定义属性选择器
		Elements abc = doc.getElementsByAttribute("abc");
		System.out.println(abc);

		System.out.println("===========");
		// 自定义属性选择器 class="class_a class_b" 中有两个CSS样式无效
		Elements class_a = doc.getElementsByAttributeValue("class", "s_name");
		System.out.println(class_a);
	}

输出

hikktn

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
打赏
4
评论
HttpClient和jsoup基础API使用

准备配置pom <dependencies>  <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId&g
复制链接

扫一扫