使用相关jar包爬取网页数据

import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;

public class Test2 {
public static void main(String args[]) {
String h = pickData(“https://wenku.baidu.com/view/026e4c20ac02de80d4d8d15abe23482fb4da02c5.html”);
Document document = Jsoup.parse(h);
//base-list switch_part
Elements words=document.getElementsByClass(“ie-fix”);
System.out.println(words.first().text());
}
private static String getExSentence(Document doc,String classes){
Elements words=doc.getElementsByClass(classes);
int i=1;
StringBuffer sb=new StringBuffer();
for(Element element:words.subList(0,10)){
sb.append(i+". “+element.getElementsByTag(“p”).text()+”\n");
i++;
}
return sb.toString();
}
private static String getChange(Document doc,String classes){
Elements words=doc.getElementsByClass(classes);
StringBuffer sb=new StringBuffer();
sb.append(words.first().getElementsByTag(“h1”).text()+"😊;
sb.append(words.first().getElementsByTag(“span”).text());
return sb.toString();
}
private static String getWord(Document doc,String classes){
Elements words=doc.getElementsByClass(classes);
StringBuffer sb=new StringBuffer();
for(Element element:words.last().children()){
sb.append(element.getElementsByTag(“span”).text()+"\n");
}
return sb.toString();
}
private static String getPhonetic(Document doc,String classes){
Elements words=doc.getElementsByClass(classes);
StringBuffer sb=new StringBuffer();
sb.append(words.first().getElementsByTag(“span”).first().text());
sb.append(" "+words.first().getElementsByTag(“span”).last().text());
return sb.toString();
}
private static String pickData(String url) {
CloseableHttpClient httpclient = HttpClients.createDefault();
try {
HttpGet httpget = new HttpGet(url);
CloseableHttpResponse response = httpclient.execute(httpget);
try {
// 获取响应实体
HttpEntity entity = response.getEntity();
// 打印响应状态
if (entity != null) {
return EntityUtils.toString(entity);
}
} catch (ParseException e) {
e.printStackTrace();
} finally {
response.close();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭连接,释放资源
try {
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}