前言:
Jsoup可以解析HTML得到Document文档对象,通常我们需要爬取的内容是文档中的某些东西而不是整个HTML内容,这时就需要使用select选择器来提取我们需要的内容,select选择器的使用类似于jquery的使用方式。注意:下面爬取的是a标签的内容,这个根据爬取的网站的升级变更而不准确。
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
public class JsoupSelector {
public static void main(String[] args) throws IOException {
Document document = Jsoup.parse(getHtmlByUrl("http://www.bingosoft.net"));
//获取class名称为head_index head_nav下面的ul 下面的li下面的a标签对象
Elements elements = document.select("[class=head_index head_nav] ul li a");
if(elements != null && !elements.isEmpty()){
for (Element element : elements) {
//获取到a标签对象的文本内容
System.out.println(element.text());
}
}
}
public static String getHtmlByUrl(String url) throws IOException {
//1.创建HttpClient对象
CloseableHttpClient aDefault = HttpClients.createDefault();
//2.指定请求方式
HttpGet httpGet = new HttpGet(url);
//3.可选的,封装请求参数
//3.1封装请求体(可选)
httpGet.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
httpGet.setHeader("Accept-Encoding","gzip, deflate");
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9");
httpGet.setHeader("Cache-Control","max-age=0");
httpGet.setHeader("Connection","keep-alive");
httpGet.setHeader("Host","www.bingosoft.net");
httpGet.setHeader("Upgrade-Insecure-Requests","1");
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36");
//4.发送请求
CloseableHttpResponse execute = aDefault.execute(httpGet);
//5.获取数据
//5.1获取状态码
if(execute.getStatusLine().getStatusCode()==200){
String html = EntityUtils.toString(execute.getEntity(), "UTF-8");
return html;
}
return null;
}
}