爬虫基础之Jsoup使用select选择器

前言：

Jsoup可以解析HTML得到Document文档对象，通常我们需要爬取的内容是文档中的某些东西而不是整个HTML内容，这时就需要使用select选择器来提取我们需要的内容，select选择器的使用类似于jquery的使用方式。注意：下面爬取的是a标签的内容，这个根据爬取的网站的升级变更而不准确。

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;

public class JsoupSelector {
    public static void main(String[] args) throws IOException {
        Document document = Jsoup.parse(getHtmlByUrl("http://www.bingosoft.net"));
        //获取class名称为head_index head_nav下面的ul 下面的li下面的a标签对象
        Elements elements = document.select("[class=head_index head_nav] ul li a");
        if(elements != null && !elements.isEmpty()){
            for (Element element : elements) {
                //获取到a标签对象的文本内容
                System.out.println(element.text());
            }
        }
    }

    public static String getHtmlByUrl(String url) throws IOException {
        //1.创建HttpClient对象
        CloseableHttpClient aDefault = HttpClients.createDefault();
        //2.指定请求方式
        HttpGet httpGet = new HttpGet(url);
        //3.可选的，封装请求参数
        //3.1封装请求体（可选）
        httpGet.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        httpGet.setHeader("Accept-Encoding","gzip, deflate");
        httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9");
        httpGet.setHeader("Cache-Control","max-age=0");
        httpGet.setHeader("Connection","keep-alive");
        httpGet.setHeader("Host","www.bingosoft.net");
        httpGet.setHeader("Upgrade-Insecure-Requests","1");
        httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36");
        //4.发送请求
        CloseableHttpResponse execute = aDefault.execute(httpGet);
        //5.获取数据
        //5.1获取状态码
        if(execute.getStatusLine().getStatusCode()==200){
            String html = EntityUtils.toString(execute.getEntity(), "UTF-8");
            return html;
        }
        return null;
    }
}