HttpClient学习笔记

HttpClient 是 Apache Jakarta Common 下的子项目，用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。HttpClient 已经应用在很多的项目中。爬虫主要是用HttpClient模拟浏览器请求第三方站点url，然后响应，获取网页数据，然后用Jsoup来提取我们需要的信息；

jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据。

package com.httpclient.jsoup;

import java.io.IOException;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


public class JsoupDemo {

    public static void main(String[] args) throws ClientProtocolException, IOException {
        CloseableHttpClient HttpClient = HttpClients.createDefault();

        HttpGet get = new HttpGet("http://www.xicidaili.com/");
        /*HttpHost proxy = new HttpHost("182.112.228.38",80);

        RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(10000).setSocketTimeout(10000).build();
        get.setConfig(config );*/
        get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0");
        CloseableHttpResponse respose = HttpClient.execute(get);
         System.out.println("状态:"+respose.getStatusLine().getStatusCode());
        HttpEntity entity = respose.getEntity();
        System.out.println("Content-Type:"+entity.getContentType().getValue());
        String content = EntityUtils.toString(entity, "utf-8");
        System.out.println("内容:"+content);
        respose.close();

        Document doc = Jsoup.parse(content);

        Elements elements= doc.getElementsByTag("title");
        System.out.println("网页标题："+elements.get(0).text());

        Element element2=doc.getElementById("header");
        String navTop=element2.text();
        System.out.println("id 为header的内容："+navTop);

        Elements postItemElements=doc.getElementsByClass("site_name"); // 根据样式名称来查询DOM
        for(Element e:postItemElements){
            System.out.println("根据样式名称来查询DOM"+e.html());
        }

        Elements widthElements=doc.getElementsByAttribute("width"); // 根据属性名来查询DOM
        for(Element e:widthElements){
            System.out.println("根据属性名来查询DOM"+e.toString());
        }

        Elements targetElements=doc.getElementsByAttributeValue("width", "11%");  // 根据属性名和属性值来查询DOM
        for(Element e:targetElements){
            System.out.println("根据属性名和属性值来查询DOM"+e.toString());
        }


        Elements linkElements=doc.select(".subtitle .country");
        for(Element e:linkElements){
            System.out.println("根据class："+e.text());
        }

        Elements hrefElements=doc.select("a[href]"); // 
        for(Element e:hrefElements){
            System.out.println("带有href属性的a元素"+e.toString());
        }

        Elements imgElements=doc.select("img[src$=.png]"); // 
        for(Element e:imgElements){
            System.out.println("查找扩展名为.png的图片DOM节点"+e.toString());
        }
    }

}