爬虫 httpClient

public class CrawerFirst {
   

    public static void main(String[] args) throws IOException {
   
        //打开浏览器  创建一个HttpClient 对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //输入网址  发起get请求 创建HTTPget对象
        HttpGet httpGet = new HttpGet("http://www.baidu.com");

        //按回车 发起请求 返回响应 ，使用HTTPClient对象发起请求

        CloseableHttpResponse response = httpClient.execute(httpGet);


        //获取响应 获取数据
        //判断状态码是否是200
        if(response.getStatusLine().getStatusCode() == 200){
   
             HttpEntity httpEntity = response.getEntity();
             String content = EntityUtils.toString(httpEntity,"utf8");

            System.out.println(content);

        }


    }
}

采集，分析 处理我们使用的数据，存储。

设置带参数的HTTPGet

 
        //创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();
        
        //设置请求地址是： http://yun.itheima.com/search?keys=java
        //创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("http://yun.itheima.com");

        //设置参数
        uriBuilder.setParameter("keys", "Java")