23 user-agent和refer请求头的作用

1. user-agent的作用，防止请求时，被资源服务器当做非法访问，拒绝服务.
2. refer: 防盗链功能，当爬取某个资源的时候，会被服务器提示不是从本网站请求的，从而被拦截

@Test
    public void test1() {
        //可关闭的httpclient的客户端，相当于打开的浏览器
        CloseableHttpClient closeableHttpClient = HttpClients.createDefault();

        String urlStr = "https://www.baidu.com/";
        //构造httpGet对象
        HttpGet httpGet = new HttpGet(urlStr);
        //设置请求头(假装成浏览器)
        httpGet.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36");
        //防盗链(当爬取某个资源的时候，会被服务器提示不是从本网站请求的，从而被拦截)
        //url要是防止防盗链的网站的url才可以
        httpGet.addHeader("Referrer","https://www.baidu.com/");
        //构造响应对象
        CloseableHttpResponse response = null;
        try{
            response = closeableHttpClient.execute(httpGet);
            //获取响应结果
            HttpEntity entity = response.getEntity();
            //对HttpEntity操作的工具类
            String toStringResult = EntityUtils.toString(entity, StandardCharsets.UTF_8);
            System.out.println("toStringResult: " + toStringResult);

            //确保流关闭
            EntityUtils.consume(entity);

        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if(closeableHttpClient != null){
                try{
                    closeableHttpClient.close();
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
            if( response != null){
                try{
                    response.close();
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
        }
    }