java爬虫学习——第一节 HttpClient使用

用于发送 HTTP 请求和接收来自通过 URI 确认的资源的 HTTP 响应

使用起点中文网测试

地址：https://www.qidian.com/all

导入HttpClient包

    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.3</version>
    </dependency>

1.GET请求

//使用方式可分为以下5个步骤
//创建HttpClient对象

//创建HttpGet请求

//发起请求

//判断响应码是不是正常响应也就是StatusCode是不是200

//释放连接

具体实现如下

public String HttpGet()throw Exception{
    //创建HttpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();

    //创建HttpGet请求
    HttpGet Get = new HttpGet("https://www.qidian.com/all");

    //发起请求,使用CloseableHttpResponse 接收返回
    CloseableHttpResponse response = null;
    try {        
        response = httpClient.execute(Get);

        //判断响应码是不是正常响应也就是StatusCode是不是200
        if (response.getStatusLine().getStatusCode() == 200) {
          
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            //打印数据长度返回页面信息
            System.out.println(content);
            return content;
        }

    } catch (Exception e) {
        e.printStackTrace();
    }finally{
        //释放连接
        if (response == null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        //关闭HttpClient
        httpClient.close();
    }

    return "";
}

 

2.GET带参数请求

使用方式和不带参的方式基本一致

地址：

//url = https://www.qidian.com/all?chanId=21&subCateId=8

public String HttpGet()throw Exception{
    //创建HttpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();

    //创建HttpGet请求
    HttpGet Get = new HttpGet("https://www.qidian.com/all?chanId=21&subCateId=8");

    //发起请求,使用CloseableHttpResponse 接收返回
    CloseableHttpResponse response = null;
    try {        
        response = httpClient.execute(Get);

        //判断响应码是不是正常响应也就是StatusCode是不是200
        if (response.getStatusLine().getStatusCode() == 200) {
          
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            //打印数据长度返回页面信息
            System.out.println(content);
            return content;
        }

    } catch (Exception e) {
        e.printStackTrace();
    }finally{
        //释放连接
        if (response == null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        //关闭HttpClient
        httpClient.close();
    }

    return "";
}

3.POST请求

使用方式类似于GET请求，不过创建的是HttpPost对象：

//url = https://www.qidian.com/all

public String HttpGet()throw Exception{
    //创建HttpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();

    //创建Post请求
    HttpPost Post = new HttpPost("https://www.qidian.com/all");

    //发起请求,使用CloseableHttpResponse 接收返回
    CloseableHttpResponse response = null;
    try {        
        response = httpClient.execute(Post);

        //判断响应码是不是正常响应也就是StatusCode是不是200
        if (response.getStatusLine().getStatusCode() == 200) {
          
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            //打印数据长度返回页面信息
            System.out.println(content);
            return content;
        }

    } catch (Exception e) {
        e.printStackTrace();
    }finally{
        //释放连接
        if (response == null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        //关闭HttpClient
        httpClient.close();
    }

    return "";
}

4.POST带参数请求

参数传递方式有点特殊，参数放到表单里进行提交

//url = https://www.qidian.com/all?chanId=21&subCateId=8

public String HttpGet()throw Exception{
    //创建HttpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();

    //创建Post请求
    HttpPost Post = new HttpPost("https://www.qidian.com/all");

    //声明存放参数的集合,多个参数可以在后边继续.add()
    List<NameValuePair> params = new ArrayList<NameValuePair>();
    params.add(new BasicNameValuePair("chanId", "21")).add(new  BasicNameValuePair("subCateId", "8"));
    //创建表单数据Entity
    UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "UTF-8");

    //设置表单Entity到httpPost请求对象中
    Post.setEntity(formEntity);


    //发起请求,使用CloseableHttpResponse 接收返回
    CloseableHttpResponse response = null;
    try {        
        response = httpClient.execute(Post);

        //判断响应码是不是正常响应也就是StatusCode是不是200
        if (response.getStatusLine().getStatusCode() == 200) {
          
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            //打印数据长度返回页面信息
            System.out.println(content);
            return content;
        }

    } catch (Exception e) {
        e.printStackTrace();
    }finally{
        //释放连接
        if (response == null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        //关闭HttpClient
        httpClient.close();
    }

    return "";
}