Java爬虫篇(一)

Java爬虫入门

1、需要的相关依赖

<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.13</version>
</dependency>

1、get传参请求http://www.baidu.com
通过实例URIBuilder，使用setParameter方法以k-v的方式设置参数，如果要设置多个参数可以使用链式写法

 //创建httpClient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //創建HttpGet对象

        URIBuilder uriBuilder = new URIBuilder("http://www.baidu.com/s");
        uriBuilder.setParameter("wd","风景");
        HttpGet httpGet = new HttpGet(uriBuilder.build());
        System.out.println("请求信息"+httpGet);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);

            if(response.getStatusLine().getStatusCode() == 200){
                System.out.println("请求成功");
                //获取响应体里的内容
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                System.out.println("请求信息"+httpGet);
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

1、post传参请求http://www.baidu.com
通过HttpPost的setsetEntity方法，entity里存放的是个集合，集合里的value是一个k-v键值对，我们采用NameValuePair的实现类BasicNameValuePair完成

 //创建httpClient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //声明list集合
        ArrayList<NameValuePair> params = new ArrayList<>();
//        加入封装的参数
        params.add(new BasicNameValuePair("wd","风景"));
//        创建一个entity
        UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(params, "utf-8");
        //創建HttpPost对象
        HttpPost httpPost = new HttpPost("http://www.baidu.com");
        httpPost.setEntity(urlEncodedFormEntity);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpPost);
            if(response.getStatusLine().getStatusCode() == 200){
                System.out.println("请求成功");
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

3、连接池-----(和数据库连接池一样，防止多次使用销毁在创建浪费资源，而是需要的时候就从连接池拿，不需要就放回连接池)

 //创建连接池管理器
        PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager();
        //设置连接池单个连接数最大为10
        manager.setDefaultMaxPerRoute(10);
        //设置连接池的连接总数最大为100
        manager.setMaxTotal(100);
        //使用连接池管理器发请求

        doGet(manager);
        doGet(manager);
    }
    public static void doGet(PoolingHttpClientConnectionManager manager){
        CloseableHttpClient client = HttpClients.custom().setConnectionManager(manager).build();
        HttpGet httpGet = new HttpGet("http://www.baidu.com");
        CloseableHttpResponse response = null;
        try {
            response = client.execute(httpGet);
            if(response.getStatusLine().getStatusCode() == 200){
                System.out.println("请求成功");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                if(response != null)
                    response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

4、请求的参数配置

 //配置请求信息
        RequestConfig config = RequestConfig.custom().setConnectTimeout(1000)//连接的最长时间
                .setConnectionRequestTimeout(500)//获取连接的最长时间
                .setSocketTimeout(10 * 1000)//数据传输的最长时间
                .build();
        //设置配置
        httpGet.setConfig(config);