爬虫工具类

爬虫工具类实现HttpGet和连接池以及对请求配置和连接池配置

@Component
public class HttpGetUtil {
    private PoolingHttpClientConnectionManager manager;

    public HttpGetUtil(){
        this.manager = new PoolingHttpClientConnectionManager();
        this.manager.setDefaultMaxPerRoute(10);
        this.manager.setMaxTotal(100);

    }

    //获取html
    public String doGetHtml(String url){
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.manager).build();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(getConfig());
        CloseableHttpResponse response = null;

        try {
            response = httpClient.execute(httpGet);
            if(response.getStatusLine().getStatusCode() == 200){
                if(response.getEntity() != null) {
                    String content = EntityUtils.toString(response.getEntity());
                    return content;
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return "";
    }

    //下载图片
    public String doGetImage(String url){
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.manager).build();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(getConfig());
        CloseableHttpResponse response = null;

        try {
            response = httpClient.execute(httpGet);
            if(response.getStatusLine().getStatusCode() == 200){

                //判断响应体是否为空
                if(response.getEntity() != null) {
                    //获取图片地址的后缀
                    String suf = url.substring(url.lastIndexOf("."));
                    //创建新的图片名
                    String imgName = UUID.randomUUID().toString()+suf;
                    //下载图片

                    FileOutputStream out = new FileOutputStream("C:\\Users\\86158\\Desktop\\京东图片\\"+imgName);
                    response.getEntity().writeTo(out);
                    return imgName;
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return "";
    }


    //请求配置
    private static RequestConfig getConfig(){
        RequestConfig conf = RequestConfig.custom().setConnectTimeout(1000)
                .setConnectionRequestTimeout(500)
                .setSocketTimeout(100 * 100).build();
        return  conf;
    }


}