httpclient+jsoup 爬虫注意事项（java）

本在在用httpclient+jsoup爬去数据，发现，有些数据返回null，断电跟入，原来是请求失败，刚开始加了httpclient重试，都不起

效果，最后，考录到电脑内存，然后并发运行调小，缓解了一些null，但是还是存在，最后，调节了一下idea编辑器的jvn参数，搞定，jvm参数，idea编辑器会占用一些资源，剩下的就是数据，数据不够是，正常是应该报错内存溢出的，但是为什么没有报错内存溢出，而是，报错网络问题，我还无从知晓，（感觉应该是内存数据是够，不过所剩无几，正好http请求需要一些内存，所以报错网络异常） 建议，电脑内存剩余两个g左右，否者你懂的，第一次写微博，如果描述的不清楚，可以直接联系我 ，见谅

还有一种是httpclient 回调，如果包含生僻字，会有乱码，解决方法是吧httpclient 的 EntityUtils.toString（）换成google CharStreams.toString() 不同处是把回调换成流 的方法，

package com.yunying.bjtimer.utils;

import com.google.common.io.CharStreams;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.protocol.HttpContext;

import javax.net.ssl.SSLException;
import javax.net.ssl.SSLHandshakeException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.net.UnknownHostException;
import java.util.concurrent.TimeUnit;

public class Test{
   //请求超时重试五次，五次还未请求成功，抛弃，详情见官网api
    public static HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {
        @Override
        public boolean retryRequest(
                IOException exception,
                int executionCount,
                HttpContext context) {
            if (executionCount >= 5) {
                // Do not retry if over max retry count
                return false;
            }

            if (exception instanceof NoHttpResponseException) {
                return true;
            }
            if (exception instanceof SSLHandshakeException) {
                return false;
            }


            if (exception instanceof InterruptedIOException) {
                // Timeout
                return false;
            }
            if (exception instanceof UnknownHostException) {
                // Unknown host
                return false;
            }
            if (exception instanceof ConnectTimeoutException) {
                // Connection refused
                return false;
            }
            if (exception instanceof SSLException) {
                // SSL handshake exception
                return false;
            }
            HttpClientContext clientContext = HttpClientContext.adapt(context);
            HttpRequest request = clientContext.getRequest();
            boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
            if (idempotent) {
                // Retry if the request is considered idempotent
                return true;
            }
            return false;
        }

    };

    /**
     * 单例 httpclient
     */
    public enum EnumHttpClient {
        SINGLETON;
        CloseableHttpClient httpclient;

        EnumHttpClient() {
            httpclient = HttpClients.custom().setRetryHandler(myRetryHandler).build();
        }

        public CloseableHttpClient httpclient() {
            return httpclient;
        }
    }
    //配置超时或是请求头 cookie等，模拟登陆，可在配置cookie
    public static void setHttpGetConfig(HttpGet httpget) {
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(60000).setConnectionRequestTimeout(10000)
                .setSocketTimeout(60000).build();
        httpget.setConfig(requestConfig);
        httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
    }

    public static String httpGetUtil(String url) {
        CloseableHttpClient httpclient = EnumHttpClient.SINGLETON.httpclient();
        CloseableHttpResponse response = null;
        String html = "";  //页面数据
        try {
            HttpGet httpget = new HttpGet(url);
            setHttpGetConfig(httpget);
            if (httpget != null) {
                try {
                    response = httpclient.execute(httpget);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            TimeUnit.MILLISECONDS.sleep(50);
            // 获取响应实体，替换下面的  html = EntityUtils.toString(entity, "GBK"); 因为生僻字会乱码
            HttpEntity entity = response.getEntity();
            InputStream is = entity.getContent();
            html = CharStreams.toString(new InputStreamReader(is, "GBK"));


            // 打印响应状态
           /* if (entity != null) {
                html = EntityUtils.toString(entity, "GBK");
            }*/
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } finally {
            try {
                if (httpclient != null) {
                    //单例模式，该处关闭取消，因为如果关闭，下面for循环执行第二次httpclient请求会空指针
                    //httpclient.close();
                }
                if (response != null) {
                    response.close();
                }
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
        return html;
    }
    //测试类
    public static void main(String[] args) {
        //循环多个网址 
        for (int i = 100; i < 105; i++) {
            //网址i从1到5000000随机一个都是有效网址
            httpGetUtil("https://jobs.51job.com/all/co"+i+".html");
        }
    }

}