Java爬取数据

HttpUtils

public class HttpUtils {
    //声明httpclient管理器对象（）
   // private static PoolingHttpClientConnectionManager cm;
    private static List<String> userAgentlist = null;
    private static RequestConfig config =null;

    static {
        //cm = new PoolingHttpClientConnectionManager();
        //cm.setMaxTotal(200);
        //cm.setDefaultMaxPerRoute(20);
        config =RequestConfig.custom()
                .setSocketTimeout(10000)
                .setConnectionRequestTimeout(10000)
                .setConnectionRequestTimeout(10000)
                .build();
        userAgentlist =new ArrayList<>();
        userAgentlist.add("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
        userAgentlist.add("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763");
        userAgentlist.add("Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
        userAgentlist.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:65.0) Gecko/20100101 Firefox/65.0");
        userAgentlist.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36");
    }

    public static String getHtml(String url){
        CloseableHttpClient httpClient = SSLClient.createSSLClient();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(config);
        httpGet.setHeader("User-Agent",userAgentlist.get(new Random().nextInt(userAgentlist.size())));
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode() == 200){
                String html ="";
                if (response.getEntity() !=null) {
                    html = EntityUtils.toString(response.getEntity(), "UTF-8");
                }
                return html;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return null;
    }
}

跳过SSL证书访问

public class SSLClient {
    public static CloseableHttpClient createSSLClient(){
        SSLContext sslContext = null;
        try {
            sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
                @Override
                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                    //通过所有证书
                    return true;
                }
            }).build();
            SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, new HostnameVerifier() {
                @OverrideSS
                public boolean verify(String s, SSLSession sslSession) {
                    //不验证hostname
                    return true;
                }
            });
            return HttpClients.custom().setSSLSocketFactory(sslSocketFactory).build();
            //如果产生了异常，就创建普通的client
        }catch (Exception e){
            e.printStackTrace();
        }
        return  HttpClients.createDefault();
    }
}

所使用的依赖

		<dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.22</version>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.3</version>
        </dependency>
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.7</version>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.7</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
        </dependency>