Java爬虫入门(1)

HttpClient-Get(不带参数)        

CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("http://www.itcast.cn/");
CloseableHttpResponse response = null;
try {
    response = httpClient.execute(httpGet);
    if (response.getStatusLine().getStatusCode()==200){
        String content = EntityUtils.toString(response.getEntity(), "GBK");
        System.out.println(content.length());
    }
} catch (IOException e) {
    e.printStackTrace();
}finally {
    try {
        response.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        httpClient.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

HttpClient-Get(带参数)  

CloseableHttpClient httpClient = HttpClients.createDefault();
URIBuilder uriBuilder = new URIBuilder("http://www.itcast.cn/search");
uriBuilder.addParameter("keys","java");
HttpGet httpGet = new HttpGet( uriBuilder.build());
CloseableHttpResponse response = null;
try {
    response = httpClient.execute(httpGet);
    if (response.getStatusLine().getStatusCode()==200){
        String content = EntityUtils.toString(response.getEntity(), "GBK");
        System.out.println(content.length());
    }
} catch (IOException e) {
    e.printStackTrace();
}finally {
    try {
        response.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        httpClient.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

   HttpClient-Post

CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("http://www.itcast.cn/");
CloseableHttpResponse response = null;
try {
    response = httpClient.execute(httpPost);
    if (response.getStatusLine().getStatusCode()==200){
        String content = EntityUtils.toString(response.getEntity(), "GBK");
        System.out.println(content.length());
    }
} catch (IOException e) {
    e.printStackTrace();
}finally {
    try {
        response.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        httpClient.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

   HttpClient-Post(带参数)

CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("http://www.itcast.cn/");
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("keys","java"));
UrlEncodedFormEntity forEntity = new UrlEncodedFormEntity(params, "utf8");
httpPost.setEntity(forEntity);
CloseableHttpResponse response = null;
try {
    response = httpClient.execute(httpPost);
    if (response.getStatusLine().getStatusCode()==200){
        String content = EntityUtils.toString(response.getEntity(), "GBK");
        System.out.println(content.length());
    }
} catch (IOException e) {
    e.printStackTrace();
}finally {
    try {
        response.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        httpClient.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

  连接池：        

PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();cm.setMaxTotal(100);//最大连接数
cm.setDefaultMaxPerRoute(10);//设置每个主机的最大连接数
CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(cm).build();

配置请求设置信息：

        

PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(100);//最大连接数
cm.setDefaultMaxPerRoute(10);//设置每个主机的最大连接数
CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(cm).build();
HttpGet httpGet = new HttpGet("http://www.itcast.cn/");
RequestConfig config = RequestConfig.custom().setConnectionRequestTimeout(1000)
        .setConnectTimeout(500)//设置获取连接的最长时间，单位是毫秒
        .setSocketTimeout(10 * 1000)//设置数据传输的最长时间，单位是毫秒
        .build();
httpGet.setConfig(config);