Java爬虫入门
1、需要的相关依赖
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
1、get传参请求http://www.baidu.com
通过实例URIBuilder,使用setParameter方法以k-v的方式设置参数,如果要设置多个参数可以使用链式写法
//创建httpClient
CloseableHttpClient httpClient = HttpClients.createDefault();
//創建HttpGet对象
URIBuilder uriBuilder = new URIBuilder("http://www.baidu.com/s");
uriBuilder.setParameter("wd","风景");
HttpGet httpGet = new HttpGet(uriBuilder.build());
System.out.println("请求信息"+httpGet);
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpGet);
if(response.getStatusLine().getStatusCode() == 200){
System.out.println("请求成功");
//获取响应体里的内容
String content = EntityUtils.toString(response.getEntity(), "utf-8");
System.out.println(content.length());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
System.out.println("请求信息"+httpGet);
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
1、post传参请求http://www.baidu.com
通过HttpPost的setsetEntity方法,entity里存放的是个集合,集合里的value是一个k-v键值对,我们采用NameValuePair的实现类BasicNameValuePair完成
//创建httpClient
CloseableHttpClient httpClient = HttpClients.createDefault();
//声明list集合
ArrayList<NameValuePair> params = new ArrayList<>();
// 加入封装的参数
params.add(new BasicNameValuePair("wd","风景"));
// 创建一个entity
UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(params, "utf-8");
//創建HttpPost对象
HttpPost httpPost = new HttpPost("http://www.baidu.com");
httpPost.setEntity(urlEncodedFormEntity);
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpPost);
if(response.getStatusLine().getStatusCode() == 200){
System.out.println("请求成功");
String content = EntityUtils.toString(response.getEntity(), "utf-8");
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
3、连接池-----(和数据库连接池一样,防止多次使用销毁在创建浪费资源,而是需要的时候就从连接池拿,不需要就放回连接池)
//创建连接池管理器
PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager();
//设置连接池单个连接数最大为10
manager.setDefaultMaxPerRoute(10);
//设置连接池的连接总数最大为100
manager.setMaxTotal(100);
//使用连接池管理器发请求
doGet(manager);
doGet(manager);
}
public static void doGet(PoolingHttpClientConnectionManager manager){
CloseableHttpClient client = HttpClients.custom().setConnectionManager(manager).build();
HttpGet httpGet = new HttpGet("http://www.baidu.com");
CloseableHttpResponse response = null;
try {
response = client.execute(httpGet);
if(response.getStatusLine().getStatusCode() == 200){
System.out.println("请求成功");
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
if(response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
4、请求的参数配置
//配置请求信息
RequestConfig config = RequestConfig.custom().setConnectTimeout(1000)//连接的最长时间
.setConnectionRequestTimeout(500)//获取连接的最长时间
.setSocketTimeout(10 * 1000)//数据传输的最长时间
.build();
//设置配置
httpGet.setConfig(config);