java爬虫 京东_java爬虫系列(二) - 京东手机信息

packagecom.ayulong.jd.util;importorg.apache.http.client.config.RequestConfig;importorg.apache.http.client.methods.CloseableHttpResponse;importorg.apache.http.client.methods.HttpGet;importorg.apache.http.impl.client.CloseableHttpClient;importorg.apache.http.impl.client.HttpClients;importorg.apache.http.impl.conn.PoolingHttpClientConnectionManager;importorg.apache.http.util.EntityUtils;importorg.springframework.stereotype.Component;importjava.io.File;importjava.io.FileOutputStream;importjava.io.IOException;importjava.io.OutputStream;importjava.util.UUID;

@Componentpublic classHttpUtils {privatePoolingHttpClientConnectionManager cm;publicHttpUtils() {this.cm = newPoolingHttpClientConnectionManager();//设置最大连接数

this.cm.setMaxTotal(100);//设置每个主机的最大连接数

this.cm.setDefaultMaxPerRoute(10);

}/*** 根据请求地址下载页面数据

*@paramurl

*@return页面数据*/

publicString doGetHtml(String url) {//获取HttpClient对象

CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.cm).build();//创建httpGet请求对象, 设置url地址

HttpGet httpGet = newHttpGet(url);//设置请求信息

httpGet.setConfig(getConfig());//设置请求头, 伪装用户

setHeaders(httpGet);

CloseableHttpResponse response= null;try{//使用HttpClient发起请求, 获取响应

response =httpClient.execute(httpGet);//解析响应, 返回结果

if (response.getStatusLine().getStatusCode() == 200) {//判断响应体Entity是否不为空, 如果不为空就可以使用EntityUtils

if (response.getEntity() != null) {

String content= EntityUtils.toString(response.getEntity(), "utf8");returncontent;

}

}

}catch(IOException e) {

e.printStackTrace();

}finally{//关闭response

if (response != null) {try{

response.close();

}catch(IOException e) {

e.printStackTrace();

}

}

}//返回空串

return "";

}/*** 下载图片

*@paramurl

*@return图片名称*/

publicString doGetImage(String url) {//获取HttpClient对象

CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.cm).build();//创建httpGet请求对象, 设置url地址

HttpGet httpGet = newHttpGet(url);//设置请求信息

httpGet.setConfig(getConfig());//设置请求头, 伪装用户

setHeaders(httpGet);

CloseableHttpResponse response= null;try{//使用HttpClient发起请求, 获取响应

response =httpClient.execute(httpGet);//解析响应, 返回结果

if (response.getStatusLine().getStatusCode() == 200) {//判断响应体Entity是否不为空, 如果不为空就可以使用EntityUtils

if (response.getEntity() != null) {//下载图片//获取图片的后缀

String extName = url.substring(url.lastIndexOf("."));//创建图片名, 重命名图片

String picName = UUID.randomUUID().toString() +extName;//下载图片//声明OutPutStream

OutputStream outputStream = new FileOutputStream(new File("D:\\ideaWorkspace\\pachong\\ayulong-crawler-jd\\src\\main\\resources\\static\\images\\" +picName));

response.getEntity().writeTo(outputStream);//返回图片名称

returnpicName;

}

}

}catch(IOException e) {

e.printStackTrace();

}finally{//关闭response

if (response != null) {try{

response.close();

}catch(IOException e) {

e.printStackTrace();

}

}

}//如果下载失败, 返回空串

return "";

}//设置请求信息

privateRequestConfig getConfig() {

RequestConfig config=RequestConfig.custom()

.setConnectTimeout(1000) //创建连接的最长时间

.setConnectionRequestTimeout(500) //获取连接的最长时间

.setSocketTimeout(10000) //数据传输的最长时间

.build();returnconfig;

}//设置请求头

private voidsetHeaders(HttpGet httpGet) {

httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");

}

}