<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.4</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.70</version>
</dependency>
yml配置
#http config
http:
maxTotal: 300
defaultMaxPerRoute: 50
connectTimeout: 1000
connectionRequestTimeout: 500
socketTimeout: 5000
staleConnectionCheckEnabled: true
remote:
news:
key: 5db0222e1111111111
url: http://v.juhe.cn/toutiao/index
contentUrl: http://v.juhe.cn/toutiao/content
categorys: top,guonei
showCategorys: 头条,国内
groups: 15
HttpCilentConfig
package com.oracle.config;
import com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter;
import lombok.Data;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.web.client.RestTemplate;
import java.nio.charset.Charset;
import java.util.List;
@Configuration
@ConfigurationProperties(prefix = "http")
@Data
public class HttpCilentConfig {
private Integer maxTotal;// 最大连接
private Integer defaultMaxPerRoute;// 每个host的最大连接
private Integer connectTimeout;// 连接超时时间
private Integer connectionRequestTimeout;// 请求超时时间
private Integer socketTimeout;// 响应超时时间
/**
* HttpClient连接池
*
* @return
*/
@Bean
public HttpClientConnectionManager httpClientConnectionManager() {
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(maxTotal);
connectionManager.setDefaultMaxPerRoute(defaultMaxPerRoute);
return connectionManager;
}
/**
* 注册RequestConfig
*
* @return
*/
@Bean
public RequestConfig requestConfig() {
return RequestConfig.custom().setConnectTimeout(connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout).setSocketTimeout(socketTimeout)
.build();
}
/**
* 注册HttpClient
*
* @param manager
* @param config
* @return
*/
@Bean
public HttpClient httpClient(HttpClientConnectionManager manager, RequestConfig config) {
return HttpClientBuilder.create().setConnectionManager(manager).setDefaultRequestConfig(config)
.build();
}
/**
* 使用连接池管理连接
*
* @param httpClient
* @return
*/
@Bean
public ClientHttpRequestFactory requestFactory(HttpClient httpClient) {
return new HttpComponentsClientHttpRequestFactory(httpClient);
}
/**
* 使用HttpClient来初始化一个RestTemplate
*
* @param requestFactory
* @return
*/
@Bean
public RestTemplate restTemplate(ClientHttpRequestFactory requestFactory) {
RestTemplate template = new RestTemplate(requestFactory);
List<HttpMessageConverter<?>> list = template.getMessageConverters();
for (HttpMessageConverter<?> mc : list) {
if (mc instanceof StringHttpMessageConverter) {
((StringHttpMessageConverter) mc).setDefaultCharset(Charset.forName("UTF-8"));
}
if (mc instanceof FastJsonHttpMessageConverter) {
((FastJsonHttpMessageConverter) mc).setDefaultCharset(Charset.forName("UTF-8"));
}
}
return template;
}
}
RemoteNewsConfig (调用测试)
package com.oracle.config;
import com.alibaba.fastjson.JSONObject;
import com.oracle.entity.vo.ResponseNewsVo;
import com.oracle.service.IRemotenewsService;
import lombok.Data;
import org.springframework.amqp.core.AmqpTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
@Configuration
@ConfigurationProperties(prefix = "remote.news")
@Data
public class RemoteNewsConfig {
private String key;
private String url;
private String[] categorys;
private String[] showCategorys;
//每个批次组获取多少数据
private Integer groups;
private String contentUrl;
@Autowired
private RestTemplate restTemplate;
@Autowired
private IRemotenewsService iRemotenewsService;
@Autowired
private AmqpTemplate amqpTemplate;
public void remoteData(){
//获取所有类型
for(int i=0;i<categorys.length;i++){
//每个类型获取 从第一页到最后一页
for(int j=1;j<=1;j++){
ResponseEntity<String> responseEntity = restTemplate.getForEntity(url + "?key="+key+"&type="+categorys[i]+"&page="+j, String.class);
String jsonData = responseEntity.getBody();
ResponseNewsVo remoteNewsVo = JSONObject.parseObject(jsonData, ResponseNewsVo.class);
iRemotenewsService.remoteBatchNewsSourceData(remoteNewsVo.getResult().getData());
}
}
}
/* public void NewsContent(){
//遍历所有类型
for(int i=0;i<showCategorys.length;i++){
Wrapper<Remotenews> wrapper = new QueryWrapper<>();
((QueryWrapper<Remotenews>) wrapper).eq("category", showCategorys[i]);
((QueryWrapper<Remotenews>) wrapper).eq("status", 0);
//总记录数
int count=this.iRemotenewsService.count(wrapper);
//共多少批次
int totalGroup=count%groups==0?count/groups:count/groups+1;
//默认是第一批次
int defaultGroup=1;
while (defaultGroup!=totalGroup){
//进行查询
Page<Remotenews> page = this.iRemotenewsService.page(new Page<>(defaultGroup, groups), wrapper);
//包含新闻的唯一id
List<Remotenews> remotenewsList = page.getRecords();
remotenewsList.stream().forEach(remotenews -> {
ResponseEntity<String> responseEntity = restTemplate.getForEntity(contentUrl+"?key="+key+"&uniquekey="+remotenews.getUniquekey(), String.class);
String jsonData = responseEntity.getBody();
NewsContent newsContent = JSONObject.parseObject(jsonData, NewsContent.class);
//当新闻内容获取到后 将内容直接拼接到 新闻列表中 然后把数据放入mq
remotenews.setContent(newsContent.getResult().getContent());
this.amqpTemplate.convertAndSend("remoteNewsExchange", "news",JSONObject.toJSONString(remotenews));
remotenews.setStatus(1);
this.iRemotenewsService.updateById(remotenews);
});
defaultGroup++;
}
}
}*/
}