pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.wei.plugs</groupId>
<artifactId>httpclient</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>war</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.6.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
</dependencies>
</project>
处理类
package com.wei.plugs.processor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@Component("processor")
public class Processor {
private Logger logger = LoggerFactory.getLogger(Processor.class);
public void processOne() {
logger.info("====processOne任务开始!!====");
try {
int concurrent = 50;//线程条数控制
int pageSize = 1000;//每次获取数据的数量
int querySpaceTime = 1000;//获取数据间隔时间
int totalSize = 21235;//总数据量
String uri = "https://www.baidu.com";
ExecutorService executor = Executors.newCachedThreadPool();
//总数据量
List<Integer> data = new ArrayList<Integer>();
for (int i = 0; i < totalSize; i++) {
data.add(i);
}
final Semaphore semaphore = new Semaphore(concurrent);
List<Integer> list = null;
//开始数据下标
int start = 0;
int end = 0;
//循环 直到总数据被取完为止
do {
end = start + pageSize;
if(end > data.size()) {
end = data.size();
}
list = data.subList(start, end);
final CountDownLatch countDownLatch = new CountDownLatch(list.size());
if(list == null || list.size() == 0) {
logger.info("没有查询到processOne需要处理的数据!");
return;
}
for (int i = 0; i < list.size(); i++) {
executor.execute(new ThreadOne(semaphore, countDownLatch,uri));
}
countDownLatch.await();//线程阻塞,直到锁为0才释放,继续向下执行
start += pageSize;
logger.info("lsit-size:===="+list.size());
Thread.sleep(querySpaceTime);
} while (list.size() == pageSize);
executor.shutdown();
} catch (Exception e) {
logger.error("processOne处理数据时出现错误,错误信息:{}", e.getMessage());
}
}
}
线程类
package com.wei.plugs.processor;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.wei.plugs.util.HttpClientUtil;
/**
* 线程
*/
public class ThreadOne implements Runnable {
private Logger logger = LoggerFactory.getLogger(ThreadOne.class);
private Semaphore semaphore;
private CountDownLatch countDownLatch;
private String uri;
public ThreadOne(Semaphore semaphore,CountDownLatch countDownLatch,String uri) {
super();
this.semaphore = semaphore;
this.countDownLatch =countDownLatch;
this.uri = uri;
}
@Override
public void run() {
try {
semaphore.acquire();
// 发送请求
HttpClientUtil.sendGet(uri);
semaphore.release();
} catch (Exception e) {
logger.error("processOne线程内出现问题,错误信息:{}", e.getMessage());
} finally{
countDownLatch.countDown();
}
}
}
并发可用httpClient工具类
package com.wei.plugs.util;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* httpclient请求工具类
* httpclient 连接池
*/
public class HttpClientUtil {
private static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class);
private static PoolingHttpClientConnectionManager cm = null;
private static CloseableHttpClient httpClient = null;
//默认content 类型
private static final String DEFAULT_CONTENT_TYPE = "application/json";
/**
* 连接池设置
*/
//长连接时间保持设置
private static final int HTTP_DEFAULT_KEEP_TIME = 60000;
/*
* 1、MaxtTotal是整个池子的大小; 2、DefaultMaxPerRoute是根据连接到的主机对MaxTotal的一个细分;比如:
* MaxtTotal=400 DefaultMaxPerRoute=200
* 而我只连接到https://www.baidu.com时,到这个主机的并发最多只有200;而不是400; 而我连接到https://www.baidu.com 和
* http://qq.com时,到每个主机的并发最多只有200;即加起来是400(但不能超过400);
* 所以起作用的设置是DefaultMaxPerRoute。
*/
private static final int DEFAULT_MAX_PER_ROUTE = 1000;
//设置连接池的大小
private static final int MAX_TOTAL = 1000;
/**
* 请求相关超时时间设置
*/
//设置链接超时
private static final int CONNECTION_TIMEOUT = 5000;
//设置等待数据超时时间
private static final int SOCKET_TIMEOUT = 15000;
/**
* 初始化连接池
*/
private static synchronized void initPools() {
if (httpClient == null) {
cm = new PoolingHttpClientConnectionManager();
cm.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
cm.setMaxTotal(MAX_TOTAL);
httpClient = HttpClients.custom().setKeepAliveStrategy(defaultStrategy).setConnectionManager(cm).build();
}
}
/**
* 长连接保持设置
* Http connection keepAlive 设置
*/
private static ConnectionKeepAliveStrategy defaultStrategy = new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
int keepTime = HTTP_DEFAULT_KEEP_TIME;
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase("timeout")) {
try {
return Long.parseLong(value) * 1000;
} catch (Exception e) {
e.printStackTrace();
logger.error("format KeepAlive timeout exception, exception:" + e.toString());
}
}
}
return keepTime * 1000;
}
};
/**
* 执行http post请求 默认采用Content-Type:application/json,Accept:application/json
* @param uri 请求地址
* @param data 请求数据
* @return 返回是数据
*/
public static String sendPost(String uri, String data) {
long startTime = System.currentTimeMillis();
HttpEntity httpEntity = null;
HttpEntityEnclosingRequestBase method = null;
String responseBody = "";
try {
if (httpClient == null) {
initPools();
}
method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME, DEFAULT_CONTENT_TYPE);
method.setEntity(new StringEntity(data));
HttpContext context = HttpClientContext.create();
CloseableHttpResponse httpResponse = httpClient.execute(method, context);
httpEntity = httpResponse.getEntity();
if (httpEntity != null) {
responseBody = EntityUtils.toString(httpEntity, "UTF-8");
}
} catch (Exception e) {
if(method != null){
method.abort();
}
e.printStackTrace();
logger.error(
"execute post request exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):"
+ (System.currentTimeMillis() - startTime));
} finally {
if (httpEntity != null) {
try {
EntityUtils.consumeQuietly(httpEntity);
} catch (Exception e) {
e.printStackTrace();
logger.error(
"close response exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):"
+ (System.currentTimeMillis() - startTime));
}
}
}
return responseBody;
}
/**
* 执行GET 请求
* @param uri 请求链接
* @return 请求结果
*/
public static String sendGet(String uri) {
long startTime = System.currentTimeMillis();
HttpEntity httpEntity = null;
HttpRequestBase method = null;
String responseBody = "";
try {
if (httpClient == null) {
initPools();
}
method = getRequest(uri, HttpGet.METHOD_NAME, DEFAULT_CONTENT_TYPE);
HttpContext context = HttpClientContext.create();
CloseableHttpResponse httpResponse = httpClient.execute(method, context);
httpEntity = httpResponse.getEntity();
if (httpEntity != null) {
responseBody = EntityUtils.toString(httpEntity, "UTF-8");
logger.info("请求URL: "+uri+" 返回状态码:"+httpResponse.getStatusLine().getStatusCode());
}
} catch (Exception e) {
if(method != null){
method.abort();
}
e.printStackTrace();
logger.error("execute get request exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):"
+ (System.currentTimeMillis() - startTime));
} finally {
if (httpEntity != null) {
try {
EntityUtils.consumeQuietly(httpEntity);
} catch (Exception e) {
e.printStackTrace();
logger.error("close response exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):"
+ (System.currentTimeMillis() - startTime));
}
}
}
return responseBody;
}
/**
* 创建请求
* @param uri 请求url
* @param methodName 请求的方法类型
* @param contentType contentType类型
* @return
*/
private static HttpRequestBase getRequest(String uri, String methodName, String contentType) {
HttpRequestBase method = null;
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(SOCKET_TIMEOUT).setConnectTimeout(CONNECTION_TIMEOUT)
.setConnectionRequestTimeout(CONNECTION_TIMEOUT).setExpectContinueEnabled(false).build();
if (HttpPut.METHOD_NAME.equalsIgnoreCase(methodName)) {
method = new HttpPut(uri);
} else if (HttpPost.METHOD_NAME.equalsIgnoreCase(methodName)) {
method = new HttpPost(uri);
} else if (HttpGet.METHOD_NAME.equalsIgnoreCase(methodName)) {
method = new HttpGet(uri);
} else {
method = new HttpPost(uri);
}
if (StringUtils.isBlank(contentType)) {
contentType = DEFAULT_CONTENT_TYPE;
}
method.addHeader("Content-Type", contentType);
method.addHeader("Accept", contentType);
method.setConfig(requestConfig);
return method;
}
public static CloseableHttpClient getHttpClient() {
return httpClient;
}
public static PoolingHttpClientConnectionManager getHttpConnectionManager() {
return cm;
}
}