新开发了一个汇总功能,为了组合数据的方便,所以接口提供非常小的粒度;但是因汇总是有多个时间、地理跨度、且查询数据量比较大,结果是200个请求查询时间非常缓慢,达到了413854毫秒,龟速!!!
研究了下HttpClient的多线程调用,经多线程实现后,200个请求耗时61382毫秒,效果非常明显。以下是封装的HttpClientThreadUtil.java工具类:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
|
import
com
.
alibaba
.
druid
.
support
.
logging
.
Log
;
import
com
.
alibaba
.
druid
.
support
.
logging
.
LogFactory
;
import
org
.
apache
.
http
.
HttpEntity
;
import
org
.
apache
.
http
.
NameValuePair
;
import
org
.
apache
.
http
.
client
.
config
.
RequestConfig
;
import
org
.
apache
.
http
.
client
.
entity
.
UrlEncodedFormEntity
;
import
org
.
apache
.
http
.
client
.
methods
.
CloseableHttpResponse
;
import
org
.
apache
.
http
.
client
.
methods
.
HttpGet
;
import
org
.
apache
.
http
.
client
.
methods
.
HttpPost
;
import
org
.
apache
.
http
.
impl
.
client
.
CloseableHttpClient
;
import
org
.
apache
.
http
.
impl
.
client
.
HttpClients
;
import
org
.
apache
.
http
.
impl
.
conn
.
PoolingHttpClientConnectionManager
;
import
org
.
apache
.
http
.
message
.
BasicNameValuePair
;
import
org
.
apache
.
http
.
protocol
.
BasicHttpContext
;
import
org
.
apache
.
http
.
protocol
.
HttpContext
;
import
org
.
apache
.
http
.
util
.
EntityUtils
;
import
java
.
io
.
IOException
;
import
java
.
io
.
UnsupportedEncodingException
;
import
java
.
util
.
ArrayList
;
import
java
.
util
.
List
;
import
java
.
util
.
Map
;
import
java
.
util
.
concurrent
.
Callable
;
import
java
.
util
.
concurrent
.
ConcurrentHashMap
;
/**
* Description:
* Author: CharlesSong
* Date : 2015/6/4
*/
public
class
HttpClientTreadUtil
{
private
static
Log
log
=
LogFactory
.
getLog
(
HttpClientTreadUtil
.
class
)
;
/**
* reqList中Map的格式{url:'请求地址',params:{请求参数},encode:'编码'}
* @param reqList
* @return
*/
public
static
String
[
]
threadPost
(
List
<
Map
<
String
,
Object
>>
reqList
)
{
System
.
out
.
println
(
"post共执行"
+
reqList
.
size
(
)
+
"个请求"
)
;
String
[
]
results
=
new
String
[
reqList
.
size
(
)
]
;
PoolingHttpClientConnectionManager
cm
=
new
PoolingHttpClientConnectionManager
(
)
;
//设置线程数最大100,如果超过100为请求个数
cm
.
setMaxTotal
(
reqList
.
size
(
)
>
100
?
reqList
.
size
(
)
:
100
)
;
CloseableHttpClient
httpclient
=
HttpClients
.
custom
(
)
.
setConnectionManager
(
cm
)
.
build
(
)
;
try
{
PostThread
[
]
postThreads
=
new
PostThread
[
reqList
.
size
(
)
]
;
for
(
int
i
=
0
;
i
<
reqList
.
size
(
)
;
i
++
)
{
Map
<
String
,
Object
>
req
=
reqList
.
get
(
i
)
;
HttpPost
post
=
new
HttpPost
(
(
String
)
req
.
get
(
"url"
)
)
;
postThreads
[
i
]
=
new
PostThread
(
httpclient
,
post
,
(
Map
<
String
,
Object
>
)
req
.
get
(
"params"
)
,
(
String
)
req
.
get
(
"encode"
)
,
i
+
1
)
;
}
//执行线程
for
(
PostThread
pt
:
postThreads
)
{
pt
.
start
(
)
;
}
//设置所有线程执行完毕之后再执行后续代码
for
(
PostThread
pt
:
postThreads
)
{
pt
.
join
(
)
;
}
for
(
int
i
=
0
;
i
<
reqList
.
size
(
)
;
i
++
)
{
results
[
i
]
=
postThreads
[
i
]
.
call
(
)
;
}
}
catch
(
Exception
e
)
{
e
.
printStackTrace
(
)
;
log
.
debug
(
"多线程post方法异常:"
+
e
.
getMessage
(
)
)
;
}
finally
{
try
{
httpclient
.
close
(
)
;
}
catch
(
IOException
e
)
{
e
.
printStackTrace
(
)
;
}
}
System
.
out
.
println
(
reqList
.
size
(
)
+
"个线程的结果为:"
+
results
.
length
+
"个,明细:::"
+
results
)
;
return
results
;
}
/**
* reqList中Map的格式{url:'请求地址',params:{请求参数},encode:'编码'}
* @param reqList
* @return
*/
public
static
String
[
]
threadGet
(
List
<
Map
<
String
,
Object
>>
reqList
)
{
System
.
out
.
println
(
"get共执行"
+
reqList
.
size
(
)
+
"个请求"
)
;
String
[
]
results
=
new
String
[
reqList
.
size
(
)
]
;
PoolingHttpClientConnectionManager
cm
=
new
PoolingHttpClientConnectionManager
(
)
;
//设置线程数最大100,如果超过100为请求个数
cm
.
setMaxTotal
(
reqList
.
size
(
)
>
100
?
reqList
.
size
(
)
:
100
)
;
CloseableHttpClient
httpclient
=
HttpClients
.
custom
(
)
.
setConnectionManager
(
cm
)
.
build
(
)
;
try
{
GetThread
[
]
getThreads
=
new
GetThread
[
reqList
.
size
(
)
]
;
for
(
int
i
=
0
;
i
<
reqList
.
size
(
)
;
i
++
)
{
Map
<
String
,
Object
>
req
=
reqList
.
get
(
i
)
;
HttpGet
get
=
new
HttpGet
(
(
String
)
req
.
get
(
"url"
)
)
;
getThreads
[
i
]
=
new
GetThread
(
httpclient
,
get
,
i
+
1
)
;
}
//执行线程
for
(
GetThread
gt
:
getThreads
)
{
gt
.
start
(
)
;
}
//设置所有线程执行完毕之后再执行后续代码
for
(
GetThread
gt
:
getThreads
)
{
gt
.
join
(
)
;
}
for
(
int
i
=
0
;
i
<
reqList
.
size
(
)
;
i
++
)
{
results
[
i
]
=
getThreads
[
i
]
.
call
(
)
;
}
}
catch
(
Exception
e
)
{
e
.
printStackTrace
(
)
;
log
.
debug
(
"多线程get方法异常:"
+
e
.
getMessage
(
)
)
;
}
finally
{
try
{
httpclient
.
close
(
)
;
}
catch
(
IOException
e
)
{
e
.
printStackTrace
(
)
;
}
}
return
results
;
}
/**
* 实现Callable回调接口
*/
static
class
PostThread
extends
Thread
implements
Callable
<String>
{
private
final
CloseableHttpClient
httpClient
;
private
final
HttpContext
context
;
private
final
HttpPost
httppost
;
private
final
int
id
;
private
String
result
=
null
;
public
PostThread
(
CloseableHttpClient
httpClient
,
HttpPost
httppost
,
Map
<
String
,
Object
>
params
,
String
encode
,
int
id
)
throws
UnsupportedEncodingException
{
//设置超时时间
RequestConfig
requestConfig
=
RequestConfig
.
custom
(
)
.
setConnectionRequestTimeout
(
60000
)
.
setConnectTimeout
(
60000
)
.
setSocketTimeout
(
60000
)
.
build
(
)
;
httppost
.
setConfig
(
requestConfig
)
;
List
<NameValuePair>
pairs
=
null
;
if
(
params
!=
null
&&
!
params
.
isEmpty
(
)
)
{
pairs
=
new
ArrayList
<NameValuePair>
(
params
.
size
(
)
)
;
for
(
Map
.
Entry
<
String
,
Object
>
entry
:
params
.
entrySet
(
)
)
{
Object
value
=
entry
.
getValue
(
)
;
if
(
value
!=
null
)
{
pairs
.
add
(
new
BasicNameValuePair
(
entry
.
getKey
(
)
,
value
.
toString
(
)
)
)
;
}
}
}
if
(
pairs
!=
null
&&
pairs
.
size
(
)
>
0
)
{
encode
=
encode
==
null
?
"UTF-8"
:
encode
;
httppost
.
setEntity
(
new
UrlEncodedFormEntity
(
pairs
,
encode
)
)
;
}
this
.
httpClient
=
httpClient
;
this
.
context
=
new
BasicHttpContext
(
)
;
this
.
httppost
=
httppost
;
this
.
id
=
id
;
}
@Override
public
void
run
(
)
{
try
{
CloseableHttpResponse
response
=
httpClient
.
execute
(
httppost
,
context
)
;
try
{
// get the response body as an array of bytes
HttpEntity
entity
=
response
.
getEntity
(
)
;
if
(
entity
!=
null
)
{
result
=
EntityUtils
.
toString
(
entity
)
;
System
.
out
.
println
(
id
+
":::执行结果:::"
+
result
)
;
}
}
finally
{
response
.
close
(
)
;
}
}
catch
(
Exception
e
)
{
log
.
error
(
id
+
" - error: "
+
e
)
;
}
}
@Override
public
String
call
(
)
throws
Exception
{
return
result
;
}
}
/**
* A thread that performs a GET.
*/
static
class
GetThread
extends
Thread
implements
Callable
<String>
{
private
final
CloseableHttpClient
httpClient
;
private
final
HttpContext
context
;
private
final
HttpGet
httpget
;
private
final
int
id
;
private
String
result
=
null
;
public
GetThread
(
CloseableHttpClient
httpClient
,
HttpGet
httpget
,
int
id
)
{
this
.
httpClient
=
httpClient
;
this
.
context
=
new
BasicHttpContext
(
)
;
this
.
httpget
=
httpget
;
this
.
id
=
id
;
}
/**
* Executes the GetMethod and prints some status information.
*/
@Override
public
void
run
(
)
{
try
{
System
.
out
.
println
(
id
+
" - about to get something from "
+
httpget
.
getURI
(
)
)
;
CloseableHttpResponse
response
=
httpClient
.
execute
(
httpget
,
context
)
;
try
{
System
.
out
.
println
(
id
+
" - get executed"
)
;
// get the response body as an array of bytes
HttpEntity
entity
=
response
.
getEntity
(
)
;
if
(
entity
!=
null
)
{
result
=
EntityUtils
.
toString
(
entity
)
;
}
}
finally
{
response
.
close
(
)
;
}
}
catch
(
Exception
e
)
{
System
.
out
.
println
(
id
+
" - error: "
+
e
)
;
}
}
@Override
public
String
call
(
)
throws
Exception
{
return
result
;
}
}
}
|
注意:
1. 需要HttpClient 4.3.X以上版本支持
2. threadGet方法未测试,threadPost方法实测没有发现问题