java实现大文件下载,基于http方式,控件神马的就不说了。
思路:下载文件无非要读取文件然后写文件,主要这两个步骤,主要难点:
1.读文件,就是硬盘到内存的过程,由于jdk内存限制,不能读的太大。
2.写文件,就是响应到浏览器端的过程,http协议是短链接,如果写文件太慢,时间过久,会造成浏览器死掉。
知识点:
1.org.apache.http.impl.client.CloseableHttpClient 模拟httpClient客户端发送http请求,可以控制到请求文件的字节位置。
2.BufferedInputStream都熟悉,用它接受请求来的流信息缓存。
3.RandomAccessFile文件随机类,可以向文件写入指定位置的流信息。
基于以上信息,我的实现思路就是首先判断下载文件大小,配合多线程分割定制http请求数量和请求内容,响应到写入到RandomAccessFile指定位置中。在俗点就是大的http分割成一个个小的http请求,相当于每次请求一个网页。
废话不说,上代码。
DownLoadManagerTest类:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
package
xxxx;
import
java.io.File;
import
java.io.IOException;
import
java.io.RandomAccessFile;
import
java.net.HttpURLConnection;
import
java.net.URL;
import
java.util.concurrent.CountDownLatch;
import
org.apache.commons.lang.exception.ExceptionUtils;
import
org.apache.http.impl.client.CloseableHttpClient;
import
org.apache.http.impl.client.HttpClients;
import
org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import
org.junit.After;
import
org.junit.Before;
import
org.junit.Test;
import
org.junit.runner.RunWith;
import
org.slf4j.Logger;
import
org.slf4j.LoggerFactory;
import
org.springframework.beans.factory.annotation.Autowired;
import
org.springframework.core.task.TaskExecutor;
import
org.springframework.test.context.ActiveProfiles;
import
org.springframework.test.context.ContextConfiguration;
import
org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests;
import
org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
*
* 文件下载管理类
*/
@RunWith
(SpringJUnit4ClassRunner.
class
)
@ActiveProfiles
(
"test"
)
@ContextConfiguration
(locations={
"classpath:test/applicationContext.xml"
})
public
class
DownLoadManagerTest
extends
AbstractTransactionalJUnit4SpringContextTests{
private
static
final
Logger LOGGER = LoggerFactory.getLogger(DownLoadManagerTest.
class
);
/**
*
* 每个线程下载的字节数
*/
private
long
unitSize =
1000
*
1024
;
@Autowired
private
TaskExecutor taskExecutor;
private
CloseableHttpClient httpClient;
private
Long starttimes;
private
Long endtimes;
@Before
public
void
setUp()
throws
Exception
{
starttimes = System.currentTimeMillis();
System.out.println(
"测试开始...."
);
}
@After
public
void
tearDown()
throws
Exception
{
endtimes = System.currentTimeMillis();
System.out.println(
"测试结束!!"
);
System.out.println(
"********************"
);
System.out.println(
"下载总耗时:"
+(endtimes-starttimes)/
1000
+
"s"
);
System.out.println(
"********************"
);
}
public
DownLoadManagerTest() {
System.out.println(
"初始化测试类...."
);
PoolingHttpClientConnectionManager cm =
new
PoolingHttpClientConnectionManager();
cm.setMaxTotal(
100
);
httpClient = HttpClients.custom().setConnectionManager(cm).build();
}
/**
*
* 启动多个线程下载文件
*/
@Test
public
void
doDownload()
throws
IOException {
String remoteFileUrl=
"http://{host}:{port}/{project}/xx.xml"
;
String localPath=
"E://test//"
;
String fileName =
new
URL(remoteFileUrl).getFile();
System.out.println(
"远程文件名称:"
+fileName);
fileName = fileName.substring(fileName.lastIndexOf(
"/"
) +
1
,
fileName.length()).replace(
"%20"
,
" "
);
System.out.println(
"本地文件名称:"
+fileName);
long
fileSize =
this
.getRemoteFileSize(remoteFileUrl);
this
.createFile(localPath+System.currentTimeMillis()+fileName, fileSize);
Long threadCount = (fileSize/unitSize)+(fileSize % unitSize!=
0
?
1
:
0
);
long
offset =
0
;
CountDownLatch end =
new
CountDownLatch(threadCount.intValue());
if
(fileSize <= unitSize) {
// 如果远程文件尺寸小于等于unitSize
DownloadThreadTest downloadThread =
new
DownloadThreadTest(remoteFileUrl,
localPath+fileName, offset, fileSize,end,httpClient);
taskExecutor.execute(downloadThread);
}
else
{
// 如果远程文件尺寸大于unitSize
for
(
int
i =
1
; i < threadCount; i++) {
DownloadThreadTest downloadThread =
new
DownloadThreadTest(
remoteFileUrl, localPath+fileName, offset, unitSize,end,httpClient);
taskExecutor.execute(downloadThread);
offset = offset + unitSize;
}
if
(fileSize % unitSize !=
0
) {
// 如果不能整除,则需要再创建一个线程下载剩余字节
DownloadThreadTest downloadThread =
new
DownloadThreadTest(remoteFileUrl, localPath+fileName, offset, fileSize - unitSize * (threadCount-
1
),end,httpClient);
taskExecutor.execute(downloadThread);
}
}
try
{
end.await();
}
catch
(InterruptedException e) {
LOGGER.error(
"DownLoadManager exception msg:{}"
,ExceptionUtils.getFullStackTrace(e));
e.printStackTrace();
}
// System.out.println("111111");
LOGGER.debug(
"下载完成!{} "
,localPath+fileName);
//return localPath+fileName;
}
/**
*
* 获取远程文件尺寸
*/
private
long
getRemoteFileSize(String remoteFileUrl)
throws
IOException {
long
fileSize =
0
;
HttpURLConnection httpConnection = (HttpURLConnection)
new
URL(
remoteFileUrl).openConnection();
httpConnection.setRequestMethod(
"HEAD"
);
int
responseCode = httpConnection.getResponseCode();
if
(responseCode >=
400
) {
LOGGER.debug(
"Web服务器响应错误!"
);
return
0
;
}
String sHeader;
for
(
int
i =
1
;; i++) {
sHeader = httpConnection.getHeaderFieldKey(i);
if
(sHeader !=
null
&& sHeader.equals(
"Content-Length"
)) {
System.out.println(
"文件大小ContentLength:"
+ httpConnection.getContentLength());
fileSize = Long.parseLong(httpConnection
.getHeaderField(sHeader));
break
;
}
}
return
fileSize;
}
/**
*
* 创建指定大小的文件
*/
private
void
createFile(String fileName,
long
fileSize)
throws
IOException {
File newFile =
new
File(fileName);
RandomAccessFile raf =
new
RandomAccessFile(newFile,
"rw"
);
raf.setLength(fileSize);
raf.close();
}
public
TaskExecutor getTaskExecutor() {
return
taskExecutor;
}
public
void
setTaskExecutor(TaskExecutor taskExecutor) {
this
.taskExecutor = taskExecutor;
}
}
|
DownloadThreadTest类:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
package
xxxx;
import
java.io.BufferedInputStream;
import
java.io.File;
import
java.io.IOException;
import
java.io.RandomAccessFile;
import
java.util.concurrent.CountDownLatch;
import
org.apache.commons.lang.exception.ExceptionUtils;
import
org.apache.http.client.ClientProtocolException;
import
org.apache.http.client.methods.CloseableHttpResponse;
import
org.apache.http.client.methods.HttpGet;
import
org.apache.http.impl.client.CloseableHttpClient;
import
org.apache.http.protocol.BasicHttpContext;
import
org.apache.http.protocol.HttpContext;
import
org.slf4j.Logger;
import
org.slf4j.LoggerFactory;
/**
*
* 负责文件下载的类
*/
public
class
DownloadThreadTest
extends
Thread {
private
static
final
Logger LOGGER = LoggerFactory
.getLogger(DownloadThreadTest.
class
);
/**
*
* 待下载的文件
*/
private
String url =
null
;
/**
*
* 本地文件名
*/
private
String fileName =
null
;
/**
*
* 偏移量
*/
private
long
offset =
0
;
/**
*
* 分配给本线程的下载字节数
*/
private
long
length =
0
;
private
CountDownLatch end;
private
CloseableHttpClient httpClient;
private
HttpContext context;
/**
*
* @param url
* 下载文件地址
*
* @param fileName
* 另存文件名
*
* @param offset
* 本线程下载偏移量
*
* @param length
* 本线程下载长度
*
*
*
* @author Angus.wang
*
* */
public
DownloadThreadTest(String url, String file,
long
offset,
long
length,
CountDownLatch end, CloseableHttpClient httpClient) {
this
.url = url;
this
.fileName = file;
this
.offset = offset;
this
.length = length;
this
.end = end;
this
.httpClient = httpClient;
this
.context =
new
BasicHttpContext();
LOGGER.debug(
"偏移量="
+ offset +
";字节数="
+ length);
}
public
void
run() {
try
{
HttpGet httpGet =
new
HttpGet(
this
.url);
httpGet.addHeader(
"Range"
,
"bytes="
+
this
.offset +
"-"
+ (
this
.offset +
this
.length -
1
));
CloseableHttpResponse response = httpClient.execute(httpGet,
context);
;
BufferedInputStream bis =
new
BufferedInputStream(response
.getEntity().getContent());
byte
[] buff =
new
byte
[
1024
];
int
bytesRead;
File newFile =
new
File(fileName);
RandomAccessFile raf =
new
RandomAccessFile(newFile,
"rw"
);
while
((bytesRead = bis.read(buff,
0
, buff.length)) != -
1
) {
raf.seek(
this
.offset);
raf.write(buff,
0
, bytesRead);
this
.offset =
this
.offset + bytesRead;
}
raf.close();
bis.close();
}
catch
(ClientProtocolException e) {
LOGGER.error(
"DownloadThread exception msg:{}"
,ExceptionUtils.getFullStackTrace(e));
}
catch
(IOException e) {
LOGGER.error(
"DownloadThread exception msg:{}"
,ExceptionUtils.getFullStackTrace(e));
}
finally
{
end.countDown();
LOGGER.info(end.getCount() +
" is go on!"
);
System.out.println(end.getCount() +
" is go on!"
);
}
}
}
|
application.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
|
<bean id=
"taskExecutor"
class
=
"org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor"
>
<!-- 线程池活跃的线程数 -->
<property name=
"corePoolSize"
value=
"5"
/>
<!-- 线程池最大活跃的线程数 -->
<property name=
"maxPoolSize"
value=
"10"
/>
<!-- 队列的最大容量 -->
<property name=
"queueCapacity"
value=
"600"
/>
</bean>
<bean id=
"downLoadManager"
class
=
"xx.DownLoadManagerTest"
>
<property name=
"taskExecutor"
ref=
"taskExecutor"
/>
</bean>
|
测试运行,500M,我这网速得半个小时左右。要想下载更大的文件,只要jdk内存够大,就无限更改队列最大容量吧。
如果不同意见,欢迎各位大神指正。
尊重原创,多谢作者分享,转载自:http://my.oschina.net/zmf/blog/336961