多线程断点续传实现

设计的几个要点:

 

1. 把每个下载文件切成若干个块(Block)，然后得到一个位图，用来标记每个块的下载情况，并保存到文件里，用于实现断点续传。

2. HTTP Header里增加Range，如果服务器返回Cotent-Range 说明服务器支持文件定位，可以实现多线程下载。

[c-sharp]  view plain  copy

1. /** 
2.  * 专注互联网,分享创造价值 
3.  *  maoxiang@gmail.com 
4.  *  2010-3-30下午04:40:06 
5.  */  
6. package common.http;  
7.   
8. import java.io.BufferedInputStream;  
9. import java.io.ByteArrayOutputStream;  
10. import java.io.File;  
11. import java.io.FileInputStream;  
12. import java.io.FileOutputStream;  
13. import java.io.IOException;  
14. import java.io.RandomAccessFile;  
15. import java.net.HttpURLConnection;  
16. import java.net.InetSocketAddress;  
17. import java.net.Proxy;  
18. import java.net.SocketAddress;  
19. import java.net.URL;  
20. import java.util.Iterator;  
21. import java.util.List;  
22. import java.util.Map;  
23. import java.util.concurrent.atomic.AtomicLong;  
24.   
25. import org.apache.commons.codec.binary.Base64;  
26. import org.apache.commons.logging.Log;  
27. import org.apache.commons.logging.LogFactory;  
28.   
29. /** 
30.  * 一个多线程支持断点续传的工具类<br/> 
31.  * 2010-03 用Htpp Component重写 
32.  */  
33. public class HttpDownloader {  
34.   
35.     private final Log log = LogFactory.getLog(getClass().getName());  
36.     private int threads = 5; // 总共的线程数  
37.     private int maxThreads = 10; // 最大的线程数  
38.     private String destUrl; // 目标的URL  
39.     private String savePath; // 保存的路径  
40.     private File lockFile;// 用来保存进度的文件  
41.     private String userAgent = "jHttpDownload";  
42.     private boolean useProxy = false;  
43.     private String proxyServer;  
44.     private int proxyPort;  
45.     private String proxyUser;  
46.     private String proxyPassword;  
47.     private int blockSize = 1024 * 4; // 4K 一个块  
48.     // 1个位代表一个块,，用来标记是否下载完成  
49.     private byte[] blockSet;  
50.     private int blockPage; // 每个线程负责的大小  
51.     private int blocks;  
52.     private boolean running; // 是否运行中,避免线程不能释放  
53.     // =======下载进度信息  
54.     private long beginTime;  
55.     private AtomicLong downloaded = new AtomicLong(0); // 已下载的字节数/  
56.     private long fileLength; // 总的字节数  
57.     // 监控线程,用来保存进度和汇报进度  
58.     private MonitorThread monitorThread = new MonitorThread();  
59.   
60.     public HttpDownloader(String destUrl, String savePath, int threads) {  
61.         this.threads = threads;  
62.         this.destUrl = destUrl;  
63.         this.savePath = savePath;  
64.     }  
65.   
66.     public HttpDownloader(String destUrl, String savePath) {  
67.         this(destUrl, savePath, 5);  
68.     }  
69.   
70.     /** 
71.      * 开始下载 
72.      */  
73.     public boolean download() {  
74.         log.info("下载文件" + destUrl + ",保存路径=" + savePath);  
75.         beginTime = System.currentTimeMillis();  
76.         boolean ok = false;  
77.         try {  
78.             File saveFile = new File(savePath);  
79.             lockFile = new File(savePath + ".lck");  
80.             if (lockFile.exists() && !lockFile.canWrite()) {  
81.                 throw new Exception("文件被锁住，或许已经在下载中了");  
82.             }  
83.             File parent = saveFile.getParentFile();  
84.             if (!parent.exists()) {  
85.                 log.info("创建目录=" + parent.getAbsolutePath());  
86.             }  
87.             if (!parent.canWrite()) {  
88.                 throw new Exception("保存目录不可写");  
89.             }  
90.             if (saveFile.exists()) {  
91.                 if (!saveFile.canWrite()) {  
92.                     throw new Exception("保存文件不可写,无法继续下载");  
93.                 }  
94.                 log.info("检查之前下载的文件");  
95.                 if (lockFile.exists()) {  
96.                     log.info("加载之前下载进度");  
97.                     loadPrevious();  
98.                 }  
99.             } else {  
100.                 lockFile.createNewFile();  
101.             }  
102.             // 1初始化httpClient  
103.             HttpURLConnection httpUrlConnection = getHttpConnection(0);  
104.             String contentLength = httpUrlConnection  
105.                     .getHeaderField("Content-Length");  
106.             if (contentLength != null) {  
107.                 try {  
108.                     fileLength = Long.parseLong(contentLength);  
109.                 } catch (Exception e) {  
110.                 }  
111.             }  
112.             log.info("下载文件的大小:" + fileLength);  
113.             if (fileLength <= 0) {  
114.                 // 不支持多线程下载,采用单线程下载  
115.                 log.info("服务器不能返回文件大小，采用单线程下载");  
116.                 threads = 1;  
117.             }  
118.             if (httpUrlConnection.getHeaderField("Content-Range") == null) {  
119.                 log.info("服务器不支持断点续传");  
120.                 threads = 1;  
121.             } else {  
122.                 log.info("服务器支持断点续传");  
123.             }  
124.             //  
125.             if (fileLength > 0 && parent.getFreeSpace() < fileLength) {  
126.                 throw new Exception("磁盘空间不够");  
127.             }  
128.             if (fileLength > 0) {  
129.                 int i = (int) (fileLength / blockSize);  
130.                 if (fileLength % blockSize > 0) {  
131.                     i++;  
132.                 }  
133.                 blocks = i;               
134.             } else {  
135.                 // 一个块             
136.                 blocks = 1;  
137.             }  
138.             if (blockSet != null) {  
139.                 log.info("检查文件，是否能够续传");  
140.                 if (blockSet.length != fileLength / (8l * blockSize)) {  
141.                     log.info("文件大小已改变，需要重新下载");  
142.                     blockSet = null;  
143.                 }  
144.             }  
145.             if (blockSet == null) {  
146.                 blockSet = BitUtil.createBit(blocks);  
147.             }  
148.             log.info("文件的块数:" + blocks + "," + blockSet.length);  
149.             if (threads > maxThreads) {  
150.                 log.info("超过最大线程数，使用最大线程数");  
151.                 threads = maxThreads;  
152.             }  
153.             blockPage = blocks / threads; // 每个线程负责的块数  
154.             log.info("分配线程。线程数量=" + threads + ",块总数=" + blocks + ",总字节数="  
155.                     + fileLength + ",每块大小=" + blockSize + ",块/线程=" + blockPage);  
156.             // 检查  
157.             running = true;  
158.             // 创建一个线程组,方便观察和调试  
159.             ThreadGroup downloadGroup = new ThreadGroup("download");  
160.             for (int i = 0; i < threads; i++) {  
161.                 int begin = i * blockPage;  
162.                 int end = (i + 1) * blockPage;  
163.                 if (i == threads - 1 && blocks % threads > 0) {  
164.                     // 如果最后一个线程，有余数，需要修正  
165.                     end = blocks;  
166.                 }  
167.                 // 扫描每个线程的块是否有需要下载的  
168.                 boolean needDownload = false;  
169.                 for (int j = begin; j < end; j++) {  
170.                     if (!BitUtil.getBit(blockSet, j)) {  
171.                         needDownload = true;  
172.                         break;  
173.                     }  
174.                 }  
175.                 if (!needDownload) {  
176.                     log.info("所有块已经下载完毕.Begin=" + begin + ",End=" + end);  
177.                 }  
178.                 // 启动下载其他线程  
179.                 DownloadThread downloadThread = new DownloadThread(  
180.                         downloadGroup, i, begin, end);  
181.                 downloadThread.start();  
182.             }  
183.             monitorThread.setStop(false);  
184.             monitorThread.start();  
185.             // 也可以用Thread.join 实现等待进程完成  
186.             while (downloadGroup.activeCount() > 0) {  
187.                 Thread.sleep(2000);  
188.             }  
189.             ok = true;  
190.         } catch (Exception e) {  
191.             e.printStackTrace();  
192.             log.error(e);  
193.         } finally {  
194.             // closeHttpClient();  
195.         }  
196.         if (ok) {  
197.             log.info("删除进度文件:" + lockFile.getAbsolutePath());  
198.             lockFile.delete();  
199.         }  
200.         monitorThread.setStop(true);  
201.         log.info("下载完成，耗时:"  
202.                 + getTime((System.currentTimeMillis() - beginTime) / 1000));  
203.         return ok;  
204.     }  
205.   
206.     private void loadPrevious() throws Exception {  
207.         ByteArrayOutputStream outStream = new ByteArrayOutputStream();  
208.         FileInputStream inStream = new FileInputStream(lockFile);  
209.         byte[] buffer = new byte[4096];  
210.         int n = 0;  
211.         while (-1 != (n = inStream.read(buffer))) {  
212.             outStream.write(buffer, 0, n);  
213.         }  
214.         outStream.close();  
215.         inStream.close();  
216.         blockSet = outStream.toByteArray();  
217.         log.info("之前的文件大小应该是:" + blockSet.length * 8l * blockSize + ",一共有:"  
218.                 + blockSet.length + "块");  
219.     }  
220.   
221.     private HttpURLConnection httpConnection0; // 用来快速返回,减少一次连接  
222.   
223.     private HttpURLConnection getHttpConnection(long pos) throws IOException {  
224.         if (pos == 0 && httpConnection0 != null) {  
225.             return httpConnection0;  
226.         }  
227.         URL url = new URL(destUrl);  
228.         log.debug("开始一个Http请求连接。Url=" + url + "定位:" + pos + "/n");  
229.         // 默认的会处理302请求  
230.         HttpURLConnection.setFollowRedirects(false);  
231.         HttpURLConnection httpConnection = null;  
232.         if (useProxy) {  
233.             log.debug("使用代理进行连接.ProxyServer=" + proxyServer + ",ProxyPort="  
234.                     + proxyPort);  
235.             SocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);  
236.             Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);  
237.             httpConnection = (HttpURLConnection) url.openConnection(proxy);  
238.             if (proxyUser != null && proxyPassword != null) {  
239.                 String encoded = new String(Base64.encodeBase64(new String(  
240.                         proxyUser + ":" + proxyPassword).getBytes()));  
241.                 httpConnection.setRequestProperty("Proxy-Authorization",  
242.                         "Basic " + encoded);  
243.             }  
244.         } else {  
245.             httpConnection = (HttpURLConnection) url.openConnection();  
246.         }  
247.         httpConnection.setRequestProperty("User-Agent", userAgent);  
248.         httpConnection.setRequestProperty("RANGE", "bytes=" + pos + "-");  
249.         int responseCode = httpConnection.getResponseCode();  
250.         log.debug("服务器返回:" + responseCode);  
251.         Map<String, List<String>> headers = httpConnection.getHeaderFields();  
252.         Iterator<String> iterator = headers.keySet().iterator();  
253.         while (iterator.hasNext()) {  
254.             String key = iterator.next();  
255.             String value = "";  
256.             for (String v : headers.get(key)) {  
257.                 value = ";" + v;  
258.             }  
259.             log.debug(key + "=" + value);  
260.         }  
261.         if (responseCode < 200 || responseCode >= 400) {  
262.             throw new IOException("服务器返回无效信息:" + responseCode);  
263.         }  
264.         if (pos == 0) {  
265.             httpConnection0 = httpConnection;  
266.         }  
267.         return httpConnection;  
268.     }  
269.   
270.     public void returnConnection(HttpURLConnection connecton) {  
271.         if (connecton != null)  
272.             connecton.disconnect();  
273.         connecton = null;  
274.     }  
275.   
276.     private String getDesc() {  
277.         long downloadBytes = downloaded.longValue();  
278.   
279.         return String  
280.                 .format(  
281.                         "已下载/总大小=%s/%s(%s),速度:%s,耗时:%s,剩余大小:%d",  
282.                         getFileSize(downloadBytes),  
283.                         getFileSize(fileLength),  
284.                         getProgress(fileLength, downloadBytes),  
285.                         getFileSize(downloadBytes  
286.                                 / ((System.currentTimeMillis() - beginTime) / 1000 + 1)),  
287.                         getTime((System.currentTimeMillis() - beginTime) / 1000),  
288.                         fileLength - downloadBytes);  
289.     }  
290.   
291.     private String getFileSize(long totals) {  
292.         // 计算文件大小  
293.         int i = 0;  
294.         String j = "BKMGT";  
295.         float s = totals;  
296.         while (s > 1024) {  
297.             s /= 1024;  
298.             i++;  
299.         }  
300.         return String.format("%.2f", s) + j.charAt(i);  
301.     }  
302.   
303.     private String getProgress(long totals, long read) {  
304.         if (totals == 0)  
305.             return "0%";  
306.         return String.format("%d", read * 100 / totals) + "%";  
307.     }  
308.   
309.     private String getTime(long seconds) {  
310.         int i = 0;  
311.         String j = "秒分时天";  
312.         long s = seconds;  
313.         String result = "";  
314.         while (s > 0) {  
315.             if (s % 60 > 0) {  
316.                 result = String.valueOf(s % 60) + (char) j.charAt(i) + result;  
317.             }  
318.             s /= 60;  
319.             i++;  
320.         }  
321.         return result;  
322.     }  
323.   
324.     /** 
325.      * 一个下载线程. 
326.      */  
327.     private class DownloadThread extends Thread {  
328.   
329.         private RandomAccessFile destFile; // 用来实现保存的随机文件  
330.         private int id = 0;  
331.         private int blockBegin = 0; // 开始块  
332.         private int blockEnd = 0; // 结束块  
333.         private long pos;// 绝对指针  
334.   
335.         private String getThreadName() {  
336.             return "DownloadThread-" + id + "=>";  
337.         }  
338.   
339.         public DownloadThread(ThreadGroup group, int id, int blockBegin,  
340.                 int blockEnd) throws Exception {  
341.             super(group, "downloadThread-" + id);  
342.             this.id = id;  
343.             this.blockBegin = blockBegin;  
344.             this.blockEnd = blockEnd;  
345.             this.pos = 1l * blockBegin * blockSize; // 转换为长整型  
346.             destFile = new RandomAccessFile(savePath, "rw");  
347.         }  
348.   
349.         public void run() {  
350.             BufferedInputStream inputStream = null;  
351.             try {  
352.                 log.info(getThreadName() + "下载线程." + this.toString());  
353.                 log.info(getThreadName() + ":定位文件位置.Pos=" + 1l * blockBegin  
354.                         * blockSize);  
355.                 destFile.seek(1l * blockBegin * blockSize);  
356.                 log.info(getThreadName() + ":开始下载.[ " + blockBegin + " - "  
357.                         + blockEnd + "]");  
358.   
359.                 HttpURLConnection httpConnection = getHttpConnection(pos);  
360.                 inputStream = new BufferedInputStream(httpConnection  
361.                         .getInputStream());  
362.                 byte[] b = new byte[blockSize];  
363.                 while (blockBegin < blockEnd) {  
364.                     if (!running) {  
365.                         log.info(getThreadName() + ":停止下载.当前块:" + blockBegin);  
366.                         return;  
367.                     }  
368.                     log.debug(getThreadName() + "下载块=" + blockBegin);  
369.                     int counts = 0; // 已下载字节数  
370.                     if (BitUtil.getBit(blockSet, blockBegin)) {  
371.                         log.debug(getThreadName() + ":块下载已经完成=" + blockBegin);  
372.                         destFile.skipBytes(blockSize);  
373.                         int skips = 0;  
374.                         while (skips < blockSize) {  
375.                             skips += inputStream.skip(blockSize - skips);  
376.                         }  
377.                         downloaded.addAndGet(blockSize);  
378.   
379.                     } else {  
380.                         while (counts < blockSize) {  
381.                             int read = inputStream.read(b, 0, blockSize  
382.                                     - counts);  
383.                             if (read < 0)  
384.                                 break;  
385.                             counts += read;  
386.                             destFile.write(b, 0, read);  
387.                             downloaded.addAndGet(read);  
388.                         }  
389.                         BitUtil.setBit(blockSet, blockBegin, true); // 标记已经下载完成  
390.                     }  
391.                     blockBegin++;  
392.                 }  
393.                 httpConnection.disconnect();  
394.                 log.info(getThreadName() + "下载完成.");  
395.                 return;  
396.             } catch (Exception e) {  
397.                 log.error(getThreadName() + "下载错误:" + e.getMessage());  
398.                 e.printStackTrace();  
399.             } finally {  
400.                 try {  
401.                     if (inputStream != null)  
402.                         inputStream.close();  
403.                 } catch (Exception te) {  
404.                     log.error(te);  
405.                 }  
406.                 try {  
407.                     if (destFile != null)  
408.                         destFile.close();  
409.                 } catch (Exception te) {  
410.                     log.error(te);  
411.                 }  
412.             }  
413.         }  
414.     }  
415.   
416.     // 监控线程,并保存进度，方便下次断点续传  
417.     private class MonitorThread extends Thread {  
418.         boolean stop = false;  
419.   
420.         public void setStop(boolean stop) {  
421.             this.stop = stop;  
422.         }  
423.   
424.         public void run() {  
425.             FileOutputStream saveStream = null;  
426.             try {  
427.                 while (running && !stop) {  
428.                     saveStream = new FileOutputStream(lockFile);  
429.                     log.info(getDesc());  
430.                     // 保存进度  
431.                     saveStream.write(blockSet, 0, blockSet.length);  
432.                     sleep(1000);  
433.                     saveStream.close();  
434.                 }  
435.             } catch (Exception e) {  
436.                 e.printStackTrace();  
437.                 log.error(e);  
438.             } finally {  
439.   
440.             }  
441.         }  
442.     }  
443.   
444.     // 用来操作位的工具  
445.     private static class BitUtil {  
446.         public static byte[] createBit(int len) {  
447.             int size = len / Byte.SIZE;  
448.             if (len % Byte.SIZE > 0) {  
449.                 size++;  
450.             }  
451.             return new byte[size];  
452.         }  
453.   
454.         /** 取出某位，是0 还是1 */  
455.         public static boolean getBit(byte[] bits, int pos) {  
456.             int i = pos / Byte.SIZE;  
457.             int b = bits[i];  
458.             int j = pos % Byte.SIZE;  
459.             byte c = (byte) (0x80 >>> (j - 1));  
460.             return b == c;  
461.         }  
462.   
463.         /** 设置某位，是0 还是1 */  
464.         public static void setBit(byte[] bits, int pos, boolean flag) {  
465.             int i = pos / Byte.SIZE;  
466.             byte b = bits[i];  
467.             int j = pos % Byte.SIZE;  
468.             byte c = (byte) (0x80 >>> (j - 1));  
469.             if (flag) {  
470.                 bits[i] = (byte) (b | c);  
471.             } else {  
472.                 c = (byte) (0xFF ^ c);  
473.                 bits[i] = (byte) (b & c);  
474.             }  
475.         }  
476.     }  
477. }  

写了一个用Http Component组件实现的，不过目前版本，4.0.1 在连接池管理上貌似有问题，连接不能正确关闭，有兴趣的可以调试看看

 

[c-sharp]  view plain  copy

1. /** 
2.  * 专注互联网,分享创造价值 
3.  *  maoxiang@gmail.com 
4.  *  2010-3-30下午04:40:06 
5.  */  
6. package common.http;  
7.   
8. import java.io.BufferedInputStream;  
9. import java.io.ByteArrayOutputStream;  
10. import java.io.File;  
11. import java.io.FileInputStream;  
12. import java.io.FileOutputStream;  
13. import java.io.IOException;  
14. import java.io.RandomAccessFile;  
15. import java.util.concurrent.atomic.AtomicLong;  
16.   
17. import javax.net.ssl.SSLHandshakeException;  
18.   
19. import org.apache.commons.logging.Log;  
20. import org.apache.commons.logging.LogFactory;  
21. import org.apache.http.Header;  
22. import org.apache.http.HttpEntityEnclosingRequest;  
23. import org.apache.http.HttpHost;  
24. import org.apache.http.HttpRequest;  
25. import org.apache.http.HttpResponse;  
26. import org.apache.http.NoHttpResponseException;  
27. import org.apache.http.auth.AuthScope;  
28. import org.apache.http.auth.UsernamePasswordCredentials;  
29. import org.apache.http.client.HttpRequestRetryHandler;  
30. import org.apache.http.client.methods.HttpGet;  
31. import org.apache.http.conn.ClientConnectionManager;  
32. import org.apache.http.conn.params.ConnManagerParams;  
33. import org.apache.http.conn.params.ConnPerRoute;  
34. import org.apache.http.conn.params.ConnRoutePNames;  
35. import org.apache.http.conn.routing.HttpRoute;  
36. import org.apache.http.conn.scheme.PlainSocketFactory;  
37. import org.apache.http.conn.scheme.Scheme;  
38. import org.apache.http.conn.scheme.SchemeRegistry;  
39. import org.apache.http.impl.client.DefaultHttpClient;  
40. import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;  
41. import org.apache.http.params.BasicHttpParams;  
42. import org.apache.http.params.CoreProtocolPNames;  
43. import org.apache.http.params.HttpParams;  
44. import org.apache.http.protocol.BasicHttpContext;  
45. import org.apache.http.protocol.ExecutionContext;  
46. import org.apache.http.protocol.HttpContext;  
47.   
48. /** 
49.  * 一个多线程支持断点续传的工具类<br/> 
50.  * 2010-03 用Htpp Component重写 
51.  */  
52. public class HttpDownloader1 {  
53.   
54.     private final Log log = LogFactory.getLog(getClass().getName());  
55.     private int threads = 5; // 总共的线程数  
56.     private int maxThreads = 10; // 最大的线程数  
57.     private String destUrl; // 目标的URL  
58.     private String savePath; // 保存的路径  
59.     private File lockFile;// 用来保存进度的文件  
60.     private String userAgent = "jHttpDownload";  
61.     private boolean useProxy = false;  
62.     private String proxyServer;  
63.     private int proxyPort;  
64.     private String proxyUser;  
65.     private String proxyPassword;  
66.     private int blockSize = 1024 * 4; // 4K 一个块  
67.     // 1个位代表一个块,，用来标记是否下载完成  
68.     private byte[] blockSet;  
69.     private int blockPage; // 每个线程负责的大小  
70.     private int blocks;  
71.     private boolean running; // 是否运行中,避免线程不能释放  
72.     private DefaultHttpClient httpClient;//  
73.     // =======下载进度信息  
74.     private long beginTime;  
75.     private AtomicLong downloaded = new AtomicLong(0); // 已下载的字节数/  
76.     private long fileLength; // 总的字节数  
77.     // 监控线程,用来保存进度和汇报进度  
78.     private MonitorThread monitorThread = new MonitorThread();  
79.   
80.     public HttpDownloader1(String destUrl, String savePath, int threads) {  
81.         this.threads = threads;  
82.         this.destUrl = destUrl;  
83.         this.savePath = savePath;  
84.     }  
85.   
86.     public HttpDownloader1(String destUrl, String savePath) {  
87.         this(destUrl, savePath, 5);  
88.     }  
89.   
90.     /** 
91.      * 开始下载 
92.      */  
93.     public boolean download() {  
94.         log.info("下载文件" + destUrl + ",保存路径=" + savePath);  
95.         beginTime = System.currentTimeMillis();  
96.         boolean ok = false;  
97.         try {  
98.             File saveFile = new File(savePath);  
99.             lockFile = new File(savePath + ".lck");  
100.             if (lockFile.exists() && !lockFile.canWrite()) {  
101.                 throw new Exception("文件被锁住，或许已经在下载中了");  
102.             }  
103.             File parent = saveFile.getParentFile();  
104.             if (!parent.exists()) {  
105.                 log.info("创建目录=" + parent.getAbsolutePath());  
106.             }  
107.             if (!parent.canWrite()) {  
108.                 throw new Exception("保存目录不可写");  
109.             }  
110.             if (saveFile.exists()) {  
111.                 if (!saveFile.canWrite()) {  
112.                     throw new Exception("保存文件不可写,无法继续下载");  
113.                 }  
114.                 log.info("检查之前下载的文件");  
115.                 if (lockFile.exists()) {  
116.                     log.info("加载之前下载进度");  
117.                     loadPrevious();  
118.                 }  
119.             } else {  
120.                 lockFile.createNewFile();  
121.             }  
122.             // 1初始化httpClient  
123.             setupHttpClient();  
124.             HttpResponse response = getResponse(0);  
125.             Header length = response.getFirstHeader("Content-Length");  
126.             if (length != null) {  
127.                 try {  
128.                     fileLength = Long.parseLong(length.getValue());  
129.                 } catch (Exception e) {  
130.                 }  
131.             }  
132.             log.info("下载文件的大小:" + fileLength);  
133.             if (fileLength <= 0) {  
134.                 // 不支持多线程下载,采用单线程下载  
135.                 log.info("服务器不能返回文件大小，采用单线程下载");  
136.                 threads = 1;  
137.             }  
138.             if (response.getFirstHeader("Content-Range") == null) {  
139.                 log.info("服务器不支持断线续传");  
140.                 threads = 1;  
141.             } else {  
142.                 log.info("服务器支持断点续传");  
143.             }  
144.             if (blockSet != null) {  
145.                 log.info("检查文件，是否能够续传");  
146.                 if (blockSet.length * 8l * blockSize < fileLength) {  
147.                     log.info("文件大小已改变，需要重新下载");  
148.                     blockSet = null;  
149.                 }  
150.             }  
151.             //  
152.             if (fileLength > 0 && parent.getFreeSpace() < fileLength) {  
153.                 throw new Exception("磁盘空间不够");  
154.             }  
155.             if (fileLength > 0) {  
156.                 int i = (int) (fileLength / blockSize);  
157.                 if (fileLength % blockSize > 0) {  
158.                     i++;  
159.                 }  
160.                 blocks = i;  
161.                 log.info("文件的块数:" + blocks);  
162.                 blockSet = BitUtil.createBit(blocks);  
163.             } else {  
164.                 // 一个块  
165.                 blocks = 1;  
166.             }  
167.             blockPage = blocks / threads; // 每个线程负责的块数  
168.             log.info("分配线程。线程数量=" + threads + ",块总数=" + blocks + ",总字节数="  
169.                     + fileLength + ",每块大小=" + blockSize + ",块/线程=" + blockPage);  
170.             // 检查  
171.             running = true;  
172.             ThreadGroup downloadGroup = new ThreadGroup("download");  
173.             for (int i = 0; i < threads; i++) {  
174.                 int begin = i * blockPage;  
175.                 int end = (i + 1) * blockPage;  
176.                 if (i == threads - 1 && blocks % threads > 0) {  
177.                     // 如果最后一个线程，有余数，需要修正  
178.                     end = blocks;  
179.                 }  
180.                 // 扫描每个线程的块是否有需要下载的  
181.                 boolean needDownload = false;  
182.                 for (int j = begin; j < end; j++) {  
183.                     if (!BitUtil.getBit(blockSet, j)) {  
184.                         needDownload = true;  
185.                         break;  
186.                     }  
187.                 }  
188.                 if (!needDownload) {  
189.                     log.info("所有块已经下载完毕.Begin=" + begin + ",End=" + end);  
190.   
191.                 }  
192.                 // 启动下载其他线程  
193.                 DownloadThread downloadThread = new DownloadThread(  
194.                         downloadGroup, i, begin, end);  
195.                 downloadThread.start();  
196.             }  
197.             monitorThread.setStop(false);  
198.             monitorThread.start();  
199.             while (downloadGroup.activeCount() > 0) {  
200.                 Thread.sleep(2000);  
201.             }  
202.             ok = true;  
203.         } catch (Exception e) {  
204.             e.printStackTrace();  
205.             log.error(e);  
206.         } finally {  
207.             // closeHttpClient();  
208.             if (ok) {  
209.                 log.info("删除进度文件:" + lockFile.getAbsolutePath());  
210.                 lockFile.delete();  
211.             }  
212.             httpClient = null;  
213.         }  
214.         monitorThread.setStop(true);  
215.         log.info("下载完成，耗时:"  
216.                 + getTime((System.currentTimeMillis() - beginTime) / 1000));  
217.         return ok;  
218.     }  
219.   
220.     private void loadPrevious() throws Exception {  
221.         ByteArrayOutputStream outStream = new ByteArrayOutputStream();  
222.         FileInputStream inStream = new FileInputStream(lockFile);  
223.         byte[] buffer = new byte[4096];  
224.         int n = 0;  
225.         while (-1 != (n = inStream.read(buffer))) {  
226.             outStream.write(buffer, 0, n);  
227.         }  
228.         outStream.close();  
229.         inStream.close();  
230.         blockSet = outStream.toByteArray();  
231.         log.debug("之前的文件大小应该是:" + blockSet.length * 8l * blockSize);  
232.     }  
233.   
234.     private void setupHttpClient() throws Exception {  
235.         HttpParams params = new BasicHttpParams();  
236.         SchemeRegistry schemeRegistry = new SchemeRegistry();  
237.         schemeRegistry.register(new Scheme("http", PlainSocketFactory  
238.                 .getSocketFactory(), 80));  
239.         ClientConnectionManager cm = new ThreadSafeClientConnManager(params,  
240.                 schemeRegistry);  
241.         httpClient = new DefaultHttpClient(cm, params);  
242.         httpClient.getParams().setParameter(CoreProtocolPNames.USER_AGENT,  
243.                 userAgent);  
244.         ConnPerRoute defaultConnPerRoute = new ConnPerRoute() {  
245.   
246.             public int getMaxForRoute(HttpRoute route) {  
247.                 return maxThreads;  
248.             }  
249.   
250.         };  
251.         httpClient.getParams().setParameter(  
252.                 ConnManagerParams.MAX_CONNECTIONS_PER_ROUTE,  
253.                 defaultConnPerRoute);  
254.         // 设置重试机制  
255.         httpClient.setHttpRequestRetryHandler(myRetryHandler);  
256.         if (useProxy) {  
257.             log.info("采用代理服务器=" + proxyServer + ",端口=" + proxyPort);  
258.             final HttpHost proxy = new HttpHost(proxyServer, proxyPort, "http");  
259.             httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,  
260.                     proxy);  
261.             if (proxyUser != null && proxyPassword != null) {  
262.                 httpClient.getCredentialsProvider().setCredentials(  
263.                         AuthScope.ANY,  
264.                         new UsernamePasswordCredentials(proxyUser,  
265.                                 proxyPassword));  
266.             }  
267.         }  
268.     }  
269.   
270.     private HttpResponse response0; // 用来快速返回,减少一次连接  
271.   
272.     private HttpResponse getResponse(long pos) throws Exception {  
273.         if (pos == 0 && response0 != null) {  
274.             return response0;  
275.         }  
276.         HttpGet httpget = new HttpGet(destUrl);  
277.         HttpContext localContext = new BasicHttpContext();  
278.         // 实现多线程下载的核心，也可以用来实现断点续传  
279.         httpget.addHeader("RANGE", "bytes=" + pos + "-");  
280.         HttpResponse response = httpClient.execute(httpget, localContext);  
281.         if (pos == 0) {  
282.             response0 = response;  
283.         }  
284.         return response;  
285.     }  
286.   
287.     // 重试机制  
288.     private HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {  
289.   
290.         public boolean retryRequest(IOException exception, int executionCount,  
291.                 HttpContext context) {  
292.             if (executionCount >= 5) {  
293.                 return false;  
294.             }  
295.             if (exception instanceof NoHttpResponseException) {  
296.                 return true;  
297.             }  
298.             if (exception instanceof SSLHandshakeException) {  
299.                 return false;  
300.             }  
301.             HttpRequest request = (HttpRequest) context  
302.                     .getAttribute(ExecutionContext.HTTP_REQUEST);  
303.             boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);  
304.             if (idempotent) {  
305.                 return true;  
306.             }  
307.             return false;  
308.         }  
309.     };  
310.   
311.     private String getDesc() {  
312.         long downloadBytes = downloaded.longValue();  
313.   
314.         return String.format("已下载/总大小=%s/%s(%s),速度:%s,耗时:%s,剩余大小:%d",  
315.                 getFileSize(downloadBytes), getFileSize(fileLength),  
316.                 getProgress(fileLength, downloadBytes),  
317.                 getFileSize(downloadBytes  
318.                         / ((System.currentTimeMillis() - beginTime) / 1000)),  
319.                 getTime((System.currentTimeMillis() - beginTime) / 1000),  
320.                 fileLength - downloadBytes);  
321.     }  
322.   
323.     private String getFileSize(long totals) {  
324.         // 计算文件大小  
325.         int i = 0;  
326.         String j = "BKMGT";  
327.         float s = totals;  
328.         while (s > 1024) {  
329.             s /= 1024;  
330.             i++;  
331.         }  
332.         return String.format("%.2f", s) + j.charAt(i);  
333.     }  
334.   
335.     private String getProgress(long totals, long read) {  
336.         if (totals == 0)  
337.             return "0%";  
338.         return String.format("%d", read * 100 / totals) + "%";  
339.     }  
340.   
341.     private String getTime(long seconds) {  
342.         int i = 0;  
343.         String j = "秒分时天";  
344.         long s = seconds;  
345.         String result = "";  
346.         while (s > 0) {  
347.             if (s % 60 > 0) {  
348.                 result = String.valueOf(s % 60) + (char) j.charAt(i) + result;  
349.             }  
350.             s /= 60;  
351.             i++;  
352.         }  
353.         return result;  
354.     }  
355.   
356.     /** 
357.      * 一个下载线程. 
358.      */  
359.     private class DownloadThread extends Thread {  
360.   
361.         private RandomAccessFile destFile; // 用来实现保存的随机文件  
362.         private int id = 0;  
363.         private int blockBegin = 0; // 开始块  
364.         private int blockEnd = 0; // 结束块  
365.         private long pos;// 绝对指针  
366.   
367.         private String getThreadName() {  
368.             return "DownloadThread-" + id + "=>";  
369.         }  
370.   
371.         public DownloadThread(ThreadGroup group, int id, int blockBegin,  
372.                 int blockEnd) throws Exception {  
373.             super(group, "downloadThread-" + id);  
374.             this.id = id;  
375.             this.blockBegin = blockBegin;  
376.             this.blockEnd = blockEnd;  
377.             this.pos = 1l * blockBegin * blockSize; // 转换为长整型  
378.             destFile = new RandomAccessFile(savePath, "rw");  
379.         }  
380.   
381.         public void run() {  
382.             BufferedInputStream inputStream = null;  
383.             try {  
384.                 log.info(getThreadName() + "下载线程." + this.toString());  
385.                 log.info(getThreadName() + ":定位文件位置.Pos=" + 1l * blockBegin  
386.                         * blockSize);  
387.                 destFile.seek(1l * blockBegin * blockSize);  
388.                 log.info(getThreadName() + ":开始下载.[ " + blockBegin + " - "  
389.                         + blockEnd + "]");  
390.   
391.                 HttpResponse response = getResponse(pos);  
392.                 inputStream = new BufferedInputStream(response.getEntity()  
393.                         .getContent());  
394.                 byte[] b = new byte[blockSize];  
395.                 while (blockBegin < blockEnd) {  
396.                     if (!running) {  
397.                         log.info(getThreadName() + ":停止下载.当前块:" + blockBegin);  
398.                         return;  
399.                     }  
400.                     log.debug(getThreadName() + "下载块=" + blockBegin);  
401.                     int counts = 0; // 已下载字节数  
402.                     if (BitUtil.getBit(blockSet, blockBegin)) {  
403.                         log.debug(getThreadName() + ":块下载已经完成=" + blockBegin);  
404.                         destFile.skipBytes(blockSize);  
405.                         int skips = 0;  
406.                         while (skips < blockSize) {  
407.                             skips += inputStream.skip(blockSize - skips);  
408.                         }  
409.                         downloaded.addAndGet(blockSize);  
410.   
411.                     } else {  
412.                         while (counts < blockSize) {  
413.                             int read = inputStream.read(b, 0, blockSize  
414.                                     - counts);  
415.                             if (read < 0)  
416.                                 break;  
417.                             counts += read;  
418.                             destFile.write(b, 0, read);  
419.                             downloaded.addAndGet(read);  
420.                         }  
421.                         BitUtil.setBit(blockSet, blockBegin, true); // 标记已经下载完成  
422.                     }  
423.                     blockBegin++;  
424.                 }  
425.                 response.getEntity().consumeContent();  
426.                 log.info(getThreadName() + "下载完成.");  
427.                 return;  
428.             } catch (Exception e) {  
429.                 log.error(getThreadName() + "下载错误:" + e.getMessage());  
430.                 e.printStackTrace();  
431.             } finally {  
432.                 try {  
433.                     if (inputStream != null)  
434.                         inputStream.close();  
435.                 } catch (Exception te) {  
436.                     log.error(te);  
437.                 }  
438.                 try {  
439.                     if (destFile != null)  
440.                         destFile.close();  
441.                 } catch (Exception te) {  
442.                     log.error(te);  
443.                 }  
444.             }  
445.         }  
446.     }  
447.   
448.     // 监控线程,并保存进度，方便下次断点续传  
449.     private class MonitorThread extends Thread {  
450.         boolean stop = false;  
451.   
452.         public void setStop(boolean stop) {  
453.             this.stop = stop;  
454.         }  
455.   
456.         public void run() {  
457.             FileOutputStream saveStream = null;  
458.             try {  
459.                 saveStream = new FileOutputStream(lockFile);  
460.                 while (running && !stop) {  
461.                     log.info(getDesc());  
462.                     // 保存进度  
463.                     saveStream.write(blockSet);  
464.                     sleep(1000);  
465.                 }  
466.             } catch (Exception e) {  
467.                 log.error(e);  
468.             } finally {  
469.                 if (saveStream != null) {  
470.                     try {  
471.                         saveStream.close();  
472.                     } catch (Exception e) {  
473.                         log.error(e);  
474.                     }  
475.                 }  
476.             }  
477.         }  
478.     }  
479.   
480.     // 用来操作位的工具  
481.     private static class BitUtil {  
482.         public static byte[] createBit(int len) {  
483.             int size = len / Byte.SIZE;  
484.             if (len % Byte.SIZE > 0) {  
485.                 size++;  
486.             }  
487.             return new byte[size];  
488.         }  
489.   
490.         /** 取出某位，是0 还是1 */  
491.         public static boolean getBit(byte[] bits, int pos) {  
492.             int i = pos / Byte.SIZE;  
493.             int b = bits[i];  
494.             int j = pos % Byte.SIZE;  
495.             byte c = (byte) (0x80 >>> (j - 1));  
496.             return b == c;  
497.         }  
498.   
499.         /** 设置某位，是0 还是1 */  
500.         public static void setBit(byte[] bits, int pos, boolean flag) {  
501.             int i = pos / Byte.SIZE;  
502.             byte b = bits[i];  
503.             int j = pos % Byte.SIZE;  
504.             byte c = (byte) (0x80 >>> (j - 1));  
505.             if (flag) {  
506.                 bits[i] = (byte) (b | c);  
507.             } else {  
508.                 c = (byte) (0xFF ^ c);  
509.                 bits[i] = (byte) (b & c);  
510.             }  
511.         }  
512.     }  
513. }  

 

使用测试

[c-sharp]  view plain  copy

1. /** 
2.  * 专注互联网,分享创造价值 
3.  *  maoxiang@gmail.com 
4.  *  2010-3-31下午10:55:16 
5.  */  
6. package common.http;  
7.   
8. public class TestDownloader {  
9.   
10.     public static void main(String[] args) throws Exception {  
11.         String url = "http://apache.etoak.com/tomcat/tomcat-6/v6.0.26/bin/apache-tomcat-6.0.26.zip";  
12.         String saveFile = "f:/tomcat.zip";  
13.         HttpDownloader downloader = new HttpDownloader(url, saveFile);  
14.         downloader.download();  
15.         System.out.println("finsihed");  
16.   
17.     }  
18. }