java多线程网页下载代码

1.使用了java.util.concurrent包里的线程池，可以飙升到满带宽，在100M带宽上，可以达到10MB/s。

2.使用了java.nio里的channels，性能比自己缓冲有一些提高。

 1 import java.io.FileOutputStream;
 2 import java.io.InputStream;
 3 import java.net.URL;
 4 import java.net.URLConnection;
 5 import java.nio.channels.Channels;
 6 import java.nio.channels.FileChannel;
 7 import java.nio.channels.ReadableByteChannel;
 8 import java.util.Calendar;
 9 import java.util.concurrent.Callable;
10 import java.util.concurrent.ExecutorService;
11 import java.util.concurrent.Executors;
12 
13 public class HttpDownloader implements Callable<String> {
14     URLConnection connection;
15     FileChannel outputChann;
16     public static volatile int count = 0;
17 
18     public static void main(String[] args) throws Exception {
19 
20         ExecutorService poll = Executors.newFixedThreadPool(100);
21 
22         for (int i = 0; i < 100; i++) {
23             Calendar now = Calendar.getInstance();
24             String fileName = "../data/" + now.get(Calendar.YEAR) + "年"
25                     + (now.get(Calendar.MONTH) + 1) + "月"
26                     + now.get(Calendar.DAY_OF_MONTH) + "日--" + i + ".txt";
27             poll.submit(new HttpDownloader("http://www.sina.com",
28                     (new FileOutputStream(fileName)).getChannel()));
29         }
30 
31         poll.shutdown();
32 
33         long start = System.currentTimeMillis();
34         while (!poll.isTerminated()) {
35             Thread.sleep(1000);
36             System.out.println("已运行"
37                     + ((System.currentTimeMillis() - start) / 1000) + "秒，"
38                     + HttpDownloader.count + "个任务还在运行");
39         }
40     }
41 
42     public HttpDownloader(String url, FileChannel fileChannel) throws Exception {
43         synchronized (HttpDownloader.class) {
44             count++;
45         }
46         connection = (new URL(url)).openConnection();
47         this.outputChann = fileChannel;
48     }
49 
50     @Override
51     public String call() throws Exception {
52         connection.connect();
53         InputStream inputStream = connection.getInputStream();
54         ReadableByteChannel rChannel = Channels.newChannel(inputStream);
55         outputChann.transferFrom(rChannel, 0, Integer.MAX_VALUE);
56         // System.out.println(Thread.currentThread().getName() + " completed!");
57         inputStream.close();
58         outputChann.close();
59         synchronized (HttpDownloader.class) {
60             count--;
61         }
62         return null;
63     }
64 }

 

 1 import java.io.FileOutputStream;
 2 import java.io.InputStream;
 3 import java.net.URL;
 4 import java.net.URLConnection;
 5 import java.nio.channels.Channels;
 6 import java.nio.channels.FileChannel;
 7 import java.nio.channels.ReadableByteChannel;
 8 import java.util.Calendar;
 9 import java.util.concurrent.Callable;
10 import java.util.concurrent.ExecutorService;
11 import java.util.concurrent.Executors;
12 
13 public class HttpDownloader implements Callable<String> {
14     URLConnection connection;
15     FileChannel outputChann;
16     public static volatile int count = 0;
17 
18     public static void main(String[] args) throws Exception {
19 
20         ExecutorService poll = Executors.newFixedThreadPool(100);
21 
22         for (int i = 0; i < 100; i++) {
23             Calendar now = Calendar.getInstance();
24             String fileName = "../data/" + now.get(Calendar.YEAR) + "年"
25                     + (now.get(Calendar.MONTH) + 1) + "月"
26                     + now.get(Calendar.DAY_OF_MONTH) + "日--" + i + ".txt";
27             poll.submit(new HttpDownloader("http://www.sina.com",
28                     (new FileOutputStream(fileName)).getChannel()));
29         }
30 
31         poll.shutdown();
32 
33         long start = System.currentTimeMillis();
34         while (!poll.isTerminated()) {
35             Thread.sleep(1000);
36             System.out.println("已运行"
37                     + ((System.currentTimeMillis() - start) / 1000) + "秒，"
38                     + HttpDownloader.count + "个任务还在运行");
39         }
40     }
41 
42     public HttpDownloader(String url, FileChannel fileChannel) throws Exception {
43         synchronized (HttpDownloader.class) {
44             count++;
45         }
46         connection = (new URL(url)).openConnection();
47         this.outputChann = fileChannel;
48     }
49 
50     @Override
51     public String call() throws Exception {
52         connection.connect();
53         InputStream inputStream = connection.getInputStream();
54         ReadableByteChannel rChannel = Channels.newChannel(inputStream);
55         outputChann.transferFrom(rChannel, 0, Integer.MAX_VALUE);
56         // System.out.println(Thread.currentThread().getName() + " completed!");
57         inputStream.close();
58         outputChann.close();
59         synchronized (HttpDownloader.class) {
60             count--;
61         }
62         return null;
63     }
64 }