基于netty4的文件下载

在使用过程中发现这种方式下载文件，是在将文件加载至内存后再往服务端发送，会随之文件的大小而占用内存。我测试中下载2G的文件，发现内存升高，下载10G文件时无法下载。

客户端：

[java]  view plain  copy

1. public class DownLoadClient {  
2.     private StringBuffer resultBuffer = new StringBuffer();  
3.     private EventLoopGroup group = null;  
4.     private HttpDataFactory factory = null;  
5.       
6.     private ChannelFuture future = null;  
7.       
8.     private Object waitObject = new Object();  
9.       
10.     private String downFileName = null;  
11.       
12.     public DownLoadClient(String host, int port) throws Exception {  
13.         this.group = new NioEventLoopGroup();  
14.         this.factory = new DefaultHttpDataFactory(DefaultHttpDataFactory.MINSIZE);  
15.           
16.         Bootstrap b = new Bootstrap();  
17.         b.option(ChannelOption.TCP_NODELAY, true);  
18.         b.option(ChannelOption.SO_RCVBUF, 1048576*200);  
19.         b.option(ChannelOption.SO_KEEPALIVE, true);  
20.           
21.         b.group(group).channel(NioSocketChannel.class);  
22.         b.handler(new DownLoadClientIntializer());  
23.           
24.         this.future = b.connect(host, port).sync();  
25.     }  
26.       
27.     public void downLoadFile(String fileName) {  
28.           
29.         if(fileName == null || "".equals(fileName) || fileName.indexOf(".") == -1) {  
30.             System.out.println("下载的文件名未正确指定...");  
31.             return;  
32.         }  
33.         this.downFileName = fileName;  
34.         try {  
35.             HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, fileName);  
36.               
37.             Channel channel = this.future.channel();  
38.             if(channel.isActive() && channel.isWritable()) {  
39.                 channel.writeAndFlush(request);  
40.             }  
41.             channel.closeFuture().sync();  
42.         } catch (Exception e) {  
43.             e.printStackTrace();  
44.         }  
45.     }  
46.       
47.     public void shutdownClient() {  
48.         // 等待数据的传输通道关闭  
49.         group.shutdownGracefully();  
50.         factory.cleanAllHttpDatas();  
51.     }  
52.       
53.     public boolean isCompleted() {  
54.         while(waitObject != null) {  
55.             //当通道处于开通和活动时，处于等待  
56.         }  
57.         if(resultBuffer.length() > 0) {  
58.             if("200".equals(resultBuffer.toString())) {  
59.                 resultBuffer.setLength(0);  
60.                 return true;  
61.             }  
62.         }  
63.           
64.         return false;  
65.     }  
66.       
67.     private class DownLoadClientIntializer extends ChannelInitializer<SocketChannel> {  
68.         @Override  
69.         protected void initChannel(SocketChannel ch) throws Exception {  
70.             ChannelPipeline pipeline = ch.pipeline();  
71.               
72.             pipeline.addLast("decoder", new HttpResponseDecoder());  
73.             pipeline.addLast("encoder", new HttpRequestEncoder());    
74.             pipeline.addLast("chunkedWriter", new ChunkedWriteHandler());    
75.   
76.             pipeline.addLast("dispatcher", new DownLoadClientHandler());  
77.         }  
78.     }  
79.       
80.     private class DownLoadClientHandler extends SimpleChannelInboundHandler<HttpObject> {  
81.         private boolean readingChunks = false;  
82.         private File downloadFile = null;  
83.         private FileOutputStream fOutputStream = null;  
84.           
85.         private int succCode = 200;  
86.           
87.         protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg)  
88.                 throws Exception {  
89.             if (msg instanceof HttpResponse) {  
90.                 HttpResponse response = (HttpResponse) msg;  
91.                   
92.                 succCode = response.getStatus().code();  
93.                   
94.                 if (succCode == 200) {  
95.                     setDownLoadFile();  
96.                     readingChunks = true;  
97.                 }  
98.             }  
99.                   
100.             if (msg instanceof HttpContent) {  
101.                 HttpContent chunk = (HttpContent) msg;  
102.                 if (chunk instanceof LastHttpContent) {  
103.                     readingChunks = false;  
104.                 }   
105.                   
106.                 ByteBuf buffer = chunk.content();  
107.                 byte[] dst = new byte[buffer.readableBytes()];  
108.                 if(succCode == 200) {  
109.                     while(buffer.isReadable()) {  
110.                         buffer.readBytes(dst);  
111.                         fOutputStream.write(dst);  
112.                     }  
113.                       
114.                     if (null != fOutputStream) {  
115.                         fOutputStream.flush();  
116.                     }  
117.                 }  
118.             }  
119.               
120.             if (!readingChunks) {  
121.                 if (null != fOutputStream) {  
122.                     fOutputStream.flush();  
123.                     fOutputStream.close();  
124.                       
125.                     downloadFile = null;  
126.                     fOutputStream = null;  
127.                       
128.                     resultBuffer.append(succCode);  
129.                 }  
130.                 ctx.channel().close();  
131.             }  
132.         }  
133.           
134.         private void setDownLoadFile() throws Exception {  
135.             if(null == fOutputStream) {  
136.                 downloadFile = DBTools.getDownlaodFile(downFileName);  
137.                 if(!downloadFile.exists()) {  
138.                     downloadFile.createNewFile();  
139.                 }  
140.                 fOutputStream = new FileOutputStream(downloadFile);  
141.             }  
142.         }  
143.   
144.         @Override  
145.         public void channelInactive(ChannelHandlerContext ctx) throws Exception {  
146.             waitObject = null;  
147.         }  
148.   
149.         @Override  
150.         public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)  
151.                 throws Exception {  
152.             resultBuffer.setLength(0);  
153.             resultBuffer.append(500);  
154.             System.out.println("管道异常：" + cause.getMessage());  
155.             cause.printStackTrace();  
156.             ctx.channel().close();  
157.         }  
158.     }  
159. }  

 

服务端：

[java]  view plain  copy

1. public class DBServer extends Thread {  
2.       
3.     //单实例  
4.     private static DBServer dbServer = null;  
5.       
6.     //定时调度的周期实例  
7.     private static Scheduler sched = null;  
8.       
9.     private EventLoopGroup bossGroup = null;  
10.     private EventLoopGroup workerGroup = null;  
11.     //创建实例  
12.     public static DBServer newBuild() {  
13.         if(dbServer == null) {  
14.             dbServer = new DBServer();  
15.         }  
16.         return dbServer;  
17.     }  
18.       
19.     public void run() {  
20.         try {  
21.             startServer();  
22.         } catch(Exception e) {  
23.             System.out.println("数据服务启动出现异常："+e.toString());  
24.             e.printStackTrace();  
25.         }  
26.     }  
27.       
28.     private void startServer() throws Exception {  
29.         bossGroup = new NioEventLoopGroup();  
30.         workerGroup = new NioEventLoopGroup();  
31.           
32.         try {  
33.             ServerBootstrap b = new ServerBootstrap();  
34.               
35.             b.group(bossGroup, workerGroup);  
36.               
37.             b.option(ChannelOption.TCP_NODELAY, true);  
38.             b.option(ChannelOption.SO_TIMEOUT, 60000);  
39.             b.option(ChannelOption.SO_SNDBUF, 1048576*200);  
40.               
41.             b.option(ChannelOption.SO_KEEPALIVE, true);  
42.               
43.             b.channel(NioServerSocketChannel.class);  
44.             b.childHandler(new DBServerInitializer());  
45.   
46.             // 服务器绑定端口监听  
47.             ChannelFuture f = b.bind(DBConfig.curHost.getIp(), DBConfig.curHost.getPort()).sync();  
48.               
49.             System.out.println("数据服务："+DBConfig.curHost.getServerHost()+"启动完成...");  
50.             // 监听服务器关闭监听  
51.             f.channel().closeFuture().sync();  
52.         } finally {  
53.             bossGroup.shutdownGracefully();  
54.             workerGroup.shutdownGracefully();  
55.         }  
56.     }  
57. }  

 

[java]  view plain  copy

1. public class DBServerHandler extends SimpleChannelInboundHandler<HttpObject> {  
2.       
3.     private static final HttpDataFactory factory = new DefaultHttpDataFactory(DefaultHttpDataFactory.MINSIZE);  
4.       
5.     private String uri = null;  
6.       
7.     private HttpRequest request = null;  
8.       
9.     private HttpPostRequestDecoder decoder;  
10.       
11.     //message、download、upload  
12.     private String type = "message";  
13.       
14.     public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";  
15.     public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";  
16.     public static final int HTTP_CACHE_SECONDS = 60;  
17.       
18.     static {  
19.         DiskFileUpload.baseDirectory = DBConfig.curHost.getZipPath();  
20.     }  
21.   
22.     @Override  
23.     public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {  
24.         if (msg instanceof HttpRequest) {  
25.             request = (HttpRequest) msg;  
26.               
27.             uri = sanitizeUri(request.getUri());  
28.               
29.             //文件下载  
30.             if (request.getMethod() == HttpMethod.GET) {  
31.                 type = "download";  
32.                 File file = DBTools.getDownlaodFile(uri);  
33.                 if (file.isHidden() || !file.exists() || !file.isFile()) {  
34.                     writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, "下载的文件不存在...");  
35.                     return;  
36.                 }  
37.                   
38.                 // 随机文件读取，这种方式速度快  
39.                 RandomAccessFile raf = null;  
40.                 try {  
41.                     raf = new RandomAccessFile(file, "r");  
42.                 } catch(FileNotFoundException e) {  
43.                     writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, e.toString());  
44.                     return;  
45.                 }  
46.                   
47.                 writeDownLoadResponse(ctx, raf, file);  
48.             }  
49.     }  
50.           
51.         if (decoder != null && msg instanceof HttpContent) {  
52.             HttpContent chunk = (HttpContent) msg;  
53.               
54.             try {  
55.                 decoder.offer(chunk);  
56.             } catch (Exception e) {  
57.                 e.printStackTrace();  
58.                 writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, e.toString());  
59.                 ctx.channel().close();  
60.                 return;  
61.             }  
62.               
63.             readHttpDataChunkByChunk();  
64.               
65.             if (chunk instanceof LastHttpContent) {  
66.                 writeResponse(ctx.channel(), HttpResponseStatus.OK, "");  
67.                 reset();  
68.                 return;  
69.             }  
70.         }  
71.     }  
72.   
73.     private String sanitizeUri(String uri) {  
74.         try {  
75.             uri = URLDecoder.decode(uri, "UTF-8");  
76.         } catch(UnsupportedEncodingException e) {  
77.             try {  
78.                 uri = URLDecoder.decode(uri, "ISO-8859-1");  
79.             } catch(UnsupportedEncodingException e1) {  
80.                 throw new Error();  
81.             }  
82.         }  
83.   
84.         return uri;  
85.     }  
86.   
87.     private void reset() {  
88.         request = null;  
89.   
90.         //销毁decoder释放所有的资源  
91.         decoder.destroy();  
92.           
93.         decoder = null;  
94.     }  
95.   
96.     /** 
97.      * 通过chunk读取request，获取chunk数据 
98.      * @throws IOException  
99.      */  
100.     private void readHttpDataChunkByChunk() throws IOException {  
101.         try {  
102.             while (decoder.hasNext()) {  
103.                   
104.                 InterfaceHttpData data = decoder.next();  
105.                 if (data != null) {  
106.                     try {  
107.                         writeHttpData(data);  
108.                     } finally {  
109.                         data.release();  
110.                     }  
111.                 }  
112.             }  
113.         } catch (EndOfDataDecoderException e1) {  
114.             System.out.println("end chunk");  
115.         }  
116.     }  
117.   
118.     private void writeHttpData(InterfaceHttpData data) throws IOException {  
119.         if (data.getHttpDataType() == HttpDataType.FileUpload) {  
120.             FileUpload fileUpload = (FileUpload) data;  
121.             if (fileUpload.isCompleted()) {  
122.                   
123.                 StringBuffer fileNameBuf = new StringBuffer();  
124.                 fileNameBuf.append(DiskFileUpload.baseDirectory)  
125.                            .append(uri);  
126.   
127.                 fileUpload.renameTo(new File(fileNameBuf.toString()));  
128.             }  
129.         } else if (data.getHttpDataType() == HttpDataType.Attribute) {  
130.             Attribute attribute = (Attribute) data;  
131.             if(CommonParam.DOWNLOAD_COLLECTION.equals(attribute.getName())) {  
132.                 SynchMessageWatcher.newBuild().getMsgQueue().add(attribute.getValue());  
133.             }  
134.         }  
135.     }  
136.       
137.     private void writeDownLoadResponse(ChannelHandlerContext ctx, RandomAccessFile raf, File file) throws Exception {  
138.         long fileLength = raf.length();  
139.           
140.         //判断是否关闭请求响应连接  
141.         boolean close = HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(CONNECTION))  
142.                 || request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)  
143.                 && !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(request.headers().get(CONNECTION));  
144.           
145.         HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);  
146.         HttpHeaders.setContentLength(response, fileLength);  
147.           
148.         setContentHeader(response, file);  
149.           
150.         if (!close) {  
151.             response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);  
152.         }  
153.           
154.         ctx.write(response);  
155.         System.out.println("读取大小："+fileLength);  
156.           
157.         final FileRegion region = new DefaultFileRegion(raf.getChannel(), 0, 1000);  
158.         ChannelFuture writeFuture = ctx.write(region, ctx.newProgressivePromise());  
159.         writeFuture.addListener(new ChannelProgressiveFutureListener() {  
160.             public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {  
161.                 if (total < 0) {  
162.                     System.err.println(future.channel() + " Transfer progress: " + progress);  
163.                 } else {  
164.                     System.err.println(future.channel() + " Transfer progress: " + progress + " / " + total);  
165.                 }  
166.             }  
167.   
168.             public void operationComplete(ChannelProgressiveFuture future) {  
169.             }  
170.         });  
171.           
172.         ChannelFuture lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);  
173.         if(close) {  
174.             raf.close();  
175.             lastContentFuture.addListener(ChannelFutureListener.CLOSE);  
176.         }  
177.     }  
178.       
179.     private static void setContentHeader(HttpResponse response, File file) {  
180.         MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();  
181.         response.headers().set(CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath()));  
182.           
183.         SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);  
184.         dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));  
185.   
186.         // Date header  
187.         Calendar time = new GregorianCalendar();  
188.         response.headers().set(DATE, dateFormatter.format(time.getTime()));  
189.   
190.         // Add cache headers  
191.         time.add(Calendar.SECOND, HTTP_CACHE_SECONDS);  
192.         response.headers().set(EXPIRES, dateFormatter.format(time.getTime()));  
193.         response.headers().set(CACHE_CONTROL, "private, max-age=" + HTTP_CACHE_SECONDS);  
194.         response.headers().set(LAST_MODIFIED, dateFormatter.format(new Date(file.lastModified())));  
195.     }  
196.       
197.     private void writeResponse(Channel channel, HttpResponseStatus httpResponseStatus, String returnMsg) {  
198.         String resultStr = "节点【"+DBConfig.curHost.getServerHost()+"】";  
199.         if(httpResponseStatus.code() == HttpResponseStatus.OK.code()) {  
200.             resultStr += "正常接收";  
201.             if("message".equals(type)) {  
202.                 resultStr += "字符串。";  
203.             } else if("upload".equals(type)) {  
204.                 resultStr += "上传文件。";  
205.             } else if("download".equals(type)) {  
206.                 resultStr += "下载文件名。";  
207.             }  
208.         } else if(httpResponseStatus.code() == HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) {  
209.             resultStr += "接收";  
210.             if("message".equals(type)) {  
211.                 resultStr += "字符串";  
212.             } else if("upload".equals(type)) {  
213.                 resultStr += "上传文件";  
214.             } else if("download".equals(type)) {  
215.                 resultStr += "下载文件名";  
216.             }  
217.             resultStr += "的过程中出现异常："+returnMsg;  
218.         }  
219.         //将请求响应的内容转换成ChannelBuffer.  
220.         ByteBuf buf = copiedBuffer(resultStr, CharsetUtil.UTF_8);  
221.   
222.         //判断是否关闭请求响应连接  
223.         boolean close = HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(CONNECTION))  
224.                 || request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)  
225.                 && !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(request.headers().get(CONNECTION));  
226.           
227.         //构建请求响应对象  
228.         FullHttpResponse response = new DefaultFullHttpResponse(  
229.                 HttpVersion.HTTP_1_1, httpResponseStatus, buf);  
230.         response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8");  
231.   
232.         if (!close) {  
233.             //若该请求响应是最后的响应，则在响应头中没有必要添加'Content-Length'  
234.             response.headers().set(CONTENT_LENGTH, buf.readableBytes());  
235.         }  
236.           
237.         //发送请求响应  
238.         ChannelFuture future = channel.writeAndFlush(response);  
239.         //发送请求响应操作结束后关闭连接  
240.         if (close) {  
241.             future.addListener(ChannelFutureListener.CLOSE);  
242.         }  
243.     }  
244.   
245.     @Override  
246.     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {  
247.         cause.getCause().printStackTrace();  
248.         writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, "数据文件通过过程中出现异常："+cause.getMessage().toString());  
249.         ctx.channel().close();  
250.     }  
251. }  

 

[java]  view plain  copy

1. public class DBServerInitializer extends ChannelInitializer<SocketChannel> {  
2.   
3.     @Override  
4.     public void initChannel(SocketChannel ch) {  
5.         ChannelPipeline pipeline = ch.pipeline();  
6.           
7.         pipeline.addLast("decoder", new HttpRequestDecoder());  
8.         pipeline.addLast("encoder", new HttpResponseEncoder());  
9.           
10.         pipeline.addLast("deflater", new HttpContentCompressor());  
11.   
12.         pipeline.addLast("handler", new DBServerHandler());  
13.     }  
14. }