NettyExample 1.2 Http文件的上传和下载
Netty服务端。
注意:如果文件过大需要调整
HttpObjectAggregator的初始化参数,数据传输的大小
public class NettyHttpServer {
private static Logger log = LoggerFactory.getLogger(NettyHttpServer.class);
static final boolean SSL = System.getProperty("ssl") != null;
static final int PORT = Integer.parseInt(System.getProperty("port", SSL ? "8443" : "8080"));
public void start(int port) throws Exception {
EventLoopGroup bossGroup = new NioEventLoopGroup();
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class) // (3)
.childHandler(new ChannelInitializer<SocketChannel>() { // (4)
@Override
public void initChannel(SocketChannel ch) throws Exception {
// server端发送的是httpResponse,所以要使用HttpResponseEncoder进行编码
ch.pipeline().addLast(new HttpResponseEncoder());
// server端接收到的是httpRequest,所以要使用HttpRequestDecoder进行解码
ch.pipeline().addLast(new HttpRequestDecoder());
ch.pipeline().addLast(new HttpObjectAggregator(65536*100));
ch.pipeline().addLast(new HttpServerInboundHandler());
}
}).option(ChannelOption.SO_BACKLOG, 128) // (5)
.childOption(ChannelOption.SO_KEEPALIVE, true); // (6)
ChannelFuture f = b.bind(port).sync();
f.channel().closeFuture().sync();
} finally {
workerGroup.shutdownGracefully();
bossGroup.shutdownGracefully();
}
}
public static void main(String[] args) throws Exception {
NettyHttpServer server = new NettyHttpServer();
log.info("Http Server listening on 8844 ...");
server.start(8844);
}
}
服务端处理handler。
public class HttpServerInboundHandler extends ChannelInboundHandlerAdapter {
private static Logger logger = LoggerFactory.getLogger(HttpServerInboundHandler.class);
private Gson gson = new Gson();
public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";
public static final int HTTP_CACHE_SECONDS = 60;
private static String PDF_SOURCE_DIR = PDFConfig.SOURCE_DIR;
private static String PDF_PARSE_DIR = PDFConfig.PARSE_DIR;
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
HashMap<String, String> params = new HashMap<String, String>();
String taskId = UUID.randomUUID().toString().replaceAll("-", "");
System.out.println(msg.getClass().toString());
if (msg instanceof FullHttpRequest) {
FullHttpRequest fullrequest = (FullHttpRequest) msg;
String uri = fullrequest.uri(); // 获取请求uri
logger.info("【url :{} taskid;{}】", uri, taskId);
System.out.println(fullrequest.headers().get("messageType")); // 获取头部信息
if (uri.contains("download")) {
SendFile(pb.getParseFileName(), ctx, fullrequest); //发送文件
} else {
params = RequestParse.parseHttp(fullrequest, taskId); // 解析get/post请求参数(对post请求中的文件数据进行保存)
String res = "I am OK";
String fileName = params.get("fileName");
if (StringUtils.isBlank(fileName)) {
res = "file Name is Blank";
}
PdfParseBean pb = PdfToolsUtil.PdfParse(fileName, "txt", taskId, true, "TEXT_LEFT");
SendFile(pb.getParseFileName(), ctx, fullrequest); //发送文件
return;
}
// ctx.flush();
}
}
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
ctx.flush();
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
logger.error(cause.getMessage());
ctx.close();
}
private static void setDateAndCacheHeaders(HttpResponse response, File fileToCache) {
SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));
// Date header
Calendar time = new GregorianCalendar();
response.headers().set(HttpHeaderNames.DATE, dateFormatter.format(time.getTime()));
// Add cache headers
time.add(Calendar.SECOND, HTTP_CACHE_SECONDS);
response.headers().set(HttpHeaderNames.EXPIRES, dateFormatter.format(time.getTime()));
response.headers().set(HttpHeaderNames.CACHE_CONTROL, "private, max-age=" + HTTP_CACHE_SECONDS);
response.headers().set(HttpHeaderNames.LAST_MODIFIED,
dateFormatter.format(new Date(fileToCache.lastModified())));
}
private static void setContentTypeHeader(HttpResponse response, File file) {
MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
response.headers().set(HttpHeaderNames.CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath()));
}
public void SendFile(String fileName, ChannelHandlerContext ctx, FullHttpRequest request) throws IOException {
String filePath = PDF_PARSE_DIR + fileName;
File file = new File(filePath);
RandomAccessFile raf;
try {
raf = new RandomAccessFile(file, "r");
} catch (FileNotFoundException ignore) {
logger.error("【获取解析文件失败】:{}", file);
return;
}
long fileLength = raf.length();
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
HttpUtil.setContentLength(response, fileLength);
setContentTypeHeader(response, file);
setDateAndCacheHeaders(response, file);
response.headers().set("Content-Disposition", "attachment; filename=" + URLEncoder.encode(fileName));
if (HttpUtil.isKeepAlive(request)) {
response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
}
ctx.write(response);
ChannelFuture sendFileFuture;
ChannelFuture lastContentFuture;
if (ctx.pipeline().get(SslHandler.class) == null) {
sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength),
ctx.newProgressivePromise());
lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
} else {
sendFileFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)),
ctx.newProgressivePromise());
lastContentFuture = sendFileFuture;
}
sendFileFuture.addListener(new ChannelProgressiveFutureListener() {
@Override
public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
if (total < 0) { // total unknown
System.err.println(future.channel() + " Transfer progress: " + progress);
} else {
System.err.println(future.channel() + " Transfer progress: " + progress + " / " + total);
}
}
@Override
public void operationComplete(ChannelProgressiveFuture future) {
System.err.println(future.channel() + " Transfer complete.");
}
});
if (!HttpUtil.isKeepAlive(request)) {
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
}
// raf.close();
//不能将raf.close。否则会导致文件块传输过程中Netty客户端无法获取lastHttpContent导致无法获取完整的文件。浏览器下载文件无法下载完整的文件。
while(!request.release()); //防止内存溢出,需要释放掉request中bytebuf数据块,否则会因为不断上传文件导致内存溢出
if (httpChunkedInput!=null) {
try {
httpChunkedInput.close();
} catch (Exception e) {
logger.error("httpChunkedInput.close() error ",e);
}
}
}
}
对Http请求参数的解析,获取FileUpLoad保存文件到服务端。
public class RequestParse {
private static Logger logger = LoggerFactory.getLogger(RequestParse.class);
private static String PDF_SOURCE_DIR = PDFConfig.SOURCE_DIR;
private static String PDF_PARSE_DIR = PDFConfig.PARSE_DIR;
private static Gson gson = new Gson();
static{
DiskFileUpload.baseDirectory = PDF_SOURCE_DIR;
}
/**
* 解析Netty HttpRequest的请求参数,GET/POST的解析结果都为一个Map
* @param request
* @param taskId 本次请求taskId
* @return HashMap<String, String>
*/
public static HashMap<String, String> parseHttp(FullHttpRequest request, String taskId) throws Exception {
HashMap<String, String> params = new HashMap<String, String>();
HttpMethod method = request.method();
String uri = request.uri();
System.out.println(request.getClass().toString());
if (HttpMethod.GET.equals(method)) {
........
} else if (HttpMethod.POST.equals(method)) {
HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(new DefaultHttpDataFactory(false), request);
List<InterfaceHttpData> postData = decoder.getBodyHttpDatas(); //
for (InterfaceHttpData data : postData) {
if (data.getHttpDataType() == HttpDataType.Attribute) {
Attribute attribute = (Attribute) data;
params.put(attribute.getName(), attribute.getValue());
}
else if(data.getHttpDataType() == HttpDataType.FileUpload) {
FileUpload fileUpload = (FileUpload) data;
String fileName = taskId + "-" + fileUpload.getFilename();
if(fileUpload.isCompleted()) {
//保存到磁盘
File dir = new File(DiskFileUpload.baseDirectory);
if (!dir.exists() || !dir.isDirectory()) {
dir.mkdirs();
}
StringBuffer fileNameBuf = new StringBuffer();
fileNameBuf.append(DiskFileUpload.baseDirectory).append(fileName);
fileUpload.renameTo(new File(fileNameBuf.toString()));
}
params.put("fileName", fileName);
}
}
logger.info("【POST 接受参数】:{}", gson.toJson(params));
} else {
throw new MethodNotSupportedException("not sopport such method. please use CET or POST");
}
return params;
}
}
Netty客户端
public class MyUploadClient {
static String Filepath = "D:/pdf/×ST八钢2016年年度报告.pdf";
public void connect(String host, int port) throws Exception {
EventLoopGroup workerGroup = new NioEventLoopGroup();
HttpDataFactory factory = new DefaultHttpDataFactory(DefaultHttpDataFactory.MINSIZE);
try {
Bootstrap b = new Bootstrap();
b.group(workerGroup);
b.channel(NioSocketChannel.class);
b.option(ChannelOption.SO_KEEPALIVE, true);
b.handler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast("codec", new HttpClientCodec());
pipeline.addLast("inflater", new HttpContentDecompressor());
pipeline.addLast("chunkedWriter", new ChunkedWriteHandler());
pipeline.addLast(new HttpClientInboundHandler());
}
});
// Start the client.
ChannelFuture f = b.connect(host, port).sync(); //
String uriSting = new URI("http://127.0.0.1:8844/").toASCIIString();
ChannelFuture future = b.connect(SocketUtils.socketAddress(host, port));
Channel channel = future.sync().channel();
//不能使用FullHttpRequest
HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST,uriSting);
HttpPostRequestEncoder bodyRequestEncoder =
new HttpPostRequestEncoder(factory, request, true); // true => multipart
HttpHeaders headers1 = request.headers();
headers1.set(HttpHeaderNames.HOST, host);
headers1.set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
headers1.set(HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderValues.GZIP + "," + HttpHeaderValues.DEFLATE);
headers1.set(HttpHeaderNames.ACCEPT_CHARSET, "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
headers1.set(HttpHeaderNames.ACCEPT_LANGUAGE, "fr");
headers1.set(HttpHeaderNames.USER_AGENT, "Netty Simple Http Client side");
headers1.set(HttpHeaderNames.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
File file = new File(Filepath);
bodyRequestEncoder.addBodyFileUpload("myfile", file, "application/x-zip-compressed", false);
// finalize request
bodyRequestEncoder.finalizeRequest();
// send request
channel.write(request);
// test if request was chunked and if so, finish the write
if (bodyRequestEncoder.isChunked()) {
channel.write(bodyRequestEncoder);
}
channel.flush();
// Now no more use of file representation (and list of HttpData)
bodyRequestEncoder.cleanFiles();
// Wait for the server to close the connection.
channel.closeFuture().sync();
} finally {
workerGroup.shutdownGracefully();
}
}
public static void main(String[] args) throws Exception {
MyUploadClient client = new MyUploadClient();
client.connect("127.0.0.1", 8844);
}
}
Client端对文件的保存。
文件通过Chunk块传输。每获取的HttpContent是文件的一部分,所以需要对每个数据块进行保存,得到一个完成的文件数据流。
注意:
【1】如何确定文件传输已经完成。HttpContent 的类型instanceof 等于LastHttpContent时。说明传输已经完成。
【2】使用对数据块进行保存的时候,在toString过程中导致byte[]被切分。数据块不完成导致了文件部分出现乱码。(前后两个数据块之间间隔的文字)
原因是使用了一下代码。导致转为String的时候byte[]不完整导致中文乱码,再从String转回byte[]时数据就缺失了
ByteBuf buffer = chunk.content();
byte[] vArray = buffer.toString(CharsetUtil.UTF_8).getBytes(); //Netty的ByteBuf转String,最后一个中文可能会乱码(byte[]不全)
ans = byteMerger(ans, vArray);
//以下是正确的使用规范
ByteBuf buffer = chunk.content();
byte[] vArray = new byte[buffer.capacity()];
buffer.readBytes(vArray); //读取ByteBuf的数据。
ans = byteMerger(ans, vArray);
public class HttpClientInboundHandler extends ChannelInboundHandlerAdapter {
private byte[] ans = {};
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (msg instanceof HttpResponse) {
HttpResponse response = (HttpResponse) msg;
System.out.println("CONTENT_TYPE:" + response.headers().get(HttpHeaderNames.CONTENT_TYPE));
}
if (msg instanceof HttpContent) {
HttpContent chunk = (HttpContent) msg;
ByteBuf buffer = chunk.content();
byte[] vArray = new byte[buffer.capacity()];
buffer.readBytes(vArray);
ans = byteMerger(ans, vArray);
if (chunk instanceof LastHttpContent) {
save(ans, "/pdf/", "xST八钢2016年年度报告.txt");
}
}
}
private void save(byte[] fileByte, String download_path, String filename) throws Exception {
FileOutputStream fileOutputStream = null;
String fileName = (System.currentTimeMillis()) + "-" + filename;
String fileFullPath = download_path + fileName;
try {
fileOutputStream = new FileOutputStream(fileFullPath);
fileOutputStream.write(fileByte);
} catch (Exception e) {
throw e;
} finally {
if (fileOutputStream != null) {
fileOutputStream.close();
}
}
}
public static byte[] byteMerger(byte[] byte_1, byte[] byte_2) {
byte[] byte_3 = new byte[byte_1.length + byte_2.length];
System.arraycopy(byte_1, 0, byte_3, 0, byte_1.length);
System.arraycopy(byte_2, 0, byte_3, byte_1.length, byte_2.length);
return byte_3;
}
}