依赖库:
1,jetty(提供http方式接口)
2,mongodb的java驱动(访问mongodb存取文件)
3,thumbnailator包,进行缩略图生成
4,commons-fileupload包及commons-io包用于处理文件上传
架构图:
入口代码如下:
package com.ciaos.vfs; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.nio.SelectChannelConnector; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.thread.QueuedThreadPool; public class VfsServer { public static void main(String[] args) throws Exception { SelectChannelConnector connector = new SelectChannelConnector(); connector.setHost("127.0.0.1"); connector.setPort(8080); connector.setThreadPool(new QueuedThreadPool(200)); Server server = new Server(); server.setConnectors(new Connector[]{connector}); ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); context.setContextPath("/"); server.setHandler(context); context.addServlet(new ServletHolder(new VfsFileServlet()), "/file"); server.start(); server.join(); } }
业务代码如下:
package com.ciaos.vfs; import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.UnknownHostException; import java.util.List; import javax.imageio.ImageIO; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import net.coobird.thumbnailator.Thumbnails; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.bson.types.ObjectId; import com.mongodb.DB; import com.mongodb.Mongo; import com.mongodb.gridfs.GridFS; import com.mongodb.gridfs.GridFSDBFile; import com.mongodb.gridfs.GridFSInputFile; public class VfsFileServlet extends HttpServlet { /** * */ private static final long serialVersionUID = 5586455171943232770L; private String tempPath = null; private Mongo mongo = null; final private String MongoDbIp = "127.0.0.1"; final private Integer MongoDbPort = 27017; @Override public void init() throws ServletException { // TODO Auto-generated method stub tempPath = "/tmp/"; try { mongo=new Mongo(MongoDbIp, MongoDbPort); } catch (UnknownHostException e) { // TODO Auto-generated catch block e.printStackTrace(); System.exit(-1); } super.init(); } @Override protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // TODO Auto-generated method stub String fileId = req.getParameter("id"); String MongoDbName = req.getParameter("type"); if(fileId == null || MongoDbName == null){ resp.sendError(400,"bad request"); return; } DB db=mongo.getDB(MongoDbName); GridFS gridFS=new GridFS(db); try{ gridFS.remove(new ObjectId(fileId)); } catch(IllegalArgumentException ex){ resp.sendError(404,"file not found"); return; } } @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // TODO Auto-generated method stub String fileId = req.getParameter("id"); String MongoDbName = req.getParameter("type"); if(fileId == null || MongoDbName == null){ resp.sendError(400,"bad request"); return; } DB db=mongo.getDB(MongoDbName); GridFS gridFS=new GridFS(db); GridFSDBFile mongofile = null; try{ mongofile = gridFS.find(new ObjectId(fileId)); } catch(IllegalArgumentException ex){ resp.sendError(404,"file not found"); return; } if(mongofile!=null){ resp.setHeader("Content-type", mongofile.getContentType()); resp.setHeader("Content-Disposition", "attachment; filename= "+mongofile.getFilename()); mongofile.put("viewtimes", (Integer)mongofile.get("viewtimes")+1); ServletOutputStream out = resp.getOutputStream(); String dtype = req.getParameter("act"); if(dtype!=null && dtype.equals("preview")){ Integer width = 0, height = 0; try{ width = Integer.parseInt(req.getParameter("w")); height = Integer.parseInt(req.getParameter("h")); BufferedImage image = ImageIO.read(mongofile.getInputStream()); width = Math.min(width,image.getWidth()); height = Math.min(height,image.getHeight()); Thumbnails.of(mongofile.getInputStream()) .size(width, height) .toOutputStream(out); } catch(NumberFormatException ex){ resp.sendError(400,"bad request"); return; } }else{ InputStream input = mongofile.getInputStream(); byte[] buffer = new byte[64*1024]; for(;;) { int count = input.read(buffer); if (count < 0) break; out.write(buffer,0,count); } } out.flush(); out.close(); mongofile.save(); }else{ resp.sendError(500,"internal server error"); } } @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // TODO Auto-generated method stub doPut(req, resp); } @Override protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // TODO Auto-generated method stub req.setCharacterEncoding("utf-8"); DiskFileItemFactory factory = new DiskFileItemFactory(); factory.setRepository(new File(tempPath)); factory.setSizeThreshold(8 * 1024 * 1024); ServletFileUpload upload = new ServletFileUpload(factory); try { List<FileItem> list = (List<FileItem>) upload.parseRequest(req); for (FileItem item : list) { if (item.isFormField()) { } else { String value = item.getName(); int start = value.lastIndexOf("\\"); String filename = value.substring(start + 1); String MongoDbName = req.getParameter("type"); if(MongoDbName == null){ resp.sendError(400,"bad request"); return; } DB db=mongo.getDB(MongoDbName); GridFS gridFS=new GridFS(db); GridFSInputFile mongofile=gridFS.createFile(item.getInputStream()); mongofile.put("viewtimes",0); mongofile.setContentType(item.getContentType()); mongofile.put("filename", filename); mongofile.save(); resp.getWriter().println(mongofile); resp.getWriter().flush(); } } } catch (Exception e) { e.printStackTrace(); resp.sendError(500,"internal server error"); } } }
运行方法一(eclipse很简单不用介绍,linux命令行运行方法,大于8M文件通过/tmp/目录缓存):
javac -cp .:../libs/* com/speakbang/vfs/*.java -Xlint:deprecation
java -cp .:../libs/* com/speakbang/vfs/VfsServer
运行方法二(只需要VfsFileServlet.java,集成到jetty配置文件)
root:~/jetty/webapps # ls -R * ROOT: WEB-INF classes com/ciaos/vfs/VfsFileServlet.class lib commons-fileupload-1.3.1.jar commons-io-2.4.jar mongo-java-driver-2.11.4.jar thumbnailator-0.4.7.jar src com/ciaos/vfs/VfsFileServlet.java web.xml
web.xml
<?xml version="1.0" encoding="ISO-8859-1"?> <web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" version="2.5"> <servlet> <servlet-name>VfsServlet</servlet-name> <servlet-class>com.ciaos.vfs.VfsFileServlet</servlet-class> <load-on-startup>0</load-on-startup> </servlet> <servlet-mapping> <servlet-name>VfsServlet</servlet-name> <url-pattern>/file</url-pattern> </servlet-mapping> </web-app>
cd webapps/ROOT/WEB-INF/src/
javac -cp .:../lib/*:../../../../lib/* com/ciaos/vfs/VfsFileServlet.java -Xlint:deprecation
mv com/ciaos/vfs/VfsFileServlet.class ../classes/com/ciaos/vfs/
cd ../../../../
java -jar start.jar -Xms128m -Xmx256m
(bin/jetty.sh start|restart|stop)
运行方法三(将class文件打成jar包,放到lib目录下,可以不用放置src和classes目录及源码了)
cd classes/
jar cvf vfs.jar com
cd ..
mv classes/vfs.jar lib/
rm -rf classes src
cd ../../
java -jar start.jar
注意文件不能直接操作db.fs.files.remove()删除,fs.chunks这个collection里面才是包含文件内容的,正确的删除方法使用如下脚本或者调用DELETE接口,删除完毕后用db.repairDatabase()释放磁盘空间:
var year=2014 var month=3 var day=15 use test var items = db.fs.files.find({uploadDate:{$lte:new Date(year+"/"+month+"/"+day)}}) items.forEach(function(item) { db.fs.chunks.remove({files_id: item._id}) db.fs.files.remove({_id:item._id}) print(item._id + " deleted") }) //运行方法: bin/mongo < tool/delete.js //root:~/mongodb$ bin/mongo < tool/delete.js //MongoDB shell version: 2.4.9 //connecting to: test //switched to db test //532259170cf2c0bcb68e1e72 deleted //532259190cf2c0bcb68e1e76 deleted //5322591a0cf2c0bcb68e1e7a deleted //bye
使用方法:
上传:curl -F "action=upload" -F "Filedata=@a.png;type=image/png" -v "http://127.0.0.1:8080/file?type=test" { "_id" : { "$oid" : "531c5fe6744e22f4c22eeef5"} , "chunkSize" : 262144 , "length" : 1033834 , "md5" : "04bc7c0988ecc04f93cfa96f239dda99" , "filename" : "a.png" , "contentType" : null , "viewtimes" : 0 , "uploadDate" : { "$date" : "2014-03-09T12:34:46.347Z"} , "aliases" : null } 下载:curl -o /dev/null -v http://127.0.0.1:8080/file?type=test&id=531c5fe6744e22f4c22eeef5&act=preview&w=200&h=200 删除:curl -X DELETE -v "http://127.0.0.1:8080/file?type=test&id=531c5fe6744e22f4c22eeef5"
Java代码上传方法(支持多文件上传)
package com.test; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.List; /** * This utility class provides an abstraction layer for sending multipart HTTP * POST requests to a web server. * @author www.codejava.net * */ public class MultipartUtility { private final String boundary; private static final String LINE_FEED = "\r\n"; private HttpURLConnection httpConn; private String charset; private OutputStream outputStream; private PrintWriter writer; /** * This constructor initializes a new HTTP POST request with content type * is set to multipart/form-data * @param requestURL * @param charset * @throws IOException */ public MultipartUtility(String requestURL, String charset) throws IOException { this.charset = charset; // creates a unique boundary based on time stamp boundary = "===" + System.currentTimeMillis() + "==="; URL url = new URL(requestURL); httpConn = (HttpURLConnection) url.openConnection(); httpConn.setUseCaches(false); httpConn.setDoOutput(true); // indicates POST method httpConn.setDoInput(true); httpConn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); outputStream = httpConn.getOutputStream(); writer = new PrintWriter(new OutputStreamWriter(outputStream, charset), true); } /** * Adds a form field to the request * @param name field name * @param value field value */ public void addFormField(String name, String value) { writer.append("--" + boundary).append(LINE_FEED); writer.append("Content-Disposition: form-data; name=\"" + name + "\"") .append(LINE_FEED); writer.append("Content-Type: text/plain; charset=" + charset).append( LINE_FEED); writer.append(LINE_FEED); writer.append(value).append(LINE_FEED); writer.flush(); } /** * Adds a upload file section to the request * @param fieldName name attribute in <input type="file" name="..." /> * @param uploadFile a File to be uploaded * @throws IOException */ public void addFilePart(String fieldName, File uploadFile) throws IOException { String fileName = uploadFile.getName(); writer.append("--" + boundary).append(LINE_FEED); writer.append( "Content-Disposition: form-data; name=\"" + fieldName + "\"; filename=\"" + fileName + "\"") .append(LINE_FEED); writer.append( "Content-Type: " + URLConnection.guessContentTypeFromName(fileName)) .append(LINE_FEED); writer.append("Content-Transfer-Encoding: binary").append(LINE_FEED); writer.append(LINE_FEED); writer.flush(); FileInputStream inputStream = new FileInputStream(uploadFile); byte[] buffer = new byte[4096]; int bytesRead = -1; while ((bytesRead = inputStream.read(buffer)) != -1) { outputStream.write(buffer, 0, bytesRead); } outputStream.flush(); inputStream.close(); writer.append(LINE_FEED); writer.flush(); } /** * Adds a header field to the request. * @param name - name of the header field * @param value - value of the header field */ public void addHeaderField(String name, String value) { writer.append(name + ": " + value).append(LINE_FEED); writer.flush(); } /** * Completes the request and receives response from the server. * @return a list of Strings as response in case the server returned * status OK, otherwise an exception is thrown. * @throws IOException */ public List<String> finish() throws IOException { List<String> response = new ArrayList<String>(); writer.append(LINE_FEED).flush(); writer.append("--" + boundary + "--").append(LINE_FEED); writer.close(); // checks server's status code first int status = httpConn.getResponseCode(); if (status == HttpURLConnection.HTTP_OK) { BufferedReader reader = new BufferedReader(new InputStreamReader( httpConn.getInputStream())); String line = null; while ((line = reader.readLine()) != null) { response.add(line); } reader.close(); httpConn.disconnect(); } else { throw new IOException("Server returned non-OK status: " + status); } return response; } public static void main(String[] args) { String charset = "UTF-8"; File uploadFile1 = new File("c:/1.jpg"); File uploadFile2 = new File("c:/2.jpg"); String requestURL = "http://localhost:8080/file?type=test"; try { MultipartUtility multipart = new MultipartUtility(requestURL, charset); /* multipart.addHeaderField("User-Agent", "CodeJava"); multipart.addHeaderField("Test-Header", "Header-Value"); multipart.addFormField("description", "Cool Pictures"); multipart.addFormField("keywords", "Java,upload,Spring"); */ multipart.addFilePart("fileUpload", uploadFile1); multipart.addFilePart("fileUpload", uploadFile2); List<String> response = multipart.finish(); for (String line : response) { System.out.println(line); } //output //533e27560cf2f5c3e3d77748 //533e27560cf2f5c3e3d7774a } catch (IOException ex) { System.err.println(ex); } } }