使用jetty和mongodb做个简易文件系统

使用jetty和mongodb做个简易文件系统 - ciaos

依赖库:

1,jetty(提供http方式接口)

2,mongodb的java驱动(访问mongodb存取文件)

3,thumbnailator包,进行缩略图生成

4,commons-fileupload包及commons-io包用于处理文件上传

入口代码如下:

package com.ciaos.vfs;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;

public class VfsServer {
    public static void main(String[] args) throws Exception {
   Server server = new Server(8080);  
 
   ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
   context.setContextPath("/");  
 
   server.setHandler(context);  
 
   context.addServlet(new ServletHolder(new VfsFileServlet()), "/file");
 
   server.start();
   server.join();
    }
}

业务代码如下:

package com.ciaos.vfs;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.UnknownHostException;
import java.util.List;

import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;  
import javax.servlet.http.HttpServletRequest;  
import javax.servlet.http.HttpServletResponse;  

import net.coobird.thumbnailator.Thumbnails;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.bson.types.ObjectId;

import com.mongodb.DB;
import com.mongodb.Mongo;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSDBFile;
import com.mongodb.gridfs.GridFSInputFile;

public class VfsFileServlet extends HttpServlet {

  /**
   * 
   */
  private static final long serialVersionUID = 5586455171943232770L;
  
  private String tempPath = null;
  private Mongo mongo = null;
  
  final private String MongoDbIp = "192.168.1.106";
  final private Integer MongoDbPort = 27017;
  
  final private String MongoDbName = "chat";
  
  @Override
  public void init() throws ServletException {
    // TODO Auto-generated method stub
    
    tempPath = "uploads/";
    try {
      mongo=new Mongo(MongoDbIp, MongoDbPort);
    } catch (UnknownHostException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
      System.exit(-1);
    }
    super.init();
  }

  @Override
  protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    // TODO Auto-generated method stub
    String fileId = req.getParameter("id");  
    DB db=mongo.getDB(MongoDbName);
    GridFS gridFS=new GridFS(db);
    try{
      gridFS.remove(new ObjectId(fileId));
    }
    catch(IllegalArgumentException ex){
      resp.sendError(404,"file not found");
      return;
    }
  }

  @Override
  protected void doGet(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    // TODO Auto-generated method stub
    String fileId = req.getParameter("id");
    
    DB db=mongo.getDB(MongoDbName);
    GridFS gridFS=new GridFS(db);
    
    GridFSDBFile mongofile = null;
    try{
      mongofile = gridFS.find(new ObjectId(fileId));
    }
    catch(IllegalArgumentException ex){
      resp.sendError(404,"file not found");
    }
    
    if(mongofile!=null){
      resp.setHeader("Content-type", "application/octet-stream");
      resp.setHeader("Content-Disposition", "attachment; filename= "+mongofile.getFilename());
      mongofile.put("viewtimes", (Integer)mongofile.get("viewtimes")+1);
      ServletOutputStream out = resp.getOutputStream();
      

      String dtype = req.getParameter("type");
      if(dtype!=null && dtype.equals("preview")){
        Integer width = 0, height = 0;
        try{
          width = Integer.parseInt(req.getParameter("w"));
          height = Integer.parseInt(req.getParameter("h"));
          Thumbnails.of(mongofile.getInputStream())   
              .size(width, height)  
              .toOutputStream(out);
        }
        catch(NumberFormatException ex){
          resp.sendError(400,"bad request");
          return;
        }
      }else{
        InputStream input = mongofile.getInputStream();
        byte[] buffer = new byte[64*1024];
        for(;;)
        {
          int count = input.read(buffer);
          if (count < 0)
            break;
          out.write(buffer,0,count);
        }
      }

      out.flush();
      out.close();
      mongofile.save();
    }else{
      resp.sendError(500,"internal server error");
    }
  }

  @Override
  protected void doPost(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    // TODO Auto-generated method stub
    doPut(req, resp);
  }

  @Override
  protected void doPut(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    // TODO Auto-generated method stub
    req.setCharacterEncoding("utf-8");
    DiskFileItemFactory factory = new DiskFileItemFactory();
    factory.setRepository(new File(tempPath));
    factory.setSizeThreshold(1024 * 1024);
    ServletFileUpload upload = new ServletFileUpload(factory);
    try {
      List<FileItem> list = (List<FileItem>) upload.parseRequest(req);
      for (FileItem item : list) {
        if (item.isFormField()) {
        } else {
          String value = item.getName();
          int start = value.lastIndexOf("\\");
          String filename = value.substring(start + 1);	
          
          DB db=mongo.getDB(MongoDbName);
          GridFS gridFS=new GridFS(db);
          
          GridFSInputFile  mongofile=gridFS.createFile(item.getInputStream());
          mongofile.put("viewtimes",0);
          
          mongofile.put("filename", filename);
          mongofile.save();
          
          resp.getWriter().println(mongofile);  
          resp.getWriter().flush();  
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
      resp.sendError(500,"internal server error");
    }
  }
}

运行方法(eclipse很简单不用介绍,linux命令行下面不用创建uploads文件夹也可使用):

javac -cp .:../libs/* com/speakbang/vfs/*.java -Xlint:deprecation
java -cp .:../libs/* com/speakbang/vfs/VfsServer

测试方法:

上传:curl -F "action=upload" -F "Filedata=@a.png" -v "http://127.0.0.1:8080/file"
{ "_id" : { "$oid" : "531c5fe6744e22f4c22eeef5"} , "chunkSize" : 262144 , "length" : 1033834 , "md5" : "04bc7c0988ecc04f93cfa96f239dda99" , "filename" : "a.png" , "contentType" :  null  , "viewtimes" : 0 , "uploadDate" : { "$date" : "2014-03-09T12:34:46.347Z"} , "aliases" :  null }
下载:curl -o /dev/null -v http://127.0.0.1:8080/file?id=531c5fe6744e22f4c22eeef5&type=preview&w=200&h=200

删除:curl -X DELETE -v "http://127.0.0.1:8080/file?id=531c5fe6744e22f4c22eeef5"

第二个类似的参考文档:

使用jetty和mongodb实现简易网盘接口

依赖库:

1,jetty(提供http方式接口)

2,mongodb的java驱动(访问mongodb存取文件)

3,thumbnailator包,进行缩略图生成

4,commons-fileupload包及commons-io包用于处理文件上传

架构图:

入口代码如下:

复制代码
package com.ciaos.vfs;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.nio.SelectChannelConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.thread.QueuedThreadPool;

public class VfsServer {
    public static void main(String[] args) throws Exception {
 
        
        SelectChannelConnector connector = new SelectChannelConnector();
        connector.setHost("127.0.0.1");
        connector.setPort(8080);
        connector.setThreadPool(new QueuedThreadPool(200));
        
        Server server = new Server();  
        server.setConnectors(new Connector[]{connector}); 
        
        ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
        context.setContextPath("/");  
 
        server.setHandler(context);  
 
        context.addServlet(new ServletHolder(new VfsFileServlet()), "/file");
 
        server.start();
        server.join();
    }
}
复制代码

业务代码如下:

复制代码
package com.ciaos.vfs;

import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.UnknownHostException;
import java.util.List;

import javax.imageio.ImageIO;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;  
import javax.servlet.http.HttpServletRequest;  
import javax.servlet.http.HttpServletResponse;  

import net.coobird.thumbnailator.Thumbnails;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.bson.types.ObjectId;

import com.mongodb.DB;
import com.mongodb.Mongo;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSDBFile;
import com.mongodb.gridfs.GridFSInputFile;

public class VfsFileServlet extends HttpServlet {

    /**
     * 
     */
    private static final long serialVersionUID = 5586455171943232770L;
    
    private String tempPath = null;
    private Mongo mongo = null;
    
    final private String MongoDbIp = "127.0.0.1";
    final private Integer MongoDbPort = 27017;
    
    @Override
    public void init() throws ServletException {
        // TODO Auto-generated method stub
        
        tempPath = "/tmp/";
        try {
            mongo=new Mongo(MongoDbIp, MongoDbPort);
        } catch (UnknownHostException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            System.exit(-1);
        }
        super.init();
    }

    @Override
    protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {
        // TODO Auto-generated method stub
        String fileId = req.getParameter("id");
        String MongoDbName = req.getParameter("type");
        if(fileId == null || MongoDbName == null){
            resp.sendError(400,"bad request");
            return;
        }
        DB db=mongo.getDB(MongoDbName);
        GridFS gridFS=new GridFS(db);
        try{
            gridFS.remove(new ObjectId(fileId));
        }
        catch(IllegalArgumentException ex){
            resp.sendError(404,"file not found");
            return;
        }
    }

    @Override
    protected void doGet(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {
        // TODO Auto-generated method stub
        String fileId = req.getParameter("id");
        String MongoDbName = req.getParameter("type");
        if(fileId == null || MongoDbName == null){
            resp.sendError(400,"bad request");
            return;
        }
        
        DB db=mongo.getDB(MongoDbName);
        GridFS gridFS=new GridFS(db);
        
        GridFSDBFile mongofile = null;
        try{
            mongofile = gridFS.find(new ObjectId(fileId));
        }
        catch(IllegalArgumentException ex){
            resp.sendError(404,"file not found");
            return;
        }
        
        if(mongofile!=null){
            resp.setHeader("Content-type", mongofile.getContentType());
            resp.setHeader("Content-Disposition", "attachment; filename= "+mongofile.getFilename());
            mongofile.put("viewtimes", (Integer)mongofile.get("viewtimes")+1);
            ServletOutputStream out = resp.getOutputStream();
            

            String dtype = req.getParameter("act");
            if(dtype!=null && dtype.equals("preview")){
                Integer width = 0, height = 0;
                try{
                    width = Integer.parseInt(req.getParameter("w"));
                    height = Integer.parseInt(req.getParameter("h"));
                    
                    BufferedImage image = ImageIO.read(mongofile.getInputStream());
                    width = Math.min(width,image.getWidth());
                    height = Math.min(height,image.getHeight());
                    
                    Thumbnails.of(mongofile.getInputStream())   
                            .size(width, height)  
                            .toOutputStream(out);
                }
                catch(NumberFormatException ex){
                    resp.sendError(400,"bad request");
                    return;
                }
            }else{
                InputStream input = mongofile.getInputStream();
                byte[] buffer = new byte[64*1024];
                for(;;)
                {
                    int count = input.read(buffer);
                    if (count < 0)
                        break;
                    out.write(buffer,0,count);
                }
            }

            out.flush();
            out.close();
            mongofile.save();
        }else{
            resp.sendError(500,"internal server error");
        }
    }

    @Override
    protected void doPost(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {
        // TODO Auto-generated method stub
        doPut(req, resp);
    }

    @Override
    protected void doPut(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {
        // TODO Auto-generated method stub
        req.setCharacterEncoding("utf-8");
        DiskFileItemFactory factory = new DiskFileItemFactory();
        factory.setRepository(new File(tempPath));
        factory.setSizeThreshold(8 * 1024 * 1024);
        ServletFileUpload upload = new ServletFileUpload(factory);
        try {
            List<FileItem> list = (List<FileItem>) upload.parseRequest(req);
            for (FileItem item : list) {
                if (item.isFormField()) {
                } else {
                    String value = item.getName();
                    int start = value.lastIndexOf("\\");
                    String filename = value.substring(start + 1);    
                    
                    String MongoDbName = req.getParameter("type");
                    if(MongoDbName == null){
                        resp.sendError(400,"bad request");
                        return;
                    }
                    DB db=mongo.getDB(MongoDbName);
                    GridFS gridFS=new GridFS(db);
                  
                    GridFSInputFile  mongofile=gridFS.createFile(item.getInputStream());
                    mongofile.put("viewtimes",0);
                    mongofile.setContentType(item.getContentType());
                    mongofile.put("filename", filename);
                    mongofile.save();
                    
                    resp.getWriter().println(mongofile);  
                    resp.getWriter().flush();  
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            resp.sendError(500,"internal server error");
        }
    }
}
复制代码

运行方法一(eclipse很简单不用介绍,linux命令行运行方法,大于8M文件通过/tmp/目录缓存):

javac -cp .:../libs/* com/speakbang/vfs/*.java -Xlint:deprecation
java -cp .:../libs/* com/speakbang/vfs/VfsServer

运行方法二(只需要VfsFileServlet.java,集成到jetty配置文件)

复制代码
root:~/jetty/webapps # ls -R *
ROOT:
    WEB-INF
        classes
            com/ciaos/vfs/VfsFileServlet.class
        lib
            commons-fileupload-1.3.1.jar  commons-io-2.4.jar  mongo-java-driver-2.11.4.jar  thumbnailator-0.4.7.jar
        src
            com/ciaos/vfs/VfsFileServlet.java
        web.xml
复制代码

web.xml

复制代码
<?xml version="1.0" encoding="ISO-8859-1"?>
<web-app xmlns="http://java.sun.com/xml/ns/javaee"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
        version="2.5">
        <servlet>
        <servlet-name>VfsServlet</servlet-name>
        <servlet-class>com.ciaos.vfs.VfsFileServlet</servlet-class>
        <load-on-startup>0</load-on-startup>
        </servlet>

        <servlet-mapping>
        <servlet-name>VfsServlet</servlet-name>
        <url-pattern>/file</url-pattern>
        </servlet-mapping>
</web-app>
复制代码
cd webapps/ROOT/WEB-INF/src/
javac -cp .:../lib/*:../../../../lib/* com/ciaos/vfs/VfsFileServlet.java -Xlint:deprecation
mv com/ciaos/vfs/VfsFileServlet.class ../classes/com/ciaos/vfs/
cd ../../../../
java -jar start.jar -Xms128m -Xmx256m
(bin/jetty.sh start|restart|stop)

运行方法三(将class文件打成jar包,放到lib目录下,可以不用放置src和classes目录及源码了)

复制代码
cd classes/
jar cvf vfs.jar com
cd ..
mv classes/vfs.jar lib/
rm -rf classes src
cd ../../
java -jar start.jar
复制代码

注意文件不能直接操作db.fs.files.remove()删除,fs.chunks这个collection里面才是包含文件内容的,正确的删除方法使用如下脚本或者调用DELETE接口,删除完毕后用db.repairDatabase()释放磁盘空间:

复制代码
var year=2014
var month=3
var day=15

use test

var items = db.fs.files.find({uploadDate:{$lte:new Date(year+"/"+month+"/"+day)}})
items.forEach(function(item) {
        db.fs.chunks.remove({files_id: item._id})
        db.fs.files.remove({_id:item._id})
        print(item._id + " deleted")
})

//运行方法: bin/mongo < tool/delete.js
//root:~/mongodb$ bin/mongo < tool/delete.js
//MongoDB shell version: 2.4.9
//connecting to: test
//switched to db test
//532259170cf2c0bcb68e1e72 deleted
//532259190cf2c0bcb68e1e76 deleted
//5322591a0cf2c0bcb68e1e7a deleted
//bye
复制代码

使用方法:

 

上传:curl -F "action=upload" -F "Filedata=@a.png;type=image/png" -v "http://127.0.0.1:8080/file?type=test"
{ "_id" : { "$oid" : "531c5fe6744e22f4c22eeef5"} , "chunkSize" : 262144 , "length" : 1033834 , "md5" : "04bc7c0988ecc04f93cfa96f239dda99" , "filename" : "a.png" , "contentType" :  null  , "viewtimes" : 0 , "uploadDate" : { "$date" : "2014-03-09T12:34:46.347Z"} , "aliases" :  null }
下载:curl -o /dev/null -v http://127.0.0.1:8080/file?type=test&id=531c5fe6744e22f4c22eeef5&act=preview&w=200&h=200
删除:curl -X DELETE -v "http://127.0.0.1:8080/file?type=test&id=531c5fe6744e22f4c22eeef5"

 

Java代码上传方法(支持多文件上传)

复制代码
package com.test;
 
import java.io.BufferedReader;
 
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
 
/**
 * This utility class provides an abstraction layer for sending multipart HTTP
 * POST requests to a web server.
 * @author www.codejava.net
 *
 */
public class MultipartUtility {
    private final String boundary;
    private static final String LINE_FEED = "\r\n";
    private HttpURLConnection httpConn;
    private String charset;
    private OutputStream outputStream;
    private PrintWriter writer;
 
    /**
     * This constructor initializes a new HTTP POST request with content type
     * is set to multipart/form-data
     * @param requestURL
     * @param charset
     * @throws IOException
     */
    public MultipartUtility(String requestURL, String charset)
            throws IOException {
        this.charset = charset;
         
        // creates a unique boundary based on time stamp
        boundary = "===" + System.currentTimeMillis() + "===";
         
        URL url = new URL(requestURL);
        httpConn = (HttpURLConnection) url.openConnection();
        httpConn.setUseCaches(false);
        httpConn.setDoOutput(true); // indicates POST method
        httpConn.setDoInput(true);
        httpConn.setRequestProperty("Content-Type",
                "multipart/form-data; boundary=" + boundary);
        outputStream = httpConn.getOutputStream();
        writer = new PrintWriter(new OutputStreamWriter(outputStream, charset),
                true);
    }
 
    /**
     * Adds a form field to the request
     * @param name field name
     * @param value field value
     */
    public void addFormField(String name, String value) {
        writer.append("--" + boundary).append(LINE_FEED);
        writer.append("Content-Disposition: form-data; name=\"" + name + "\"")
                .append(LINE_FEED);
        writer.append("Content-Type: text/plain; charset=" + charset).append(
                LINE_FEED);
        writer.append(LINE_FEED);
        writer.append(value).append(LINE_FEED);
        writer.flush();
    }
 
    /**
     * Adds a upload file section to the request
     * @param fieldName name attribute in <input type="file" name="..." />
     * @param uploadFile a File to be uploaded
     * @throws IOException
     */
    public void addFilePart(String fieldName, File uploadFile)
            throws IOException {
        String fileName = uploadFile.getName();
        writer.append("--" + boundary).append(LINE_FEED);
        writer.append(
                "Content-Disposition: form-data; name=\"" + fieldName
                        + "\"; filename=\"" + fileName + "\"")
                .append(LINE_FEED);
        writer.append(
                "Content-Type: "
                        + URLConnection.guessContentTypeFromName(fileName))
                .append(LINE_FEED);
        writer.append("Content-Transfer-Encoding: binary").append(LINE_FEED);
        writer.append(LINE_FEED);
        writer.flush();
 
        FileInputStream inputStream = new FileInputStream(uploadFile);
        byte[] buffer = new byte[4096];
        int bytesRead = -1;
        while ((bytesRead = inputStream.read(buffer)) != -1) {
            outputStream.write(buffer, 0, bytesRead);
        }
        outputStream.flush();
        inputStream.close();
         
        writer.append(LINE_FEED);
        writer.flush();    
    }
 
    /**
     * Adds a header field to the request.
     * @param name - name of the header field
     * @param value - value of the header field
     */
    public void addHeaderField(String name, String value) {
        writer.append(name + ": " + value).append(LINE_FEED);
        writer.flush();
    }
     
    /**
     * Completes the request and receives response from the server.
     * @return a list of Strings as response in case the server returned
     * status OK, otherwise an exception is thrown.
     * @throws IOException
     */
    public List<String> finish() throws IOException {
        List<String> response = new ArrayList<String>();
 
        writer.append(LINE_FEED).flush();
        writer.append("--" + boundary + "--").append(LINE_FEED);
        writer.close();
 
        // checks server's status code first
        int status = httpConn.getResponseCode();
        if (status == HttpURLConnection.HTTP_OK) {
            BufferedReader reader = new BufferedReader(new InputStreamReader(
                    httpConn.getInputStream()));
            String line = null;
            while ((line = reader.readLine()) != null) {
                response.add(line);
            }
            reader.close();
            httpConn.disconnect();
        } else {
            throw new IOException("Server returned non-OK status: " + status);
        }
 
        return response;
    }
    
    public static void main(String[] args) {
        
        String charset = "UTF-8";
        File uploadFile1 = new File("c:/1.jpg");
        File uploadFile2 = new File("c:/2.jpg");
        String requestURL = "http://localhost:8080/file?type=test";
 
        try {
            MultipartUtility multipart = new MultipartUtility(requestURL, charset);
/*             
            multipart.addHeaderField("User-Agent", "CodeJava");
            multipart.addHeaderField("Test-Header", "Header-Value");
             
            multipart.addFormField("description", "Cool Pictures");
            multipart.addFormField("keywords", "Java,upload,Spring");
*/             
            multipart.addFilePart("fileUpload", uploadFile1);
            multipart.addFilePart("fileUpload", uploadFile2);

            List<String> response = multipart.finish();
             
            for (String line : response) {
                System.out.println(line);
            }
            //output
            //533e27560cf2f5c3e3d77748
            //533e27560cf2f5c3e3d7774a
            
        } catch (IOException ex) {
            System.err.println(ex);
        }
    }
}
复制代码


1.http://www.tuicool.com/articles/YBZj2i

2.http://www.cnblogs.com/ciaos/p/3590662.html



  • 1
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值