NoSQL MongoDB之nginx-gridfs

下载nginx-gridfs.git

[root@ck01 ~]# git clone https://github.com/mdirolf/nginx-gridfs.git\
[root@ck01 ~]# cd nginx-gridfs/
[root@ck01 ~]# git checkout v0.8
[root@ck01 ~]# git submodule init
[root@ck01 ~]# git submodule update

下载nginx

[root@ck01 ~]# wget http://nginx.org/download/nginx-1.21.1.tar.gz
[root@ck01 ~]# tar -zxf nginx-1.21.1.tar.gz
[root@ck01 ~]# cd nginx-1.21.1

下载nginx依赖

[root@ck01 ~]# yum install -y openssl-devel pcre-devel

编译安装nginx

[root@ck01 ~]# cd nginx-1.21.1
[root@ck01 nginx-1.21.1]# ./configure --add-module=/root/nginx-gridfs --prefix=/usr/local/nginx-1.21.1 --with-cc-opt=-Wno-error
[root@ck01 nginx-1.21.1]#  make 
[root@ck01 nginx-1.21.1]#  make install

nginx的安装文件是在/usr/local/nginx-1.21.1目录下结构如下

[root@ck01 nginx-1.21.1]# tree
.
├── client_body_temp
├── conf
│   ├── fastcgi.conf
│   ├── fastcgi.conf.default
│   ├── fastcgi_params
│   ├── fastcgi_params.default
│   ├── koi-utf
│   ├── koi-win
│   ├── mime.types
│   ├── mime.types.default
│   ├── nginx.conf
│   ├── nginx.conf.default
│   ├── scgi_params
│   ├── scgi_params.default
│   ├── uwsgi_params
│   ├── uwsgi_params.default
│   └── win-utf
├── fastcgi_temp
├── html
│   ├── 50x.html
│   └── index.html
├── logs
│   ├── access.log
│   ├── error.log
│   └── nginx.pid
├── proxy_temp
├── sbin
│   └── nginx
├── scgi_temp
└── uwsgi_temp

9 directories, 21 files

注意：有些文文件是运行时候产生的可能和初始化的时候产生的文件目录有些许差异

修改nginx.conf支持gridfs文件

#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }


        location /gridfs/ {
              gridfs jiangzz
              root_collection=fs
              field=_id
              type=objectid;
              mongo 192.168.198.148:27021;
      }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

配置参数说明：

gridfs：nginx识别插件的名字
jiangzz：数据库名称
[root_collection]: 选择collection，如root_collection=fs， mongod就会去找fs.files与fs.chunks两个块，默认是fs
[field]：查询字段，保证mongdb里有这个字段名，支持_id, filename, 可省略, 默认是_id
[type]：解释field的数据类型，支持objectid, int, string, 可省略, 默认是int
[user]：用户名, 可省略
[pass]：密码, 可省略
mongo：mongodb url mongo名称 地址：端口
更多详情：https://gitee.com/mirrors/nginx-gridfs

GridFS + SpingBoot集成

• 添加依赖

   <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.0.5.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-mongodb</artifactId>
        </dependency>

        <dependency>
            <groupId>org.mongodb</groupId>
            <artifactId>mongodb-driver</artifactId>
            <version>3.6.4</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

• 添加yml

spring:
  data:
    mongodb:
      host: 192.168.198.148
      database: jiangzz
      port: 27021

这里使用的是MongoDB的路由服务器，后端是mongodb集群

• 开启chunks的分片配置

[root@ck01 nginx-1.21.1]# cd
[root@ck01 ~]# mongo --host 192.168.198.148 --port 27021
MongoDB shell version v3.6.23
connecting to: mongodb://192.168.198.148:27021/?gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2016d5a4-e76a-454c-9544-00048af02f68") }
MongoDB server version: 3.6.23
Server has startup warnings:
2021-08-18T20:06:04.756+0800 I CONTROL  [main]
2021-08-18T20:06:04.757+0800 I CONTROL  [main] ** WARNING: Access control is not enabled for the database.
2021-08-18T20:06:04.757+0800 I CONTROL  [main] **          Read and write access to data and configuration is unrestricted.
2021-08-18T20:06:04.757+0800 I CONTROL  [main] ** WARNING: You are running this process as the root user, which is not recommended.
2021-08-18T20:06:04.757+0800 I CONTROL  [main]
mongos> use jiangzz
switched to db jiangzz
mongos> sh.enableSharding("jiangzz")
{
        "ok" : 1,
        "operationTime" : Timestamp(1629302783, 4),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1629302783, 4),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}
mongos> db.printShardingStatus()
--- Sharding Status ---
  sharding version: {
        "_id" : 1,
        "minCompatibleVersion" : 5,
        "currentVersion" : 6,
        "clusterId" : ObjectId("611cf68a1be51ad35844e911")
  }
  shards:
        {  "_id" : "shard1",  "host" : "shard1/192.168.198.146:27017,192.168.198.147:27017,192.168.198.148:27017",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/192.168.198.146:27018,192.168.198.147:27018,192.168.198.148:27018",  "state" : 1 }
        {  "_id" : "shard3",  "host" : "shard3/192.168.198.146:27019,192.168.198.147:27019,192.168.198.148:27019",  "state" : 1 }
  active mongoses:
        "3.6.23" : 1
  autosplit:
        Currently enabled: yes
  balancer:
        Currently enabled:  yes
        Currently running:  no
        Failed balancer rounds in last 5 attempts:  0
        Migration Results for the last 24 hours:
                682 : Success
  databases:
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }
                config.system.sessions
                        shard key: { "_id" : 1 }
                        unique: false
                        balancing: true
                        chunks:
                                shard1  342
                                shard2  341
                                shard3  341
                        too many chunks to print, use verbose if you want to force print
        {  "_id" : "jiangzz",  "primary" : "shard2",  "partitioned" : true }

mongos> db.fs.chunks.createIndex( { files_id : 1 , n : 1 } )
{
        "raw" : {
                "shard2/192.168.198.146:27018,192.168.198.147:27018,192.168.198.148:27018" : {
                        "createdCollectionAutomatically" : false,
                        "numIndexesBefore" : 1,
                        "numIndexesAfter" : 2,
                        "ok" : 1
                }
        },
        "ok" : 1,
        "operationTime" : Timestamp(1629302985, 1),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1629302985, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}

参考：https://docs.mongodb.com/v3.6/core/gridfs/#sharding-gridfs

• 编写文件上传下载测试逻辑

@SpringBootTest(classes = {MongoDBGridFSTemplateApplication.class})
@RunWith(SpringRunner.class)
public class TestMongoDBClient {
   @Autowired
   private GridFsTemplate gridFsTemplate;
    @Test
    public void testUploadFile() throws FileNotFoundException {
        String filePath="C:\\Users\\jiangzhongzhou\\Desktop\\aa.png";
        InputStream inputStream=new FileInputStream(filePath);
        DBObject dbObject = BasicDBObjectBuilder.start().add("expireAt", new Date().getTime())
                .add("owner", "jiangzz").add("dd","vvv").get();

        ObjectId objectId = gridFsTemplate.store(inputStream, "张海峰.pdf", dbObject);
        System.out.println("存储文件信息");
    }
    @Test
    public void testQueryFile(){
        GridFSFindIterable gridFSFiles = gridFsTemplate.find(new Query(whereFilename().is("张海峰.pdf")));
        for (GridFSFile gridFSFile : gridFSFiles) {
            System.out.println(gridFSFile.getFilename()+" "+gridFSFile.getObjectId());
        }
    }
    @Test
    public void testDownloadFile(){
        GridFSFile fsFile = gridFsTemplate.findOne(new Query(Criteria.where("_id").is("612de6c1f51bb90694b9f345")));
        System.out.println(fsFile.getFilename()+"\t"+fsFile.getMD5());
    }

    @Test
    public void testDeleteFile(){
         gridFsTemplate.delete(new Query(Criteria.where("metadata.owner").is("jiangzz")));
    }

}