分布式文件系统
存储互联网海量非结构化数据的存储需求。
FastDFS
文件存储、文件同步和文件访问,以及搞容量和负载均衡。主要是存储海量非结构化的中小文件,建议单个文件大小范围在 [ 4KB , 500MB ] 之间。
特性
优点:
对文件hash处理,避免重复存储
支持http,可基于内置Web Server或外部Web Server(Nginx)
在线扩容
缺点:
缺乏安全性
角色
Tracker cluster
跟踪服务器,主要做调度工作,起到均很的作用;负责storage server和group,每个storage 在启动后会连接Tracker,告知自己所属group等信息,并保持周期性心跳。
Storage Server
存储服务器,主要提供容量和备份服务;以group为单位,每个group内可以有多态storage server,数据互为备份。
Client
客户端,上传下载数据的服务器,也就是我们的自己的项目部署在的服务器。
环境搭建
环境准备
yum install git gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl-devel wget vim -y
https://github.com/happyfish100/
从仓库下载:
fastdfs
libfastcommon
fastdfs-nginx-module
安装fastdfs和libfastcommon
# 将fastdfs和libfastcommon传入/usr/local/soft,并分别解压
unzip xxxxx.zip
# 将解压后的fastdfs和libfastcommon,并分别编译。
./make.sh && ./make.sh install
# 在fastdf要执行这个供nginx访问使用
cp /usr/local/soft/fastdfs-master/conf/http.conf /etc/fdfs/
cp /usr/local/soft/fastdfs-master/conf/mime.types /etc/fdfs/
安装fastdfs-nginx-module
# 将fastdfs-nginx-module传入/usr/local/soft,并解压
# 移动配置文件
cp /usr/local/soft/fastdfs-nginx-module-master/src/mod_fastdfs.conf /etc/fdfs/
安装nginx
wget http://nginx.org/download/nginx-1.15.4.tar.gz
tar -zxvf nginx-1.15.4.tar.gz
# 添加fastdfs-nginx-module模块
./configure --add-module=/usr/local/soft/fastdfs-nginx-module-master/src/
## 若出现error: the HTTP rewrite module requires the PCRE library.
yum -y install pcre-devel
## 若出现./configure: error: the HTTP cache module requires md5 functionsfrom OpenSSL library. You can either disable the module by using--without-http-cache option, or install the OpenSSL library into the system,or build the OpenSSL library statically from the source with nginx by using--with-http_ssl_module --with-openssl=<path> options.
yum -y install openssl openssl-devel
# 编译
make && make install
单机部署
设置局域网下的域名
vim /etc/hosts
# 192.168.38.129 wwy.com
修改tracker配置
mkdir -p /data/fastdfs
vim /etc/fdfs/tracker.conf
# 需要修改的内容如下
# tracker服务端口
port=22122
# 存储日志和数据的根目录
base_path=/data/fastdfs
修改storage配置
vim /etc/fdfs/storage.conf
# 需要求改内容如下
# storage服务端口
port=23000
# 存储日志和数据的根目录
base_path=/data/fastdfs
# 第一个存储目录,也就是M00,若是store_path1则是M01
store_path0=/data/fastdfs
# tracker服务器IP和端口
tracker_server=wwy.com:22122
# http访问文件的端口(默认8888,看情况修改,和nginx保持一致)
http.server_port=8888
启动tracker和storage服务
# 打开对应端口,或者关闭防火墙
systemctl disable firewalld.service
# 启动tracker
/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf
# 查看是否启动成功
ps -ef|grep tracker
# 启动storage
/usr/bin/fdfs_storaged /etc/fdfs/storage.conf
# 查看是否启动成功
ps -ef|grep storage
# 重启storage
/usr/bin/fdfs_storaged /etc/fdfs/storage.conf restart
client测试
vim /etc/fdfs/client.conf
# 需要求改的内容
base_path= /data/fastdfs
# tracker服务器IP和端口
tracker_server=wwy.com:22122
# 保存后测试,返回ID表示成功 如:group1/M00/00/00/wKgmgWG019SAajURAAEyiVzLIwo621.jpg
# 上传文件
fdfs_upload_file /etc/fdfs/client.conf /usr/dlrb.jpg
# 删除文件
fdfs_delete_file /etc/fdfs/client.conf group1/M00/00/00/wKgmgWG019SAajURAAEyiVzLIwo621.jpg
group1/M00/00/00/wKgmgWG02QqAIxmVAAEyiVzLIwo283.jpg
配置nginx访问
vim /etc/fdfs/mod_fastdfs.conf
# 需要修改的内容如下
# tracker服务器IP和端口
tracker_server=wwy.com:22122
url_have_group_name=true
store_path0=/data/fastdfs
# 配置nginx.config
vim /usr/local/nginx/conf/nginx.conf
# 添加如下配置
server{
listen 8888;#该端口为storage.conf中的http.server_port相同
server_name localhost;
location ~/group[0-9]/ {
ngx_fastdfs_module;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
启动nginx
# 启动nginx
/usr/local/nginx/sbin/nginx
# 重启nginx
/usr/local/nginx/sbin/nginx -s reload
# 停止nginx
/usr/local/nginx/sbin/nginx -s stop
下次再次启动
# 启动tracker
/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf
# 查看是否启动成功
ps -ef|grep tracker
# 启动storage
/usr/bin/fdfs_storaged /etc/fdfs/storage.conf
# 查看是否启动成功
ps -ef|grep storage
# 启动nginx
/usr/local/nginx/sbin/nginx
集群部署
设置局域网下的域名
vim /etc/hosts
# 192.168.38.121 fastdfs1.com
# 192.168.38.122 fastdfs2.com
# 192.168.38.123 fastdfs3.com
tracker 配置
vim /etc/fdfs/tracker.conf
# 需要修改的内容如下
# tracker服务端口(默认22122一般不修改)
port=22122
# 存储日志和数据的根目录
base_path=/data/fastdfs
storage配置
vim /etc/fdfs/storage.conf
# 需要求改的内容如下
# storage服务端口(默认23000,一般不修改)
port=23000
# 数据和日志文件存储根目录
base_path=/data/fastdfs
# 第一个存储目录
store_path0=/data/fastdfs
# 服务器1
tracker_server=fastdfs1.com:22122
# 服务器2
tracker_server=fastdfs2.com:22122
# 服务器3
tracker_server=fastdfs3.com:22122
# http访问文件端口(默认8888,看情况修改,和nginx中保持一致)
http.server_port=8888
clint测试
vim /etc/fdfs/client.conf
# 需要修改内容如下
base_path=/data/fastdfs
# 服务器1
tracker_server=fastdfs1.com:22122
# 服务器2
tracker_server=fastdfs2.com:22122
# 服务器3
tracker_server=fastdfs3.com:22122
# 保存后测试返回ID表示成功
fdfs_upload_file /etc/fdfs/client.conf /usr/dlrb.jpg
配置nginx服务
vim /etc/fdfs/mod_fastdfs.conf
# 需要修改的内容如下
# 服务器1
tracker_server=fastdfs1.com:22122
# 服务器2
tracker_server=fastdfs2.com:22122
# 服务器3
tracker_server=fastdfs3.com:22122
url_have_group_name=true
store_path0=/data/fastdfs
# 配置nginx.config
vim /usr/local/nginx/conf/nginx.conf
# 添加如下配置
server{
listen 8888;#该端口为storage.conf中的http.server_port相同
server_name localhost;
location ~/group[0-9]/ {
ngx_fastdfs_module;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
检测集群
/usr/bin/fdfs_monitor /etc/fdfs/storage.conf
# 会显示有几台服务器,有3台就会显示 Storage 1-storage 3的详细信息
Java整合FastDFS(官方)
pom.xml
<!--如果仓库没有,可以使用源码编译成jar包然后导入 https://github.com/happyfish100/fast-client-java -->
<dependency>
<groupId>com.taoyuanx</groupId>
<artifactId>fastdfs-client-java</artifactId>
<version>1.29-RELEASE</version>
</dependency>
配置文件fdfs_client.conf
tracker_server = 192.168.38.129:22122
下载文件
# 注意若是报错:org.csource.common.MyException: connect to server 192.168.38.129:23000 fail,java.net.SocketTimeoutException: connect timed out
# 查看防火墙
firewall-cmd --state
# 打开防火墙
systemctl start firewalld.service
# 打开端口
firewall-cmd --zone=public --add-port=8888/tcp --permanent
firewall-cmd --zone=public --add-port=22122/tcp --permanent
firewall-cmd --zone=public --add-port=23000/tcp --permanent
systemctl restart firewalld.service
firewall-cmd --reload
try {
//1.加载配置文件 tracker(ip:port)
ClientGlobal.init("fdfs_client.conf");
//2.创建一个TrackerClient对象
TrackerClient trackerClient = new TrackerClient();
//3.使用TrackerClient对象获得TrackerServer对象
TrackerServer trackerServer = trackerClient.getTrackerServer();
//4.创建一个TrackerClient对象。trackerServer和StorageServer俩个参数
StorageClient storageClient = new StorageClient(trackerServer, null);
String path = System.getProperty("user.dir")+ File.separator+"dlrb.jpg";
//5.使用StorageClient对象下载文件
storageClient.download_file("group1",
"M00/00/00/wKgmgWG02QqAIxmVAAEyiVzLIwo283.jpg",path);
} catch (IOException e) {
e.printStackTrace();
} catch (MyException e) {
e.printStackTrace();
}
上传文件
try {
//1.加载配置文件
ClientGlobal.init("fdfs_client.conf");
//2.创建一个TrackerClient对象
TrackerClient trackerClient = new TrackerClient();
//3.使用trackerClient对象获取一个trackerServer对象
TrackerServer trackerServer = trackerClient.getTrackerServer();
//4.创建一个TrackerClient对象。trackerServer和StorageServer俩个参数
StorageClient storageClient = new StorageClient(trackerServer, null);
String path = System.getProperty("user.dir")+ File.separator+"1080P.mp4";
//5.使用storageClient对象上传文件
String[] webpath = storageClient.upload_file(path, "mp4", null);
for (String s : webpath) {
System.out.println(s);
}
} catch (IOException e) {
e.printStackTrace();
} catch (MyException e) {
e.printStackTrace();
}
//group1
//M00/00/00/wKgmgWG09uiAFVkQAPoSjXF9Bw8981.mp4
Java整合FastDFS(好用)
在官方的java客户端基础上做了重构。
特性:
对部分代码加入了单元测试。
支持服务端的连接池管理
支持上传图片时候检查图片格式,并且自动生成缩略图
在springBoot当中自动导入依赖
https://github.com/tobato/FastDFS_Client
pom.xml
<!-- https://mvnrepository.com/artifact/com.github.tobato/fastdfs-client -->
<dependency>
<groupId>com.github.tobato</groupId>
<artifactId>fastdfs-client</artifactId>
<version>1.27.2</version>
</dependency>
application.yml
# 分布式文件系统FDFS配置
fdfs:
so-timeout: 1500 #读取事件
connect-timeout: 600 #连接超时时间
# 缩略图生成参数
thumb-image:
width: 150
height: 150
tracker-list: #Tracker服务配置地址列表
- 192.168.38.129:22122
# - 192.168.38.121:22122
# - 192.168.38.122:22122
pool:
# 从池中借出的对象的最大数目(配置-1表示不限制)
max-total: -1
# 获取连接时的最大等待毫秒数(默认配置5秒)
max-wait-millis: 5000
#每个key最大连接数
max-total-per-key: 50
#每个key对应的连接池最大空闲连接数
max-idle-per-key: 10
#每个key对应的连接池最小空闲连接数
min-idle-per-key: 5
upload:
base-url: http://wwy.com:8888/
注
文件上传使用的是
大文件上传出现异常:
.SizeLimitException:the requested was rejected because its size (xxxx) exceeds the configured maximum (xxxx)
// 方案一:(创建配置类)
@Configuration
public class FileConfig {
@Bean
public MultipartConfigElement multipartConfigElement(){
MultipartConfigFactory factory = new MultipartConfigFactory();
factory.setMaxRequestSize(DataSize.ofBytes(200*1048576L));
factory.setMaxFileSize(DataSize.ofBytes(200*1048576L));
return factory.createMultipartConfig();
}
}
# 方案二:(yml配置)
spring:
servlet:
multipart:
max-file-size: 200MB
max-request-size: 200MB
上传文件
前端
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>文件上传</title>
</head>
<body>
<h1>文件上传测试</h1>
<hr/>
<form class="" method="post" enctype="multipart/form-data" action="/upload">
<input type="file" class="file" name="file">
<input type="submit" value="上传">
</form>
</body>
</html>
后端(Controller)
@RestController
public class UploadController {
@Autowired
private UploadService uploadService;
@RequestMapping("/upload")
public Map<String, Object> upload(MultipartFile file) {
Map<String, Object> map = new HashMap<>();
String path = this.uploadService.uploadImage(file);
map.put("path", path);
String mini_path = path.replace(".jpg", "_150x150.jpg");
map.put("mini_path",mini_path);
return map;
}
}
后端(Service)
@Service
public class UploadService {
@Autowired
private FastFileStorageClient fastFileStorageClient;
@Value("${upload.base-url}")
private String baseUrl;
public String uploadImage(MultipartFile file){
try {
//校验图片
BufferedImage image = ImageIO.read(file.getInputStream());
if (image == null || image.getWidth() == 0 || image.getHeight() ==0){
throw new RuntimeException("上传文件不是图片");
}
} catch (IOException e) {
throw new RuntimeException("文件校验内容失败"+e.getMessage());
}
try {
//获取扩展名
String extension = StringUtils.substringAfterLast(file.getOriginalFilename(), ".");
//上传缩略图
StorePath storePath = fastFileStorageClient.uploadImageAndCrtThumbImage(file.getInputStream(),file.getSize(),extension,null);
//返回路径
return baseUrl+storePath.getFullPath();
} catch (IOException e) {
throw new RuntimeException("文件上传失败"+e.getMessage());
}
}
}
下载图片(下载到服务器上)
@RestController
public class DownloadFile {
@Autowired
private FastFileStorageClient fastFileStorageClient;
@RequestMapping("/download")
public void download(){
try {
byte[] bytes = fastFileStorageClient.downloadFile("group1","M00/00/00/wKgmgWG02QqAIxmVAAEyiVzLIwo283.jpg",new DownloadByteArray());
FileOutputStream stream = new FileOutputStream("dlrb_150x150.jpg");
stream.write(bytes);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
下载图片(下载到前端页面)
<form method="get" enctype="multipart/form-data" action="/downloadtouser">
<input type="submit" value="下载">
</form>
//已经指定好图片名字了
@RestController
public class DownloadFile {
@Autowired
private FastFileStorageClient fastFileStorageClient;
@RequestMapping("/downloadtouser")
public void downloadtouser(HttpServletResponse response){
try {
byte[] bytes = fastFileStorageClient.downloadFile("group1","M00/00/00/wKgmgWG02QqAIxmVAAEyiVzLIwo283.jpg",new DownloadByteArray());
InputStream in = new ByteArrayInputStream(bytes);
OutputStream outputStream = new BufferedOutputStream(response.getOutputStream());
//创建存放文件内容的数组
byte[] buff = new byte[1024];
//所读取的内容使用n来接收
int n;
//当没有读取完时,继续读取,循环
while ((n = in.read(buff)) != -1) {
//将字节数组的数据全部写入到输出流中
outputStream.write(buff, 0, n);
}
if ((n = in.read(buff)) == -1) {
//强制将缓存区的数据进行输出
outputStream.flush();
//关流
outputStream.close();
in.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}