使用docker部署fastdfs集群版

一、前言

本文档说明在node01和node02两台主机上安装部署FastDFS双节点。
node01 ip：198.168.1.121 安装tracker1，storage1
node02 ip：198.168.1.122 安装tracker2，storage2

若要扩展更多tracker或storage节点，也是类似操作。

二、拉取fastdfs docker镜像

docker pull morunchang/fastdfs

三、安装fastdfs tracker

3.1 在node01和node02上安装tracker

在node01和node02上分别执行以下操作
注意：以下操作指令在node01中执行，在node02操作时，将tracker1改为tracker2

1）创建文件夹

	mkdir -p /apps/fastdfs/tracker1/data /apps/fastdfs/tracker1/conf

2）写入/apps/fastdfs/tracker1/conf/tracker.conf

	disabled=false
	bind_addr=
	port=22122
	connect_timeout=30
	network_timeout=30
	base_path=/data/fast_data
	max_connections=256
	accept_threads=1
	work_threads=4
	store_lookup=2
	store_group=group1
	store_server=0
	store_path=0
	download_server=0
	reserved_storage_space = 10%
	log_level=info
	run_by_group=
	run_by_user=
	allow_hosts=*
	sync_log_buff_interval = 10
	check_active_interval = 120
	thread_stack_size = 64KB
	storage_ip_changed_auto_adjust = true
	storage_sync_file_max_delay = 86400
	storage_sync_file_max_time = 300
	use_trunk_file = false 
	slot_min_size = 256
	slot_max_size = 16MB
	trunk_file_size = 64MB
	trunk_create_file_advance = false
	trunk_create_file_time_base = 02:00
	trunk_create_file_interval = 86400
	trunk_create_file_space_threshold = 20G
	trunk_init_check_occupying = false
	trunk_init_reload_from_binlog = false
	trunk_compress_binlog_min_interval = 0
	use_storage_id = false
	storage_ids_filename = storage_ids.conf
	id_type_in_filename = ip
	store_slave_file_use_link = false
	rotate_error_log = false
	error_log_rotate_time=00:00
	rotate_error_log_size = 0
	log_file_keep_days = 0
	use_connection_pool = false
	connection_pool_max_idle_time = 3600
	http.server_port=8080
	http.check_alive_interval=30
	http.check_alive_type=tcp
	http.check_alive_uri=/status.html

3）运行tracker1的docker容器

docker run -d --name tracker1 --net=host --restart always \
-v /etc/localtime:/etc/localtime \
-v /apps/fastdfs/tracker1/data:/data/fast_data/ \
-v /apps/fastdfs/tracker1/conf/tracker.conf:/etc/fdfs/tracker.conf \
morunchang/fastdfs sh tracker.sh

4）查看docker日志是否启动正常

docker logs tracker1

5）开启防火墙端口：22122

firewall-cmd --zone=public --add-port=22122/tcp --permanent
firewall-cmd --reload

四、安装fastdfs storage

4.1 在node01和node02上安装storage

在node01和node02上分别执行以下操作
注意：以下操作指令在node01中执行，在node02操作时，将storage1改为storage2

1）创建文件夹

mkdir -p /apps/fastdfs/storage1/data /apps/fastdfs/storage1/conf

2）写入/apps/fastdfs/storage1/conf/storage.conf

disabled=false
group_name=group1
bind_addr=
client_bind=true
port=23002
connect_timeout=30
network_timeout=30
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fast_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fast_data
subdir_count_per_path=256
tracker_server=198.168.1.121:22122
tracker_server=198.168.1.122:22122
log_level=debug
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = true
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=9101

3）写入/apps/fastdfs/storage1/conf/nginx.conf

worker_processes  1;
error_log  /data/fast_data/logs/nginx-error.log;

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /data/fast_data/logs/nginx-access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    server {
        listen       9101;
        server_name  localhost;

        location / {
            root   html;
            index  index.html index.htm;
        }

        location ~ /group1/M00 {
                    root /data/fast_data/data;
                    ngx_fastdfs_module;
        }

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

4）写入/apps/fastdfs/storage1/conf/mod_fastdfs.conf

connect_timeout=30
network_timeout=30
base_path=/data/fast_data
load_fdfs_parameters_from_tracker=true
storage_sync_file_max_delay = 86400
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=198.168.1.121:22122
tracker_server=198.168.1.122:22122
storage_server_port=23002
group_name=group1
url_have_group_name = true
store_path_count=1
store_path0=/data/fast_data
log_level=info
log_filename=
response_mode=proxy
if_alias_prefix=
flv_support = true
flv_extension = flv
group_count = 0

5）写入/apps/fastdfs/storage1/conf/storage.sh

#!/bin/sh
/data/fastdfs/storage/fdfs_storaged /etc/fdfs/storage.conf
/etc/nginx/sbin/nginx
tail -f /data/fast_data/logs/storaged.log

6）写入/apps/fastdfs/storage1/conf/client.conf

# connect timeout in seconds
# default value is 30s
connect_timeout=30

# network timeout in seconds
# default value is 30s
network_timeout=30

# the base path to store log files
base_path=/data/fastdfs/test

# tracker_server can ocur more than once, and tracker_server format is
#  "host:port", host can be hostname or ip address

tracker_server=198.168.1.121:22122
tracker_server=198.168.1.122:22122

#standard log level as syslog, case insensitive, value list:
### emerg for emergency
### alert
### crit for critical
### error
### warn for warning
### notice
### info
### debug
log_level=info

# if use connection pool
# default value is false
# since V4.05
use_connection_pool = false

# connections whose the idle time exceeds this time will be closed
# unit: second
# default value is 3600
# since V4.05
connection_pool_max_idle_time = 3600

# if load FastDFS parameters from tracker server
# since V4.05
# default value is false
load_fdfs_parameters_from_tracker=false

# if use storage ID instead of IP address
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# default value is false
# since V4.05
use_storage_id = false

# specify storage ids filename, can use relative or absolute path
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# since V4.05
storage_ids_filename = storage_ids.conf

#HTTP settings
http.tracker_server_port=80

#use "#include" directive to include HTTP other settiongs
##include http.conf

7）运行docker容器

docker run -d --name storage1 --net=host --restart always \
-v /etc/localtime:/etc/localtime \
-v /apps/fastdfs/storage1/data:/data/fast_data/ \
-v /apps/fastdfs/storage1/conf/storage.sh:/storage.sh \
-v /apps/fastdfs/storage1/conf/storage.conf:/etc/fdfs/storage.conf \
-v /apps/fastdfs/storage1/conf/nginx.conf:/etc/nginx/conf/nginx.conf  \
-v /apps/fastdfs/storage1/conf/mod_fastdfs.conf:/etc/fdfs/mod_fastdfs.conf \
-v /apps/fastdfs/storage1/conf/client.conf:/data/fastdfs/conf/client.conf \
morunchang/fastdfs sh storage.sh

8）查看docker日志是否正常

docker logs storage1

9）开启防火墙端口:23002、9101

firewall-cmd --zone=public --add-port=23002/tcp --add-port=9101/tcp --permanent
firewall-cmd --reload

五、验证fastdfs是否安装成功

说明：以下操作在node01中进行。

5.1查看FastDFS监控状态

docker exec -it storage1 fdfs_monitor /data/fastdfs/conf/client.conf

参数说明：
tracker_server_count：2 --表示2个Tracker Server
tracker server is 198.168.1.121:22122 --表示Leader Tracker
group count: 1 --表示有1个group
group name = group1 --组名称是group1
storage server count = 2 --组内有2个storage
active server count = 2 --活动的storage有2个
storage server port = 23002 --storage的端口
storage HTTP port = 9101 --storage的文件访问端口
store path count = 1 --storage只挂了一个存储目录
total_upload_count = 11 --总共上传了多少个文件
total_upload_bytes = 691405 --总共上传了多少字节
success_upload_bytes = 691405 --成功上传了多少字节
total_download_count = 2 --总共下载了多少文件（使用java客户端）

5.2上传文件到FastDFS

docker exec -it storage1 bash
cd /data/fastdfs/conf
fdfs_test client.conf upload anti-steal.jpg

文件上传成功后，会提示文件访问地址，同时会自动同步到storage2，因此也可以通过storage2的IP，访问文件。

由于storage1的http端口配置为9101，所以真正的访问地址是：
（示例地址）http://198.168.1.121:9101/group1/M00/00/00/xowDSFyCEcyAVz4gAABdrZgsqUU908.jpg
storage2的访问地址是：
（示例地址）http://198.168.1.122:9101/group1/M00/00/00/xowDSFyCEcyAVz4gAABdrZgsqUU908.jpg

5.3下载文件

curl http://198.168.1.121:9101/group1/M00/00/00/xowDSFyCEcyAVz4gAABdrZgsqUU908.jpg
curl http://198.168.1.122:9101/group1/M00/00/00/xowDSFyCEcyAVz4gAABdrZgsqUU908.jpg
能下载文件就表示fastdfs同步文件成功，也表示fastdfs安装配置成功。

5.4统计Storage文件数量

进入storage的data目录，执行以下指令

cd /apps/fastdfs/storage1/data/data
ls -lR|grep "^-"|wc -l

或者进入data目录下的sync目录，执行以下指令

cat binlog.000 | wc -l

binlog.000保存了storage中所有文件名。