SDFS安装扩容替换

sdfs服务端安装手册

机器信息

主机名IP地址说明
nbuspreweb20710.237.194.127搭建
nbuspreweb20610.237.194.126搭建
nbuspreweb20510.237.194.124扩容
nbuspreweb20410.237.194.125扩容
nbuspreweb20310.237.194.123替换

版本下载

在所有需要安装SDFS的虚机/tmp目录下执行下述命令,下方简称为虚机

cat /etc/redhat-release
sdfs534.tar.gz
sdfs529.tar.gz
#centos7 redhat7 使用sdfs534
#centos6 redhat6使用sdfs529

服务的版本安装

在所有虚机上执行下述操作

1.解压版本包

tar zxvf SDFS.tar.gz 
cd SDFS
sh install-sdfs.sh 	#会有参数提示
#Usage:install-sdfs.sh [options]
#Valid options are:
#    -h  help
#    -t  server or client
#      [server]
#        -d  disk device name
#        -m  mount point
#        -f  force install
#      [client]
#        -i  server ip
#        -v  volume name
#        -m  mount point
#        -f  force install
#Example:
#    sh install-sdfs.sh -t server -d /dev/vdb -m /mnt/data
#    sh install-sdfs.sh -t server -d /dev/vdb -m /mnt/data -f
#    sh install-sdfs.sh -t client -i 10.1.1.1 -v test_vol -m /mnt/sdfs
#    sh install-sdfs.sh -t client -i 10.1.1.1 -v test_vol -m /mnt/sdfs -f
sh install-sdfs.sh -t server -d /dev/datavg/datalv -m /data -f
#其中:/dev/vdb通过fdisk –l命令查看
#/mnt/data为挂载点,自定义一个目录
#eth0对外IP地址所在的网口,sdfs534不需要这个
#另外两个参数使用示例中默认即可
# echo $?,如果返回0表示安装成功,如果不是返回0,联系开发人员解决

2.所有虚机安装成功后,在其中一台上执行下述操作

#gluster peer probe <ip>		//添加peers,其中,ip为其他所有虚机的ip(不包括本机),命令全部执行成功后如下所示
gluster peer probe 10.237.194.126	#执行的很慢
gluster peer status
# gluster peer status
#Number of Peers: 1

#Hostname: 10.237.194.126
#Port: 24007
#Uuid: 7d68d3c3-c975-4649-b619-941f4db63712
#State: Peer in Cluster (Connected)
#gluster volume create <volume_name> replica 2 <ip1>:/mnt/data/br1 …  //建卷
gluster volume create machao_vol replica 2 10.237.194.127:/data/br1 10.237.194.126:/data/br1	#执行的也会比较慢
#<volume_name>为卷名,自定义。建议为服务ID
#Replica后数字为副本个数,虚机个数需要为副本个数的整数倍
#<ip1>:/mnt/data/br1,为虚机IP和brick目录,需要填写所有的IP组合,创建卷成功后如下所示。
gluster vol info
#Volume Name: machao_vol
#Type: Replicate
#Volume ID: 34d43652-bd90-4f25-a001-cdb968986f81
#Status: Created
#Number of Bricks: 1 x 2 = 2
#Transport-type: tcp
#Bricks:
#Brick1: 10.237.194.127:/data/br1
#Brick2: 10.237.194.126:/data/br1
gluster vol status		#建卷成功后查看状态
#Volume machao_vol is not started
ps -ef | grep glusterd
# ps -ef | grep glusterd
#root      7549     1  0 14:04 ?        00:00:00 /usr/sbin/glusterd -p /run/glusterd.pid
gluster vol start machao_vol		#执行启动卷
#volume start: machao_vol: success

sdfs客户端版本安装

版本下载

在所有需要安装SDFS的虚机/tmp目录下执行下述命令,下方简称为虚机

cat /etc/redhat-release
sdfs534.tar.gz
sdfs529.tar.gz
#centos7 redhat7 使用sdfs534
#centos6 redhat6使用sdfs529

在客户端虚机上执行下述操作

1.解压缩版本包

tar zxvf SDFS.tar.gz
cd SDFS
#sh src/install.sh client <server_ip> <volume_name> <mountpath>
#<server_ip>为其中一台server的IP地址
#<volume_name>为卷名
#<mountpath>为本地挂载点,可定义一个目录,如/mnt/gfsclient
# echo $?,如果返回0表示安装成功,如果不是返回0,联系开发人员解决
sh install.sh -t client  -i 10.237.194.127 -v machao_vol -m /mnt/sdfs -f

sdfs分布式节点扩容

扩容前准备

检查glusterd和brick日志是否存在错误(常见关键字:" E “和” W ")

/var/log/glusterfs/etc-glusterfs-glusterd.vol.log
/var/log/glusterfs/bricks/*.log

执行命令,检查所有Brick状态是否正常,Rebalance操作是否已完成

[root@nbuspreweb207 ~] gluster vol status
Status of volume: machao_vol
Gluster process                                         Port    Online  Pid
------------------------------------------------------------------------------
Brick 10.237.194.127:/data/br1                          49152   Y       10354
Brick 10.237.194.126:/data/br1                          49152   Y       11203
NFS Server on localhost                                 2049    Y       18111
Self-heal Daemon on localhost                           N/A     Y       18119
NFS Server on 10.237.194.126                            2049    Y       31932
Self-heal Daemon on 10.237.194.126                      N/A     Y       31940
 
           Task                                      ID         Status
           ----                                      --         ------
      Rebalance    37e7bb56-01a8-4b16-ba77-8fa20fe04c9d      completed

检查基本业务

通过客户端执行写、读、删、列目录操作,没有明显卡顿

版本下载

cat /etc/redhat-release
sdfs534.tar.gz
sdfs529.tar.gz
#centos7 redhat7 使用sdfs534
#centos6 redhat6使用sdfs529
systemctl start glusterd
systemctl status glusterd	#检查服务

扩容

在服务端集群的任意一台机器上执行(非扩容节点)

gluster peer probe 10.237.194.124
gluster peer probe 10.237.194.125
gluster volume add-brick machao_vol replica 2 10.237.194.124:/data/br1 10.237.194.125:/data/br1
gluster volume status
Status of volume: machao_vol
#Gluster process                                         Port    Online  Pid
#------------------------------------------------------------------------------
#Brick 10.237.194.127:/data/br1                          49152   Y       10354
#Brick 10.237.194.126:/data/br1                          49152   Y       11203
#Brick 10.237.194.124:/data/br1                          49152   Y       31017
#Brick 10.237.194.125:/data/br1                          49152   Y       31286
#NFS Server on localhost                                 2049    Y       31297
#Self-heal Daemon on localhost                           N/A     Y       31305
#NFS Server on 10.237.194.127                            2049    Y       18111
#Self-heal Daemon on 10.237.194.127                      N/A     Y       18119
#NFS Server on 10.237.194.124                            2049    Y       31028
#Self-heal Daemon on 10.237.194.124                      N/A     Y       31036
#NFS Server on 10.237.194.126                            2049    Y       31932
#Self-heal Daemon on 10.237.194.126                      N/A     Y       31940
# 
#           Task                                      ID         Status
#           ----                                      --         ------
#      Rebalance    37e7bb56-01a8-4b16-ba77-8fa20fe04c9d      completed
gluster volume rebalance machao_vol fix-layout start	#均衡
gluster volume rebalance machao_vol stop				#停止均衡
gluster volume rebalance machao_vol status				#均衡进度
#                                    Node Rebalanced-files          size       scanned      failures       skipped         status run time in secs
#                               ---------      -----------   -----------   -----------   -----------   -----------   ------------   --------------
#                               localhost                0        0Bytes             0             0             0      completed             0.00
#                              localhost                0        0Bytes             0             0             0      completed             0.00
#                               localhost                0        0Bytes             0             0             0      completed             0.00
#                               localhost                0        0Bytes             0             0             0      completed             0.00
#                          10.237.194.126                0        0Bytes             0             0             0      completed             0.00
#volume rebalance: machao_vol: success: 

扩容后检查

检查glusterd和brick日志是否存在错误(常见关键字:" E “和” W ")

/var/log/glusterfs/etc-glusterfs-glusterd.vol.log
/var/log/glusterfs/bricks/*.log

检查基本业务情况

通过客户端执行写、读、删、列目录操作,没有明显卡顿

通知业务自行测试,确认无影响

sdfs分布式节点替换

更换IP替换

分布式机器信息(替换10.237.194.126节点)

Status of volume: machao_vol
Gluster process                                         Port    Online  Pid
------------------------------------------------------------------------------
Brick 10.237.194.127:/data/br1                          49152   Y       10354
Brick 10.237.194.126:/data/br1                          49152   Y       11203
Brick 10.237.194.124:/data/br1                          49152   Y       31017
Brick 10.237.194.125:/data/br1                          49152   Y       31286
NFS Server on localhost                                 2049    Y       31932
Self-heal Daemon on localhost                           N/A     Y       31940
NFS Server on 10.237.194.127                            2049    Y       18111
Self-heal Daemon on 10.237.194.127                      N/A     Y       18119
NFS Server on 10.237.194.125                            2049    Y       31297
Self-heal Daemon on 10.237.194.125                      N/A     Y       31305
NFS Server on 10.237.194.124                            2049    Y       31028
Self-heal Daemon on 10.237.194.124                      N/A     Y       31036
 
           Task                                      ID         Status
           ----                                      --         ------
      Rebalance    37e7bb56-01a8-4b16-ba77-8fa20fe04c9d      completed
systemctl stop glusterd
10.237.194.126节点掉线
Status of volume: machao_vol
Gluster process                                         Port    Online  Pid
------------------------------------------------------------------------------
Brick 10.237.194.127:/data/br1                          49152   Y       10354
Brick 10.237.194.124:/data/br1                          49152   Y       31017
Brick 10.237.194.125:/data/br1                          49152   Y       31286
NFS Server on localhost                                 2049    Y       18111
Self-heal Daemon on localhost                           N/A     Y       18119
NFS Server on 10.237.194.124                            2049    Y       31028
Self-heal Daemon on 10.237.194.124                      N/A     Y       31036
NFS Server on 10.237.194.125                            2049    Y       31297
Self-heal Daemon on 10.237.194.125                      N/A     Y       31305
 
           Task                                      ID         Status
           ----                                      --         ------
      Rebalance    37e7bb56-01a8-4b16-ba77-8fa20fe04c9d      completed

新机器安装sdfs服务端并检查服务

[root@nbuspreweb203 sdfs534]# systemctl status glusterd
● glusterd.service - GlusterFS an clustered file-system server
   Loaded: loaded (/usr/lib/systemd/system/glusterd.service; enabled; vendor preset: disabled)
   Active: active (running) since Mon 2022-10-31 11:41:00 CST; 1min 55s ago
  Process: 32498 ExecStart=/usr/sbin/glusterd -p /run/glusterd.pid (code=exited, status=0/SUCCESS)
 Main PID: 32499 (glusterd)
   CGroup: /system.slice/glusterd.service
           └─32499 /usr/sbin/glusterd -p /run/glusterd.pid

Oct 31 11:40:59 nbuspreweb203 systemd[1]: Starting GlusterFS an clustered file-system server...
Oct 31 11:41:00 nbuspreweb203 systemd[1]: Started GlusterFS an clustered file-system server.
[root@nbuspreweb203 sdfs534]# systemctl start glusterd
[root@nbuspreweb203 sdfs534]# ps -ef | grep glusterd
root     32499     1  0 11:40 ?        00:00:00 /usr/sbin/glusterd -p /run/glusterd.pid
root     32587 32070  0 11:43 pts/0    00:00:00 grep --color=auto glusterd

查看目录

[root@nbuspreweb203 sdfs534]# df -Th
Filesystem                  Type      Size  Used Avail Use% Mounted on
/dev/mapper/systemvg-rootlv xfs        10G   77M   10G   1% /
devtmpfs                    devtmpfs  1.9G     0  1.9G   0% /dev
tmpfs                       tmpfs     1.9G   12K  1.9G   1% /dev/shm
tmpfs                       tmpfs     1.9G   17M  1.9G   1% /run
tmpfs                       tmpfs     1.9G     0  1.9G   0% /sys/fs/cgroup
/dev/mapper/systemvg-usrlv  xfs        10G  1.3G  8.8G  13% /usr
/dev/vda1                   xfs       497M  151M  347M  31% /boot
/dev/mapper/systemvg-tmplv  xfs       2.0G   48M  2.0G   3% /tmp
/dev/mapper/systemvg-homelv xfs       2.0G   33M  2.0G   2% /home
/dev/mapper/systemvg-varlv  xfs       8.0G  1.4G  6.7G  18% /var
/dev/mapper/systemvg-optlv  xfs        33G   39M   33G   1% /opt
tmpfs                       tmpfs     380M     0  380M   0% /run/user/0
/dev/vdb                    xfs        50G   33M   50G   1% /data

在集群中一个节点上执行 gluster peer probe [新节点ip地址]

在10.237.194.123节点上执行
[root@nbuspreweb207 ~]# gluster peer probe 10.237.194.123
peer probe: success

在新节点上执行gluster volume sync [配对的ip] [卷名]

配置文件有记录匹对的节点信息
cat /var/lib/glusterd/vols/machao_vol/machao_vol-fuse.vol
volume machao_vol-client-0
    type protocol/client
    option transport-type tcp
    option remote-subvolume /data/br1
    option remote-host 10.237.194.127
end-volume

volume machao_vol-client-1
    type protocol/client
    option transport-type tcp
    option remote-subvolume /data/br1
    option remote-host 10.237.194.126
end-volume

volume machao_vol-replicate-0				#client-0和client-1是匹对的
    type cluster/replicate
    subvolumes machao_vol-client-0 machao_vol-client-1
end-volume
或者直接gluster vol info
[root@nbuspreweb207 ~]# gluster vol info
 
Volume Name: machao_vol
Type: Distributed-Replicate
Volume ID: 34d43652-bd90-4f25-a001-cdb968986f81
Status: Started
Number of Bricks: 2 x 2 = 4
Transport-type: tcp
Bricks:
Brick1: 10.237.194.127:/data/br1			#1跟2是匹对的
Brick2: 10.237.194.126:/data/br1
Brick3: 10.237.194.124:/data/br1
Brick4: 10.237.194.125:/data/br1

替换节点上执行
[root@nbuspreweb203 ~]# gluster volume sync 10.237.194.127 machao_vol
Sync volume may make data inaccessible while the sync is in progress. Do you want to continue? (y/n) y
volume sync: success

执行gluster volume replace-brick [卷名] [故障brick名] [扩容新brick名]commit force(新brick名必须和旧的brick不一样)

mkdir -p /data/br2
gluster volume replace-brick machao_vol 10.237.194.126:/data/br1 10.237.194.123:/data/br2 commit force
[root@nbuspreweb203 ~]# gluster volume replace-brick machao_vol 10.237.194.126:/data/br1 10.237.194.123:/data/br2 commit force
volume replace-brick: success: replace-brick commit successful

执行gluster volume heal [卷名] full,检查status状态

gluster volume heal machao_vol full
[root@nbuspreweb203 ~]# gluster volume heal machao_vol full
Launching Heal operation on volume machao_vol has been successful
Use heal info commands to check status

执行gluster peer detach [旧节点ip]

gluster peer detach 10.237.194.126
[root@nbuspreweb207 ~]# gluster peer detach 10.237.194.126
peer detach: success

检查分布式信息

[root@nbuspreweb207 ~]# gluster vol info
Volume Name: machao_vol
Type: Distributed-Replicate
Volume ID: 34d43652-bd90-4f25-a001-cdb968986f81
Status: Started
Number of Bricks: 2 x 2 = 4
Transport-type: tcp
Bricks:
Brick1: 10.237.194.127:/data/br1
Brick2: 10.237.194.123:/data/br2
Brick3: 10.237.194.124:/data/br1
Brick4: 10.237.194.125:/data/br1
[root@nbuspreweb207 ~]# gluster vol status
Status of volume: machao_vol
Gluster process                                         Port    Online  Pid
------------------------------------------------------------------------------
Brick 10.237.194.127:/data/br1                          49152   Y       10354
Brick 10.237.194.123:/data/br2                          49152   Y       1560
Brick 10.237.194.124:/data/br1                          49152   Y       31017
Brick 10.237.194.125:/data/br1                          49152   Y       31286
NFS Server on localhost                                 2049    Y       31127
Self-heal Daemon on localhost                           N/A     Y       31131
NFS Server on 10.237.194.124                            2049    Y       2956
Self-heal Daemon on 10.237.194.124                      N/A     Y       2960
NFS Server on 10.237.194.125                            2049    Y       3254
Self-heal Daemon on 10.237.194.125                      N/A     Y       3258
NFS Server on 10.237.194.123                            2049    Y       1562
Self-heal Daemon on 10.237.194.123                      N/A     Y       1571
 
           Task                                      ID         Status
           ----                                      --         ------
      Rebalance    37e7bb56-01a8-4b16-ba77-8fa20fe04c9d    not started
[root@nbuspreweb207 ~]# cat /var/lib/glusterd/vols/machao_vol/machao_vol-fuse.vol
volume machao_vol-client-0
    type protocol/client
    option transport-type tcp
    option remote-subvolume /data/br1
    option remote-host 10.237.194.127
end-volume

volume machao_vol-client-1
    type protocol/client
    option transport-type tcp
    option remote-subvolume /data/br2
    option remote-host 10.237.194.123
end-volume

volume machao_vol-client-2
    type protocol/client
    option transport-type tcp
    option remote-subvolume /data/br1
    option remote-host 10.237.194.124
end-volume

volume machao_vol-client-3
    type protocol/client
    option transport-type tcp
    option remote-subvolume /data/br1
    option remote-host 10.237.194.125
end-volume

volume machao_vol-replicate-0
    type cluster/replicate
    subvolumes machao_vol-client-0 machao_vol-client-1
end-volume

volume machao_vol-replicate-1
    type cluster/replicate
    subvolumes machao_vol-client-2 machao_vol-client-3
end-volume

volume machao_vol-dht
    type cluster/distribute
    subvolumes machao_vol-replicate-0 machao_vol-replicate-1
end-volume

volume machao_vol-write-behind
    type performance/write-behind
    subvolumes machao_vol-dht
end-volume

volume machao_vol-read-ahead
    type performance/read-ahead
    subvolumes machao_vol-write-behind
end-volume

volume machao_vol-io-cache
    type performance/io-cache
    subvolumes machao_vol-read-ahead
end-volume

volume machao_vol-quick-read
    type performance/quick-read
    subvolumes machao_vol-io-cache
end-volume

volume machao_vol-open-behind
    type performance/open-behind
    subvolumes machao_vol-quick-read
end-volume

volume machao_vol-md-cache
    type performance/md-cache
    subvolumes machao_vol-open-behind
end-volume

volume machao_vol
    type debug/io-stats
    option count-fop-hits off
    option latency-measurement off
    subvolumes machao_vol-md-cache
end-volume

不更换IP替换

安装gluster版本,启动gluster进程

在集群中一个节点上执行 gluster peer probe [新节点ip地址]

在新节点上执行gluster volume sync [配对的ip] [卷名]

执行gluster volume replace-brick [卷名] [brick名] [新brick名]commit force(新brick名必须和旧的brick不一样)

执行gluster volume heal [卷名] full,检查status状态

执行gluster peer detach [旧节点ip]

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 1
    评论
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值