1.mfs的安装部署与基础配置
实验环境:
server1: 172.25.66.1 master
server2: 172.25.66.2 chunkserver
server3: 172.25.66.3 chunkserver
物理机: 172.25.66.250 client
配置主节点:
安装包:
moosefs-cgi-3.0.103-1.rhsystemd.x86_64.rpm
moosefs-cgiserv-3.0.103-1.rhsystemd.x86_64.rpm
moosefs-cli-3.0.103-1.rhsystemd.x86_64.rpm
moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm
1.下载并安装moosefs-master
[root@server1 ~]# ls
moosefs-cgi-3.0.103-1.rhsystemd.x86_64.rpm
moosefs-cgiserv-3.0.103-1.rhsystemd.x86_64.rpm
moosefs-cli-3.0.103-1.rhsystemd.x86_64.rpm
moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm
[root@server1 ~]# yum install -y *
2.添加解析
[root@server1 ~]# vim /etc/hosts
#################
172.25.66.1 server1 mfsmaster
[root@server1 ~]# scp /etc/hosts server2:/etc/hosts
[root@server1 ~]# scp /etc/hosts server3:/etc/hosts
3.开启服务
[root@server1 ~]# systemctl start moosefs-master
[root@server1 ~]# ps aux | grep mfsmaster
mfs 2190 0.4 31.9 350148 324368 ? S< 20:55 0:00 mfsmaster start
root 2197 0.0 0.0 112648 956 pts/0 R+ 20:58 0:00 grep --color=auto mfsmaster
[root@server1 ~]# netstat -antlp
4.开启监控
[root@server1 ~]# systemctl start moosefs-cgiserv
[root@server1 ~]# ps aux
[root@server1 ~]# netstat -antlp
4.网页测试
在网页上输入: 172.25.66.1:9425
配置从结点:
安装包:
moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm
在server2上:
1.下载并安装moosefs-chunkserver
[root@server2 ~]# ls
moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm
[root@server2 ~]# yum install -y *
2.更改文件
[root@server2 ~]# cd /etc/mfs/
[root@server2 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@server2 mfs]# vim mfshdd.cfg
###################
/mnt/chunk1
3.创建目录更改所属用户组
[root@server2 mfs]# mkdir /mnt/chunk1
[root@server2 mfs]# ll -d /mnt/chunk1/
drwxr-xr-x 2 root root 6 Apr 3 21:14 /mnt/chunk1/
[root@server2 mfs]# cat /etc/passwd | grep mfs
mfs:x:997:993:MooseFS:/var/lib/mfs:/sbin/nologin
#更改目录的所属用户和组
[root@server2 mfs]# chown mfs.mfs /mnt/chunk1
[root@server2 mfs]# ll -d /mnt/chunk1/
drwxr-xr-x 2 mfs mfs 6 Apr 3 21:14 /mnt/chunk1/
4.开启服务
[root@server2 mfs]# systemctl start moosefs-chunkserver
[root@server2 mfs]# ps aux | grep mfschunkserver
mfs 2190 0.8 13.4 298620 136492 ? S<l 21:16 0:00 mfschunkserver start
root 2202 0.0 0.0 112648 960 pts/0 R+ 21:17 0:00 grep --color=auto mfschunkserver
[root@server2 mfs]# netstat -antlp
在server3上:(操作同server2)
1.安装moosefs
[root@server3 ~]# ls
moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm
[root@server3 ~]# yum install -y *
2.更改文件
[root@server3 ~]# cd /etc/mfs/
[root@server3 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@server3 mfs]# vim mfshdd.cfg
###################
/mnt/chunk2
3.创建目录更改所属用户组
[root@server3 mfs]# mkdir /mnt/chunk2
[root@server3 mfs]# ll -d /mnt/chunk2/
drwxr-xr-x 2 root root 6 Apr 3 21:22 /mnt/chunk2/
[root@server3 mfs]# cat /etc/passwd | grep mfs
mfs:x:997:993:MooseFS:/var/lib/mfs:/sbin/nologin
[root@server3 mfs]# chown mfs.mfs /mnt/chunk2
[root@server3 mfs]# ll -d /mnt/chunk2/
drwxr-xr-x 2 mfs mfs 6 Apr 3 21:22 /mnt/chunk2/
4.开启服务
[root@server3 mfs]# systemctl start moosefs-chunkserver
[root@server3 mfs]# ps aux | grep mfschunkserver
mfs 2209 2.5 13.4 298620 136488 ? S<l 21:23 0:00 mfschunkserver start
root 2220 0.0 0.0 112648 960 pts/0 R+ 21:23 0:00 grep --color=auto mfschunkserver
[root@server3 mfs]# netstat -antlp
测试:
配置客户端:
安装包:
moosefs-client-3.0.103-1.rhsystemd.x86_64.rpm
1.下载并安装moosefs-client
[root@foundation66 ~]# ls
moosefs-client-3.0.103-1.rhsystemd.x86_64.rpm
[root@foundation66 ~]# yum install -y *
2.更改文件
[root@foundation66 ~]# cd /etc/mfs/
[root@foundation66 mfs]# vim mfsmount.cfg
3.创建目录
[root@foundation66 mfs]# mkdir /mnt/mfs
[root@foundation66 mfs]# cd /mnt/mfs/
[root@foundation66 mfs]# ls
[root@foundation66 mfs]# mfsmount
can't resolve master hostname and/or portname (mfsmaster:9421)
4.添加解析
#必须先添加解析才能挂载
[root@foundation66 mfs]# vim /etc/hosts
########################
172.25.66.1 server1 mfsmaster
5.自动挂载
[root@foundation66 mfs]# mfsmount
mfsmaster accepted connection with parameters: read-write,restricted_ip,admin ; root mapped to root:root
[root@foundation66 mfs]# df
数据存储:
[root@foundation66 ~]# mkdir /mnt/mfs/dir1
[root@foundation66 ~]# mkdir /mnt/mfs/dir2
[root@foundation66 ~]# cd /mnt/mfs/
[root@foundation66 mfs]# ls
dir1 dir2
#查看数据存储服务器
[root@foundation66 mfs]# mfsgetgoal dir1
dir1: 2
[root@foundation66 mfs]# mfsgetgoal dir2
dir2: 2
[root@foundation66 mfs]# mfssetgoal -r 1 dir1/
dir1/:
inodes with goal changed: 1
inodes with goal not changed: 0
inodes with permission denied: 0
[root@foundation66 mfs]# mfsgetgoal dir1
dir1: 1
[root@foundation66 mfs]# cp /etc/passwd dir1
[root@foundation66 mfs]# cp /etc/fstab dir2
[root@foundation66 mfs]# cd dir1
[root@foundation66 dir1]# ls
passwd
[root@foundation66 dir1]# mfsfileinfo passwd
[root@foundation66 dir1]# cd ../dir2/
[root@foundation66 dir2]# ls
fstab
[root@foundation66 dir2]# mfsfileinfo fstab
离散存储:
[root@foundation66 dir1]# dd if=/dev/zero of=bigfile bs=1M count=100
100+0 records in
100+0 records out
104857600 bytes (105 MB) copied, 0.226445 s, 463 MB/s
[root@foundation66 dir1]# ls
bigfile passwd
[root@foundation66 dir1]# mfsfileinfo bigfile
[root@foundation66 dir1]# mfsgettrashtime .
.: 86400
1.模拟误删文件
[root@foundation66 dir1]# ls
file1 passwd
[root@foundation66 dir1]# rm -rf passwd
[root@foundation66 dir1]# ls
file1
恢复:
[root@foundation66 dir1]# cd /mnt/mfs/
[root@foundation66 mfs]# mkdir /mnt/mfsmeta
[root@foundation66 mfs]# mfsmount -m /mnt/mfsmeta/
mfsmaster accepted connection with parameters: read-write,restricted_ip
[root@foundation66 mfs]# df
[root@foundation66 mfs]# cd /mnt/mfsmeta/trash/
[root@foundation66 trash]# find -name *passwd*
./004/00000004|dir1|passwd
[root@foundation66 trash]# mv 004/00000004\|dir1\|passwd undel/
测试:
[root@foundation66 trash]# cd /mnt/mfs/dir1/
[root@foundation66 dir1]# ls
file1 passwd
2.模拟从结点宕机
#模拟从结点:server2宕机
[root@server2 mfs]# systemctl stop moosefs-chunkserver
[root@foundation66 dir2]# mfsfileinfo fstab
[root@foundation66 dir2]# cd ../dir1
[root@foundation66 dir1]# mfsfileinfo passwd
#模拟从结点:server3也宕机
[root@server3 ~]# systemctl stop moosefs-chunkserver
[root@foundation66 dir1]# cd ../dir2
[root@foundation66 dir2]# mfsfileinfo fstab
恢复:
#开启服务
[root@server2 mfs]# systemctl start moosefs-chunkserver
[root@server3 ~]# systemctl start moosefs-chunkserver
#此时便可恢复正常
[root@foundation66 dir1]# mfsfileinfo passwd
passwd:
chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
copy 1: 172.25.66.2:9422 (status:VALID)
[root@foundation66 dir1]# cd ../dir2
[root@foundation66 dir2]# mfsfileinfo fstab
fstab:
chunk 0: 0000000000000004_00000001 / (id:4 ver:1)
copy 1: 172.25.66.2:9422 (status:VALID)
copy 2: 172.25.66.3:9422 (status:VALID)
3.模拟主结点宕机
(1)正常关闭
[root@server1 ~]# systemctl stop moosefs-master
[root@foundation66 dir1]# df
恢复:
#重新开启即可恢复
[root@server1 ~]# systemctl start moosefs-master
[root@foundation66 dir1]# df
(2)非正常关闭
[root@sevrer1 ~]# ps aux | grep mfsmaster
mfs 11720 0.7 77.2 604812 585560 ? S< 12:42 0:02 /usr/sbin/mfsmaster start
root 11728 0.0 0.1 112648 956 pts/0 R+ 12:49 0:00 grep --color=auto mfsmaster
[root@sevrer1 ~]# kill -9 11720
#发现此时再开启服务时便会报错
[root@server1 ~]# systemctl start moosefs-master
Job for moosefs-master.service failed because the control process exited with error code. See "systemctl status moosefs-master.service" and "journalctl -xe" for details.
[root@server1 ~]# journalctl -xe
恢复:
[root@server1 ~]# vim /usr/lib/systemd/system/moosefs-master.service
[root@server1 ~]# systemctl daemon-reload
[root@server1 ~]# systemctl start moosefs-master
[root@server1 ~]# systemctl status moosefs-master
2.mfs的高可用
实验环境:
server1: 172.25.66.1 master
server2: 172.25.66.2 chunkserver
server3: 172.25.66.3 chunkserver
server4: 172.25.66.4 master
配置server1:
1配置高可用yum源
[root@foundation66 ~]# cd /var/www/html/rhel7.3/
[root@foundation66 rhel7.3]# ls
addons GPL LiveOS release-notes RPM-GPG-KEY-redhat-release
EFI images media.repo repodata TRANS.TBL
EULA isolinux Packages RPM-GPG-KEY-redhat-beta
[root@foundation66 rhel7.3]# cd addons/
[root@foundation66 addons]# ls
HighAvailability ResilientStorage
[root@foundation66 addons]# pwd
/var/www/html/rhel7.3/addons
[root@server1 ~]# vim /etc/yum.repos.d/yum.repo
#################
[HighAvailability]
name=HighAvailability
baseurl=http://172.25.66.254/rhel7.3/addons/HighAvailability
gpgcheck=0
[ResilientStorage]
name=ResilientStorage
baseurl=http://172.25.66.254/rhel7.3/addons/ResilientStorage
gpgcheck=0
[root@server1 ~]# yum clean all
[root@server1 ~]# yum repolist
2.安装pacemaker和corosync
[root@server1 ~]# yum install -y pacemaker corosync
3.启动pcsd
[root@server1 ~]# yum install -y pcs bash-*
[root@server1 ~]# systemctl start pcsd
[root@server1 ~]# systemctl enable pcsd
Created symlink from /etc/systemd/system/multi-user.target.wants/pcsd.service to /usr/lib/systemd/system/pcsd.service.
4.设定用户密码
[root@server1 ~]# passwd hacluster
配置server4:
1.安装moosefs-master
[root@server1 ~]# scp moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm root@172.25.66.4:
[root@server4 ~]# ls
moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm
[root@server4 ~]# yum install -y *
2.更改启动文件
#必须与server1结点的完全相同
[root@server4 ~]# vim /usr/lib/systemd/system/moosefs-master.service
#重新加载
[root@server4 ~]# systemctl daemon-reload
3.配置yum源
[root@server1 ~]# scp /etc/yum.repos.d/yum.repo server4:/etc/yum.repos.d/yum.repo
[root@server4 ~]# yum clean all
[root@server4 ~]# yum repolist
4.安装pacemaker与corosync
[root@server4 ~]# yum install -y pacemaker corosync
5.启动pcsd
[root@server4 ~]# yum install pcs bash-* -y
[root@server4 ~]# systemctl start pcsd
[root@server4 ~]# systemctl enable pcsd
Created symlink from /etc/systemd/system/multi-user.target.wants/pcsd.service to /usr/lib/systemd/system/pcsd.service.
6.设定用户密码
[root@server4 ~]# passwd hacluster
在server1上:
1.免密
[root@server1 ~]# ssh-keygen
[root@server1 ~]# ssh-copy-id server1
[root@server1 ~]# ssh-copy-id server4
测试免密:
[root@server1 ~]# ssh server1
Last login: Fri Apr 5 14:25:09 2019 from server1
[root@server1 ~]# exit
logout
Connection to server1 closed.
[root@server1 ~]# ssh server4
Last login: Fri Apr 5 13:46:57 2019 from foundation66.localdomain
[root@server4 ~]# exit
logout
Connection to server4 closed.
2.集群认证
[root@server1 ~]# pcs cluster auth server1 server4
[root@server1 ~]# pcs cluster setup --name mycluster server1 server4
[root@server1 ~]# pcs cluster start --all
server4: Starting Cluster...
server1: Starting Cluster...
[root@server1 ~]# pcs status nodes
Pacemaker Nodes:
Online: server1 server4
Standby:
Maintenance:
Offline:
Pacemaker Remote Nodes:
Online:
Standby:
Maintenance:
Offline:
[root@server1 ~]# corosync-cfgtool -s
Printing ring status.
Local node ID 1
RING ID 0
id = 172.25.66.1
status = ring 0 active with no faults
[root@server1 ~]# pcs status corosync
Membership information
----------------------
Nodeid Votes Name
1 1 server1 (local)
2 1 server4
[root@server1 ~]# pcs status
[root@server1 ~]# crm_verify -L -V
[root@server1 ~]# pcs property set stonith-enabled=false
[root@server1 ~]# crm_verify -L -V
[root@server1 ~]# pcs status
3.添加vip
[root@server1 ~]# pcs resource create vip ocf:heartbeat:IPaddr2 ip=172.25.66.100 cidr_netmask=32 op monitor interval=30s
[root@server1 ~]# pcs resource
vip (ocf::heartbeat:IPaddr2): Started server1
[root@server1 ~]# ip addr
#监控
[root@server4 ~]# crm_mon
配置iscsi文件系统共享设备:
在server2上:
1.添加磁盘
[root@server2 ~]# fdisk -l
2.共享磁盘
[root@server2 ~]# targetcli
/> cd /backstores/block
/backstores/block> ls
/backstores/block> create my_disk1 /dev/sdb
/backstores/block> cd my_disk1
/backstores/block/my_disk1> cd /iscsi
/iscsi> create iqn.2019-04.com.example:server2
/iscsi> cd iqn.2019-04.com.example:server2/
/iscsi/iqn.20...ample:server2> cd tpg1/luns
/iscsi/iqn.20...er2/tpg1/luns> create /backstores/block/my_disk1
/iscsi/iqn.20...er2/tpg1/luns> cd ..
/iscsi/iqn.20...:server2/tpg1> cd acls
/iscsi/iqn.20...er2/tpg1/acls> create iqn.2019-04.com.example:client
/iscsi/iqn.20...er2/tpg1/acls> exit
在server1上:
[root@server1 ~]# systemctl stop moosefs-master
[root@server1 ~]# yum install -y iscsi-*
[root@server1 ~]# vim /etc/iscsi/initiatorname.iscsi
#################
InitiatorName=iqn.2019-04.com.example:client
#发现设备
[root@server1 ~]# iscsiadm -m discovery -t st -p 172.25.66.2
172.25.66.2:3260,1 iqn.2019-04.com.example:server2
#登陆
[root@server1 ~]# iscsiadm -m node -l
注意:如果discovery发现错误则执行以下命令即可
[root@server1 ~]# iscsiadm -m node -o delete
测试:
[root@server1 ~]# fdisk -l
创建分区:
[root@server1 ~]# fdisk /dev/sdb
格式化:
[root@server1 ~]# mkfs.xfs /dev/sdb1
挂载:
[root@server1 ~]# mount /dev/sdb1 /media
[root@server1 ~]# df
拷贝数据:
[root@server1 ~]# cp /etc/* /media
模拟数据丢失:
[root@server1 ~]# umount /media/
[root@server1 ~]# dd if=/dev/zero of=/dev/sdb bs=512 count=1
[root@server1 ~]# mount /dev/sdb1 /media
mount: special device /dev/sdb1 does not exist
恢复:
[root@server1 ~]# fdisk /dev/sdb
[root@server1 ~]# mount /dev/sdb1 /media
[root@server1 ~]# df
测试:数据恢复
[root@server1 ~]# cd /media/
[root@server1 media]# ls
//
[root@server1 media]# pwd
/media
[root@server1 media]# rm -rf *
[root@server1 media]# cd
[root@server1 ~]# umount /media/
[root@server1 ~]# mount /dev/sdb1 /mnt
[root@server1 ~]# df
[root@server1 ~]# cd /var/lib/mfs/
[root@server1 mfs]# ls
changelog.1.mfs changelog.7.mfs metadata.mfs.back stats.mfs
changelog.5.mfs changelog.8.mfs metadata.mfs.back.1
changelog.6.mfs metadata.crc metadata.mfs.empty
[root@server1 mfs]# cp -p * /mnt
[root@server1 mfs]# cd /mnt/
[root@server1 mnt]# ls
changelog.1.mfs changelog.7.mfs metadata.mfs.back stats.mfs
changelog.5.mfs changelog.8.mfs metadata.mfs.back.1
changelog.6.mfs metadata.crc metadata.mfs.empty
[root@server1 mnt]# ll -d /mnt/
drwxr-xr-x 2 root root 236 Apr 5 16:35 /mnt/
[root@server1 mnt]# chown mfs.mfs /mnt/
[root@server1 mnt]# ll -d /mnt/
drwxr-xr-x 2 mfs mfs 236 Apr 5 16:35 /mnt/
[root@server1 ~]# mount /dev/sdb1 /var/lib/mfs
[root@server1 ~]# df
[root@server1 ~]# ll -d /var/lib/mfs/
drwxr-xr-x 2 mfs mfs 236 Apr 5 16:35 /var/lib/mfs/
测试master是否能使用
[root@server1 ~]# systemctl start moosefs-master
[root@server1 ~]# systemctl stop moosefs-master
在server4上:
[root@server4 ~]# systemctl stop moosefs-master
[root@server4 ~]# yum install -y iscsi-*
[root@server4 ~]# vim /etc/iscsi/initiatorname.iscsi
#################
InitiatorName=iqn.2019-04.com.example:client
[root@server4 ~]# iscsiadm -m discovery -t st -p 172.25.66.2
172.25.66.2:3260,1 iqn.2019-04.com.example:server2
[root@server4 ~]# iscsiadm -m node -l
测试:
[root@server4 ~]# fdisk -l
[root@server4 ~]# mount /dev/sdb1 /var/lib/mfs/
[root@server4 ~]# systemctl start moosefs-master