Corosync+pacemaker+pcs+drbd实现mariadb的高可用

最新推荐文章于 2022-09-09 11:33:21 发布
石兴稳
最新推荐文章于 2022-09-09 11:33:21 发布
阅读量743
点赞数
分类专栏： openstack
本文链接：https://blog.csdn.net/zuopiezia/article/details/78803968
版权
openstack 专栏收录该内容
35 篇文章 3 订阅
订阅专栏


       环境：10.10.7.11   
           10.10.7.12

磁盘2个
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0     11:0    1 1024M  0 rom  
vda    253:0    0   60G  0 disk 
├─vda1 253:1    0   54G  0 part /
└─vda2 253:2    0    6G  0 part [SWAP]
vdb    253:16   0  100G  0 disk 

修改主机名
hostnamectl set-hostname drbd11
hostnamectl set-hostname drbd12

创建lvm（每个节点都需执行）
[root@drbd11 ~]# yum install lvm2 -y

[root@drbd11 ~]# pvcreate /dev/vdb
  Physical volume "/dev/vdb" successfully created.
[root@drbd11 ~]# vgcreate data /dev/vdb
  Volume group "data" successfully created
[root@drbd11 ~]# lvcreate --size 10G --name mysql data
[root@drbd11 ~]# lsblk 
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0     11:0    1 1024M  0 rom  
vda    253:0    0   60G  0 disk 
├─vda1 253:1    0   54G  0 part /
└─vda2 253:2    0    6G  0 part [SWAP]
vdb    253:16   0  100G  0 disk 
[root@drbd11 ~]# lvcreate --size 10G --name mysql data
  Logical volume "mysql" created.
[root@drbd11 ~]# lsblk 
NAME         MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0           11:0    1 1024M  0 rom  
vda          253:0    0   60G  0 disk 
├─vda1       253:1    0   54G  0 part /
└─vda2       253:2    0    6G  0 part [SWAP]
vdb          253:16   0  100G  0 disk 
└─data-mysql 252:0    0   10G  0 lvm  
[root@drbd11 ~]# 
 
关闭防火墙（每个节点都需执行）
[root@drbd12 ~]# setenforce 0
x/config d12 ~]# sed -i.bak "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinu 
[root@drbd12 ~]# less /etc/selinux/config

# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected. 
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted 


[root@drbd12 ~]# systemctl disable firewalld.service
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
Removed symlink /etc/systemd/system/basic.target.wants/firewalld.service.
[root@drbd12 ~]# systemctl stop firewalld.service
[root@drbd12 ~]# iptables --flush
[root@drbd12 ~]# 

修改hosts(两个都需要修改)
[root@drbd11 ~]# echo -e "10.10.7.11 drbd11 \n10.10.7.12 drbd12" >> /etc/hosts

[root@drbd11 ~]# less /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.10.7.11 drbd11 
10.10.7.12 drbd12

设置ntp 和时区(两个都需要修改)
[root@drbd12 ~]# timedatectl set-timezone Asia/Shanghai
[root@drbd12 ~]# date
Wed Aug  2 09:46:13 CST 2017
[root@drbd12 ~]# yum install chrony -y

[root@drbd12 ~]# systemctl start chronyd

[root@drbd12 ~]# systemctl enable chronyd

ssh 无密码访问(两个都需要修改)

[root@drbd12 ~]# ssh-keygen 
[root@drbd12 ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub root@drbd11
[root@drbd12 ~]# ssh drbd11
Last login: Tue Aug  1 17:02:07 2017
[root@drbd11 ~]# ssh drbd12
Last login: Tue Aug  1 17:02:02 2017
[root@drbd12 ~]# exit
logout
Connection to drbd12 closed.

第二部分

安装drbd软件（各个节点）

[root@drbd11 ~]# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
[root@drbd11 ~]# rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
Retrieving http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
Preparing...                          ################################# [100%]
Updating / installing...
   1:elrepo-release-7.0-3.el7.elrepo  ################################# [100%]
[root@drbd11 ~]# yum install -y kmod-drbd84 drbd84-utils

配置文件介绍

 复制配置文件
[root@drbd12 etc]# cp /etc/drbd.conf /etc/drbd.conf.20170802.bk
[root@drbd12 etc]# cp /etc/drbd.d/global_common.conf /etc/drbd.d/global_common.conf.20170802.bk
[root@drbd12 etc]# 


[root@drbd11 drbd.d]# vi global_common.conf
# DRBD is the result of over a decade of development by LINBIT.
# In case you need professional services for DRBD or have
# feature requests visit http://www.linbit.com

global {
        usage-count no; #是否参加DRBD使用统计，默认为yes。官方统计drbd的装机量
        # minor-count dialog-refresh disable-ip-verification
        # cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
}

common {
        handlers {
                # These are EXAMPLE handlers only.
                # They may have severe implications,
                # like hard resetting the node under certain circumstances.
                # Be careful when chosing your poison.

                 pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
                 pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
                 local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
                # fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
                # split-brain "/usr/lib/drbd/notify-split-brain.sh root";
                # out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
                # before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
                # after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
        }

        startup {
                # wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
        }

        options {
                # cpu-mask on-no-data-accessible
        }

        disk {
                on-io-error detach;#配置I/O错误处理策略为分离
                # size on-io-error fencing disk-barrier disk-flushes
                # disk-drain md-flushes resync-rate resync-after al-extents
                # c-plan-ahead c-delay-target c-fill-target c-max-rate
                # c-min-rate disk-timeout
        }

        net {
                # protocol timeout max-epoch-size max-buffers unplug-watermark
                # connect-int ping-int sndbuf-size rcvbuf-size ko-count
                # allow-two-primaries cram-hmac-alg shared-secret after-sb-0pri
                # after-sb-1pri after-sb-2pri always-asbp rr-conflict
                # ping-timeout data-integrity-alg tcp-cork on-congestion
                # congestion-fill congestion-extents csums-alg verify-alg
                # use-rle
        }
}

注释： on-io-error 策略可能为以下选项之一 
detach 分离：这是默认和推荐的选项，如果在节点上发生底层的硬盘I/O错误，它会将设备运行在Diskless无盘模式下 
pass_on：DRBD会将I/O错误报告到上层，在主节点上，它会将其报告给挂载的文件系统，但是在此节点上就往往忽略（因此此节点上没有可以报告的上层） 
-local-in-error：调用本地磁盘I/O处理程序定义的命令；这需要有相应的local-io-error调用的资源处理程序处理错误的命令；这就给管理员有足够自由的权力命令命令或是脚本调用local-io-error处理I/O错误 
c、定义一个资源 
新建/etc/drbd.d/MySQL.res并写入下列内容


[root@drbd11 ~]# less /etc/drbd.d/MySQL.res 
resource mysql { #资源名称
protocol C; #使用协议
meta-disk internal;
device /dev/drbd1; #DRBD设备名称
syncer {
verify-alg sha1;# 加密算法
}
net {
allow-two-primaries;
}
on drbd11 {
disk /dev/data/mysql; #drbd1使用的磁盘分区为"mysql"
address 10.10.7.11:7789; #设置DRBD监听地址与端口
}
on drbd12 {
disk /dev/data/mysql;
address 10.10.7.12:7789;
}
}
[root@drbd11 ~]# 


将配置文件scp到另一个节点
[root@drbd11 drbd.d]# scp -rp /etc/drbd.d/* drbd12:/etc/drbd.d/
global_common.conf                                                                                                                                        100% 2080     2.0KB/s   00:00    
global_common.conf.20170802.bk                                                                                                                            100% 2062     2.0KB/s   00:00    
MySQL.res                                                                                                                                                 100%  380     0.4KB/s   00:00    

启用drbd

[root@drbd11 drbd.d]# drbdadm create-md mysql
initializing activity log
NOT initializing bitmap
Writing meta data...
New drbd meta data block successfully created.
[root@drbd11 drbd.d]# lsblk 
NAME         MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0           11:0    1 1024M  0 rom  
vda          253:0    0   60G  0 disk 
├─vda1       253:1    0   54G  0 part /
└─vda2       253:2    0    6G  0 part [SWAP]
vdb          253:16   0  100G  0 disk 
└─data-mysql 252:0    0   10G  0 lvm  
[root@drbd11 drbd.d]# modprobe drbd

[root@drbd11 drbd.d]# drbdadm up mysql

[root@drbd11 drbd.d]# drbdadm -- --force primary mysql

[root@drbd11 drbd.d]# lsblk 
NAME         MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0           11:0    1 1024M  0 rom  
vda          253:0    0   60G  0 disk 
├─vda1       253:1    0   54G  0 part /
└─vda2       253:2    0    6G  0 part [SWAP]
vdb          253:16   0  100G  0 disk 
└─data-mysql 252:0    0   10G  0 lvm  
  └─drbd1    147:1    0   10G  0 disk 

确认配置
[root@drbd11 drbd.d]# cat /proc/drbd 
version: 8.4.9-1 (api:1/proto:86-101)
GIT-hash: 9976da086367a2476503ef7f6b13d4567327a280 build by akemi@Build64R7, 2016-12-04 01:08:48

 1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r----s
    ns:0 nr:0 dw:0 dr:912 al:8 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:10485404

 在另一台节点上启用

[root@drbd11 ~]# ssh drbd12 "drbdadm create-md mysql"
NOT initializing bitmap
initializing activity log
Writing meta data...
New drbd meta data block successfully created.
[root@drbd11 ~]# ssh drbd12 "modprobe drbd"
[root@drbd11 ~]# ssh drbd12 "drbdadm up mysql"


在drbd11 上挂在磁盘


[root@drbd11 ~]# mkfs.xfs /dev/drbd1
meta-data=/dev/drbd1             isize=512    agcount=4, agsize=655338 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=0, sparse=0
data     =                       bsize=4096   blocks=2621351, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=2560, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

[root@drbd11 ~]# lsblk 
NAME         MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0           11:0    1 1024M  0 rom  
vda          253:0    0   60G  0 disk 
├─vda1       253:1    0   54G  0 part /
└─vda2       253:2    0    6G  0 part [SWAP]
vdb          253:16   0  100G  0 disk 
└─data-mysql 252:0    0   10G  0 lvm  
  └─drbd1    147:1    0   10G  0 disk 
[root@drbd11 ~]# mkdir /var/lib/mysql
[root@drbd11 ~]# mount /dev/drbd1 /var/lib/mysql

[root@drbd11 ~]# mount

/dev/drbd1 on /var/lib/mysql type xfs (rw,relatime,seclabel,attr2,inode64,noquota)

[root@drbd11 ~]# lsblk 
NAME         MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0           11:0    1 1024M  0 rom  
vda          253:0    0   60G  0 disk 
├─vda1       253:1    0   54G  0 part /
└─vda2       253:2    0    6G  0 part [SWAP]
vdb          253:16   0  100G  0 disk 
└─data-mysql 252:0    0   10G  0 lvm  
  └─drbd1    147:1    0   10G  0 disk /var/lib/mysql
[root@drbd11 ~]# 


三、安装集群软件(各个节点需要安装)
3.1安装相关软件包
yum install -y pacemaker pcs psmisc policycoreutils-python

[root@drbd11 ~]# systemctl status pcsd
● pcsd.service - PCS GUI and remote configuration interface
   Loaded: loaded (/usr/lib/systemd/system/pcsd.service; disabled; vendor preset: disabled)
   Active: inactive (dead)
[root@drbd11 ~]# systemctl start pcsd
[root@drbd11 ~]# systemctl status pcsd
● pcsd.service - PCS GUI and remote configuration interface
   Loaded: loaded (/usr/lib/systemd/system/pcsd.service; disabled; vendor preset: disabled)
   Active: active (running) since Wed 2017-08-02 11:07:40 CST; 1s ago
 Main PID: 27226 (pcsd)
   CGroup: /system.slice/pcsd.service
           └─27226 /usr/bin/ruby /usr/lib/pcsd/pcsd > /dev/null &

Aug 02 11:07:39 drbd11 systemd[1]: Starting PCS GUI and remote configuration interface...
Aug 02 11:07:40 drbd11 systemd[1]: Started PCS GUI and remote configuration interface.
[root@drbd11 ~]# systemctl enable pcsd
Created symlink from /etc/systemd/system/multi-user.target.wants/pcsd.service to /usr/lib/systemd/system/pcsd.service.
[root@drbd11 ~]# systemctl status pcsd
● pcsd.service - PCS GUI and remote configuration interface
   Loaded: loaded (/usr/lib/systemd/system/pcsd.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2017-08-02 11:07:40 CST; 12s ago
 Main PID: 27226 (pcsd)
   CGroup: /system.slice/pcsd.service
           └─27226 /usr/bin/ruby /usr/lib/pcsd/pcsd > /dev/null &

Aug 02 11:07:39 drbd11 systemd[1]: Starting PCS GUI and remote configuration interface...
Aug 02 11:07:40 drbd11 systemd[1]: Started PCS GUI and remote configuration interface.
[root@drbd11 ~]# 

修改用户hacluster的密码

[root@drbd11 ~]# echo sursen@2015 |passwd --stdin hacluster
Changing password for user hacluster.
passwd: all authentication tokens updated successfully.

[root@drbd11 ~]# ssh drbd12 -- 'echo sursen@2015 |passwd --stdin hacluster'
Changing password for user hacluster.
passwd: all authentication tokens updated successfully.
[root@drbd11 ~]# 

四、安装mysql（各个节点）  4.1、安装软件
# yum install epel* -y
# yum install mariadb mariadb-server MySQL-python

4.2、禁止mysql开机启动
# chkconfig  mariadb off

配置corosync 
5.1认证各个节点，并创建集群（注意如果有代理记得取消）
[root@drbd12 ~]# pcs cluster auth drbd11 drbd12
Username: hacluster
Password: 
drbd11: Authorized
drbd12: Authorized

[root@drbd12 ~]# pcs cluster setup --name mycluster drbd11 drbd12
Destroying cluster on nodes: drbd11, drbd12...
drbd11: Stopping Cluster (pacemaker)...
drbd12: Stopping Cluster (pacemaker)...
drbd12: Successfully destroyed cluster
drbd11: Successfully destroyed cluster

Sending cluster config files to the nodes...
drbd11: Succeeded
drbd12: Succeeded

Synchronizing pcsd certificates on nodes drbd11, drbd12...
drbd11: Success
drbd12: Success

Restarting pcsd on the nodes in order to reload the certificates...
drbd11: Success
drbd12: Success
[root@drbd12 ~]# 
[root@drbd12 ~]# 


5.2启动集群
[root@drbd11 ~]# pcs cluster start --all
drbd12: Starting Cluster...
drbd11: Starting Cluster...
5.3验证corosync的安装
[root@drbd11 ~]# corosync-cfgtool -s
Printing ring status.
Local node ID 1
RING ID 0
        id      = 10.10.7.11
        status  = ring 0 active with no faults
5.4查看接入成员
[root@drbd11 ~]# corosync-cmapctl |grep members
runtime.totem.pg.mrp.srp.members.1.config_version (u64) = 0
runtime.totem.pg.mrp.srp.members.1.ip (str) = r(0) ip(10.10.7.11) 
runtime.totem.pg.mrp.srp.members.1.join_count (u32) = 1
runtime.totem.pg.mrp.srp.members.1.status (str) = joined
runtime.totem.pg.mrp.srp.members.2.config_version (u64) = 0
runtime.totem.pg.mrp.srp.members.2.ip (str) = r(0) ip(10.10.7.12) 
runtime.totem.pg.mrp.srp.members.2.join_count (u32) = 1
runtime.totem.pg.mrp.srp.members.2.status (str) = joined
[root@drbd11 ~]# 
[root@drbd11 ~]# 

5.5查看corosync状态
[root@drbd11 ~]# pcs status corosync

Membership information
----------------------
    Nodeid      Votes Name
         1          1 drbd11 (local)
         2          1 drbd12
[root@drbd11 ~]# 
[root@drbd11 ~]# 
5.6检查pacemaker的安装
[root@drbd11 ~]# ps axf
  PID TTY      STAT   TIME COMMAND
    2 ?        S      0:00 [kthreadd]
    3 ?        S      0:00  \_ [ksoftirqd/0]
    4 ?        S      0:00  \_ [kworker/0:0]
    5 ?        S<     0:00  \_ [kworker/0:0H]
    6 ?        S      0:00  \_ [kworker/u32:0]
    7 ?        S      0:00  \_ [migration/0]
    8 ?        S      0:00  \_ [rcu_bh]
    9 ?        S      0:09  \_ [rcu_sched]
   10 ?        S      0:00  \_ [watchdog/0]

5.7检查pcs status  

[root@drbd11 ~]# 
[root@drbd11 ~]# pcs status
Cluster name: mycluster
WARNING: no stonith devices and stonith-enabled is not false
Stack: corosync
Current DC: drbd12 (version 1.1.15-11.el7_3.5-e174ec8) - partition with quorum
Last updated: Wed Aug  2 11:21:20 2017          Last change: Wed Aug  2 11:20:08 2017 by hacluster via crmd on drbd12

2 nodes and 0 resources configured

Online: [ drbd11 drbd12 ]

No resources


Daemon Status:
  corosync: active/disabled
  pacemaker: active/disabled
  pcsd: active/enabled
[root@drbd11 ~]# 
[root@drbd11 ~]# 

5.8查看系统中error（stonith除外）
[root@drbd11 ~]# journalctl |grep -i error
Aug 01 16:59:10 localhost.localdomain firewalld[681]: ERROR: UNKNOWN_INTERFACE: 'eth0' is not in any zone
Aug 02 09:55:17 drbd11 dracut[16004]: -rw-r--r--   1 root     root          191 Nov  6  2016 usr/lib/kbd/consolefonts/ERRORS
Aug 02 09:55:21 drbd11 dracut[16004]: lrwxrwxrwx   1 root     root           22 Aug  2 09:55 usr/lib64/libgpg-error.so.0 -> libgpg-error.so.0.10.0
Aug 02 09:55:21 drbd11 dracut[16004]: -rwxr-xr-x   1 root     root        19384 Aug  2 09:55 usr/lib64/libgpg-error.so.0.10.0
[root@drbd11 ~]# 


六、配置集群（任选一个节点）