mysql的mha+keepalived读写分离

master：192.128.232.129
slave01：192.128.232.130
slave02：192.128.232.131
keepalied01: 192.128.232.127
keepalied02: 192.128.232.128

1：mysql的配置

[root@mysql-129 ~]# cat /etc/my.cnf
[client]
user=root
password=1111aaA_          #几个实例的密码一定要设置这个。否则无法停止。

[mysql]
prompt=(\u@\h) [\d]\_
no-auto-rehash

[mysqld_multi]
mysqld = /usr/local/mysql/bin/mysqld_safe
mysqladmin = /usr/local/mysql/bin/mysqladmin
log = /usr/local/mysql/mysqld_multi.log

[mysqld57]
skip-name-resolve
port = 3306
server-id = 129
basedir = /usr/local/mysql
datadir = /data/mysql_data/data
socket = /tmp/mysql.sock
log_error = /data/mysql_data/logs/error.log
event_scheduler = 1
slow_query_log = 1
slow_query_log_file = slow.log
long_query_time = 2
sql_mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION"
binlog_format = row
transaction-isolation = READ-COMMITTED
log_bin=/data/mysql_data/logbin/mysql-binlog
#log_bin=/data/mysql_data/binlog_data/
binlog_rows_query_log_events = 1
slow_query_log=ON
slow_query_log_file=/data/mysql_data/logs/mysql-slow.log
long_query_time=2

#session memory
sort_buffer_size = 64M
tmp_table_size = 64M

#replication
slave_parallel_workers = 16
slave_parallel_type = LOGICAL_CLOCK

#rpl_semi
plugin_load="rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so"
rpl_semi_sync_master_enabled=1
rpl_semi_sync_slave_enabled=1
rpl_semi_sync_master_timeout=100000
rpl_semi_sync_master_wait_for_slave_count = 1
rpl_semi_sync_master_wait_point = AFTER_SYNC

#innodb
innodb_log_file_size = 4G
innodb_log_buffer_size = 16M
innodb_log_files_in_group=2
innodb_buffer_pool_size = 1024M
innodb_buffer_pool_instances = 16
innodb_online_alter_log_max_size = 512M
innodb_write_io_threads = 16
innodb_page_size = 16384
innodb_flush_method = O_DIRECT
innodb_io_capacity = 4000
innodb_page_cleaners = 4
innodb_fast_shutdown = 1
innodb_flush_neighbors = 0
innodb_purge_threads = 8
innodb_lock_wait_timeout = 300
innodb_autoinc_lock_mode = 2
innodb_status_output_locks = 1
innodb_print_all_deadlocks = 1

#relay
relay_log_recovery = 1
relay_log_info_repository = table
master_info_repository = table
log_slave_updates

#gtid
gtid-mode = ON
enforce-gtid-consistency = ON

2：每个mysql节点都部署node，选择131为mha管理节点

[root@mysql-131 ~]# cat /etc/mha/app1.conf
[server default]
log_level=debug
manager_log=/usr/local/mha/app1/manager.log
manager_workdir=/usr/local/mha/app1.log
master_binlog_dir=/data/mysql_data/
master_ip_failover_script=/usr/local/bin/master_ip_failover
master_ip_online_change_script=/usr/local/bin/master_ip_online_change
user=mha
password=mha!123
ping_interval=3
ping_type=INSERT
remote_workdir=/tmp
repl_password=repl123
repl_user=repl
secondary_check_script=/usr/local/bin/masterha_secondary_check -s 192.128.232.129 -s 192.128.232.130 -s 192.128.232.131
ssh_user=root

[server1]
candidate_master=1
check_repl_delay=0
hostname=192.128.232.129
port=3306
ssh_port=22

[server2]
candidate_master=1
check_repl_delay=0
hostname=192.128.232.130
port=3306
ssh_port=22

[server3]
candidate_master=1
hostname=192.128.232.131
no_master=1
port=3306
ssh_port=22

3: 配置mysql的主从切换脚本

[root@localhost ~]# cat /usr/local/bin/master_ip_failover
#!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';

use Getopt::Long;
use MHA::DBHelper;

my (
  $command,        $ssh_user,         $orig_master_host,
  $orig_master_ip, $orig_master_port, $new_master_host,
  $new_master_ip,  $new_master_port,  $new_master_user,
  $new_master_password
);
my $vip = '192.128.232.132/24';     #写vip
my $key = "1";
my $ssh_start_vip = "/sbin/ifconfig eth0:$key $vip";
my $ssh_stop_vip = "/sbin/ifconfig eth0:$key down";

GetOptions(
  'command=s'             => \$command,
  'ssh_user=s'            => \$ssh_user,
  'orig_master_host=s'    => \$orig_master_host,
  'orig_master_ip=s'      => \$orig_master_ip,
  'orig_master_port=i'    => \$orig_master_port,
  'new_master_host=s'     => \$new_master_host,
  'new_master_ip=s'       => \$new_master_ip,
  'new_master_port=i'     => \$new_master_port,
  'new_master_user=s'     => \$new_master_user,
  'new_master_password=s' => \$new_master_password,
);

exit &main();

sub main {
  if ( $command eq "stop" || $command eq "stopssh" ) {

    # $orig_master_host, $orig_master_ip, $orig_master_port are passed.
    # If you manage master ip address at global catalog database,
    # invalidate orig_master_ip here.
    my $exit_code = 1;
    eval {

      # updating global catalog, etc
      $exit_code = 0;
    };
    if ($@) {
      warn "Got Error: $@\n";
      exit $exit_code;
    }
    exit $exit_code;
  }
    elsif ( $command eq "start" ) {

        # all arguments are passed.
        # If you manage master ip address at global catalog database,
        # activate new_master_ip here.
        # You can also grant write access (create user, set read_only=0, etc) here.
        my $exit_code = 10;
        eval {
            print "Enabling the VIP - $vip on the new master - $new_master_host \n";
            &start_vip();               
            &stop_vip();                
            $exit_code = 0;
        };
        if ($@) {
            warn $@;
            exit $exit_code;
        }
        exit $exit_code;
    }
    elsif ( $command eq "status" ) {
        print "Checking the Status of the script.. OK \n";
        `ssh $ssh_user\@$orig_master_host \" $ssh_start_vip \"`;
        exit 0;
    }
    else {
        &usage();
        exit 1;
    }
}

sub start_vip() {
    `ssh $ssh_user\@$new_master_host \" $ssh_start_vip \"`;
}
# A simple system call that disable the VIP on the old_master
sub stop_vip() {
   `ssh $ssh_user\@$orig_master_host \" $ssh_stop_vip \"`;
}

sub usage {
  print
"Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n";
}
 

4: 启动mha
[root@mysql-131 ~]# nohup masterha_manager --ignore_last_failover --remove_dead_master_conf --conf=/etc/mha/app1.conf 2>1&

5：每个mysql节点都执行，绑定读vip到lo网卡上。

[root@mysql-129 ~]# cat /etc/init.d/realserver.sh
#!/bin/bash
# description: Config realserver lo and apply noarp

SNS_VIP=192.128.232.133   #读vip

case "$1" in
start)
       ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
       /sbin/route add -host $SNS_VIP dev lo:0
       echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
       echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
       echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
       echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
       sysctl -p >/dev/null 2>&1
       echo "RealServer Start OK"
       ;;

stop)
       ifconfig lo:0 down
       route del $SNS_VIP >/dev/null 2>&1
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
       echo "RealServer Stoped"
       ;;

*)

       echo "Usage: $0 {start|stop}"
       exit 1
esac
exit 0
 

6: keepalived的主配置都是一模一样的。

[root@keepalived-127 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   router_id LVS_MASTER
}
vrrp_instance Msyql_Lb {
    state BACKUP            #全部节点设置BACKUP,以优先级高暂时为master。
    interface eth0          #对应主机的网卡名称
    virtual_router_id 251   #id号要一致
    priority 100
    advert_int 1
    nopreempt               #当优先级高的主机宕机后，再次恢复时，不会抢夺vip，防止业务频繁切换。

    authentication {
        auth_type PASS
        auth_pass 11111111
    }

    virtual_ipaddress {
        192.128.232.133/24  #vip地址
    }
}

virtual_server 192.128.232.133 3306 {      # 定义虚拟服务器，地址与上面的virtual_ipaddress相同
    delay_loop 6                           # 健康检查时间间隔，3秒
    lb_algo wrr                            # 负载均衡调度算法：rr|wrr|lc|wlc|sh|dh|lblc
    lb_kind DR                             # 负载均衡转发规则：NAT|DR|TUN
    nat_mask 255.255.255.0
   # persistence_timeout 50                # 会话保持时间5秒，动态服务建议开启
    protocol TCP                           # 转发协议protocol，一般有tcp和udp两种

    real_server 192.128.232.129 3306 {
        weight 1                           # 权重越大负载分越大，0表示失效
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 3306
        }
    }

    real_server 192.128.232.130 3306 {
        weight 1                           # 权重越大负载分越大，0表示失效
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 3306
        }
    }

    real_server 192.128.232.131 3306 {
        weight 2                           # 权重越大负载分越大，0表示失效
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 3306
        }
    }
}

7: keepalived的备配置都是一模一样的
[root@keepalived-128 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   router_id LVS_MASTER
}
vrrp_instance Msyql_Lb {
    state BACKUP    #全部节点设置BACKUP,以优先级高暂时为master。
    interface eth0      #对应主机的网卡名称
    virtual_router_id 251   #id号要一致
    priority 90             #优先级低于master
    advert_int 1
    nopreempt     #当优先级高的主机宕机后，再次恢复时，不会抢夺vip，防止业务频繁切换。

    authentication {
        auth_type PASS
        auth_pass 11111111
    }

    virtual_ipaddress {
        192.128.232.133/24    #vip地址
    }
}

virtual_server 192.128.232.133 3306 {# 定义虚拟服务器，地址与上面的virtual_ipaddress相同
    delay_loop 6                  # 健康检查时间间隔，3秒
    lb_algo wrr                    # 负载均衡调度算法：rr|wrr|lc|wlc|sh|dh|lblc
    lb_kind DR                    # 负载均衡转发规则：NAT|DR|TUN
    nat_mask 255.255.255.0
   # persistence_timeout 50        # 会话保持时间5秒，动态服务建议开启
    protocol TCP                  # 转发协议protocol，一般有tcp和udp两种

    real_server 192.128.232.129 3306 {
        weight 1                  # 权重越大负载分越大，0表示失效
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 3306
        }
    }

    real_server 192.128.232.130 3306 {
        weight 1                  # 权重越大负载分越大，0表示失效
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 3306
        }
    }

    real_server 192.128.232.131 3306 {
        weight 2                  # 权重越大负载分越大，0表示失效
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 3306
        }
    }
}

8: ipvsadm转发
[root@keepalived-127 ~]# watch ipvsadm -Ln --stats                                                                           
Every 2.0s: ipvsadm -ln --stats                                                                                                                                                         Tue Jun  8 11:50:22 2021

IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
  -> RemoteAddress:Port
TCP  192.128.232.133:3306                5       45        0     3740        0
  -> 192.128.232.129:3306                1        9        0      748        0
  -> 192.128.232.130:3306                1        9        0      748        0
  -> 192.128.232.131:3306                3       27        0     2244        0

9: 测试，在一个客户端机器上，通过读vip实现负载均衡。
[root@keepalived-128 ~]# while true;do mysql -h 192.128.232.133 -u repl -prepl123 -e 'show variables like "hostname"';sleep 2;done
mysql: [Warning] Using a password on the command line interface can be insecure.
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| hostname      | mysql-131 |
+---------------+-----------+
mysql: [Warning] Using a password on the command line interface can be insecure.
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| hostname      | mysql-130 |
+---------------+-----------+
mysql: [Warning] Using a password on the command line interface can be insecure.
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| hostname      | mysql-129 |
+---------------+-----------+
mysql: [Warning] Using a password on the command line interface can be insecure.
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| hostname      | mysql-131 |
+---------------+-----------+
mysql: [Warning] Using a password on the command line interface can be insecure.
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| hostname      | mysql-131 |
+---------------+-----------+
mysql: [Warning] Using a password on the command line interface can be insecure.
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| hostname      | mysql-130 |
+---------------+-----------+