HA-LNMP高可用架构搭建WEB

1.1架构图示

1.1.1主机节点规划

nginx1 3G mysql-master-A centos 172.20.10.105
nginx2 3G mysql-master-B centos 172.20.10.106

LVS1 2G 172.20.10.100
LVS2 2G 172.20.10.101

Haproxy1 2G 172.20.10.103 NFS
Haproxy2 2G 172.20.10.104

2.1部署数据库

mysql-master-A 172.20.10.105
mysql-master-B 172.20.10.106
数据库的高可用这里采用的是双主模式。所以数据库的部署配置是一样的，这里复用后端的Nginx服务器

[root@nginx-1 ~]#yum install vim gcc gcc-c++ wget autoconf net-tools lrzsz iotop lsof 
 iotop bash-completion curl policycoreutils openssh-server openssh-clients postfix libaio -y

[root@nginx-1 src]#tar xf  mysql-5.6.46-linux-glibc2.12-x86_64.tar.gz 

[root@nginx-1 src]#ln -sv /usr/local/src/mysql-5.6.46-linux-glibc2.12-x86_64 /usr/local/mysql
‘/usr/local/mysql’ -> ‘/usr/local/src/mysql-5.6.46-linux-glibc2.12-x86_64’

[root@nginx-1 src]#useradd mysql -s /sbin/nologin 

[root@nginx-1 src]#mkdir -pv /data/mysql /var/lib/mysql
mkdir: created directory ‘/data/mysql’
mkdir: created directory ‘/var/lib/mysql’

[root@nginx-1 src]#chown -R mysql.mysql /data/   /var/lib/mysql 

[root@nginx-1 src]#/usr/local/mysql/scripts/mysql_install_db --user=mysql --datadir=/data/mysql --basedir=/usr/local/mysql #运行初始化安装脚本

[root@nginx-1 src]#cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld

[root@nginx-1 src]#chmod a+x /etc/init.d/mysqld 

[root@nginx-1 mysql]#cd /usr/local/mysql/
[root@nginx-1 mysql]#vim /etc/my.cnf
[root@nginx-1 mysql]#cat /etc/my.cnf
[mysqld]
datadir=/data/mysql
socket=/data/mysql/mysql.sock
symbolic-links=0
user=mysql
innodb_file_per_table=1
max_connections=200

[client]
port=3306
socket=/var/lib/mysql/mysql.sock

[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/tem/mysql.sock

!includedir /etc/my.cnf.d

[root@nginx-1 mysql]#/etc/init.d/mysqld start
Starting MySQL. SUCCESS! 

[root@nginx-1 mysql]#ln -sv /data/mysql/mysql.sock /var/lib/mysql/mysql.sock
‘/var/lib/mysql/mysql.sock’ -> ‘/data/mysql/mysql.sock’

[root@nginx-1 mysql]#/usr/local/mysql/bin/mysql #进入到数据库

mysql> CREATE DATABASE wordpress;
Query OK, 1 row affected (0.00 sec)

mysql> GRANT ALL PRIVILEGES ON wordpress.* TO "wordpress"@"192.168.32.%" IDENTIFIED BY "123456";
Query OK, 0 rows affected (0.00 sec)

mysql> GRANT ALL PRIVILEGES ON wordpress.* TO "wordpress"@"nginx-1" IDENTIFIED BY "123456"; #注意2个数据库hostname不一样
Query OK, 0 rows affected (0.00 sec)

mysql> FLUSH PRIVILEGES;
Query OK, 0 rows affected (0.00 sec)

mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| wordpress          |
+--------------------+
5 rows in set (0.00 sec)

3.1部署NGINX

nginx高可用部署2台web服务器配置保持一样

[root@nginx-1]# yum install -y vim lrzsz tree screen psmisc lsof tcpdump wget ntpdate gcc
gcc-c++ glibc glibc-devel pcre pcre-devel openssl openssl-devel systemd-devel net-tools
iotop bc zip unzip zlib-devel bash-completion nfs-utils automake libxml2 libxml2-devel
libxslt libxslt-devel perl perl-ExtUtils-Embed

[root@nginx-1]# cd /usr/local/src/
[root@nginx-1 src]# wget https://nginx.org/download/nginx-1.16.1.tar.gz
[root@s1 src]# tar xf nginx-1.16.1.tar.gz

#编译安装NGINX
[root@s1 src]# cd nginx-1.16.1
[root@nginx-1 nginx-1.16.1]#./configure --prefix=/apps/nginx \ 
 --user=www \
 --group=www \
 --with-http_ssl_module \
 --with-http_v2_module \
 --with-http_realip_module \
 --with-http_stub_status_module \
 --with-http_gzip_static_module \
 --with-pcre \
 --with-stream \
 --with-stream_ssl_module \
 --with-stream_realip_module
 [root@nginx-1 nginx-1.16.1]#make
 [root@nginx-1 nginx-1.16.1]#make install
 [root@nginx-1 nginx-1.16.1]]#useradd nginx
 

[root@nginx-1 src]#grep -v "#" /apps/nginx/conf/nginx.conf | grep -v "^$"
user  nginx;
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;
    server {
        listen       80;
        server_name  www.duanxin.io;
        location / {
            root   /data/nginx/wordpress;  #访问目录
            index  index.php index.html index.htm;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
        location ~ \.php$ {
	    root /data/nginx/wordpress;
	    fastcgi_pass 127.0.0.1:9000;
	    fastcgi_index index.php;
	    fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
	    include fastcgi_params;
       }
    }
}

[root@nginx-1 src]#/apps/nginx/sbin/nginx -s reload #重新加载配置文件

[root@nginx-1 src]#/apps/nginx/sbin/nginx -t #语法检测
nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /apps/nginx/conf/nginx.conf test is successful

[root@nginx-1 src]#/apps/nginx/sbin/nginx #开启nginx
[root@nginx-1 src]#ss -ntl  #查看80端口是否打开
State       Recv-Q Send-Q            Local Address:Port                           Peer 
LISTEN      0      128                           *:80                                        *:*                  

4.1部署PHP

PHP部署在172.20.10.105和172.20.10.106的WEB服务器中，2台服务器中PHP部署一致

下载php-7.3.10.tar.gz安装包

[root@nginx-1 src]#tar xf php-7.3.10.tar.gz

[root@nginx-1 src]#cd php-7.3.10/
[root@nginx-1 php-7.3.10]#./configure --prefix=/apps/php --enable-fpm --with-fpm-user=nginx --with-fpm-group=nginx --with-pear --with-curl --with-png-dir --with-freetype-dir --with-iconv --with-mhash --with-zlib --with-xmlrpc --with-xsl --with-openssl --with-mysqli --with-pdo-mysql --disable-debug --enable-zip --enable-sockets --enable-soap --enable-inline-optimization --enable-xml --enable-ftp --enable-exif --enable-wddx --enable-bcmath --enable-calendar --enable-shmop --enable-dba --enable-sysvsem --enable-sysvshm --enable-sysvmsg

[root@nginx-1 php-7.3.10]#make -j 2
[root@nginx-1 php-7.3.10]#make install

4.1.1依赖解决

error: cURL version 7.15.5 or later is required to compile php with cURL support
[root@nginx-1 src]#yum -y install curl-devel

error: Please reinstall the libzip distribution
[root@nginx-1 src]#tar xf cmake-3.6.0-Linux-x86_64.tar.gz
[root@nginx-1 src]#export PATH=$PATH:/usr/local/src/cmake-3.6.0-Linux-x86_64/bin #增加临时PATH

[root@nginx-1 src]#yum remove  libzip -y
[root@nginx-1 src]#tar xf libzip-1.5.1.tar.gz 
[root@nginx-1 src]#cd libzip-1.5.1/
[root@nginx-1 libzip-1.5.1]#mkdir build
[root@nginx-1 libzip-1.5.1]#cd  build
[root@nginx-1 build]#cmake ..
[root@nginx-1 build]#make && make install

error: off_t undefined; check your library configuration
[root@nginx-1 php-7.3.10]#echo '/usr/local/lib64                                                        
/usr/local/lib
/usr/lib
/usr/lib64'>>/etc/ld.so.conf
[root@nginx-1 php-7.3.10]#ldconfig -v 

4.1.2准备PHP配置文件

[root@nginx-1 php-7.3.10]#cd /apps/php/etc/php-fpm.d/
[root@nginx-1 php-fpm.d]#cp www.conf.default www.conf
[root@nginx-1 php-fpm.d]#vim www.conf

[root@nginx-1 php-fpm.d]#grep -v ";" www.conf | grep -v "^$"
[www]
user = nginx
group = nginx
listen = 127.0.0.1:9000
listen.allowed_clients = 127.0.0.1
pm = dynamic
pm.max_children = 10
pm.start_servers = 5
pm.min_spare_servers = 5
pm.max_spare_servers = 10
pm.status_path = /pm_status
ping.path = /ping
ping.response = pong
access.log = log/$pool.access.log
slowlog = log/$pool.log.slow

[root@nginx-1 php-fpm.d]#cp /usr/local/src/php-7.3.10/php.ini-production /apps/php/etc/php.ini
[root@nginx-1 php-fpm.d]#useradd nginx -s /sbin/nologin -u 1001

[root@nginx-1 php-fpm.d]mkdir /apps/php/log/

[root@nginx-1 php-fpm.d]cd /apps/php/etc/
[root@nginx-1 etc]cp php-fpm.conf.default php-fpm.conf

注意点：
nginx服务中的2台机器中的id必须保持一致，不一致就配置如下操作

[root@nginx-1 php-fpm.d]#userdel -rf nginx 
userdel: user nginx is currently used by process 28134
[root@nginx-1 php-fpm.d]#useradd nginx -u 2020
[root@nginx-1 php-fpm.d]#id nginx
uid=2020(nginx) gid=2020(nginx) groups=2020(nginx)

4.1.3启动并验证php-fpm：

[root@nginx-1 etc]#/apps/php/sbin/php-fpm -t #检测php配置文件
[11-Apr-2020 00:42:29] NOTICE: configuration file /apps/php/etc/php-fpm.conf test is successful

[root@nginx-1 etc]#/apps/php/sbin/php-fpm -c /apps/php/etc/php.ini #-c指定开机启动文件
[root@nginx-1 etc]#ps -ef | grep php-fpm
root      63958      1  0 00:42 ?        00:00:00 php-fpm: master process (/apps/php/etc/php-fpm.conf)
nginx     63959  63958  0 00:42 ?        00:00:00 php-fpm: pool www
nginx     63960  63958  0 00:42 ?        00:00:00 php-fpm: pool www
nginx     63961  63958  0 00:42 ?        00:00:00 php-fpm: pool www
nginx     63962  63958  0 00:42 ?        00:00:00 php-fpm: pool www
nginx     63963  63958  0 00:42 ?        00:00:00 php-fpm: pool www
root      63969  39163  0 00:42 pts/1    00:00:00 grep --color=auto php-fpm
[root@nginx-1 etc]#netstat -tanlp | grep php-fpm
tcp        0      0 127.0.0.1:9000          0.0.0.0:*               LISTEN      63958/php-fpm: mast 

4.1.4验证PHP是否编译成功

[root@nginx-1 etc]#mkdir /data/nginx/wordpress -p
[root@nginx-1 etc]#cat /data/nginx/wordpress/index.php 
<?php
phpinfo();
?>

5.1HAproxy部署

HAproxy部署在机器节点172.20.10.103和172.20.10.104上

root@Haproxy-1:~# apt install nfs-server -y
root@Haproxy-1:~# vim /etc/haproxy/haproxy.cfg

#在最后添加
listen mysql-3306
  bind 192.168.32.103:3306 #本机监听后端服务器105的3306端口
  mode tcp
  server mysql 192.168.32.105:3306 check inter 3s fall 3 rise 5
root@Haproxy-1:~# systemctl restart haproxy
root@Haproxy-1:~# ss -ntl #查看80端口是否起来

在192.168.32.105的nginx-1机器安装了mysql的机器中测试是否能通过HAproxy是否能够完成调度访问

[root@nginx-1 ]#/usr/local/mysql/bin/mysql -uwordpress -p123456 -h192.168.32.103
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql>
#能够进入数据库表示能够正常调度到后端服务器

6.1部署wordpress

[root@nginx-1 ~]#cd /data/nginx/wordpress/
[root@nginx-1 wordpress]#tar xf wordpress-5.3-zh_CN.tar.gz 
[root@nginx-1 wordpress]#mv wordpress/* .
[root@nginx-1 wordpress]#mv wordpress/ /opt

[root@nginx-1 wordpress]#cp wp-config-sample.php wp-config.php 
[root@nginx-1 wordpress]#vim wp-config.php
define( 'DB_NAME', 'wordpress' );

/** MySQL数据库用户名 */
define( 'DB_USER', 'wordpress' );

/** MySQL数据库密码 */
define( 'DB_PASSWORD', '123456' );

/** MySQL主机 */
define( 'DB_HOST', '192.168.32.103' ); #到时候可以换成VIP，这里暂时先用Haproxy机器103测试

/** 创建数据表时默认的文字编码 */
define( 'DB_CHARSET', 'utf8' );

/** 数据库整理类型。如不确定请勿更改 */
define( 'DB_COLLATE', '' );

window主机临时解析地址

C:\Windows\System32\drivers\etc\hosts
192.168.32.105 www.duanxin.io

访问www.duanxin.io


访问数据库：

[root@nginx-1 wordpress]#/usr/local/mysql/bin/mysql 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> use wordpress;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> show database;

mysql> show tables;
+-----------------------+
| Tables_in_wordpress   |
+-----------------------+
| wp_commentmeta        |
| wp_comments           |
| wp_links              |
| wp_options            |
| wp_postmeta           |
| wp_posts              |
| wp_term_relationships |
| wp_term_taxonomy      |
| wp_termmeta           |
| wp_terms              |
| wp_usermeta           |
| wp_users              |
+-----------------------+
12 rows in set (0.00 sec)

注意：上传图片等需要对应的权限，对于现在的nginx的用户是nginx（2020） 对于php也是nginx用户

[root@nginx-1 wordpress]#chown nginx.nginx /data/nginx/wordpress/ -R
[root@nginx-1 wordpress]#ll
total 192
-rw-r--r-- 1 nginx nginx   420 Apr 11 01:25 index.php
-rw-r--r-- 1 nginx nginx 19935 Apr 11 01:25 license.txt
-rw-r--r-- 1 nginx nginx  7005 Apr 11 01:25 readme.html
-rw-r--r-- 1 nginx nginx  6939 Apr 11 01:25 wp-activate.php
-rw-r--r-- 1 nginx nginx   369 Apr 11 01:25 wp-blog-header.php
-rw-r--r-- 1 nginx nginx  2283 Apr 11 01:25 wp-comments-post.php
-rw-r--r-- 1 nginx nginx  3039 Apr 11 01:25 wp-config.php
-rw-r--r-- 1 nginx nginx  2776 Apr 11 01:25 wp-config-sample.php
-rw-r--r-- 1 nginx nginx  3955 Apr 11 01:25 wp-cron.php
-rw-r--r-- 1 nginx nginx  2504 Apr 11 01:25 wp-links-opml.php
-rw-r--r-- 1 nginx nginx  3326 Apr 11 01:25 wp-load.php
-rw-r--r-- 1 nginx nginx 47007 Apr 11 01:25 wp-login.php
-rw-r--r-- 1 nginx nginx  8483 Apr 11 01:25 wp-mail.php
-rw-r--r-- 1 nginx nginx 19120 Apr 11 01:25 wp-settings.php
-rw-r--r-- 1 nginx nginx 31112 Apr 11 01:25 wp-signup.php
-rw-r--r-- 1 nginx nginx  4764 Apr 11 01:25 wp-trackback.php
-rw-r--r-- 1 nginx nginx  3150 Apr 11 01:25 xmlrpc.php

部署成功。但是以上依旧是单机部署，没有形成高可用机构的LNMP

6.1.2 拷贝wordpress文件到master-B

[root@nginx-1 wordpress]#scp -r ./ 192.168.32.106:/data/nginx/wordpress/

7.1NFS挂载

这里复用Haproxy-1的172.20.10.103的主机节点

root@Haproxy-1:~# apt install nfs-server -y
root@Haproxy-1:~# mkdir /data/wordpress -p

root@Haproxy-1:~# vim /etc/exports 
/data/wordpress *(rw,no_root_squash)

root@Haproxy-1:~# systemctl restart nfs-server
root@Haproxy-1:~# systemctl enable nfs-server

这里需要注意的是 在数据挂载之前先要把数据备份到其他地方之后才挂载，不然数据可能会丢失

#在其他安装了NFS服务器中查看是否共享
root@Haproxy-2:~# showmount -e 192.168.32.103
Export list for 192.168.32.103:
/data/wordpress *

7.1.1nginx-web服务器配置

nginx-web服务器都需要配置NFS进行挂载。用来存储静态文件，wordpress 的静态文件放在/data/nginx/wordpress/wp-content/uploads的目录下面

[root@nginx-1 wordpress]#yum install nfs-utils -y
[root@nginx-1 wordpress]#mount -t nfs 192.168.32.103:/data/wordpress /data/nginx/wordpress/wp-content/uploads

[root@nginx-1 wordpress]#vim /etc/fstab #开机自动挂载
192.168.32.103:/data/wordpress /data/nginx/wordpress/wp-content/uploads nfs defaults,_netdev 0 0  #这里注意要添加_netdev

[root@nginx-1 wordpress]#mount -a
[root@nginx-1 wordpress]#df -TH
Filesystem                     Type      Size  Used Avail Use% Mounted on
devtmpfs                       devtmpfs  1.5G     0  1.5G   0% /dev
tmpfs                          tmpfs     1.5G     0  1.5G   0% /dev/shm
tmpfs                          tmpfs     1.5G   11M  1.5G   1% /run
tmpfs                          tmpfs     1.5G     0  1.5G   0% /sys/fs/cgroup
/dev/sda2                      xfs       108G  5.4G  102G   6% /
/dev/sda5                      xfs        54G  457M   54G   1% /data
/dev/sda1                      xfs       1.1G  185M  880M  18% /boot
tmpfs                          tmpfs     289M     0  289M   0% /run/user/0
192.168.32.103:/data/wordpress nfs4      106G  4.4G   96G   5% /data/nginx/wordpress/wp-content/uploads

用nginx-1：192.168.32.105在wordpress的媒体库中上传一张图片
然后测试把window的hosts改为192.168.32.106 看看是否能访问到数据

#修改window中解析文件

C:\Windows\System32\drivers\etc\hosts
192.168.32.106 www.duanxin.io

访问www.duanxin.io

访问成功，说明挂载没问题

8.1LVS部署高可用

LVS1 主机节点172.20.10.100
LVS2 主机节点 172.20.10.101
在这2个节点中都要安装ipvsadm和keepalived

# apt install ipvsadm keepalived -y
# find / -name "keepalived*"
# cp /usr/share/doc/keepalived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf
# vim /etc/keepalived/keepalived.conf #LVS-1中的keepalived配置文件
vrrp_instance VI_1 {
    state MASTER #标识为master
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 58 #修改不同id  这里是Ubantu 所以不用加iptable规则
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.248 dev eth0 label eth0:0 #配置VIP
    }
}

root@LVS-1:~# systemctl restart keepalived
root@LVS-1:~# systemctl enable keepalived

root@LVS-1:~# ifconfig  #检测是否有192.168.32.248的VIP生成
eth0:0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.32.248  netmask 255.255.255.255  broadcast 0.0.0.0
        ether 00:0c:29:75:30:12  txqueuelen 1000  (Ethernet)

root@LVS-1:~# scp /etc/keepalived/keepalived.conf 192.168.32.102:/etc/keepalived/keepalived.conf #从LVS-1服务器中拷贝到LVS-2中

root@LVS-2:~# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state BACKUP  #名称
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 68 #id名称
    priority 80 #优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.248 dev eth0 label eth0:0
    }
}
root@LVS-2:~# systemctl restart keepalived
root@LVS-2:~# systemctl enable keepalived

#测试VIP的迁移
root@LVS-1:~# systemctl stop keepalived #停止
root@LVS-2:~# ifconfig  #看是否会把VIP飘过来
eth0:0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.32.248  netmask 255.255.255.255  broadcast 0.0.0.0
        ether 00:0c:29:7a:10:5d  txqueuelen 1000  (Ethernet)

8.1.1LVS_DR绑定脚本

#cat lvs_dr.sh 
#!/bin/bash
LVS_VIP=192.168.32.248
#source /etc/rc.d/init.d/functions
case "$1" in
start)
	/sbin/ifconfig lo:0 $LVS_VIP netmask 255.255.255.255 broadcast $LVS_VIP
	/sbin/route add -host $LVS_VIP dev lo:0
	echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
	echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
	echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
	echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
	sysctl -p >/dev/null 2>&1
	echo "Realserver start OK"
	;;
stop)
	/sbin/ifconfig lo:0 down
        /sbin/route del $LVS_VIP >/dev/null 2>&1
	echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
	echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
	echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
	echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
	echo "Realserver stopd"
	;;
*)
	echo "Usage: $(basename $0) start|stop"
	exit 1
	;;
esac
exit 0
#bash lvs_dr.sh start

8.1.2LVS_Keepalived配置

root@LVS-1:~# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 58
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.248 dev eth0 label eth0:0
    }
}

virtual_server 192.168.32.248 80 { #虚拟地址VIP
    delay_loop 6
    lb_algo wrr
    lb_kind DR
    persistence timeout 120
    protocol TCP

    real_server 192.168.32.105 80{ #调度到的后端服务器192.168.32.105
	weight 1
	TCP_CHECK {
	connect_timeout 5
	nb_get_retry 3
	 connect_port 80
        }
   }
    real_server 192.168.32.106 80 {  #调度到的后端服务器192.168.32.106
        weight 1
        TCP_CHECK {
        connect_timeout 5
        nb_get_retry 3
         connect_port 80
        }
   }
}

# systemctl restart keepalived
# systemctl enable keepalived

root@LVS-1:~# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.32.248:80 wrr
  -> 192.168.32.105:80            Route   1      0          0         
  -> 192.168.32.106:80            Route   1      0          0   

在LVS-2中的keepalived配置一样
后端服务器配置完成

测试：把window的hosts解析到192.168.32.248进行测试

C:\Windows\System32\drivers\etc\hosts
192.168.32.248 www.duanxin.io

通过上面的结构实现了以下的一种高可用的架构：

用户直接通过Keepalived服务的VIP和LVS的lo绑定。从而访问后端的NGINGX服务器，但是访问返回时不会再一次经过LVS，而是直接通过NGINX服务的网关路由就直接给用户返回请求的信息报文。

9.1实现架构lvs向haproxy访问调度

9.1.1扩展架构演示

扩展架构演示主要是在上一层基础之上在中间加了一层Haproxy调度
在实际的生产中并没有上面意义，这里只是为了更加好的理解LNMP高可用架构的优化

Haproxy机器192.168.32.103和192.168.32.104都部署安装了haproxy和keepalived

root@Haproxy-1:~# vim lvs_dr.sh  # haproxy机器绑定lo
#!/bin/bash
LVS_VIP=192.168.32.248
#source /etc/rc.d/init.d/functions
case "$1" in
start)
        /sbin/ifconfig lo:0 $LVS_VIP netmask 255.255.255.255 broadcast $LVS_VIP
        /sbin/route add -host $LVS_VIP dev lo:0
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        sysctl -p >/dev/null 2>&1
        echo "Realserver start OK"
        ;;
stop)
        /sbin/ifconfig lo:0 down
        /sbin/route del $LVS_VIP >/dev/null 2>&1
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo "Realserver stopd"
        ;;
*)
        echo "Usage: $(basename $0) start|stop"
        exit 1
        ;;
esac
exit 0
root@Haproxy-1:~# bash lvs_dr.sh start
root@Haproxy-1:~# ifconfig 
lo:0: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 192.168.32.248  netmask 255.255.255.255
        loop  txqueuelen 1000  (Local Loopback)

让LVS中的后端服务器的调度改为调度到Haproxy上，这里LVS的2台主机节点都要配置
root@LVS-1:~# vim /etc/keepalived/keepalived.conf  #修改地址调度到haproxy服务器中
37    real_server 192.168.32.103 80 { #把之前的105改为103
45    real_server 192.168.32.104 80 { #把之前的106改为104

root@LVS-1:~## systemctl restart keepalived
root@LVS-1:~# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.32.248:80 wrr
  -> 192.168.32.103:80            Route   1      0          0         
  -> 192.168.32.104:80            Route   1      0          0   

root@LVS-2:~# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.32.248:80 wrr
  -> 192.168.32.103:80            Route   1      0          0         
  -> 192.168.32.104:80            Route   1      0          0   

配置haproxy配置文件

root@Haproxy-1:~# vim /etc/haproxy/haproxy.cfg
listen mysql-3306
  bind 192.168.32.103:3306
  mode tcp
  server mysql 192.168.32.105:3306 check inter 3s fall 3 rise 5

listen nginx-80
  bind 192.168.32.103:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5
root@Haproxy-1:~# systemctl restart haproxy

让全部的Haproxy后端调度到nginx服务器中
root@Haproxy-1:~# vim /etc/haproxy/haproxy.cfg
listen nginx-80
  bind 192.168.32.104:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5

root@Haproxy-1:~# systemctl restart haproxy

让haproxy的端口监听后端的服务器地址为nginx-1和nginx-2

现在单独用192.168.32.103和192.168.32.104服务器单独访问是可以正常访问wordpress的

但是通过LVS先调度到Haproxy，再让Haproxy调度到后端的nginx服务器则失败

这里的VIP调度到的服务器是192.168.32.103和192.168.32.104，但是访问VIP失败

访问VIP

root@Haproxy-2:~# vim /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1
root@Haproxy-2:~# sysctl -p

root@Haproxy-2:~# sysctl -a |grep net.ipv4.ip_nonlocal_bind  #修改参数为1
net.ipv4.ip_nonlocal_bind = 1  

root@Haproxy-1:~# vim /etc/haproxy/haproxy.cfg
listen mysql-3306
  bind 192.168.32.103:3306
  mode tcp
  server mysql 192.168.32.105:3306 check inter 3s fall 3 rise 5

listen nginx-80
  bind 192.168.32.103:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5
root@Haproxy-1:~# systemctl restart haproxy

root@Haproxy-2:~# vim /etc/haproxy/haproxy.cfg
listen nginx-80
  bind 192.168.32.104:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5

root@Haproxy-2:~# systemctl restart haproxy

# ss -ntl #查看192.168.32.248：80是否监听

访问VIP

但是单独访问haproxy服务器仍然调度不到后端服务器


这里是因为VIP绑定的是haproxy的lo回环网卡的地址 回环网卡只能在本机重复循迹报文。但是无法调度到后端服务器
这里需要在调度VIP的时候追加绑定一个自己本身的地址监听后端服务器的端口地址

root@Haproxy:~# vim /etc/haproxy/haproxy.cfg
listen nginx-80
  bind 192.168.32.248:80,192.168.32.103:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5
root@Haproxy:~# systemctl restart haproxy

10.1优化架构

这里优化架构直接在haproxy中配合keepalived的VIP直接对后端服务器（Nginx）的调用
不用LVS绑定lo和VIP来访问

root@Haproxy:~# apt install keepalived -y
root@Haproxy:~# find / -name "keepalived*"
root@Haproxy:~# cp /usr/share/doc/keepalived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf

root@Haproxy:~# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
  
global_defs {
   notification_email {
     acassen
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 88
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.249 dev eth0 label eth0:0
    }
}

root@Haproxy-1:~# systemctl restart keepalived
root@Haproxy-1:~# ifconfig 

eth0:0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.32.249  netmask 255.255.255.255  broadcast 0.0.0.0
        ether 00:0c:29:9f:2d:49  txqueuelen 1000  (Ethernet)

root@Haproxy-2:~# vim /etc/keepalived/keepalived.conf 

! Configuration File for keepalived
  
global_defs {
   notification_email {
     acassen
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state BACKUP #标识
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 89  #不同id即可
    priority 80 #优先级比master小
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.249 dev eth0 label eth0:0
    }
}
root@Haproxy-2:~# systemctl restart keepalived

在haproxy的配置文件中增加一个访问入口以及修改数据库的访问地址
在配置文件最后增加后端服务器

root@Haproxy:~# vim /etc/haproxy/haproxy.cfg

listen mysql-3306
  bind 192.168.32.249:3306  #修改192.168.32.105为VIP地址
  mode tcp
  server mysql 192.168.32.105:3306 check inter 3s fall 3 rise 5

listen nginx-80
  bind 192.168.32.248:80,192.168.32.103:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5

listen nginx-80-249
  bind 192.168.32.249:80
  mode tcp
  server web1 192.168.32.105:80 check inter 3s fall 3 rise 5
  server web2 192.168.32.106:80 check inter 3s fall 3 rise 5


root@Haproxy:~# systemctl restart haproxy
root@Haproxy:~#ss -ntl #查看 192.168.32.249:80  是否监听

这里报错是因为上面数据库访问地址修改成了VIP，修改数据库的访问地址 数据库服务器在Nginx服务器上复用

[root@nginx-1 ~]#vim /data/nginx/wordpress/wp-config.php 
/** MySQL主机 */
 32 define( 'DB_HOST', '192.168.32.249' );

再次访问

通过VIP192.168.32.249调度访问

10.2数据库高可用

通过VIP浮动来调度后端数据库主机。数据库之间采用双主结构复制
192.168.32.105为Master-1
192.168.32.106为Master-2

#在192.168.32.105的Master-1操作
[root@nginx-1 ~]#vim /etc/my.cnf
[mysqld]
server-id=105
log-bin
auto_increment_offset=1
auto_increment_increment=2
[root@nginx-1 ~]#systemctl restart mysql

#在192.168.32.106的Master-2操作
[root@nginx-2 ~]#vim /etc/my.cnf
[mysqld]
server-id=106
log-bin
auto_increment_offset=2
auto_increment_increment=2
[root@nginx-2 ~]#systemctl restart mysql

#在192.168.32.105的Master-1操作
[root@nginx-1 ]#/usr/local/mysql/bin/mysql
mysql> show master logs;
+--------------------+-----------+
| Log_name           | File_size |
+--------------------+-----------+
| nginx-1-bin.000001 |      865  | 
+--------------------+-----------+
1 row in set (0.24 sec)
mysql> GRANT REPLICATION SLAVE ON *.* TO 'tongbu'@'192.168.32.%' IDENTIFIED BY '123456';
Query OK, 0 rows affected (0.74 sec)

mysql> FLUSH PRIVILEGES;
Query OK, 0 rows affected (0.00 sec)

#在192.168.32.105的Master-1操作备份
[root@nginx-1 ~]#mkdir /data/backup;/usr/local/mysql/bin/mysqldump -uroot -p -A -F --single-transaction --master-data=2 | gzip > /data/backup/all-`date +%F`.sql.gz

#把完全备份的压缩包拷贝到192.168.32.106的Master-2
[root@nginx-1 backup]#scp all-2020-04-12.sql.gz 192.168.32.106:/root

#在192.168.32.106的Master-2操作
#因为导入备份库会生成大量垃圾二进制日志，所以临时关闭二进制日志
mysql> SET SQL_LOG_BIN=OFF;
Query OK, 0 rows affected (0.00 sec)

#在192.168.32.106的Master-2操作
[root@nginx-2 ~]#gzip -d all-2020-04-12.sql.gz 
[root@nginx-2 ~]#/usr/local/mysql/bin/mysql < all-2020-04-12.sql 

#在192.168.32.106的Master-2操作
[root@nginx-2 ]#/usr/local/mysql/bin/mysql
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| test1              |
| wordpress          |
+--------------------+
6 rows in set (0.03 sec)

mysql> use wordpress;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> show tables;
+-----------------------+
| Tables_in_wordpress   |
+-----------------------+
| wp_commentmeta        |
| wp_comments           |
| wp_links              |
| wp_options            |
| wp_postmeta           |
| wp_posts              |
| wp_term_relationships |
| wp_term_taxonomy      |
| wp_termmeta           |
| wp_terms              |
| wp_usermeta           |
| wp_users              |
+-----------------------+
12 rows in set (0.00 sec)

mysql> show master logs;
+--------------------+-----------+
| Log_name           | File_size |
+--------------------+-----------+
| nginx-2-bin.000001 |       753 |
| nginx-2-bin.000002 |       143 |
| nginx-2-bin.000003 |   1267096 |
+--------------------+-----------+
3 rows in set (0.00 sec)

mysql> SET SQL_LOG_BIN=ON; #导入备份数据之后打开二进制文件
Query OK, 0 rows affected (0.00 sec)

mysql> CHANGE MASTER TO
    -> MASTER_HOST='192.168.32.105',
    -> MASTER_USER='tongbu',
    -> MASTER_PASSWORD='123456',
    -> MASTER_PORT=3306,
    -> MASTER_LOG_FILE='nginx-1-bin.000003',
    -> MASTER_LOG_POS=865; #这个pos需要和最开始之前的master-1的File_size保持一致
Query OK, 0 rows affected, 2 warnings (0.01 sec)

mysql> start slave; #开启线程同步
Query OK, 0 rows affected (0.01 sec)

mysql> show slave status\G 查看slave状态
  
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 192.168.32.105
                  Master_User: tongbu
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: nginx-1-bin.000003
          Read_Master_Log_Pos: 120
               Relay_Log_File: nginx-2-relay-bin.000002
                Relay_Log_Pos: 285
        Relay_Master_Log_File: nginx-1-bin.000003
             Slave_IO_Running: Yes    
            Slave_SQL_Running: Yes
              Replicate_Do_DB: 
          Replicate_Ignore_DB: 
           Replicate_Do_Table: 
       Replicate_Ignore_Table: 
      Replicate_Wild_Do_Table: 
  Replicate_Wild_Ignore_Table: 
                   Last_Errno: 0
                   Last_Error: 
                 Skip_Counter: 0
          Exec_Master_Log_Pos: 120
              Relay_Log_Space: 460
              Until_Condition: None
               Until_Log_File: 
                Until_Log_Pos: 0
           Master_SSL_Allowed: No
           Master_SSL_CA_File: 
           Master_SSL_CA_Path: 
              Master_SSL_Cert: 
            Master_SSL_Cipher: 
               Master_SSL_Key: 
        Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
                Last_IO_Errno: 0
                Last_IO_Error: 
               Last_SQL_Errno: 0
               Last_SQL_Error: 
  Replicate_Ignore_Server_Ids: 
             Master_Server_Id: 105
                  Master_UUID: ea1b88e1-7b32-11ea-8b8d-000c29a4149e
             Master_Info_File: /data/mysql/master.info
                    SQL_Delay: 0
          SQL_Remaining_Delay: NULL
      Slave_SQL_Running_State: Slave has read all relay log; waiting for the slave I/O thread to update it
           Master_Retry_Count: 86400
                  Master_Bind: 
      Last_IO_Error_Timestamp: 
     Last_SQL_Error_Timestamp: 
               Master_SSL_Crl: 
           Master_SSL_Crlpath: 
           Retrieved_Gtid_Set: 
            Executed_Gtid_Set: 
                Auto_Position: 0
1 row in set (0.00 sec)
#一定要  Slave_IO_Running: Yes 以及 Slave_SQL_Running: Yes

#在192.168.32.105的Master-1操作

mysql> CHANGE MASTER TO
    -> MASTER_HOST='192.168.32.106',
    -> MASTER_USER='tongbu',
    -> MASTER_PASSWORD='123456',
    -> MASTER_PORT=3306,
    -> MASTER_LOG_FILE='nginx-2-bin.000003',
    -> MASTER_LOG_POS=1267096;
Query OK, 0 rows affected, 2 warnings (0.00 sec)

mysql> start slave;
Query OK, 0 rows affected (0.00 sec)

mysql> show slave status\G
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 192.168.32.106
                  Master_User: tongbu
                  Master_Port: 3306
                Connect_Retry: 10
              Master_Log_File: nginx-2-bin.000003
          Read_Master_Log_Pos: 1267096
               Relay_Log_File: nginx-1-relay-bin.000002
                Relay_Log_Pos: 285
        Relay_Master_Log_File: nginx-2-bin.000003
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
              Replicate_Do_DB: 
          Replicate_Ignore_DB: 
           Replicate_Do_Table: 
       Replicate_Ignore_Table: 
      Replicate_Wild_Do_Table: 
  Replicate_Wild_Ignore_Table: 
                   Last_Errno: 0
                   Last_Error: 
                 Skip_Counter: 0
          Exec_Master_Log_Pos: 1267096
              Relay_Log_Space: 460
              Until_Condition: None
               Until_Log_File: 
                Until_Log_Pos: 0
           Master_SSL_Allowed: No
           Master_SSL_CA_File: 
           Master_SSL_CA_Path: 
              Master_SSL_Cert: 
            Master_SSL_Cipher: 
               Master_SSL_Key: 
        Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
                Last_IO_Errno: 0
                Last_IO_Error: 
               Last_SQL_Errno: 0
               Last_SQL_Error: 
  Replicate_Ignore_Server_Ids: 
             Master_Server_Id: 106
                  Master_UUID: 846a0729-7b3a-11ea-8bbe-000c29320b76
             Master_Info_File: /data/mysql/master.info
                    SQL_Delay: 0
          SQL_Remaining_Delay: NULL
      Slave_SQL_Running_State: Slave has read all relay log; waiting for the slave I/O thread to update it
           Master_Retry_Count: 86400
                  Master_Bind: 
      Last_IO_Error_Timestamp: 
     Last_SQL_Error_Timestamp: 
               Master_SSL_Crl: 
           Master_SSL_Crlpath: 
           Retrieved_Gtid_Set: 
            Executed_Gtid_Set: 
                Auto_Position: 0
1 row in set (0.00 sec)

测试是否同步数据：

#在192.168.32.105的Master-1操作
mysql> create database testDB;
Query OK, 1 row affected (0.00 sec)

mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testDB             |
| wordpress          |
+--------------------+

#在192.168.32.106的Master-2操作
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testDB             |
| wordpress          |
+--------------------+
10 rows in set (0.00 sec)

测试停止一个数据库是否能够正常访问，如果能，则实现了数据库的双主高可用