小型分布式LNMP平台架构设计与搭建-------实现web服务器的负载均衡和前后端分离

一、项目介绍

1、项目介绍

        本项目使用九台云服务器，搭建小型分布式LNMP平台，实现一个群集，在前面部署一个负载均衡服务器，后面几台服务器完成同一业务。如果有用户进行相应业务访问时，负载均衡器根据后端哪台服务器的负载情况，决定由给哪一台去完成响应，并且一台服务器垮了，其它的服务器可以顶上来。配备mysql的主从架构和文件共享及同步系统，来确保数据的安全。

2、项目背景

        为保证服务器的高可用性、网络隔离性和数据的安全性。本次项目采用MySQL主和NFS+backup确保数据的安全性和高可用性，采用LVS-DR多网段实现集群的高可用性和网络的隔离性

二、项目环境（本架构图是NAT，后附DR多网段搭建教程）

主机系统：Ubuntu22.04.1

架构图：

 

主机IP	服务名称	版本号
eth0:192.168.10.10 eth1:172.18.0.11	firwalld+DNAT	无
eth0:10.0.0.209 eth1:192.168.10.11	LVS_NAT	无
eth0:10.0.0.201 eth1:172.18.0.10	OpeaVPN	openvpn 2.5.5
eth0:10.0.0.202	DNS+Chronyd	bind9-1:9.18.1
eth0:10.0.0.203	server2	nginx 1.18.0
eth0:10.0.0.204	server1	nginx 1.18.0
eth0:10.0.0.208	msyql_slave	MySQL8.0
eth0:10.0.0.206	mysql_master	MySQL8.0
eth0:10.0.0.205	NFS	nfs-kernel-server 1:2.6.1
eth0:10.0.0.207	BackUp	无
172.18.0.9	访问主机

 

三、安装教程

注：防火墙策略和DNAT每个公司不一样所以在此就不展示

1）mysql 安装

（1）安装吗msyql8.0（我选择的是二进制安装，包安装也可以）

执行脚本mysql_install_bin

        执行注意事项：

                1、保证3306端口没有被占用。

                2、将二进制包：mysql-8.0.31-linux-glibc2.12-x86_64.tar.xz和脚本放到同一目录下

二进制包下载网址：MySQL :: Download MySQL Community Server (Archived Versions)。

                3、如果是包安装需要检查端口是不是开放的。 

cat mysql_install_bin
#/bin/bash/
#mysql压缩包放到/root/ 下面
TarName=mysql-8.0.31-linux-glibc2.12-x86_64.tar.xz
ls ~/${TarName} ||(echo "tar of mysql unclear";exit)
. /etc/os-release
if	[ ${ID} = 'rocky' -o ${ID} = 'centos' ];then
	echo "rocky";exit
		
elif [ ${ID} = 'ubuntu'	];then
	echo "ubuntu"
else	
	echo " versions unclear"
	exit 13
fi
apt update &>/dev/null ||(echo "apt unclear";exit)
apt -y  install libtinfo5 libncurses5 libaio-dev numactl ||exit 14
groupadd mysql
useradd -r -g mysql -s /bin/false mysql 
cd /usr/local || mkdir -pv /data/mydql
tar xf ~/$TarName  -C /usr/local			||exit 18
TarName1=${TarName%.tar.xz}
ln -s ${TarName1}/ /usr/local/mysql		||exit 19
ln -s /usr/local/mysql/bin/* /usr/bin/			||exit 20
echo 'PATH=/usr/local/mysql/bin:$PATH' > /etc/profile.d/mysql.sh
. /etc/profile.d/mysql.sh
cp /etc/my.cnf{,.bak}						
cat > /etc/my.cnf << EOF
[mysqld]
server-id=`hostname -I|cut -d. -f4`
log-bin
datadir=/data/mysql
socket=/data/mysql/mysql.sock         
log-error=/data/mysql/mysql.log
pid-file=/data/mysql/mysql.pid
default_authentication_plugin=mysql_native_password
#default_authentication_plugin=caching_sha2_password
[client]
socket=/data/mysql/mysql.sock
EOF
mkdir -pv /data/mysql  
mysqld --initialize-insecure --user=mysql --datadir=/data/mysql ||exit 39
cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld
systemctl enable  mysqld
systemctl start mysql

（2）部署master

#default_authentication_plugin=mysql_native_password
mysql -e 'create user 'caojidong'@'10.0.0.%' identified by '123456';'
mysql -e 'grant replication slave on *.* to 'caojidong'@'10.0.0.%';'
cat > ~/mysql_M_S << EOF
CHANGE MASTER TO MASTER_HOST='`hostname -I`',   
MASTER_USER='caojidong',   
MASTER_PASSWORD='123456',
MASTER_PORT=3306, 
MASTER_LOG_FILE='`mysql -e 'show master status;'|grep server|awk -F' ' '{print $1}'`',
MASTER_LOG_POS=`mysql -e 'show master status;'|grep server|awk -F' ' '{print $2}'`;
EOF

mysqldump -A -F --single-transaction --master-data=1 > /backup/fullbackup_`date +%F_%T`.sql
[root@slave ~]#vim /data/fullbackup_2019-11-27_17\:41\:17.sql
CHANGE MASTER TO
MASTER_HOST='10.0.0.8',
MASTER_USER='repluser',
MASTER_PASSWORD='wang',
MASTER_PORT=3306,                                                               
          MASTER_LOG_FILE='mariadb-bin.000003', MASTER_LOG_POS=389;
scp ~/mysql_M_S 10.0.0.12:

（3）部署slave

mysql < /data/fullbackup_2019-11-27_17\:41\:17.sql
 start slave;

2） server安装（编译安装nginx）

下载网址：nginx: download

#!/bin/bash
#
#****************************************************
#Author:           caojidong
#QQ:               1549396190
#Date:             2022-12-31
#FileName:         install.nginx.sh
#cell-phone number:               13739548267
#Description:      test
#Copyright(C):     2022 All right
#***************************************************
NGINX_FILE=nginx-1.20.2
#NGINX_FILE=nginx-1.18.0
NGINX_URL=http://nginx.org/download/
TAR=.tar.gz
SRC_DIR=/usr/local/src
NGINX_INSTALL_DIR=/apps/nginx
CPUS=`lscpu |awk '/^CPU\(s\)/{print $2}'`
. /etc/os-release
color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \\033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\033[1;31m"
    SETCOLOR_WARNING="echo -en \\033[1;33m"
    SETCOLOR_NORMAL="echo -en \E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        ${SETCOLOR_SUCCESS}
        echo -n $" OK "    
    elif [ $2 = "failure" -o $2 = "1" ] ;then 
        ${SETCOLOR_FAILURE}
        echo -n $"FAILED"
    else
        ${SETCOLOR_WARNING}
        echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo
}
check () {
   [ -e ${NGINX_INSTALL_DIR} ] && { color "nginx 已安装,请卸载后再安装" 1; exit; }
    cd  ${SRC_DIR}
    if [  -e ${NGINX_FILE}${TAR} ];then
       color "相关文件已准备好" 0
    else
       color '开始下载 nginx 源码包' 0
        wget ${NGINX_URL}${NGINX_FILE}${TAR}
       [ $? -ne 0 ] && { color "下载 ${NGINX_FILE}${TAR}文件失败" 1; exit; } 
    fi
} 
install () {
   color "开始安装 nginx" 0
    if id nginx &> /dev/null;then
       color "nginx 用户已存在" 1
    else
       useradd -s /sbin/nologin -r nginx
       color "创建 nginx 用户" 0
    fi
   color "开始安装 nginx 依赖包" 0
    if [ $ID == "centos" ] ;then
    if [[ $VERSION_ID =~ ^7 ]];then
           yum -y -q install make gcc pcre-devel openssl-devel zlib-devel 
perl-ExtUtils-Embed
 elif [[ $VERSION_ID =~ ^8 ]];then
           yum -y -q install make gcc-c++ libtool pcre pcre-devel zlib zlibdevel openssl openssl-devel perl-ExtUtils-Embed 
 else
 color '不支持此系统!'  1
            exit
        fi
    elif [ $ID == "rocky" ];then
   yum -y -q install make gcc-c++ libtool pcre pcre-devel zlib zlib-devel 
openssl openssl-devel perl-ExtUtils-Embed 
 else
       apt update &> /dev/null
       apt -y install make gcc libpcre3 libpcre3-dev openssl libssl-dev zlib1gdev &> /dev/null
    fi
    cd $SRC_DIR
   tar xf ${NGINX_FILE}${TAR}
    NGINX_DIR=`echo ${NGINX_FILE}${TAR}| sed -nr 's/^(.*[0-9]).*/\1/p'`
    cd ${NGINX_DIR}
   ./configure --prefix=${NGINX_INSTALL_DIR} --user=nginx --group=nginx --withhttp_ssl_module --with-http_v2_module --with-http_realip_module --withhttp_stub_status_module --with-http_gzip_static_module --with-pcre --with-stream
--with-stream_ssl_module --with-stream_realip_module
    make -j $CPUS && make install 
   [ $? -eq 0 ] && color "nginx 编译安装成功" 0 || { color "nginx 编译安装失败,退
出!" 1 ;exit; }
    echo "PATH=${NGINX_INSTALL_DIR}/sbin:${PATH}" > /etc/profile.d/nginx.sh
    chown -R nginx.nginx ${NGINX_INSTALL_DIR}
    cat > /lib/systemd/system/nginx.service <<EOF
[Unit]
Description=The nginx HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target
[Service]
Type=forking
PIDFile=${NGINX_INSTALL_DIR}/logs/nginx.pid
ExecStartPre=/bin/rm -f ${NGINX_INSTALL_DIR}/logs/nginx.pid
ExecStartPre=${NGINX_INSTALL_DIR}/sbin/nginx -t
ExecStart=${NGINX_INSTALL_DIR}/sbin/nginx
ExecReload=/bin/kill -s HUP \$MAINPID
KillSignal=SIGQUIT
TimeoutStopSec=5
KillMode=process
PrivateTmp=true
LimitNOFILE=100000
[Install]
WantedBy=multi-user.target
EOF
   systemctl daemon-reload
   systemctl enable --now nginx &> /dev/null 
   systemctl is-active nginx &> /dev/null || { color "nginx 启动失败,退出!" 1 ; 
exit; }
   color "nginx 安装完成" 0
}
check
install

3)NFSan 安装

apt install nfs-kernel-server
mkdir -pv /data/home
useradd -d /data/home/cao -u 2000 cao
Vim /etc/exports.d/test.exports
apt install rsync -y
apt isntall 

4）backup 安装

5）DNS安装

#/bin/bash
#安装bind
apt -y install bind9 bind9-utils bind9-host bind9-dnsutils &>/dev/null
ls /etc/bind/
cat >> /etc/bind/named.conf.default-zones << EOF
zone "cao.com" {
	type master;
	file "/etc/bind/db.cao";
};
EOF
touch /etc/bind/db.cao
cat > /etc/bind/db.cao << EOF
\$TTL	604800
@	IN	SOA	cao.com  1549396190.pp.com (
			      1		; Serial
			 604800		; Refresh
			  86400		; Retry
			2419200		; Expire
			 604800 )	; Negative Cache TTL
@		IN	NS	master
master  IN	A	10.0.0.11
www		IN	A	10.0.0.123
EOF
named-checkconf && echo "语法成功" || echo "语法不对"
named-checkzone "cao.com" /etc/bind/db.cao && echo "配置成功"



#添加域名
cat >> /etc/bind/db.cao << EOF
mysqlM	IN 	A 	10.0.0.124
EOF

named-checkconf && echo "语法成功" || echo "语法不对"
named-checkzone "cao.com" /etc/bind/db.cao && echo "配置成功"

systemctl restart named

6）openvpn（执行下面脚本）

#!/bin/bash
openvpn
apt update ; apt -y install openvpn easy-rsa
dpkg -L openvpn easy-rsa
#准备相关配置文件
cp -r /usr/share/easy-rsa/ /etc/openvpn/
mv /etc/openvpn/easy-rsa/vars.example /etc/openvpn/easy-rsa/vars
vim /etc/openvpn/easy-rsa/vars
set_var EASYRSA_CA_EXPIRE 36500
set_var EASYRSA_CERT_EXPIRE 3650

#准备证书相关文件
cd /etc/openvpn/easy-rsa
#easyrsa帮助用法
./easyrsa
#初始化PKI生成PKI相关目录和文件
./easyrsa init-pki
./easyrsa build-ca nopass
caojidong
#准备服务端证书环境
./easyrsa gen-req server nopass
openvpen
#req: /etc/openvpn/easy-rsa/pki/reqs/server.req #生成请求文件
#key: /etc/openvpn/easy-rsa/pki/private/server.key #生成私钥文件
#颁发服务端证书
./easyrsa sign server server
#查看证书相关文件
cat pki/index.txt
#创建 Diffie-Hellman 密钥
./easyrsa gen-dh
#建议修改给客户端颁发证书的有效期,可适当减少,比如:90天
vim /etc/openvpn/easy-rsa/vars
set_var EASYRSA_CERT_EXPIRE 90
#创建客户端证书申请
./easyrsa gen-req wangxiaochun nopass
#req: /etc/openvpn/easy-rsa/pki/reqs/wangxiaochun.req #私钥文件
#key: /etc/openvpn/easy-rsa/pki/private/wangxiaochun.key #证书申请文件
./easyrsa sign client wangxiaochun

#将CA和服务器证书相关文件复制到服务器相应的目录
cp /etc/openvpn/easy-rsa/pki/ca.crt /etc/openvpn/server/
cp /etc/openvpn/easy-rsa/pki/issued/server.crt /etc/openvpn/server/
cp /etc/openvpn/easy-rsa/pki/private/server.key /etc/openvpn/server/
cp /etc/openvpn/easy-rsa/pki/dh.pem /etc/openvpn/server/

#将客户端私钥与证书相关文件复制到服务器相关的目录
find /etc/openvpn/easy-rsa -name "wangxiaochun.key" -o -name "wangxiaochun.crt" -o -name ca.crt
find /etc/openvpn/easy-rsa \( -name "wangxiaochun.key" -o -name \
"wangxiaochun.crt" -o -name ca.crt \) -exec cp {} \
/etc/openvpn/client/wangxiaochun \;

cp /etc/openvpn/easy-rsa/pki/private/wangxiaochun.key /etc/openvpn/client/wangxiaochun/
cp /etc/openvpn/easy-rsa/pki/issued/wangxiaochun.crt /etc/openvpn/client/wangxiaochun/
cp /etc/openvpn/easy-rsa/pki/ca.crt /etc/openvpn/client/wangxiaochun/
#服务器端配置文件说明
cd /usr/share/doc/openvpn/examples/sample-config-files/
gzip server.conf.gz -d /opt/

vim /etc/openvpn/server.conf
cat > /etc/openvpn/server.conf << EOF
port 1194
proto tcp
dev tun
ca /etc/openvpn/server/ca.crt
cert /etc/openvpn/server/server.crt
key /etc/openvpn/server/server.key 
dh /etc/openvpn/server/dh.pem
server 10.8.0.0 255.255.255.0
push "route 192.168.10.0 255.255.255.0"
keepalive 10 120
cipher AES-256-CBC
compress lz4-v2
push "compress lz4-v2"
max-clients 2048
user openvpn
group openvpn
status /var/log/openvpn/openvpn-status.log
log-append /var/log/openvpn/openvpn.log
verb 3
mute 20
EOF

useradd -r -s /sbin/nologin openvpn
mkdir /var/log/openvpn
chown openvpn.openvpn /var/log/openvpn


#启动 OpenVPN 服务
systemctl start openvpn@server
#sys日志不在错了
cat .var/log/syslog

#客户端配置
cat > /etc/openvpn/client/wangxiaochun/client.ovpn << EOF
client
dev tun
proto tcp
remote 172.18.0.240 1194
resolv-retry infinite
nobind
#persist-key
#persist-tun
ca ca.crt
cert wangxiaochun.crt
key wangxiaochun.key
remote-cert-tls server
#tls-auth ta.key 1
cipher AES-256-CBC
verb 3 
compress lz4-v2
EOF

#打包
zip /root/wangxiaochun.zip *
sz /root/wangxiaochun.zip

#开启ip forward

sysctl -p

echo 'iptables -t nat -A POSTROUTING -s 192.168.10.0/24 ! -d 192.168.10.0/24 -j MASQUERADE' >> /etc/rc.local
./etc/rc.local

7）nat设置

在10.0.0.29设置

root@docker1-50-UB:~# vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
root@docker1-50-UB:~# sysctl -p
net.ipv4.ip_forward = 1
[root@lvs-server ~]#ipvsadm -A -t 192.168.10.100:80 -s wrr 
root@docker1-50-UB:~# ipvsadm -a -t 192.168.10.100:80 -r 10.0.0.7:80 -m
root@docker1-50-UB:~# ipvsadm -a -t 192.168.10.100:80 -r 10.0.0.17:80 -m

设置成功！！！！！