centos7/redhat7 安装部署Nginx集群+Keepalived 并配置反向代理http和https 负载均衡配置

nginx离线安装包以及相关依赖链接：https://pan.baidu.com/s/1b44Vo91gD3oSz4QibokwAA 
提取码：1012

一、安装nginx
1、安装nginx相关依赖： 
备注：如果编译nginx的过程中出现报错，一般都是缺少相关依赖。
#安装PCRE：
tar -zxvf pcre-8.35.tar.gz
cd pcre-8.35
./configure
make
make install 
#安装libtool
tar -zxvf libtool-2.4.2.tar.gz
cd libtool-2.4.2
./configure
make
make install
#安装gcc和gcc-c++
将gcc和gcc-c++传到虚拟机中，进入gcc和gcc-c++目录安装：rpm -ivh *.rpm --nodeps –force

2、安装nginx
cd /usr/local
tar -zxvf  nginx压缩包
cd nginx-1.22.0
./configure --prefix=/usr/local/nginx --with-stream --with-http_ssl_module
make
make install

3、配置nginx.service服务：
vim /usr/lib/systemd/system/nginx.service
[Unit]
Description=nginx - high performance web server
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
PIDFile=/usr/local/nginx/logs/nginx.pid
ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/usr/local/nginx/sbin/nginx -s stop
PrivateTmp=true

[Install]
WantedBy=multi-user.target

4、启动nginx服务
# 配置nginx别名
vi /root/.bashrc
alias nginx='/usr/local/nginx/sbin/nginx'
生效别名配置：source /root/.bashrc
启动nginx服务并设置开机自启：systemctl start nginx && systemctl enable nginx

5.1、配置nginx配置文件（以下为代理kong的http80端口和nacos服务的配置）：
#user  nobody;
worker_processes  4;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

events {
    worker_connections  1024;
}

# 配置nginx代理其他服务器上的nacos服务

stream {
   upstream lb-nacos-client-rpc-ma{
        server   10.109.100.53:8848 weight=1;  #nacos集群节点
        server   10.109.100.54:8848 weight=1;  #nacos集群节点
        server   10.109.100.55:8848 weight=1;  #nacos集群节点
    }
    server {
        listen 8848; 
        proxy_pass lb-nacos-client-rpc-ma;
    }   
 
   upstream lb-nacos-client-rpc{
        server   10.109.100.53:9848 weight=1;
        server   10.109.100.54:9848 weight=1;
        server   10.109.100.55:9848 weight=1;
    }
    server {
        listen 9848;
        proxy_pass lb-nacos-client-rpc;
    }

    upstream lb-nacos-raft-rpc{
        server 10.109.100.53:9849 weight=1;
        server 10.109.100.54:9849 weight=1;
        server 10.109.100.55:9849 weight=1;
    }

    server {
        listen 9849;
        proxy_pass lb-nacos-raft-rpc;
    }

    upstream lb-nacos-old-raft-rpc{
        server 10.109.100.53:7848 weight=1;
        server 10.109.100.54:7848 weight=1;
        server 10.109.100.55:7848 weight=1;
    }

    server {
        listen 7848;
        proxy_pass lb-nacos-old-raft-rpc;
    }

}
http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;
    upstream lb-kong80{
        server 10.109.100.33:80 weight=1;
        server 10.109.100.34:80 weight=1;
        server 10.109.100.35:80 weight=1;
    }

    server {
        listen       80;
        server_name  localhost;
        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
             proxy_pass  http://lb-kong80;
             client_max_body_size 1024m;
             proxy_redirect                 off;
             proxy_set_header               Host $host;
             proxy_set_header               X-real-ip $remote_addr;
             proxy_set_header               X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header Upgrade       $http_upgrade;
             proxy_set_header Connection    "upgrade";
          #  root   html;
         #   index  index.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }  

    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

5.2、配置nginx配置文件（以下为代理kong的https443端口和nacos的配置，需要在nginx.conf所在目录放置证书：key和pem）：
#user  nobody;
worker_processes  4;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

events {
    worker_connections  1024;
}

stream {
   upstream lb-nacos-client-rpc-ma{
        server   10.100.111.113:8848 weight=1;
    }
    server {
        listen 8848;
        proxy_pass lb-nacos-client-rpc-ma;
    }   
 
   upstream lb-nacos-client-rpc{
        server   10.100.111.113:9848 weight=1;
    }
    server {
        listen 9848;
        proxy_pass lb-nacos-client-rpc;
    }

    upstream lb-nacos-raft-rpc{
        server 10.100.111.113:9849 weight=1;
    }

    server {
        listen 9849;
        proxy_pass lb-nacos-raft-rpc;
    }

    upstream lb-nacos-old-raft-rpc{
        server 10.100.111.113:7848 weight=1;
    }

    server {
        listen 7848;
        proxy_pass lb-nacos-old-raft-rpc;
    }

}
http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;
 
    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

    upstream lb-kong443{
        server 10.100.111.124:443 weight=1;  #使用kong网关的K8S集群worker节点
        server 10.100.111.125:443 weight=1;  #使用kong网关的K8S集群worker节点
        server 10.100.111.126:443 weight=1;  #使用kong网关的K8S集群worker节点
        server 10.100.111.127:443 weight=1;  #使用kong网关的K8S集群worker节点
    }

    # HTTPS server
    #
    server {
        listen       443 ssl;
        server_name  localhost;
        # 证书名称，放置在nginx.conf所在的目录
        ssl_certificate      443.pem;
        ssl_certificate_key  443.key;

        ssl_session_cache    shared:SSL:1m;
        ssl_session_timeout  5m;

        ssl_ciphers  HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers  on;

        location / {
             proxy_pass  https://lb-kong443;
             client_max_body_size 1024m;
             proxy_redirect                 off;
             proxy_set_header               Host $host;
             proxy_set_header               X-real-ip $remote_addr;
             proxy_set_header               X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header Upgrade       $http_upgrade;
             proxy_set_header Connection    "upgrade";
        }
    }

}

6、安装Keepalived：
yum -y install keepalived
设置开机自启：systemctl enable keepalived
vi /etc/keepalived/keepalived.conf（以下是主ng的配置文，priority为100，文件中virtual_ipaddress配置的IP为vip）：
! Configuration File for keepalived

global_defs {
    script_user root root
}

vrrp_script chk_nginx {
        script "/etc/keepalived/nginx_check.sh"
        interval 2
        weight 2

}

vrrp_instance VI_1 {
    state MASTER
    interface ens160  # ip a查看网卡名称
    virtual_router_id 61 # 主备ID必须一致，且同网段唯一
    priority 100
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
        10.109.222.164
    }
}

以下是备ng的keepalived配置文件：
! Configuration File for keepalived

global_defs {
  router_id LVS_DEVEL
}

vrrp_script chk_nginx {
  script "/etc/keepalived/nginx_check.sh"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
  state BACKUP
    interface ens160  # ip a查看网卡名称
    virtual_router_id 61  # 主备ID必须一致，且同网段唯一
    priority 90
    advert_int 1
    unicast_src_ip 10.109.214.163
    unicast_peer {
    10.109.214.162
  }
  authentication {
    auth_type PASS
      auth_pass 1111
  }
  track_script {
    chk_nginx
  }
  virtual_ipaddress {
    10.109.222.164
  }
}

vi  /etc/keepalived/nginx_check.sh

#!/bin/bash
run=`ps -C nginx --no-header | wc -l`
if [ "${run}" = "0" ]
then
  systemctl restart nginx
  sleep 3
  run=`ps -C nginx --no-header | wc -l`
  if [ "${run}" = "0" ]
  then
    systemctl stop keepalived
  fi
fi