架构:
2个节点,每个节点都按照keepalive 和nginx
1. 2个keepalive使用一个vip对外提供服务,使用备备不抢占模式
2. keepalive的作用是保证所在节点的nginx服务处于启动状态,若nginx启动失败则kill掉自身keepalive,由另一个keepalive节点提供服务
3. 2个nginx的作用是使用最小连接数模式把http(s)请求发送给2个gateway,同一个时间只有一个nginx提供服务
nginx部署
nginx安装
yum -y install gcc automake autoconf libtool
yum -y install gcc gcc-c++
下载pcre-8.42.tar.gz 到目录/usr/local/src/下 #https://sourceforge.net/projects/pcre/
tar -zxvf pcre-8.42.tar.gz
cd pcre-8.42/
./configure
make
make install
下载zlib-1.2.10.tar.gz到目录/usr/local/src/下 #http://zlib.net/fossils/
tar -zxvf zlib-1.2.10.tar.gz
cd zlib-1.2.10/
./configure
make
make install
下载openssl-1.0.2r.tar.gz 到目录/usr/local/src/下 #https://www.openssl.org/source/
tar zvxf openssl-1.0.2r.tar.gz
cd openssl-1.0.2r/
./config
make
make install
下载nginx-1.14.2.tar.gz 到目录/usr/local/src/下 #http://nginx.org/en/download.html
tar -zxvf nginx-1.14.2.tar.gz
cd nginx-1.14.2/
配置config:
./configure
--sbin-path=/usr/local/nginx/nginx
--conf-path=/usr/local/nginx/nginx.conf
--pid-path=/usr/local/nginx/nginx.pid
--with-http_ssl_module
--with-pcre=/usr/local/src/pcre-8.42
--with-zlib=/usr/local/src/zlib-1.2.10
--with-openssl=/usr/local/src/openssl-1.0.2r
make
make install
配置参数说明
worker_processes: nginx进程数,默认=1,可设置成与cpu核数相同
worker_cpu_affinity: 把进程和cpu核绑定
worker_rlimit_nofile: 要和系统的单进程打开文件数一致65535
use epoll:事件驱动类型
least_conn:负载均衡算法--最小连接数
client_max_body_size 5g; 客户端传输大小限制
nginx.conf
#user nobody;
worker_processes 8;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
error_log /var/log/nginx/error.log warn;
#pid logs/nginx.pid;
events {
use epoll;
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream servers {
least_conn;
server 100.75.0.21:7480 weight=1 max_fails=3 fail_timeout=10s;
server 100.75.0.27:7480 weight=1 max_fails=3 fail_timeout=10s;
}
# test server
server {
listen 8080;
server_name 100.75.1.16:8080;
default_type text/plain;
return 200 'hello to inter 100.69.6.27:8080';
}
server {
listen 80;
server_name 100.75.1.16:80;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
proxy_pass http://servers;
client_max_body_size 5g;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
# HTTPS server
server {
listen 443 ssl;
server_name 100.75.1.16:443;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_certificate cmft.com.pem; #ssl/server.pem;
ssl_certificate_key cmft.com.pem; #ssl/privkey.pem;
# ssl_session_cache shared:SSL:1m;
ssl_session_timeout 5m;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
location / {
root html;
index index.html index.htm;
proxy_pass http://servers;
}
}
}
日志压缩
定期压缩删除nginx产生的日志。
vim /etc/logrotate.d/nginx
/var/log/nginx/*log {
create 0644 nginx nginx
daily
rotate 10
missingok
notifempty
compress
sharedscripts
postrotate
/bin/kill -USR1 `cat /run/nginx.pid 2>/dev/null` 2>/dev/null || true
endscript
}
keepalive部署
keepalive安装
yum install httpd
yum install ipvsadm
# yum install -y openssl #在安装nginx已经编译安装了
yum install -y openssl-devel keepalived #1.3.5-6.el7
keepalive.conf
2个节点权重不一致,都是backup节点,非强制模式,每2秒钟检查一次nginx服务状态。
[root@ceph1 rhlog]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
vrrp_script chk_haproxy {
script "/app/haproxy/bin/check_haproxy.sh"
interval 3
weight 2
}
global_defs {
notification_email {
huangaj001@cmft.com
}
notification_email_from huangaj001@cmft.com
smtp_server mail.cmrh.com
smtp_connect_timeout 30
router_id gw
}
vrrp_instance VI_1 {
state BACKUP
interface br-ex
virtual_router_id 18
priority 100 #另一个节点设置90
advert_int 3
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
100.75.1.18/24
}
track_script {
chk_haproxy
}
}
nginx_check.sh
检查nginx服务状态,若非运行状态则尝试拉起,启动不成功则kill掉自己。
#!/bin/bash
A=`ps -C nginx --no-header | wc -l`
if [ $A -eq 0 ];then
/usr/local/nginx/nginx
sleep 3
if [ `ps -C nginx --no-header | wc -l` -eq 0 ];then
killall keepalived
fi
fi
测试
nginx参数测试
测试项 | 说明 |
worker_processes | nginx进程数 |
worker_cpu_affinity | nginx进程绑定指定逻辑cpu |
max_fails=3 fail_timeout=30s | 指定时间段内3次连接失败则认为gateway不可用 |
功能测试
测试项 | 说明 |
keepalive高可用测试 | 能及时拉起未启动的nginx服务 |
nginx启动失败能转移VIP | |
keepalive再次启动不会抢占原VIP | |
nginx https转http测试 |
|
|
|
nginx负载均衡测试 | 压力测试观察2个gateway处理连接数是否达到负载均衡要求 |
压力测试
测试项 | 说明 |
大量客户端并发访问集群,多个桶,多个客户端同时访问 | 观察负载均衡情况 观察是否有请求超时 |
多种类型测试:小文件读/写,大文件读写 | 观察负载均衡情况 观察是否有请求超时 |