NGINX构架实验 （nfs、nginx反代、nginx upstream、nginx fastcgi ）
172.16.31.124 nginx 反向代理图片到11.100.46.9
反向代理静态页面到static upstream组11.100.46.4 （nginx）11.100.46.7（apache）健康检查
反向代理.php到动态fastcgi server 11.100.40.124
mysql server地址11.100.40.125
设置缓存
设置cacache
安装xchache
设置buffer
设置连接参数
一、安装主服务器172.16.31.124
1、安装nginx
# rpm -ivh http://172.16.31.125/soft/nginx-filesystem-1.10.2-1.el6.noarch.rpm http://172.16.31.125/soft/nginx-all-modules-1.10.2-1.el6.noarch.rpm http://172.16.31.125/soft/nginx-1.10.2-1.el6.x86_64.rpm –nodeps
2、创建nginx配置文件
# vim /etc/nginx/nginx.conf
user nginx nginx;
pid /var/run/nginx/nginx.pid;
worker_rlimit_core 1G;
worker_rlimit_nofile 65535;
worker_processes 3;
worker_cpu_affinity 0001 0010 0100;
timer_resolution 1000ms;
worker_priority -10;
events {
accept_mutex on;
worker_connections 10240;
accept_mutex_delay 500ms;
use epoll;
}
lock_file /var/lock/nginx.lock;
daemon on;
error_log /var/log/nginx/error.log error;
master_process on;
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
include server.conf;
error_page 404 /404.html;
error_page 500 502 504 /50x.html;
error_page 503 =200 /empty.gif;
}
# vim server.conf
server {
listen 172.16.31.124:80;
server_name name2.nginx.cpe.com;
location / {
root /usr/share/html/;
}
}
3、启动并测试nginx
# nginx
二、安装上游服务器web
1、11.100.46.7
# yum -y install httpd
#service httpd start
2、11.100.46.4
# rpm -ivh http://172.16.31.125/soft/nginx-filesystem-1.10.2-1.el6.noarch.rpm http://172.16.31.125/soft/nginx-all-modules-1.10.2-1.el6.noarch.rpm http://172.16.31.125/soft/nginx-1.10.2-1.el6.x86_64.rpm –nodeps
#nginx
3、11.100.46.9
# rpm -ivh http://172.16.31.125/soft/nginx-filesystem-1.10.2-1.el6.noarch.rpm http://172.16.31.125/soft/nginx-all-modules-1.10.2-1.el6.noarch.rpm http://172.16.31.125/soft/nginx-1.10.2-1.el6.x86_64.rpm –nodeps
#nginx
三、配置upstream
1、编辑nginx.conf 在http {}创建upsteam
# vim /etc/nginx/nginx.conf
upstream dynamic {
ip_hash;
server 11.100.40.124 weight=2 max_fails=3 fail_timeout=1s;
keepalive 8;
}
#由于非商业版无法使用cookie绑定负载均衡所以使用iphash来绑定cookie。
#memcache、fscgi有时会使用长连接。
upstream static {
least_conn;
server 11.100.46.7 weight=2 max_fails=3 fail_timeout=1s;
server 11.100.46.4 max_fails=3 fail_timeout=1s;
}
#静态连接使用短连接+ least_conn调度，会考虑权重以及后端服务器负载情况。
upstream images {
# ip_hash;
least_conn;
server 11.100.46.9 weight=2 max_fails=3 fail_timeout=1s;
# server 11.100.46.4 max_fails=3 fail_timeout=1s;
# server 11.100.46.9 max_fails=3 fail_timeout=1s backup;
# sticky cookie srv_id expires=1h path=/;
}
#静态连接使用短连接+ least_conn调度，会考虑权重以及后端服务器负载情况。
#match welcome是health check的官方配置由于不是商业版无法完成测试
#sticky cookie是官方商业版标准配置方式
# match welcome {
# status 200;
# header Content-Type = text/html;
# body ~ “Welcome to nginx!”;
# }
}
2、在server{}中反向代理指向upstream组
server {
listen 172.16.31.124:80;
server_name name2.nginx.cpe.com;
location / {
proxy_pass http://dynamic;
}
location ~*\.(.jpg|gif|jpeg|png)$ {
proxy_pass http://images;
}
}
3、配置代理服务器向后端服务器传递真实ip写入log
(1)编辑文件在代理服务器上加入访问日志配置 172.16.31.124
在server.conf中server{}相应location中加入：
# vim /etc/nginx/server.conf
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
(2)下游apache服务器接受11.100.46.7传递的日志参数
配置apache日志
# vim /etc/httpd/conf/httpd.conf
LogFormat “\”%{X-Real-IP}i\” %h %l %u %t \”%r\” %>s %b \”%{Referer}i\” \”%{User-Agent}i\” \”%{X-Forwarded-For}i\”” combined
#LogFormat “%h %l %u %t \”%r\” %>s %b \”%{Referer}i\” \”%{User-Agent}i\”” combined
(3)下游nginx服务器接受11.100.46.4、11.100.46.9传递的日志参数
在nginx.conf http {}中加入log配置文件
# vim /etc/nginx/nginx.conf
log_format main ‘$http_X_Real_IP – $remote_addr [$time_local] “$request” ‘
‘$status $body_bytes_sent “$http_referer” ‘
‘”$http_user_agent” “$http_x_forwarded_for”‘;
4、编译安装fpm-php 高级配置
# wget http://172.16.31.125/soft/php-5.6.30.tar.gz
# rpm -ivh http://172.16.31.125/soft/libmcrypt-2.5.8-9.el6.x86_64.rpm
# rpm -ivh http://172.16.31.125/soft/libmcrypt-devel-2.5.8-9.el6.x86_64.rpm
# rpm -ivh http://172.16.31.125/soft/mhash-0.9.9.9-3.el6.x86_64.rpm
# rpm -ivh http://172.16.31.125/soft/mcrypt-2.6.8-10.el6.x86_64.rpm
# tar xf php-5.6.30.tar.gz
# cd php-5.6.30
./configure –prefix=/usr/local/php \
–enable-mysqlnd –with-mysql –with-mysqli \
–with-openssl \
–enable-mbstring \
–with-freetype-dir –with-jpeg-dir –with-png-dir \
–with-zlib-dir –with-libxml-dir=/usr –enable-xml \
–with-mhash –with-mcrypt \
–enable-sockets –enable-fpm \
–with-config-file-path=/etc/php –with-config-file-scan-dir=/etc/php \
–with-bz2 –with-curl
configure: error: Cannot find OpenSSL’s <evp.h>
# yum -y install openssl-devel
configure: error: Please reinstall the BZip2 distribution
# yum -y reinstall bzip2 bzip2-devel
configure: error: Please reinstall the libcurl distribution –
easy.h should be in <curl-dir>/include/curl/
# yum -y install curl curl-devel
# yum -y install openssl-devel bzip2 bzip2-devel curl curl-devel libxml2-devel
configure: error: xml2-config not found. Please check your libxml2 installation.
# yum -y install libxml2-devel
# make -j 4 && make install
启动fastcgi：
配置启动脚本
# cp sapi/fpm/init.d.php-fpm /etc/rc.d/init.d/php-fpm
按实际情况修改：
# vim /etc/rc.d/init.d/php-fpm
php_fpm_BIN=/usr/local/php/sbin/php-fpm
php_fpm_CONF=/etc/php/php-fpm.conf
php_fpm_PID=/var/run/php-fpm.pid
# chmod +x /etc/rc.d/init.d/php-fpm
# chkconfig –add php-fpm
# chkconfig php-fpm on
创建配置文件：
# mkdir -pv /etc/php
# cp php.ini-production /etc/php/php.ini
# cp /usr/local/php/etc/php-fpm.conf.default /etc/php/php-fpm.conf
# vim /etc/php/php-fpm.conf
启用如下选项：
pm.max_children = 50
pm.start_servers = 5
pm.min_spare_servers = 2
pm.max_spare_servers = 8
pid = /var/run/php-fpm.pid
user = nginx
group = nginx
listen = 11.100.40.124:9000
5、安装nfs
172.16.31.124 用户nginx信息
# id nginx
uid=987(nginx) gid=982(nginx) groups=982(nginx)
# yum -y install nfs-utils
# service rpcbind start
# service nfs start
# rpcinfo -p
100005 1 udp 53616 mountd
100005 1 tcp 45831 mountd
100005 2 udp 50841 mountd
100005 2 tcp 53544 mountd
100005 3 udp 55905 mountd
100005 3 tcp 47932 mountd
100003 2 tcp 2049 nfs
100003 3 tcp 2049 nfs
100003 4 tcp 2049 nfs
创建web，php目录
# mkdir -pv /web/phps/
# groupadd -r -g 982 nginx
# useradd -M -u 987 -g 982 -s /sbin/nologin nginx
# chmod -R u=rwx,g=rwx,o= /web/phps/
编辑配置文件加入：
# vim /etc/exports
/web/phps/ 172.16.31.124(rw,all_squash,anonuid=987,anongid=982)
导出nfs:
# exportfs -ar
# showmount -e
Export list for localhost.localdomain:
/web/phps 11.100.46.4
5、在172.16.31.124上配置挂载nfs
安装nfs服务：
# yum -y install nfs-utils rpcbind
保存两端nginx uid与gid一致
# id nginx
uid=987(nginx) gid=982(nginx) groups=982(nginx)
#挂载nfs文件夹
# mkdir -pv /web/phps/
# showmount -e 11.100.40.124
Export list for 11.100.40.124:
/web/phps 172.16.31.124
# mount -t nfs 11.100.40.124:/web/phps /web/phps/
测试nfs创建测试页面：
# sudo -u nginx vim /web/phps/index.php
<?php
phpinfo();
?>
6、配置nginx fscgi：
# vim /etc/nginx/server.conf
location ~ \.php$ {
root /var/www/html/;
index index.php index.html index.htm;
fastcgi_pass 11.100.40.124:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root/$fastcgi_script_name;
include /etc/nginx/fastcgi_params;
root /web/phps/;
add_header X-Via $upstream_addr;
add_header X-Cache $upstream_cache_status;
}
7、编辑fascgi传递参数
# vim /etc/nginx/fastcgi_params
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
fastcgi_param SERVER_SOFTWARE nginx;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param SCRIPT_NAME $fastcgi_script_name;
fastcgi_param REQUEST_URI $request_uri;
fastcgi_param DOCUMENT_URI $document_uri;
fastcgi_param DOCUMENT_ROOT $document_root;
fastcgi_param SERVER_PROTOCOL $server_protocol;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param REMOTE_PORT $remote_port;
fastcgi_param SERVER_ADDR $server_addr;
fastcgi_param SERVER_PORT $server_port;
fastcgi_param SERVER_NAME $server_name;
测试php看是否成功
8、安装mysql
(1)下载解压mysql
# wget http://mirrors.tuna.tsinghua.edu.cn/mariadb//mariadb-5.5.54/bintar-linux-x86_64/mariadb-5.5.54-linux-x86_64.tar.gz -c
# tar xf mariadb-5.5.54-linux-x86_64.tar.gz -C /usr/local/
# cd /usr/local/
# mkdir -pv /mydata/data
# groupadd -r mysql
# useradd -r mysql -g mysql
# chown mysql:mysql /mydata -R
# ln -sv mariadb-5.5.54-linux-x86_64 mysql
# cd mysql
(2)创建初始化数据库：
# scripts/mysql_install_db –user=mysql –datadir=/mydata/data/
(3)提供启动脚本：
# cp support-files/mysql.server /etc/rc.d/init.d/mysqld
# chkconfig –add mysqld
# chkconfig mysqld on
(4)提供配置文件
# mkdir -pv /etc/mysql
# cp support-files/my-large.cnf /etc/mysql/my.cnf
(5)编辑配置文件：
# vim /etc/mysql/my.cnf
在[mysqld]段加入
datadir = /mydata/data
innodb_file_per_table = on
skip_name_resolve = on
(6)配置man文档
# vim /etc/man.config
MANPATH /usr/local/mysql/man/man
(7)导入mysql头文件
# ln -sv /usr/local/mysql/include /usr/include/mysql
(8)增加环境变量
# vim /etc/profile.d/mysql.sh
export PATH=”/usr/local/mysql/bin/:${PATH}”
# yum -y install mysql
(9)初始化wordpress数据库
mysql> use msyql;
mysql> create database wordpress;
mysql> grant all privileges on wordpress to ‘wordpress’@’11.100.40.124’ identified by ‘cisco’;
9、重新编译fpm-php支持mysql
# cd /root/php-5.6.30
./configure –prefix=/usr/local/php \
–with-mysql –with-mysqli –with-pdo-mysql \
–with-openssl \
–enable-mbstring \
–enable-sysvshm \
–with-freetype-dir –with-jpeg-dir –with-png-dir \
–with-zlib-dir –with-libxml-dir=/usr –enable-xml \
–with-mhash –with-mcrypt \
–enable-sockets –enable-fpm \
–with-config-file-path=/etc/ –with-config-file-scan-dir=/etc/php.d/ \
–with-bz2 –with-curl
# make -j 4 && make install
配置启动脚本
# cp sapi/fpm/init.d.php-fpm /etc/rc.d/init.d/php-fpm
按实际情况修改：
# vim /etc/rc.d/init.d/php-fpm
php_fpm_BIN=/usr/local/php/sbin/php-fpm
php_fpm_CONF=/etc/php/php-fpm.conf
php_fpm_PID=/var/run/php-fpm.pid
# chmod +x /etc/rc.d/init.d/php-fpm
# chkconfig –add php-fpm
# chkconfig php-fpm on
创建配置文件：
# mkdir -pv /etc/php/
# cp php.ini-production /etc/php/php.ini
# cp /usr/local/php/etc/php-fpm.conf.default /etc/php/php-fpm.conf
# vim /etc/php/php-fpm.conf
启用如下选项：
pm.max_children = 50
pm.start_servers = 5
pm.min_spare_servers = 2
pm.max_spare_servers = 8
pid = /var/run/php-fpm.pid
user = nginx
group = nginx
listen = 11.100.40.124:9000
10、安装x-cache
i.安装xcache
# wget http://xcache.lighttpd.net/pub/Releases/3.2.0/xcache-3.2.0.tar.gz
# tar -xf xcache-3.2.0.tar.gz
# cd xcache-3.2.0
# /usr/local/php/bin/phpize
# ./configure –enable-xcache –with-php-config=/usr/local/php/bin/php-config
# make && make install
ii. 编辑php.ini，整合php和xcache：
首先将xcache提供的样例配置导入php.ini
# cp xcache.ini /etc/php.d/
# vim /etc/php/xcache.ini
extension = /usr/local/php/lib/php/extensions/no-debug-non-zts-20121212/xcache.so
11、nginx性能优化
# vim /etc/sysctl.conf
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096 87380 4194304
net.ipv4.tcp_wmem = 4096 16384 4194304
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 30
net.ipv4.ip_local_port_range = 1024 65000
# sysctl -p
# ulimit -n 65535
12、主配置文件参数
user nginx nginx;
pid /var/run/nginx/nginx.pid;
worker_rlimit_core 1G;
worker_rlimit_nofile 65535;
worker_processes 3;
worker_cpu_affinity 0001 0010 0100;
timer_resolution 1000ms;
worker_priority -10;
lock_file /var/lock/nginx.lock;
daemon on;
error_log /var/log/nginx/error.log error;
master_process on;
events {
accept_mutex on;
worker_connections 10240;
accept_mutex_delay 500ms;
use epoll;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
include server.conf;
tcp_nopush on;
tcp_nodelay off;
types_hash_max_size 2048;
reset_timedout_connection on;
keepalive_requests 100;
proxy_buffering on;
proxy_buffer_size 8k;
proxy_buffers 4 64k;
proxy_temp_file_write_size 128k;
proxy_max_temp_file_size 128m;
proxy_busy_buffers_size 128k;
client_body_temp_path /var/tmp/client_body_temp 1 2;
proxy_temp_path /var/tmp/proxy_temp 1 2;
proxy_pass_request_body on;
proxy_pass_request_headers on;
client_max_body_size 20m;
client_body_buffer_size 256k;
ignore_invalid_headers on;
server_names_hash_max_size 256;
server_names_hash_bucket_size 64;
client_header_buffer_size 8k;
large_client_header_buffers 4 32k;
connection_pool_size 256;
request_pool_size 64k;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
fastcgi_cache_path /cache/nginx/fastcgi_cache levels=1:2 keys_zone=test_fastcgi:100M inactive=5m;
fastcgi_temp_path /tmp/wpcache/temp;
fastcgi_cache_key “$scheme$request_method$host$request_uri”;
error_page 404 /404.html;
error_page 500 502 504 /50x.html;
error_page 503 =200 /empty.gif;
gzip on;
gzip_http_version 1.1;
gzip_vary on;
gzip_proxied any;
gzip_min_length 1024;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_proxied expired no-cache no-store private auth no_last_modified no_etag;
gzip_types text/plain application/x-javascript text/css application/xml application/json;
gzip_disable “MSIE [1-6]\.(?!.*SV1)”;
proxy_cache_path /cache/nginx/ levels=1:2:2 keys_zone=map:100m inactive=10m;
log_format main ‘$server_addr $remote_addr [$time_local] $msec+$connection ‘
‘”$request” $status $connection $request_time $body_bytes_sent “$http_referer” ‘
‘”$http_user_agent” “$http_x_forwarded_for”‘;
access_log /var/log/nginx/access.log main;
upstream dynamic {
ip_hash;
server 11.100.40.124 weight=2 max_fails=3 fail_timeout=1s;
keepalive 8;
}
upstream static {
least_conn;
server 11.100.46.7 weight=2 max_fails=3 fail_timeout=1s;
server 11.100.46.4 max_fails=3 fail_timeout=1s;
}
upstream images {
least_conn;
server 11.100.46.9 weight=2 max_fails=3 fail_timeout=1s;
}
}
13、server.conf
server {
listen 172.16.31.124:80;
server_name 172.16.31.124;
location / {
proxy_pass http://static;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
#health_check;
add_header X-Via $upstream_addr;
add_header X-Cache $upstream_cache_status;
proxy_cache map;
proxy_cache_valid 200 302 10m;
#缓存状态码为：200 302 10分钟；
proxy_cache_valid 301 1h;
#缓存状态码为：301的一小时；
proxy_cache_valid any 1m;
#缓存其它的1分钟；
proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504 http_403 http_404;
#为情况，可以使用过期缓存。
proxy_cache_revalidate on;
#缓存到期后重新检查后，重新启用；
proxy_cache_min_uses 1;
#请求最小一次就缓存；
proxy_cache_methods GET HEAD;
#缓存方法为GET HEAD的；
}
location ~*\.()$ {
proxy_pass http://images;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header X-Via $upstream_addr;
add_header X-Cache $upstream_cache_status;
}
location ~ \.php$ {
index index.php index.html index.htm;
fastcgi_pass 11.100.40.124:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root/$fastcgi_script_name;
include /etc/nginx/fastcgi_params;
root /web/phps/;
add_header X-Via $upstream_addr;
add_header X-Cache $upstream_cache_status;
fastcgi_cache test_fastcgi;
fastcgi_cache_valid 200 302 1h;
fastcgi_cache_valid 301 1d;
fastcgi_cache_valid any 1m;
}
location /wordpress/ {
index index.php index.html index.htm;
fastcgi_pass 11.100.40.124:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root/$fastcgi_script_name;
include /etc/nginx/fastcgi_params;
root /web/phps/;
fastcgi_cache test_fastcgi;
fastcgi_cache_valid 200 302 1h;
fastcgi_cache_valid 301 1d;
fastcgi_cache_valid any 1m;
}
}
14、打开php错误日志
(1)修改php-fpm.conf中配置 没有则增加
catch_workers_output = yes
error_log = /var/log/php/php-fpm.log
(2)修改php.ini中配置，没有则增加
log_errors = On
error_log = “/var/log/php/php-fpm.log”
error_reporting=E_ALL&~E_NOTICE
(10)测试fascgi连接mysql
扫一扫
1630
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
