nginx 日志分割、缓存及日志分析、统计ip

nginx日志每日生成单个文件

nginx日志分割是nginx本身自带，不需重新编译。

直接在server中添加即可。

log_format  main  '$remote_addr $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '$http_user_agent $http_x_forwarded_for $request_time $upstream_response_time $upstream_addr $upstream_status';         #定义一个日志格式main，此main可自己随机定义，必须与一下main一致

if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})") {
        set $year $1;
        set $month $2;
        set $day $3;
}   #用于日志分割时间：年月日
 access_log  /var/logs/xxxx/access/xxxxx_xx_access_$year-$month-$day.log  main;    #日志存放位置及生成名称。  main名称与日志格式名称一致。 

nginx js/css…开启文件缓存

    location ~ .*\.(js|css|ico|gif|jpg|jpeg|png|bmp|swf)$ {                                                                                                                                                                        
        expires 7d;                                                                                                                                                    
        access_log off;       
        proxy_pass   http://wwwserv;  #如果图片和网站放在一起，必须加这个转向                                                                                                                                         
    }                                                                                                                                                                  
 
  location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$  location ~匹配的意思以后面结尾的缓存30不记录
        {undefined
                expires      30d;
                access_log off;
        }

        location ~ .*\.(js|css)$            js/css结尾缓存12个小时并且不记录访问日志
        {undefined
                expires      12h;
                access_log off;
        }

日志分割命令：

split -l 行数 -b 大小

日志分析过滤 ：

$7 “页面索引” $9 状态码 $10 流量 $11 url

***访问量：***
统计ip前十：		awk '{print $1}'  xaa |sort |uniq -c |sort -nr | head -10 
				awk '{a[$1]++}END{for(i in a)print a[i],i|"sort -k1 -nr|head -n10"}' access.log
统计单日： 			grep "05/Nov/2019" xaa |awk '{print $1}' |sort |uniq -c |sort -nr | head -10
统计某个特定时间之后：awk '($4>"05/Nov/2019"){print $1}' xaa |sort |uniq -c |sort -nr | head -10
小时单位里ip连接数最多的10个时段：awk -vFS="[:]" '{gsub("-.*","",$1);num[$2" "$1]++}END{for(i in num)print i,num[i]}' log_file | sort -n -k 3 -r | head -10
统计访问最多的前10	awk '{a[$7]++}END{for(i in a)print a[i],i|"sort -k1 -nr|head -n10"}' access.log

***流量：***
统计单日流量： 	grep "05/Nov/2019" xaa  | awk '{sum+=$10}END{print sum}'
统计网站流量（G) ：cat access.log |awk '{sum+=$10} END {print sum/1024/1024/1024}'


***资源：***
指定资源统计：		awk '($7~/\.html$/){print $1 " " $7 " " $9}' xaa##第七列已html结尾的
将每个IP访问的页面数进行从小到大排序：awk '{++S[$1]} END {for (a in S) print S[a],a}' log_file | sort -n


***状态码***
按状态码： 			awk '($9>"200"){print $1 " " $7 " " $9}' xaa
统计状态码：		awk '{print $9}'  xaa |sort |uniq -c |sort -nr | head -10 
统计访问状态码为404的IP及出现次数	awk '{if($9~/404/)a[$1" "$9]++}END{for(i in a)print i,a[i]}' access.log

nginx配合时间任务、自动封堵ip shell

crontab -e 
* * * * * cd /usr/local/nginx/logs/ && sh ip_test.sh  每一分钟运行一次

#不能把别人IP一直封着吧，这里就清除掉了
echo "" > /usr/local/nginx/conf/blockip.conf

#前面最开始编写的统计数据功能
ip_list=$(awk '{print $1}' access.log | sort | uniq -cd | awk '{if($1>60)print $0}')

#判断这个变量是否为空
if test -z "$ip_list"
then
        #为空写入 11.log中，并重新启动ngnix
        echo "为空"  >> /usr/local/nginx/logs/11.log

        /usr/local/nginx/sbin/nginx -s reload

else
        #如果不为空 前面加上 deny格式和ip写入blockip.conf中
        echo "deny" $ip_list > /usr/local/nginx/conf/blockip.conf
    
        #因为前面携带了行数，所有我们需要去除掉前面的行数，写入后在读取一次
        ip_list2=$(awk '{print $3}' /usr/local/nginx/conf/blockip.conf)
        
        #最后再把读取出来的值，在次写入到blockip.conf中
        echo "deny" $ip_list2";"> /usr/local/nginx/conf/blockip.conf

        #重启ngnix
        /usr/local/nginx/sbin/nginx -s reload
        #清空之前的日志，从最新的开始截取
        echo "" > /usr/local/nginx/logs/access.log

fi 

日志json格式

log_format json_analytics escape=json '{'
                            '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
                            '"connection": "$connection", ' # connection serial number
                            '"connection_requests": "$connection_requests", ' # number of requests made in connection
                    '"pid": "$pid", ' # process pid
                    '"request_id": "$request_id", ' # the unique request id
                    '"request_length": "$request_length", ' # request length (including headers and body)
                    '"remote_addr": "$remote_addr", ' # client IP
                    '"remote_user": "$remote_user", ' # client HTTP username
                    '"remote_port": "$remote_port", ' # client port
                    '"time_local": "$time_local", '
                    '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
                    '"request": "$request", ' # full path no arguments if the request
                    '"request_uri": "$request_uri", ' # full path and arguments if the request
                    '"args": "$args", ' # args
                    '"status": "$status", ' # response status code
                    '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
                    '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
                    '"http_referer": "$http_referer", ' # HTTP referer
                    '"http_user_agent": "$http_user_agent", ' # user agent
                    '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
                    '"http_host": "$http_host", ' # the request Host: header
                    '"server_name": "$server_name", ' # the name of the vhost serving the request
                    '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
                    '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
                    '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
                    '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
                    '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
                    '"upstream_response_length": "$upstream_response_length", ' # upstream response length
                    '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
                    '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
                    '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
                    '"scheme": "$scheme", ' # http or https
                    '"request_method": "$request_method", ' # request method
                    '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
                    '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise
                    '"gzip_ratio": "$gzip_ratio", '
                    '"http_cf_ray": "$http_cf_ray",'
                    '"geoip_country_code": "$geoip_country_code"'
                    '}';

json格式简化版

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"'
                      '$request_time $upstream_addr $upstream_response_time';
    
    log_format json   '{"@timestamp":"$time_iso8601",'
                      '"clientip":"$remote_addr",'
                      '"clientip2":"$proxy_protocol_addr",'
                      '"responsetime":$request_time,'
                      '"upstreamtime":"$upstream_response_time",'
                      '"upstreamhost":"$upstream_addr",'
                      '"domain":"$host",'
                      '"url":"$uri",'
                      '"referer":"$http_referer",'
                      '"status":"$status"}';

    access_log  /var/log/nginx/access.log  json;