ELK 通过 Logtsash 收集 Nginx 访问日志

一、部署 Nginx

准备Nginx脚本，以及把安装包放到/usr/local/src/目录下
安装包下载：https://download.csdn.net/download/qq_42606357/19324277

SRC_DIR=/usr/local/src
COLOR="echo -e \\033[01;31m"
END='\033[0m'
NGINX_URL=http://nginx.org/download/
NGINX_FILE=nginx-1.18.0.tar.gz
ECHO_NGINX=echo-nginx-module-0.62.tar.gz
OPENSSL=openssl-1.1.1j.tar.gz
NGINX_INSTALL_DIR=/apps/nginx
CPUS=`lscpu |awk '/^CPU\(s\)/{print $2}'`

os(){
    if grep -Eqi "CentOS" /etc/issue || grep -Eq "CentOS" /etc/*-release;then
        rpm -q redhat-lsb-core &> /dev/null || { ${COLOR}"安装lsb_release工具"${END};yum -y install  redhat-lsb-core &> /dev/null; }
    fi
    OS_RELEASE_VERSION=`lsb_release -rs |awk -F'.' '{print $1}'`
}

check_file (){
    cd  ${SRC_DIR}
    rpm -q wget &> /dev/null || yum -y install wget &> /dev/null
    if [ ! -e ${NGINX_FILE} ];then
        ${COLOR}"缺少${NGINX_FILE}文件"${END}
        ${COLOR}'开始下载NGINX源码包'${END}
        wget ${NGINX_URL}${NGINX_FILE} || { ${COLOR}"NGINX源码包下载失败"${END}; exit; }
    elif [ ! -e ${ECHO_NGINX} ];then
        ${COLOR}"缺少${ECHO_NGINX}文件"${END}
        exit    
    elif [ ! -e ${OPENSSL} ];then
        ${COLOR}"缺少${OPENSSL}文件"${END}
        exit
    else
        ${COLOR}"相关文件已准备好"${END}
    fi
} 

install_nginx(){
    ${COLOR}"开始安装NGINX"${END}
    id nginx  &> /dev/null || { useradd -s /sbin/nologin -r  nginx; $COLOR"创建nginx用户"$END; }
    ${COLOR}"开始安装NGINX依赖包"${END}
    if [[ ${OS_RELEASE_VERSION} == 8 ]] &> /dev/null;then
        yum -y install make gcc-c++ libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel perl-ExtUtils-Embed &> /dev/null
    elif [[ ${OS_RELEASE_VERSION} == 7 ]] &> /dev/null;then
        yum -y install make gcc pcre-devel openssl-devel zlib-devel perl-ExtUtils-Embed &> /dev/null
    else
        apt update &> /dev/null;apt -y install make gcc libpcre3 libpcre3-dev openssl libssl-dev zlib1g-dev &> /dev/null
    fi
    cd $SRC_DIR
    tar xf ${NGINX_FILE} && tar xf ${ECHO_NGINX} && tar xf ${OPENSSL}
    NGINX_DIR=`echo ${NGINX_FILE}| sed -nr 's/^(.*[0-9]).*/\1/p'`
    ECHO_NGINX_DIR=`echo ${ECHO_NGINX}| sed -nr 's/^(.*[0-9]).*/\1/p'`
    OPENSSL_DIR=`echo ${OPENSSL}| sed -nr 's/^(.*[0-9][a-z]).*/\1/p'`
    cd ${NGINX_DIR}
    ./configure --prefix=${NGINX_INSTALL_DIR} --user=nginx --group=nginx --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-http_stub_status_module --with-http_gzip_static_module --with-pcre --with-stream --with-stream_ssl_module --with-stream_realip_module  --with-file-aio --add-module=${SRC_DIR}/${ECHO_NGINX_DIR} --with-openssl=${SRC_DIR}/${OPENSSL_DIR} 
    make -j $CPUS && make install 
    [ $? -eq 0 ] && $COLOR"NGINX编译安装成功"$END ||  { $COLOR"NGINX编译安装失败,退出!"$END;exit; }
    echo "PATH=${NGINX_INSTALL_DIR}/sbin:${PATH}" > /etc/profile.d/nginx.sh
    cat > /lib/systemd/system/nginx.service <<EOF
[Unit]
Description=The nginx HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
PIDFile=${NGINX_INSTALL_DIR}/logs/nginx.pid
ExecStartPre=/bin/rm -f ${NGINX_INSTALL_DIR}/logs/nginx.pid
ExecStartPre=${NGINX_INSTALL_DIR}/sbin/nginx -t
ExecStart=${NGINX_INSTALL_DIR}/sbin/nginx
ExecReload=/bin/kill -s HUP \$MAINPID
KillSignal=SIGQUIT
TimeoutStopSec=5
KillMode=process
PrivateTmp=true

[Install]
WantedBy=multi-user.target
EOF
    systemctl daemon-reload
    systemctl enable --now nginx &> /dev/null 
    systemctl is-active nginx &> /dev/null ||  { ${COLOR}"NGINX 启动失败,退出!"${END} ; exit; }
    ${COLOR}"NGINX安装完成"${END}
}

main(){
    os
    check_file
    install_nginx
}

main

二、编辑 Nginx 配置文件

把日志格式改成JOSN格式

root@logstash1:/usr/local/src# vim /apps/nginx/conf/nginx.conf    #在http模块下插入下面内容
    log_format access_json '{"@timestamp":"$time_iso8601",'
        '"host":"$server_addr",'
        '"clientip":"$remote_addr",'
        '"size":$body_bytes_sent,'
        '"responsetime":$request_time,'
        '"upstreamtime":"$upstream_response_time",'
        '"upstreamhost":"$upstream_addr",'
        '"http_host":"$host",'
        '"url":"$uri",'
        '"domain":"$host",'
        '"xff":"$http_x_forwarded_for",'
        '"referer":"$http_referer",'
        '"status":"$status"}';
    access_log /var/log/nginx/access.log access_json;

root@logstash1:/usr/local/src# mkdir /var/log/nginx/ -p

三、测试 Nginx 配置并启动服务

root@logstash1:/usr/local/src# /apps/nginx/sbin/nginx -t
nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /apps/nginx/conf/nginx.conf test is successful
root@logstash1:/usr/local/src# /apps/nginx/sbin/nginx -s reload

四、验证日志格式是否为 JSON

root@ubuntu1804:/usr/local/src# tail -f /var/log/nginx/access.log
{"@timestamp":"2021-08-28T14:02:57+00:00","host":"10.0.0.38","clientip":"10.0.0.1","size":555,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"10.0.0.38","url":"/favicon.ico","domain":"10.0.0.38","xff":"-","referer":"-","status":"404"}
{"@timestamp":"2021-08-28T14:02:57+00:00","host":"10.0.0.38","clientip":"10.0.0.1","size":555,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"10.0.0.38","url":"/favicon.ico","domain":"10.0.0.38","xff":"-","referer":"-","status":"404"}

五、编辑 Logstash 配置文件

root@web1:~# vim /etc/logstash/conf.d/nginx-log-to-es.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    type => "nginx-accesslog"
    start_position => "beginning"
    stat_interval => "3 second"
    codec => "json"
  }
  file {
    path => "/apps/nginx/logs/error.log"
    type => "nginx-errorlog"
    start_position => "beginning"
    stat_interval => "3 second"
  }
}

output {
  if [type] == "nginx-accesslog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-nginx-accesslog-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "nginx-errorlog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-nginx-errorlog-%{+YYYY.MM.dd}"
    }
  }
}

六、检测配置文件语法是否正确

root@web1:~# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log-to-es.conf -t

七、启动服务并验证

systemctl restart logstash.service

八、创建索引方便查询日志

8.1 创建访问日志索引

8.2 创建错误日志索引