nginx日志同步到elasticsearch-7.6.1（单台）

搭建环境 nginx，elk-7.6.1版本， CentOs 7 单台
启动elk， 安装nginx 到 /usr/local文件夹下。修改名字为 nginx
安装nginx 不在赘述，修改nginx的配置文件 或者 直接替换 nginx.conf

vi /usr/local/nginx/conf/nginx.conf  #复制以下内容

worker_processes  1;
error_log  logs/error.log;
error_log  logs/error.log  notice;
error_log  logs/error.log  info;
pid        logs/nginx.pid;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    #修改日志的打印格式 最好不要换行
	log_format main '{"client_ip":"$remote_addr","log_time":"$time_local","request":"$request","status":"$status","body_bytes_sent":"$body_bytes_sent","http_referer":"$http_referer","AgentVersion":"$http_user_agent","upstream_addr":"$upstream_addr","request_time":"$request_time","upstream_response_time":"$upstream_response_time"}';

    access_log  logs/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    server {
        listen       80;
        server_name  localhost;
        location / {
            root   html;
            index  index.html index.htm;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

启动nginx进行测试

# 测试 启动nginx  没问题
./usr/local/nginx/sbin/nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
#启动nginx
./usr/local/nginx/sbin/nginx

#关闭防火墙 打开浏览器访问 或者直接 curl localhost:80
systemstl stop firewalld.service

打开日志查看日志状态 格式为

cat /usr/local/nginx/logs/access.log
{
  "client_ip":"127.0.0.1",
  "log_time":"26/Apr/2021:18:22:30 +0800",
  "request":"GET / HTTP/1.1",
  "status":"200",
  "body_bytes_sent":"612",
  "http_referer":"-",
  "AgentVersion":"curl/7.29.0",
  "upstream_addr":"-",
  "request_time":"0.000",
  "upstream_response_time":"-"
}

先输出到logstash的控制台

停止 logstash服务 编写 nginx_elk.conf 配置文件 配置nginx日志的格式

vi usr/local/logstash-7.6.1/nginx_elk.conf #输入以下内容

input{
  file{
    #日志存放路径
    path => "/usr/local/nginx/logs/access.log"
    #从日志文件开头读取
    start_position => "beginning"
    #设置多长时间检测文件是否修改 默认是1s
    stat_interval => "2"
    codec => "json"
  }
}

output {  # 定义日志输出
    stdout {
        codec => rubydebug # 输出到控制台先
    }
}

再次启动 Logstash 访问 nginx

cd /usr/local/logstash-7.6.1
bin/logstash -f nginx_elk.conf --config.debug

# 访问nginx 查看打印到控制台的日志信息
{
  "client_ip" => "192.168.111.1",
  "status" => "304",
  "request_time" => "0.000",
  "http_referer" => "-",
  "AgentVersion" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
  "log_time" => "26/Apr/2021:18:40:40 +0800",
  "@timestamp" => 2021-04-26T10:40:41.709Z,
  "@version" => "1",
  "request" => "GET / HTTP/1.1",
  "upstream_addr" => "-",
  "upstream_response_time" => "-",
  "path" => "/usr/local/nginx/logs/access.log",
  "host" => "bogon",
  "body_bytes_sent" => "0"
}

此时说明咱们的nginx日志能同步到Logstash 格式是正确的，

输出到ElasticSearch

修改 nginx_elk.conf 配置文件 配置nginx日志的格式

input{
  file{
    #日志存放路径
    path => "/usr/local/nginx/logs/access.log"
    #从日志文件开头读取
    start_position => "beginning"
    #设置多长时间检测文件是否修改 默认是1s
    stat_interval => "2"
    codec => "json"
  }
}

filter {}
# 定义日志输出
output { 
    elasticsearch {
        hosts => ["192.168.111.143:9200"]     # 定义es服务器的ip，这里使用本地
        index => "nginx-log"     #定义索引名称 
    }
    stdout {
        codec => json_lines
    }
}

再次启动Logstash

cd /usr/local/logstash-7.6.1
bin/logstash -f nginx_elk.conf --config.debug
# 等待时间稍微长一点：多访问几次 nginx 在等一会发现已经有了nginx的数据 

打开 https://github.com/mobz/elasticsearch-head 下载head插件，选择下载zip 解压 打开index.html查看节点状态

#nginx-log的其中一条数据
{
  "_index": "nginx-log",
  "_type": "_doc",
  "_id": "QBrUDXkBjUFwuBXCQWu6",
  "_version": 1,
  "_score": 1,
  "_source": {
    "AgentVersion": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
    "body_bytes_sent": "0",
    "client_ip": "192.168.111.1",
    "upstream_addr": "-",
    "request_time": "0.000",
    "upstream_response_time": "-",
    "@version": "1",
    "status": "304",
    "@timestamp": "2021-04-26T10:57:57.462Z",
    "host": "bogon",
    "log_time": "26/Apr/2021:18:57:51 +0800",
    "http_referer": "-",
    "path": "/usr/local/nginx/logs/access.log",
    "request": "GET / HTTP/1.1"
  }
}

打开kibana管理页面