ELK 转换Nginx日志为Json格式并使用Logstash导入至ES

nginx配置

nginx.conf

 log_format log_json '{"remote_addr": "$remote_addr", '
                    '"ident": "-", '
                    '"user": "$remote_user", '
                    '"timestamp": "$time_local", '
                    '"request": "$request", '
                    '"status": $status, '
                    '"bytes": $body_bytes_sent, '
                    '"referer": "$http_referer", '
                    '"agent": "$http_user_agent", '
                    '"x_forwarded": "$http_x_forwarded_for"'
                    ' }';
access_log  logs/access-json.log  log_json;

logstash配置

input {
   # 从文件读取日志信息
   file {
        path => "/usr/local/nginx/logs/access-json.log" #修改为json日志文件
        type => "nginx_access_json"
        start_position => "beginning"
    }
 }
 
filter {
    json {
       source => "message"
       remove_field => ["beat","offset","tags","prospector"] #移除不需要采集的字段
    }
    date {
      match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"] #匹配timestamp字段
      target => "@timestamp"  #将匹配到的数据写到@timestamp字段中
    }
}
 
output {
      
       elasticsearch {
          hosts => ["localhost:9200"]
          index => "nginx-%{+YYYY.MM.dd}"
      }
 
}

重启即可