nginx日志采集(filebeat-＞logstash-＞es)

目录

nginx日志输出json格式配置,修改nginx.conf

filebeat配置

logstash配置

---

• nginx日志输出json格式配置,修改nginx.conf

...
http {
...
  log_format json '{"@timestamp":"$time_iso8601",'
                    '"@version":1,'
                    '"host":"$server_addr",'
                    '"client":"$remote_addr",'
                    '"size":$body_bytes_sent,'
                    '"responsetime":$request_time,'
                    '"domain":"$host",'
                    '"url":"$uri",'
                    '"status":"$status"}';
    #access.log路径                
   access_log /app/rpa/nginx/logs/access.log  json;}
...

• filebeat配置,修改filebeat.yml

  filebeat.inputs:
  
  - type: log
    enabled: true
    paths:
      - /app/rpa/nginx/logs/access.log
    ignore_older: 5m
    fields:
      log_resource: nginx
      filetype: nginx-access
    #fields_under_root: true
    tags: ["nginx"]
  
  - type: log
    enabled: true
    paths:
      - /app/rpa/nginx/logs/error.log
    ignore_older: 5m
    fields:
      log_resource: nginx
      filetype: nginx-error
    tags: ["nginx"]
  
  filebeat.config.modules:
    # Glob pattern for configuration loading
    path: ${path.config}/modules.d/*.yml
  
    # Set to true to enable config reloading
    reload.enabled: false
  
  setup.template.settings:
    index.number_of_shards: 1
  
  setup.kibana:
  
  output.logstash:
    # logstash的ip,需要按照实际配置
    hosts: ["ip:port"]
  
  processors:
    - add_host_metadata: ~
    - add_cloud_metadata: ~
    - add_docker_metadata: ~
    #- add_kubernetes_metadata: 
  

  • logstash配置,修改logstash.conf

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  beats {
    port => 5044
  }
}
filter {
  if [fields][log_resource] == "nginx"{
    ruby {
      code => '
        require "json"
        json_data = event.get("message")
        begin
          parsed_data = JSON.parse(json_data)
          event.set("requestTime", parsed_data["@timestamp"])
          event.set("host", parsed_data["host"])
          event.set("client", parsed_data["client"])
          event.set("size", parsed_data["size"])
          event.set("responsetime", parsed_data["responsetime"])
          event.set("domain", parsed_data["domain"])
          event.set("url", parsed_data["url"])
          event.set("status", parsed_data["status"])
        end
      '
    }
    # 1. 增加一个字段，计算timestamp+8小时,处理索引少8h问题
 ruby { 
    code => "event.set('index_date', event.get('@timestamp').time.localtime + 8*60*60)" 
 } 
 mutate { 
    convert => ["index_date", "string"] 
    gsub => ["index_date", "T([\S\s]*?)Z", ""]  
 }      
    fingerprint {
        source => ["host", "client", "domain", "url", "status","message"]
        target => "[@metadata][generated_id]"
        method => "MURMUR3"
        concatenate_sources => true
    }
  }
}
# elasticsearch的ip,按照实际情况配置
output {
if [fields][filetype] == "nginx-access"{
    elasticsearch {
      hosts => ["ip:port"]
      index => "nginx-access-logs-%{index_date}"
      document_id => "%{[@metadata][generated_id]}"
    }
  } else if [fields][filetype] == "nginx-error"{
    elasticsearch {
      hosts => ["ip:port"]
      index => "nginx-error-logs-%{index_date}"
      document_id => "%{[@metadata][generated_id]}"
    }
  }
}