ELK+redis+filebeat整合及报错解决

重启filebeat后，nginx访问日志会写入到logstash中，但此时logstash写入elasticsearch会报错：failed to parse field [host] of type [text] in document with id 'E0lsjW4BTdp_eLcgfhbu'
看elasticsearch日志发现此时host为一个json对象，需要变为字符串才行
修改配置，添加过滤器，把host.name赋值为host
vi config/logstash.conf
filter {
  mutate {
    rename => { "[host][name]" => "host" }
  }
}

重启后即可

最后logstash.conf:

input {
  redis {
    host => "192.168.1.110"
    port => "6379"
    password => "xxxx"
    db => 12
    key => "filebeat"
    batch_count => 500
    data_type => "list"
  }
}

filter {
  if [filetype] == "nginx_access" {
    json {
      source => "message"
      remove_field => ["message","@version","path","input","log","agent","ecs","tags"]
    }
  }
    geoip {
      source => "client_ip"
      database => "/home/ELK/GeoLite2-City_20200324/GeoLite2-City.mmdb"
    }
 
    geoip {
      source => "x_forword_ip"
      target => "x_forword_geo"
    }
 
    date {
      match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
    }
 
    useragent {
      source => "ngx_ua"
      target => "ua"
    }
 
    mutate {
      split  => { "x_forword" => ", " }
    }
    mutate {
      rename => { "[host][name]" => "host" }
  }
}

output{
  if [filetype] == "nginx_access" {
    elasticsearch {
      hosts => ["http://192.168.1.110:9200"]
      user => "elastic"
      password => "xiaoxixxx"
      index => "nginx_access-%{+YYYY.MM}"
    }
  }
  else if [filetype] == "linux_secure" {
    elasticsearch {
      hosts => ["http://192.168.1.110:9200"]
      user => "elastic"
      password => "xiaoxixxx"
      index => "linux_secure-%{+YYYY.MM}"
    }
  }
}

 

filebeat.yml:

filebeat.inputs:
- type: log
  enabled: true
  tail_files: true
  backoff: "1s"
  paths:
      - /usr/local/nginx/logs/*-access.log
  fields:
     filetype: nginx_access
  fields_under_root: true
- type: log
  enabled: true
  tail_files: true
  backoff: "1s"
  paths:
    - /var/log/secure
  fields:
     filetype: linux_secure
  fields_under_root: true
 

output.redis:
  enabled: true
  hosts: ["192.168.1.110:6379"]
  password: "xxx"
  db: 12
  key: "filebeat"
  datatype: list
  timeout: 5
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~