单个logstash文件收集多个filebeat日志

一 背景说明

我现在安装了logstash,只配置了一个文件，想同时收集nginx和java的日志，nginx要显示ip,国家城市，状态码，等，java要显示日志的具体内容。

二 nginx filebeat设置

filebeat.inputs:
# 收集 nginx 日志
- type: log
  enabled: true
  paths:
    - /var/log/nginx/*.log
  tags: ["nginx_logs"]
 # 日志是json开启这个
  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: true
  # 如果值为ture，那么fields存储在输出文档的顶级位置
  fields_under_root: true
  fields:
    app: easydong
    logtype: nginx_logs


filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
setup.kibana:
output.logstash:
  hosts: ["172.17.199.231:5044"]

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~

三 java filebeat设置

filebeat.inputs:
- type: log
  enabled: true
  paths: 
    
     -  /xs/logs/app-front1/easydong-app/app.log 
  encoding: utf-8
  tail_files: true
  
  fields:
    app: easydong
    type: applog-V1-appfront1
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after
- type: log
  enabled: true
  paths:
     -  /xs/logs/app-front1/easydong-admin/admin.log
  encoding: utf-8
  tail_files: true
  fields:
    app: easydong
    type: adminlog-V1-appfront1
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after

- type: log
  enabled: true
  paths:
                   
     -  /xs/logs/app-front2/easydong-admin/admin.log
  encoding: utf-8
  tail_files: true
  fields:
    app: easydong
    type: adminlog-V1-appfront2
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after

- type: log
  enabled: true
  paths:
     -  /xs/logs/app-front2/easydong-app/app.log
  encoding: utf-8
  tail_files: true
  fields:
    app: easydong
    type: applog-V1-appfront2
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
setup.kibana:
output.logstash:
  hosts: ["39.96.179.187:5044"]

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~

四 logstash设置

根据 if [logtype] == "nginx_logs 去判断是nginx日志还是java日志，

input {
 beats {
     port => 5044
     
  }



}


filter{
  if [type] == "adminlog-V1-appfront1"{

      grok {
         match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
         overwrite => ["message"]
     }
  }
  if [logtype] == "nginx_logs"{
       mutate {
          convert => [ "status","integer" ]
          convert => [ "size","integer" ]
          convert => [ "upstreatime","float" ]
          convert => ["[geoip][coordinates]", "float"]
          remove_field => "message"
        }

        date {
            match => [ "timestamp" ,"dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        geoip {
          source => "client"  ##日志格式里的ip来源，这里是client这个字段(client":"$remote_addr")
          target => "geoip"
          database =>"/usr/share/logstash/GeoLite2-City.mmdb"   ##### 下载GeoIP库
          add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
          add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
        }
       # mutate {
       #   remove_field => "timestamp"
       # }
    }

}



output {

    if [app] == "easydong" {
        if [type] == "applog-V1-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              password => "111111"
              user => "elastic"
              index => "applog-v1-appfront1-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "adminlog-V1-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              password => "111111"
              user => "elastic"
              index => "adminlog-v1-appfront1-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "applog-V1-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "applog-v1-appfront2-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "adminlog-V1-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "adminlog-v1-appfront2-%{+YYYY.MM.dd}"
           }
        }

        
        else if [type] == "applog-V2-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "applog-v2-appfront1-%{+YYYY.MM.dd}"
           }  
        }  
        else if [type] == "adminlog-V2-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "adminlog-v2-appfront1-%{+YYYY.MM.dd}"
           }  
        }  
        else if [type] == "applog-V2-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "applog-v2-appfront2-%{+YYYY.MM.dd}"
           }  
        }  
        else if [type] == "adminlog-V2-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "adminlog-v2-appfront2-%{+YYYY.MM.dd}"
           }  
        }  

        else if [type] == "indonesia-adminlog-1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "indonesia-adminlog-1-%{+YYYY.MM.dd}"
           }
        }
 
    if [app] == "easydong" {
          if [logtype] == "nginx_logs"{
            elasticsearch {
            hosts => ["http://172.17.199.231:9200"]
            index => "logstash-sanwenqian-nginx-%{+YYYY-MM}"
          }
      }
  }
    
  }

  stdout { codec=> rubydebug }

最后重启logstash看能否正常启动，可以去/var/log/message查看是否有新的日志产生。