filebeat发送日志到logstash转发到elasticearch的日志收集

测试通过filebeat发送日志到logstash转发到elasticearch的日志收集

logstash下载

安装包https://www.elastic.co/cn/downloads/logstash

grok匹配

一个空格都不能多严格匹配
在线工具：https://www.5axxw.com/tools/v2/grok.html#google_vignette
本地环境kibana：http://192.168.1.4:5601/app/dev_tools#/grokdebugger

示例：

bin/logstash -f job/nginx3.conf

# 文本1
== 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
# 规则1
\=\= %{IP:client_ip} \- %{USER:remote_user} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:response_bytes}

nginx3.conf

input {
  beats {
    port=>5044
    codec=>plain{
        charset=>"GBK"
    }
  }
}
filter {
  grok {
    match => {
       "message" => "\=\= %{IP:client_ip} \- %{USER:remote_user} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:response_bytes}"
    }
  }
  date {
    match => [ "timestamp", "MMM dd yyyy HH:mm:ss" ]
  }
}
output {
  stdout {
    codec => rubydebug
  }
}

filebeat

cd /Users/yyyyjinying/sortware/filebeat-8.5.2
./filebeat -e -c a.yml

a.yml

filebeat.inputs:
- type: log 
  paths:
    - "/Users/yyyyjinying/sortware/testfile/*.log"
  enabled: true
  fields:
    zjynamestatus: true
  fields_under_root: true
  multiline.type: pattern
  multiline.pattern: '^\=\='
  multiline.negate: true
  multiline.match: after
setup.template.settings:
  index.number_of_shards: 1

output.logstash:
  hosts: ["localhost:5044"]