ELK实践-Logstash

ELK版本6.7

安装

略[参考地址](https://www.elastic.co/guide/en/logstash/6.7/installing-logstash.html)

Stashing Your First Event

cd logstash-6.7.1
#-e参数启用直接参数
bin/logstash -e 'input { stdin { } } output { stdout {} }'

测试Logstash with filebeat

• 创建一个first-pipelie.yml

# filebeat->logstash
input {
    beats {
        port => "5044"
    }
}
# The filter part of this file is commented out to indicate that it is
# optional.
# filter {
#
# }
output {
    stdout { codec => rubydebug }
}

#filebeat ->logstash ->elasticsearch
input {
    beats {
        port => "5044"
    }
}
filter {
    grok {
        match => { "message" => "%{COMBINEDAPACHELOG}"}
    }
    geoip {
        source => "clientip"
    }
}
output {
    elasticsearch {
        hosts => [ "localhost:9200" ]
    }
}

• 检查配置

bin/logstash -f pipelines-filebeat-stdout.yml --config.test_and_exit

• 启动Logstash

#--config.reload.automatic 可以自动重启配置
bin/logstash -f pipelines-filebeat-stdout.yml --config.reload.automatic

与logback集成

• logback.xml中配置appender
• logstash 中input配置

input {
    tcp {
        port => 4567
        codec => json_lines
    }
}
filter {
}
output {
    elasticsearch {
        hosts => [ "localhost:9200" ]
    }
}

<appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <param name="Encoding" value="UTF-8"/>
        <destination>localhost:4567</destination>
        <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
</appender>
<!--添加日志输出-->
<root level="INFO">
        <appender-ref ref="STDOUT" />
        <appender-ref ref="stash" />
</root>

logback-prd.xml情况下，es会出现host转object的异常object mapping for [host] tried to parse field [host] as object, but found a concrete value，导致logstash 传输失败，解决方案：

filter {
     mutate {
      remove_field => [ "host" ]
    }
}